[ARVADOS] updated: 42e27f90df96881d22e365af7069f36c4538cf07
git at public.curoverse.com
git at public.curoverse.com
Thu Aug 13 21:09:29 EDT 2015
Summary of changes:
services/api/lib/salvage_collection.rb | 104 ++++++++++++++++++++++++++++++
services/api/script/salvage_collection.rb | 74 +--------------------
2 files changed, 106 insertions(+), 72 deletions(-)
create mode 100755 services/api/lib/salvage_collection.rb
via 42e27f90df96881d22e365af7069f36c4538cf07 (commit)
from 749b308236ab70cd15088fcb81d093c9c5e8d30b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 42e27f90df96881d22e365af7069f36c4538cf07
Author: radhika <radhika at curoverse.com>
Date: Thu Aug 13 21:06:06 2015 -0400
6859: Add check for ENV[ARVADOS_API_TOKEN] and handle any other issues during arv-put. Split salvalge_collection method into lib module.
diff --git a/services/api/lib/salvage_collection.rb b/services/api/lib/salvage_collection.rb
new file mode 100755
index 0000000..8781ec4
--- /dev/null
+++ b/services/api/lib/salvage_collection.rb
@@ -0,0 +1,104 @@
+module SalvageCollection
+ # Take two input parameters: a collection uuid and reason
+ # Get "src_collection" with the given uuid
+ # Create a new collection with:
+ # src_collection.manifest_text as "invalid_manifest_text.txt"
+ # Locators from src_collection.manifest_text as "salvaged_data"
+ # Update src_collection:
+ # Set src_collection.manifest_text to: ""
+ # Append to src_collection.name: " (reason; salvaged data at new_collection.uuid)"
+ # Set portable_data_hash to "d41d8cd98f00b204e9800998ecf8427e+0"
+
+ require File.dirname(__FILE__) + '/../config/environment'
+ require 'arvados/keep'
+ include ApplicationHelper
+ require 'tempfile'
+ require 'shellwords'
+
+ def self.salvage_collection uuid, reason
+ act_as_system_user do
+ if !ENV['ARVADOS_API_TOKEN'] or !ENV['ARVADOS_API_HOST']
+ $stderr.puts "Please set ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables. Exiting."
+ exit 1
+ end
+
+ src_collection = Collection.find_by_uuid uuid
+ if !src_collection
+ $stderr.puts "No collection found for #{uuid}. Returning."
+ exit 1
+ end
+
+ begin
+ src_manifest = src_collection.manifest_text || ''
+
+ # Get all the locators from the original manifest
+ locators = []
+ src_manifest.each_line do |line|
+ line.split(' ').each do |word|
+ if match = Keep::Locator::LOCATOR_REGEXP.match(word)
+ word = word.split('+')[0..1].join('+') # get rid of any hints
+ locators << word if !word.start_with?('00000000000000000000000000000000')
+ end
+ end
+ end
+ locators << 'd41d8cd98f00b204e9800998ecf8427e+0' if !locators.any?
+
+ # create new collection using 'arv-put' with original manifest_text as the data
+ temp_file = Tempfile.new('temp')
+ temp_file.write(src_manifest)
+ temp_file.close
+ new_manifest = %x(arv-put --as-stream --use-filename invalid_manifest_text.txt #{Shellwords::shellescape(temp_file.path)})
+ temp_file.unlink
+
+ if !new_manifest.present?
+ $stderr.puts "arv-put --as-stream failed for #{uuid}"
+ exit 1
+ end
+
+ words = []
+ new_manifest.split(' ').each do |word|
+ if match = Keep::Locator::LOCATOR_REGEXP.match(word)
+ word = word.split('+')[0..1].join('+') # get rid of any hints
+ words << word
+ else
+ words << word
+ end
+ end
+
+ new_manifest = words.join(' ') + "\n"
+ new_collection = Collection.new
+
+ total_size = 0
+ locators.each do |locator|
+ total_size += locator.split('+')[1].to_i
+ end
+ new_manifest += (". #{locators.join(' ')} 0:#{total_size}:salvaged_data\n")
+
+ new_collection.name = "salvaged from #{src_collection.uuid}, #{src_collection.portable_data_hash}"
+ new_collection.manifest_text = new_manifest
+ new_collection.portable_data_hash = Digest::MD5.hexdigest(new_collection.manifest_text)
+
+ created = new_collection.save!
+ raise "New collection creation failed." if !created
+
+ $stderr.puts "Salvaged manifest and data for #{uuid} are in #{new_collection.uuid}."
+ puts "Created new collection #{new_collection.uuid}"
+ rescue => error
+ $stderr.puts "Error creating collection for #{uuid}: #{error}"
+ exit 1
+ end
+
+ begin
+ # update src_collection collection name, pdh, and manifest_text
+ src_collection.name = (src_collection.name || '') + ' (' + (reason || '') + '; salvaged data at ' + new_collection.uuid + ')'
+ src_collection.manifest_text = ''
+ src_collection.portable_data_hash = 'd41d8cd98f00b204e9800998ecf8427e+0'
+ src_collection.save!
+ $stderr.puts "Collection #{uuid} emptied and renamed to #{src_collection.name.inspect}."
+ rescue => error
+ $stderr.puts "Error salvaging collection #{new_collection.uuid}: #{error}"
+ exit 1
+ end
+ end
+ end
+end
diff --git a/services/api/script/salvage_collection.rb b/services/api/script/salvage_collection.rb
index 60becf4..b70807b 100755
--- a/services/api/script/salvage_collection.rb
+++ b/services/api/script/salvage_collection.rb
@@ -11,6 +11,7 @@
# Set portable_data_hash to "d41d8cd98f00b204e9800998ecf8427e+0"
require 'trollop'
+require './lib/salvage_collection'
opts = Trollop::options do
banner ''
@@ -21,76 +22,5 @@ opts = Trollop::options do
opt :reason, "Reason for salvaging.", type: :string, required: false
end
-require File.dirname(__FILE__) + '/../config/environment'
-require 'arvados/keep'
-include ApplicationHelper
-require 'tempfile'
-require 'shellwords'
-
-def salvage_collection uuid, reason
- act_as_system_user do
- src_collection = Collection.find_by_uuid uuid
- if !src_collection
- $stderr.puts "No collection found for #{uuid}. Returning."
- exit 1
- end
-
- begin
- src_manifest = src_collection.manifest_text || ''
-
- # Get all the locators from the original manifest
- locators = []
- src_manifest.each_line do |line|
- line.split(' ').each do |word|
- if match = Keep::Locator::LOCATOR_REGEXP.match(word)
- word = word.split('+')[0..1].join('+') # get rid of any hints
- locators << word if !word.start_with?('00000000000000000000000000000000')
- end
- end
- end
- locators << 'd41d8cd98f00b204e9800998ecf8427e+0' if !locators.any?
-
- # create new collection using 'arv-put' with original manifest_text as the data
- temp_file = Tempfile.new('temp')
- temp_file.write(src_manifest)
- temp_file.close
- new_manifest = %x(arv-put --as-stream --use-filename invalid_manifest_text.txt #{Shellwords::shellescape(temp_file.path)})
- temp_file.unlink
-
- new_collection = Collection.new
-
- total_size = 0
- locators.each do |locator|
- total_size += locator.split('+')[1].to_i
- end
- new_manifest += (". #{locators.join(' ')} 0:#{total_size}:salvaged_data\n")
-
- new_collection.name = "salvaged from #{src_collection.uuid}, #{src_collection.portable_data_hash}"
- new_collection.manifest_text = new_manifest
- new_collection.portable_data_hash = Digest::MD5.hexdigest(new_manifest)
-
- created = new_collection.save!
- raise "New collection creation failed." if !created
-
- $stderr.puts "Salvaged manifest and data for #{uuid} are in #{new_collection.uuid}."
- puts "Created new collection #{created}"
- rescue => error
- $stderr.puts "Error creating collection for #{uuid}: #{error}"
- exit 1
- end
-
- begin
- # update src_collection collection name, pdh, and manifest_text
- src_collection.name = (src_collection.name || '') + ' (' + (reason || '') + '; salvaged data at ' + created + ')'
- src_collection.portable_data_hash = 'd41d8cd98f00b204e9800998ecf8427e+0'
- src_collection.save!
- $stderr.puts "Collection #{uuid} emptied and renamed to #{src_collection.name.inspect}."
- rescue => error
- $stderr.puts "Error salvaging collection #{uuid}: #{error}"
- exit 1
- end
- end
-end
-
# Salvage the collection with the given uuid
-salvage_collection opts.uuid, opts.reason
+SalvageCollection.salvage_collection opts.uuid, opts.reason
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list