[ARVADOS] updated: 7fc67e3f9bfd12058e6f3d86d995704ed8962a8b
git at public.curoverse.com
git at public.curoverse.com
Wed Aug 26 20:21:14 EDT 2015
Summary of changes:
.gitignore | 3 +
.../install-shell-server.html.textile.liquid | 95 ++++++++++++++++++
.../install-workbench-app.html.textile.liquid | 46 +++++----
.../arvados/v1/virtual_machines_controller.rb | 18 ++--
services/api/lib/salvage_collection.rb | 15 +--
.../arvados/v1/virtual_machines_controller_test.rb | 7 ++
services/api/test/unit/salvage_collection_test.rb | 59 ++++++-----
{sdk/ruby => services/login-sync}/.gitignore | 2 +-
services/login-sync/Gemfile | 7 ++
{sdk/ruby => services/login-sync}/Rakefile | 0
.../login-sync/arvados-login-sync.gemspec | 18 ++--
services/login-sync/bin/arvados-login-sync | 111 +++++++++++++++++++++
services/login-sync/test/binstub_new_user/useradd | 9 ++
services/login-sync/test/stubs.rb | 52 ++++++++++
services/login-sync/test/test_add_user.rb | 37 +++++++
15 files changed, 411 insertions(+), 68 deletions(-)
copy {sdk/ruby => services/login-sync}/.gitignore (52%)
create mode 100644 services/login-sync/Gemfile
copy {sdk/ruby => services/login-sync}/Rakefile (100%)
copy sdk/ruby/arvados.gemspec => services/login-sync/arvados-login-sync.gemspec (51%)
create mode 100755 services/login-sync/bin/arvados-login-sync
create mode 100755 services/login-sync/test/binstub_new_user/useradd
create mode 100644 services/login-sync/test/stubs.rb
create mode 100644 services/login-sync/test/test_add_user.rb
via 7fc67e3f9bfd12058e6f3d86d995704ed8962a8b (commit)
via a1379fdd1f825c22a3ada0a2c085a1b0121ce89a (commit)
via f2afe64c25d1b3e79b38972e69104431a2587935 (commit)
via cf2d5c1b966cae7d5023ff482891c4ea7e767de4 (commit)
via a9359cd6ff10866acc943889ee77ab3f971dfa4e (commit)
via 51a185e0f13ec3b2c376d3229c56d012dc91a04a (commit)
via 51e6fbab8d08a5a92fc08c552f841a84bccbe0d6 (commit)
via 8c1005ef92cffff5a4a9321f3b62da715b141c3d (commit)
via f71873ca6ec9969a90242a1c4d391770cabce1c2 (commit)
via ef8f43b830273dd54ae75f6a3823a946a455c2b3 (commit)
via 1ca973446ddf0094cd563a7af63f4e3078f5b772 (commit)
via a8b431b5cfccd36995514560f965b4943ac93c6b (commit)
via 36b1667a9b2ab888a80e84b6ecd75403c8f6782c (commit)
via 2a6a3720271a12bc8a47928ade5b2c641514ca44 (commit)
from d78a90009ca25da4f8cfa75672df0b5bd77cabfe (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7fc67e3f9bfd12058e6f3d86d995704ed8962a8b
Author: radhika <radhika at curoverse.com>
Date: Wed Aug 26 20:19:57 2015 -0400
6859: improve locator parsing to consider any terminating [[:xdigit:]]{32}; improve tests with more invalid locators.
diff --git a/services/api/lib/salvage_collection.rb b/services/api/lib/salvage_collection.rb
index 2ffe0f4..86c95c6 100755
--- a/services/api/lib/salvage_collection.rb
+++ b/services/api/lib/salvage_collection.rb
@@ -23,24 +23,24 @@ module SalvageCollection
end
end
- LOCATOR_REGEXP = /^([[:xdigit:]]{32})(\+(.*))?\z/
+ # Get all the locators from the original manifest
+ LOCATOR_REGEXP = /((.*))?([[:xdigit:]]{32})(\+(.*))?\z/
def self.salvage_collection_locator_data manifest
- # Get all the locators from the original manifest
locators = []
size = 0
manifest.each_line do |line|
line.split(' ').each do |word|
if match = LOCATOR_REGEXP.match(word)
- if match.size > 3 and match[3]
- size_str = match[3].split('+')[0]
+ if match.size == 6 and match[5]
+ size_str = match[5].split('+')[0]
if size_str.to_i.to_s == size_str
- word = match[1] + '+' + size_str # get rid of any hints
+ word = match[3] + '+' + size_str # get rid of any other hints
size += size_str.to_i
else
- word = match[1]
+ word = match[3]
end
else
- word = match[1]
+ word = match[3]
end
locators << word
end
@@ -70,6 +70,7 @@ module SalvageCollection
# create new collection using 'arv-put' with original manifest_text as the data
temp_file = Tempfile.new('temp')
temp_file.write(src_manifest)
+
temp_file.close
new_manifest = salvage_collection_arv_put "arv-put --as-stream --use-filename invalid_manifest_text.txt #{Shellwords::shellescape(temp_file.path)}"
diff --git a/services/api/test/unit/salvage_collection_test.rb b/services/api/test/unit/salvage_collection_test.rb
index 9a52bb6..96b2c3e 100644
--- a/services/api/test/unit/salvage_collection_test.rb
+++ b/services/api/test/unit/salvage_collection_test.rb
@@ -3,27 +3,30 @@ require 'salvage_collection'
# Valid manifest_text
TEST_MANIFEST = ". 341dabea2bd78ad0d6fc3f5b926b450e+85626+Ad391622a17f61e4a254eda85d1ca751c4f368da9 at 55e076ce 0:85626:brca2-hg19.fa\n. d7321a918923627c972d8f8080c07d29+82570+A22e0a1d9b9bc85c848379d98bedc64238b0b1532 at 55e076ce 0:82570:brca1-hg19.fa\n"
+TEST_MANIFEST_STRIPPED = ". 341dabea2bd78ad0d6fc3f5b926b450e+85626 0:85626:brca2-hg19.fa\n. d7321a918923627c972d8f8080c07d29+82570 0:82570:brca1-hg19.fa\n"
# This invalid manifest_text has the following flaws:
# Missing stream name with locator in it's place
-# Two invalid locators:
+# Invalid locators:
+# foofaafaafaabd78ad0d6fc3f5b926b450e+foo
+# bar-baabaabaabd78ad0d6fc3f5b926b450e
+# bad12345dae58ad0d6fc3f5b926b450e+
# 341dabea2bd78ad0d6fc3f5b926b450e+abc
-# 341dabea2bd78ad0d6fc3f5b926b450e
-# Expectation: These locators are preserved in salvaged_data
-BAD_MANIFEST = "341dabea2bd78ad0d6fc3f5b926b450e+abc 341dabea2bd78ad0d6fc3f5b926abcdf 0:85626:brca2-hg19.fa\n. abcdabea2bd78ad0d6fc3f5b926b450e+1000 0:1000:brca-hg19.fa\n . d7321a918923627c972d8f8080c07d29+2000+A22e0a1d9b9bc85c848379d98bedc64238b0b1532 at 55e076ce 0:2000:brca1-hg19.fa\n"
+# 341dabea2bd78ad0d6fc3f5b926abcdf
+# Expectation: All these locators are preserved in salvaged_data
+BAD_MANIFEST = "faaaafaafaafaabd78ad0d6fc3f5b926b450e+foo bar-baabaabaabd78ad0d6fc3f5b926b450e bad12345dae58ad0d6fc3f5b926b450e+ 341dabea2bd78ad0d6fc3f5b926b450e+abc 341dabea2bd78ad0d6fc3f5b926abcdf 0:85626:brca2-hg19.fa\n. abcdabea2bd78ad0d6fc3f5b926b450e+1000 0:1000:brca-hg19.fa\n. d7321a918923627c972d8f8080c07d29+2000+A22e0a1d9b9bc85c848379d98bedc64238b0b1532 at 55e076ce 0:2000:brca1-hg19.fa\n"
# Mock arv_put
module SalvageCollection
def self.salvage_collection_arv_put(cmd)
- file_contents = File.new(cmd.split[-1], "r").gets
-
+ file_contents = File.open(cmd.split[-1], "r").read
# simulate arv-put error when it is 'user_agreement'
if file_contents.include? 'GNU_General_Public_License'
raise("Error during arv-put")
else
". " +
- Digest::MD5.hexdigest(TEST_MANIFEST) + "+" + TEST_MANIFEST.length.to_s +
- " 0:" + TEST_MANIFEST.length.to_s + ":invalid_manifest_text.txt\n"
+ Digest::MD5.hexdigest(file_contents) + "+" + file_contents.length.to_s +
+ " 0:" + file_contents.length.to_s + ":invalid_manifest_text.txt\n"
end
end
end
@@ -69,8 +72,10 @@ class SalvageCollectionTest < ActiveSupport::TestCase
assert_equal src_collection.uuid, match[1]
# verify the new collection's manifest format
- match = new_collection.manifest_text.match /^. (.*) (.*):invalid_manifest_text.txt\n. (.*) (.*):salvaged_data/
- assert_not_nil match
+ expected_manifest = ". " + Digest::MD5.hexdigest(TEST_MANIFEST_STRIPPED) + "+" +
+ TEST_MANIFEST_STRIPPED.length.to_s + " 0:" + TEST_MANIFEST_STRIPPED.length.to_s +
+ ":invalid_manifest_text.txt\n. 341dabea2bd78ad0d6fc3f5b926b450e+85626 d7321a918923627c972d8f8080c07d29+82570 0:168196:salvaged_data\n"
+ assert_equal expected_manifest, new_collection.manifest_text
end
test "salvage collection with no uuid required argument" do
@@ -105,19 +110,28 @@ class SalvageCollectionTest < ActiveSupport::TestCase
# This test uses BAD_MANIFEST, which has the following flaws:
# Missing stream name with locator in it's place
- # Two invalid locators:
+ # Invalid locators:
+ # foo-faafaafaabd78ad0d6fc3f5b926b450e+foo
+ # bar-baabaabaabd78ad0d6fc3f5b926b450e
+ # bad12345dae58ad0d6fc3f5b926b450e+
# 341dabea2bd78ad0d6fc3f5b926b450e+abc
- # 341dabea2bd78ad0d6fc3f5b926b450e
- # Expectation: These locators are preserved in salvaged_data
+ # 341dabea2bd78ad0d6fc3f5b926abcdf
+ # Expectation: All these locators are preserved in salvaged_data
test "invalid locators preserved during salvaging" do
locator_data = SalvageCollection.salvage_collection_locator_data BAD_MANIFEST
- assert_equal true, locator_data[0].size.eql?(4)
+ assert_equal 7, locator_data[0].size
+ assert_equal false, locator_data[0].include?("foo-faafaafaabd78ad0d6fc3f5b926b450e+foo")
+ assert_equal true, locator_data[0].include?("faafaafaabd78ad0d6fc3f5b926b450e")
+ assert_equal false, locator_data[0].include?("bar-baabaabaabd78ad0d6fc3f5b926b450e")
+ assert_equal true, locator_data[0].include?("baabaabaabd78ad0d6fc3f5b926b450e")
+ assert_equal false, locator_data[0].include?("bad12345dae58ad0d6fc3f5b926b450e+")
+ assert_equal true, locator_data[0].include?("bad12345dae58ad0d6fc3f5b926b450e")
assert_equal false, locator_data[0].include?("341dabea2bd78ad0d6fc3f5b926b450e+abc")
- assert_equal true, locator_data[0].include?("341dabea2bd78ad0d6fc3f5b926b450e")
- assert_equal true, locator_data[0].include?("341dabea2bd78ad0d6fc3f5b926abcdf")
- assert_equal true, locator_data[0].include?("abcdabea2bd78ad0d6fc3f5b926b450e+1000")
- assert_equal true, locator_data[0].include?("d7321a918923627c972d8f8080c07d29+2000")
- assert_equal true, locator_data[1].eql?(1000 + 2000) # size
+ assert_equal true, locator_data[0].include?("341dabea2bd78ad0d6fc3f5b926b450e")
+ assert_equal true, locator_data[0].include?("341dabea2bd78ad0d6fc3f5b926abcdf")
+ assert_equal true, locator_data[0].include?("abcdabea2bd78ad0d6fc3f5b926b450e+1000")
+ assert_equal true, locator_data[0].include?("d7321a918923627c972d8f8080c07d29+2000")
+ assert_equal true, locator_data[1].eql?(1000 + 2000) # size
end
test "salvage a collection with invalid manifest text" do
@@ -144,10 +158,9 @@ class SalvageCollectionTest < ActiveSupport::TestCase
match = new_collection.name.match /^salvaged from (.*),.*/
assert_not_nil match
assert_equal src_collection.uuid, match[1]
-
# verify the new collection's manifest includes the bad locators
- match = new_collection.manifest_text.match /^. (.*) (.*):invalid_manifest_text.txt\n. (.*) (.*):salvaged_data/
- assert_not_nil match
- assert_includes(new_collection.manifest_text, ". 341dabea2bd78ad0d6fc3f5b926b450e 341dabea2bd78ad0d6fc3f5b926abcdf abcdabea2bd78ad0d6fc3f5b926b450e+1000 d7321a918923627c972d8f8080c07d29+2000 0:3000:salvaged_data")
+ expected_manifest = ". " + Digest::MD5.hexdigest(BAD_MANIFEST) + "+" + BAD_MANIFEST.length.to_s +
+ " 0:" + BAD_MANIFEST.length.to_s + ":invalid_manifest_text.txt\n. faafaafaabd78ad0d6fc3f5b926b450e baabaabaabd78ad0d6fc3f5b926b450e bad12345dae58ad0d6fc3f5b926b450e 341dabea2bd78ad0d6fc3f5b926b450e 341dabea2bd78ad0d6fc3f5b926abcdf abcdabea2bd78ad0d6fc3f5b926b450e+1000 d7321a918923627c972d8f8080c07d29+2000 0:3000:salvaged_data\n"
+ assert_equal expected_manifest, new_collection.manifest_text
end
end
commit a1379fdd1f825c22a3ada0a2c085a1b0121ce89a
Merge: d78a900 f2afe64
Author: radhika <radhika at curoverse.com>
Date: Wed Aug 26 18:06:17 2015 -0400
Merge branch 'master' into 6859-fix-invalid-manifests
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list