[ARVADOS] updated: d302307a4a66867419722034228823d1fc3910a6
git at public.curoverse.com
git at public.curoverse.com
Wed Feb 11 18:49:14 EST 2015
Summary of changes:
apps/workbench/Gemfile | 6 +-
apps/workbench/Gemfile.lock | 4 +-
.../workbench/app/assets/javascripts/filterable.js | 6 +-
.../app/controllers/projects_controller.rb | 4 +-
.../workbench/app/controllers/search_controller.rb | 3 +-
.../views/application/_browser_unsupported.html | 24 +++++++
.../app/views/application/_show_sharing.html.erb | 9 +++
.../jobs/_rerun_job_with_options_popup.html.erb | 6 +-
apps/workbench/app/views/layouts/body.html.erb | 3 +
apps/workbench/app/views/users/welcome.html.erb | 4 +-
apps/workbench/config/application.default.yml | 4 +-
apps/workbench/public/browser_unsupported.js | 14 ++++
.../controllers/collections_controller_test.rb | 19 +++++
.../test/integration/browser_unsupported_test.rb | 17 +++++
.../integration/filterable_infinite_scroll_test.rb | 14 ++--
.../test/integration/pipeline_instances_test.rb | 27 ++-----
apps/workbench/test/integration/projects_test.rb | 28 ++++++++
apps/workbench/test/integration_helper.rb | 17 +++--
apps/workbench/test/performance_test_helper.rb | 2 +-
apps/workbench/test/support/remove_file_api.js | 1 +
doc/api/methods/collections.html.textile.liquid | 2 +-
doc/api/methods/groups.html.textile.liquid | 9 +--
doc/api/schema/Collection.html.textile.liquid | 4 +-
doc/images/keyfeatures/chooseinputs.png | Bin 76641 -> 67586 bytes
doc/images/keyfeatures/collectionpage.png | Bin 84533 -> 68735 bytes
doc/images/keyfeatures/dashboard2.png | Bin 44396 -> 39651 bytes
doc/images/keyfeatures/graph.png | Bin 49127 -> 37727 bytes
doc/images/keyfeatures/log.png | Bin 135214 -> 94845 bytes
doc/images/keyfeatures/provenance.png | Bin 67060 -> 53567 bytes
doc/images/keyfeatures/rerun.png | Bin 73314 -> 56872 bytes
doc/images/keyfeatures/running2.png | Bin 56959 -> 40453 bytes
doc/images/keyfeatures/shared.png | Bin 62395 -> 46090 bytes
doc/images/keyfeatures/webupload.png | Bin 88061 -> 70592 bytes
doc/images/quickstart/1.png | Bin 47239 -> 36164 bytes
doc/images/quickstart/2.png | Bin 74431 -> 58616 bytes
doc/images/quickstart/3.png | Bin 80668 -> 68576 bytes
doc/images/quickstart/4.png | Bin 66686 -> 45334 bytes
doc/images/quickstart/5.png | Bin 83038 -> 66066 bytes
doc/images/quickstart/6.png | Bin 98604 -> 83813 bytes
doc/images/quickstart/7.png | Bin 77820 -> 60031 bytes
sdk/python/arvados/commands/put.py | 16 +++--
sdk/python/gittaggers.py | 20 ++++++
sdk/python/setup.py | 31 +++-----
sdk/python/tests/test_arv_put.py | 2 +-
sdk/ruby/lib/arvados/keep.rb | 5 +-
sdk/ruby/test/test_keep_manifest.rb | 66 +++++++++++++++++
services/api/Gemfile | 2 +-
services/api/Gemfile.lock | 16 ++---
.../api/app/controllers/application_controller.rb | 19 +++--
.../arvados/v1/collections_controller.rb | 7 +-
.../controllers/arvados/v1/groups_controller.rb | 76 +++++++-------------
.../app/controllers/arvados/v1/users_controller.rb | 2 +-
services/api/app/models/arvados_model.rb | 25 +++++++
services/api/app/models/collection.rb | 8 +--
services/api/app/models/database_seeds.rb | 1 +
.../db/migrate/20150123142953_full_text_search.rb | 18 +++++
...206210804_all_users_can_read_anonymous_group.rb | 12 ++++
...20150206230342_rename_replication_attributes.rb | 10 ++-
services/api/db/structure.sql | 39 ++++++++++
services/api/lib/current_api_client.rb | 12 ++++
services/api/lib/record_filters.rb | 20 +++++-
services/api/test/fixtures/collections.yml | 29 +++++++-
services/api/test/fixtures/groups.yml | 5 +-
services/api/test/fixtures/links.yml | 14 ++++
.../arvados/v1/collections_controller_test.rb | 11 +--
.../api/test/functional/arvados/v1/filters_test.rb | 41 +++++++++++
.../arvados/v1/groups_controller_test.rb | 17 ++++-
.../api/test/integration/collections_api_test.rb | 79 +++++++++++++++++++--
services/api/test/integration/groups_test.rb | 35 +++++++++
services/api/test/unit/arvados_model_test.rb | 22 ++++++
services/api/test/unit/collection_test.rb | 41 ++++++++++-
services/api/test/unit/link_test.rb | 5 ++
services/fuse/gittaggers.py | 1 +
services/fuse/setup.py | 31 +++-----
services/fuse/tests/test_mount.py | 16 +++--
services/{fuse => nodemanager}/MANIFEST.in | 0
services/nodemanager/gittaggers.py | 1 +
services/nodemanager/setup.py | 32 +++------
78 files changed, 775 insertions(+), 239 deletions(-)
create mode 100644 apps/workbench/app/views/application/_browser_unsupported.html
create mode 100644 apps/workbench/public/browser_unsupported.js
create mode 100644 apps/workbench/test/integration/browser_unsupported_test.rb
create mode 100644 apps/workbench/test/support/remove_file_api.js
create mode 100644 sdk/python/gittaggers.py
create mode 100644 services/api/db/migrate/20150123142953_full_text_search.rb
create mode 100644 services/api/db/migrate/20150206210804_all_users_can_read_anonymous_group.rb
create mode 120000 services/fuse/gittaggers.py
copy services/{fuse => nodemanager}/MANIFEST.in (100%)
create mode 120000 services/nodemanager/gittaggers.py
discards 755faba1d06331d50dc14d5982cb0ec53a3fa0a6 (commit)
discards 3de9172e76e7b8c39c5ab37adb145fdc94795f37 (commit)
discards 1361ff3051cef16a444daa89f78c358b8b677073 (commit)
discards fc0bdda63ea9596e1cd2cf28b7bf7cbdc3f68375 (commit)
discards 6f7820b3d58e321d6ed6a0daeb9ed6116123d42d (commit)
via d302307a4a66867419722034228823d1fc3910a6 (commit)
via d62b73382398808a440f15fdda2eea2e15e44282 (commit)
via 966bd97704f635315ab7ba50f23590a5fc9a97be (commit)
via ec0c0f54da513b2b8221d65d9a2c621a7d95d79e (commit)
via 353a72e637532f2641e55c79edc0de52e2dd3508 (commit)
via b80db28cdd536077e5effe6c08af079532c2059b (commit)
via f16b7abe9b1ae5967ffaab62b9c9ae3f955f44f1 (commit)
via 148ff097b57571dda1b6db063a2eca5a4eb98a35 (commit)
via d3a9326a2c92de950216fb2a88dbbc9de898e4b3 (commit)
via 6bf9ae122958b25b4a22447f67fb11cf24765d97 (commit)
via 5923d0fa912c73e3725e52c869d72793304ae44a (commit)
via 58bf2ad27c760fb7da0641b239f1871918b84a42 (commit)
via 8676d8d8fe7ea86db75fd9e6f53b07e21437cd6d (commit)
via fb181d9653d80317422e1d979697da908fa804c8 (commit)
via b599ef92fcfc25045eb6a366907555594496bfad (commit)
via 67a4825340187c05cbada61d38c12645a17acb65 (commit)
via 0215bf7b8c61d59462a476d850af999105856177 (commit)
via 34d6dc1f56b59b7c7cc3e6dc7d54053149c49bc6 (commit)
via 045bce46ede1995ed17747c48611f22c478cc82d (commit)
via d9e2de2e142fe1a79bd83064d8d9135ba44fd807 (commit)
via 710b03568da92458279db56608cba84cb5151847 (commit)
via 31e1554c4372d8206618bf7fee48323b08f24ec3 (commit)
via 9b6b5f0bd2ad96deeea2070a4eba56795bb28c1a (commit)
via 07f50aff99bbb837c9419e7a931add36d1611e2d (commit)
via ae7e8221d669b29ff3e098ac9259afb2875e9d3b (commit)
via f6089c82da72f331ba5a44874ce267b18bcaf557 (commit)
via a934fcf84acd4cc3a351fde1b6e21a0bd93757ef (commit)
via fc8e572937f2fd61bdc1e7f34a2e3f9a5cebd7ff (commit)
via d65b683af52e072b3d179b6f32edfbf37e108011 (commit)
via c9f5db97ad5d853cc2f4636d0743037f6048ceeb (commit)
via f7ec673ce72af1e076408f394b6401e4f253e703 (commit)
via b8148b3bcdfc6fe8a8b20e6a4c589b7a50e147a8 (commit)
via c882575c856e01313cf2caf2e4ead1f27bfb33ae (commit)
via 525d5d6351a0610237c52f1564dec5b77cf3af4f (commit)
via 538caa064785b645a2b8f815bf77a30192b20665 (commit)
via 9f1fafa8c7c7f3750d6769d863b82cb826d7ed6e (commit)
via 330a46e91b4ceaefba2bcfc383931eb59c77d461 (commit)
via fefce5e8e133a8fa064bbcdf31d85d41dc4a6729 (commit)
via 204f433a870e2bf1cf7af1fbe076e91f427ef05e (commit)
via 9b61792d905324a98b24224d45347082efbe5205 (commit)
via 16b720950262eb559358cf357f5098a142901665 (commit)
via e73af668c24cd259800c344c3efe8b7d769903da (commit)
via b6a7a62f4f38710f50d08a91a6a9b210700bb011 (commit)
via 0b102fac0e8d2a7d46d088b1bd8f7b27b325dd2a (commit)
via 79aca915815d298d2c20546108284627ee6cb84b (commit)
via b21b81e6623d025da4d93cbf09d523e63d2e07b0 (commit)
via aaffcb23198b4223c48092ccd30ef7152b434187 (commit)
via 20f5b178a850b029ecd501ed49e4ed0a537c1fad (commit)
via fb4921f56d1c13a86add2e59205ec32fa1f6efe4 (commit)
via 1d4a39ab3e97c031683ada9f6c98e4c7365fa414 (commit)
via f6ab9be0046a6f8d760259c1a0eba8ab7c636903 (commit)
via 231242b6378abda494f2c684995519a259cfe174 (commit)
via 1963df31ffb7e95b72e53a0ec5c891f539b6dadb (commit)
via b59b310e23b588c4007af84741d4b94bc9f595f1 (commit)
via e2da84c1fd9052791ed2b684741469570e09ea35 (commit)
via d6ec5672045b29aeaf983a78c5487ae354ccb20d (commit)
via 757212484d9da8bb8d8852bfb6870433d2b4fa97 (commit)
via f85132f1a018179b7127c199932c1f0f3e3f76d5 (commit)
via 2e9f5f1aa841972d1c6d3ff0828d774f60c28307 (commit)
via 71c05eec3e9c8e6f37f14760b04584a8d4c4372c (commit)
via f32690a4a18f85909c0a04de83ecf7819f127df8 (commit)
via df507d6cdebca220ac19dbbc5c16d18498cb852c (commit)
via 4982008e820ed48f362226c61540c18305c6acd6 (commit)
via 77daa60985c94cf4137c8a54681bb89278db8436 (commit)
via bd720586c0152ca4e7d109389bda2c0e463c76bb (commit)
via 2527b9cd7958d89a5ae0dd84856027908c48ae53 (commit)
via dd645c9e973b9b725f310513ce309fa1e1a82421 (commit)
via c9e19eb6c3c6889b55c3b63424b36f1139c9abf0 (commit)
via f8067dd18b72705f3317e85745e87cffc9e25313 (commit)
via cb79358321eff7a49dd4a3fb6e0ea448ead92597 (commit)
via cda964acdb8132d90b881e62db008c574fdd5cc4 (commit)
via 2e5ac62b550f7dd608cf133ae66ef04f801be76b (commit)
via b48e7f0c19f1a7256222c220e938832789492aa3 (commit)
via 7a71d74c538c37437e65f5d22205c224d0fe9207 (commit)
via ff49b1144f5b9f9f7624f3741f5af791073de03c (commit)
via 348801d41f0bd06582675223f07a7ef7f36ac887 (commit)
via eab43fcf2826f4416a70bef95c3ae04a77b487c9 (commit)
via 9b59cd2f10fa44f4cdbf8986b08e92bdde5a62a7 (commit)
via 1eda4774a59f46296f82231eeb80484aca70a961 (commit)
via 8cd7249b96576285388ef036d04532f72a8f1ee3 (commit)
via a11c56ef66604a9117e3db8c2fa2273c98f88b51 (commit)
via 5b70a11a08dc26b43b3ec4aef178bafe3a801b86 (commit)
via 7d76a3fbfbe15a6813df5d2d4fa111f1b8e62f9c (commit)
via a533cde870452a69fd7db28806531475aba81486 (commit)
via 981c7264123f24a1873f2692a72f012ac43e726b (commit)
via 9fbf89259b2c4313334a1a2c8f0f30e581cc932e (commit)
via 316eca14d7f7bafb2e0c24b125dee0befe5bdce6 (commit)
via eeebd1e25974beff2455c96100532aaa4dae68fb (commit)
via c430a289af3eeed00e220b5658f9d64191798b1c (commit)
via 6d93d2ecf9c3d75c2b032f0ad9689af6501570e0 (commit)
via 7349451eb29d3bd972f6f051f1cf14459b3fe14e (commit)
via 1b01104f5e4a20a3ede82b6d7250814476e23dc9 (commit)
via 7631b2c9e33a22a9b47fe3396f2f2854745a3ded (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (755faba1d06331d50dc14d5982cb0ec53a3fa0a6)
\
N -- N -- N (d302307a4a66867419722034228823d1fc3910a6)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d302307a4a66867419722034228823d1fc3910a6
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 6 21:36:39 2015 -0500
3410: Fix munge_manifest_locators!: don't skip locators that have no +hints.
Also, fix portable_manifest_text: do not add a trailing + to a locator
that has no size hint.
Portable data hash of ". d41d8cd98f00b204e9800998ecf8427e+Foo 0:0:x\n"
is md5(". d41d8cd98f00b204e9800998ecf8427e 0:0:x\n")
not md5(". d41d8cd98f00b204e9800998ecf8427e+ 0:0:x\n")
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index cd05b4b..1b07e50 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -205,7 +205,7 @@ class Collection < ArvadosModel
def self.munge_manifest_locators! manifest
# Given a manifest text and a block, yield each locator,
# and replace it with whatever the block returns.
- manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) do |word|
+ manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+\S+)?/) do |word|
if loc = Keep::Locator.parse(word.strip)
" " + yield(loc)
else
@@ -308,7 +308,11 @@ class Collection < ArvadosModel
def portable_manifest_text
portable_manifest = self[:manifest_text].dup
self.class.munge_manifest_locators!(portable_manifest) do |loc|
- loc.hash + '+' + loc.size.to_s
+ if loc.size
+ loc.hash + '+' + loc.size.to_s
+ else
+ loc.hash
+ end
end
portable_manifest
end
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 9dda9d5..6a3225e 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -119,6 +119,21 @@ class CollectionTest < ActiveSupport::TestCase
end
end
+ test 'portable data hash with missing size hints' do
+ [[". d41d8cd98f00b204e9800998ecf8427e+0+Bar 0:0:x",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x"],
+ [". d41d8cd98f00b204e9800998ecf8427e+Foo 0:0:x",
+ ". d41d8cd98f00b204e9800998ecf8427e 0:0:x"],
+ [". d41d8cd98f00b204e9800998ecf8427e 0:0:x",
+ ". d41d8cd98f00b204e9800998ecf8427e 0:0:x"],
+ ].each do |unportable, portable|
+ c = Collection.new(manifest_text: unportable)
+ assert c.valid?
+ assert_equal(Digest::MD5.hexdigest(portable)+"+#{portable.length}",
+ c.portable_data_hash)
+ end
+ end
+
[0, 2, 4, nil].each do |ask|
test "set replication_desired to #{ask.inspect}" do
Rails.configuration.default_collection_replication = 2
commit d62b73382398808a440f15fdda2eea2e15e44282
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 6 19:07:43 2015 -0500
3410: Add tests for replication attributes.
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 3ab97bf..cd05b4b 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -216,8 +216,8 @@ class Collection < ArvadosModel
def self.each_manifest_locator manifest
# Given a manifest text and a block, yield each locator.
- manifest.andand.scan(/ ([[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+))/) do |word|
- if loc = Keep::Locator.parse(word[1])
+ manifest.andand.scan(/ ([[:xdigit:]]{32}(\+\S+)?)/) do |word, _|
+ if loc = Keep::Locator.parse(word)
yield loc
end
end
@@ -331,8 +331,8 @@ class Collection < ArvadosModel
end
self.class.each_manifest_locator(manifest_text) do |loc|
if not in_old_manifest[loc.hash]
- replication_confirmed_at = nil
- replication_confirmed = nil
+ self.replication_confirmed_at = nil
+ self.replication_confirmed = nil
break
end
end
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 9ddc452..3f848f6 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -391,6 +391,48 @@ collection_with_unique_words_to_test_full_text_search:
name: collection_with_some_unique_words
description: The quick_brown_fox jumps over the lazy_dog
+replication_undesired_unconfirmed:
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-07 00:19:28.596506247 Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2015-02-07 00:19:28.596338465 Z
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ replication_desired: ~
+ replication_confirmed_at: ~
+ replication_confirmed: ~
+ updated_at: 2015-02-07 00:19:28.596236608 Z
+ uuid: zzzzz-4zz18-wjxq7uzx2m9jj4a
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: replication wantnull havenull
+
+replication_desired_2_unconfirmed:
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-07 00:21:35.050333515 Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2015-02-07 00:21:35.050189104 Z
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ replication_desired: 2
+ replication_confirmed_at: ~
+ replication_confirmed: ~
+ updated_at: 2015-02-07 00:21:35.050126576 Z
+ uuid: zzzzz-4zz18-3t236wrz4769h7x
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: replication want2 havenull
+
+replication_desired_2_confirmed_2:
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-07 00:19:28.596506247 Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2015-02-07 00:19:28.596338465 Z
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ replication_desired: 2
+ replication_confirmed_at: 2015-02-07 00:24:52.983381227 Z
+ replication_confirmed: 2
+ updated_at: 2015-02-07 00:24:52.983381227 Z
+ uuid: zzzzz-4zz18-434zv1tnnf2rygp
+ manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:3:foo 3:6:bar\n"
+ name: replication want2 have2
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 61b0557..42059e7 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -698,21 +698,17 @@ EOS
end
[1, 5, nil].each do |ask|
- test "Set replication_desired=#{ask} using redundancy attr" do
- # The Python SDK checks the Collection schema in the discovery
- # doc, then asks for 'redundancy' or 'replication_desired'
- # accordingly, so it isn't necessary to maintain backward
- # compatibility here when the attribute changes to
- # replication_desired.
+ test "Set replication_desired=#{ask.inspect}" do
+ Rails.configuration.default_collection_replication = 2
authorize_with :active
put :update, {
- id: collections(:collection_owned_by_active).uuid,
+ id: collections(:replication_undesired_unconfirmed).uuid,
collection: {
- redundancy: ask,
+ replication_desired: ask,
},
}
assert_response :success
- assert_equal (ask or 2), json_response['replication_desired']
+ assert_equal ask, json_response['replication_desired']
end
end
end
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 59f9d3d..9dda9d5 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -120,12 +120,91 @@ class CollectionTest < ActiveSupport::TestCase
end
[0, 2, 4, nil].each do |ask|
- test "replication_desired reports #{ask or 2} if redundancy is #{ask}" do
+ test "set replication_desired to #{ask.inspect}" do
+ Rails.configuration.default_collection_replication = 2
act_as_user users(:active) do
- c = collections(:collection_owned_by_active)
- c.update_attributes redundancy: ask
- assert_equal (ask or 2), c.replication_desired
+ c = collections(:replication_undesired_unconfirmed)
+ c.update_attributes replication_desired: ask
+ assert_equal ask, c.replication_desired
end
end
end
+
+ test "replication_confirmed* can be set by admin user" do
+ c = collections(:replication_desired_2_unconfirmed)
+ act_as_user users(:admin) do
+ assert c.update_attributes(replication_confirmed: 2,
+ replication_confirmed_at: Time.now)
+ end
+ end
+
+ test "replication_confirmed* cannot be set by non-admin user" do
+ act_as_user users(:active) do
+ c = collections(:replication_desired_2_unconfirmed)
+ # Cannot set just one at a time.
+ assert_raise ArvadosModel::PermissionDeniedError do
+ c.update_attributes replication_confirmed: 1
+ end
+ assert_raise ArvadosModel::PermissionDeniedError do
+ c.update_attributes replication_confirmed_at: Time.now
+ end
+ # Cannot set both at once.
+ assert_raise ArvadosModel::PermissionDeniedError do
+ c.update_attributes(replication_confirmed: 1,
+ replication_confirmed_at: Time.now)
+ end
+ end
+ end
+
+ test "replication_confirmed* can be cleared (but only together) by non-admin user" do
+ act_as_user users(:active) do
+ c = collections(:replication_desired_2_confirmed_2)
+ # Cannot clear just one at a time.
+ assert_raise ArvadosModel::PermissionDeniedError do
+ c.update_attributes replication_confirmed: nil
+ end
+ c.reload
+ assert_raise ArvadosModel::PermissionDeniedError do
+ c.update_attributes replication_confirmed_at: nil
+ end
+ # Can clear both at once.
+ c.reload
+ assert c.update_attributes(replication_confirmed: nil,
+ replication_confirmed_at: nil)
+ end
+ end
+
+ test "clear replication_confirmed* when introducing a new block in manifest" do
+ c = collections(:replication_desired_2_confirmed_2)
+ act_as_user users(:active) do
+ assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text)
+ assert_nil c.replication_confirmed
+ assert_nil c.replication_confirmed_at
+ end
+ end
+
+ test "don't clear replication_confirmed* when just renaming a file" do
+ c = collections(:replication_desired_2_confirmed_2)
+ act_as_user users(:active) do
+ new_manifest = c.signed_manifest_text.sub(':bar', ':foo')
+ assert c.update_attributes(manifest_text: new_manifest)
+ assert_equal 2, c.replication_confirmed
+ assert_not_nil c.replication_confirmed_at
+ end
+ end
+
+ test "don't clear replication_confirmed* when just deleting a data block" do
+ c = collections(:replication_desired_2_confirmed_2)
+ act_as_user users(:active) do
+ new_manifest = c.signed_manifest_text
+ new_manifest.sub!(/ \S+:bar/, '')
+ new_manifest.sub!(/ acbd\S+/, '')
+ # We really deleted a block there, right?
+ assert_operator new_manifest.length+40, :<, c.signed_manifest_text.length
+
+ assert c.update_attributes(manifest_text: new_manifest)
+ assert_equal 2, c.replication_confirmed
+ assert_not_nil c.replication_confirmed_at
+ end
+ end
end
commit 966bd97704f635315ab7ba50f23590a5fc9a97be
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 6 18:51:12 2015 -0500
3410: Add replication attributes (and rules about updating them) to model and docs.
diff --git a/doc/api/schema/Collection.html.textile.liquid b/doc/api/schema/Collection.html.textile.liquid
index 69a8dc3..9aa783c 100644
--- a/doc/api/schema/Collection.html.textile.liquid
+++ b/doc/api/schema/Collection.html.textile.liquid
@@ -30,11 +30,10 @@ Each collection has, in addition to the usual "attributes of Arvados resources":
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
-|locator|string|||
-|portable_data_hash|string|||
|name|string|||
-|redundancy|number|||
-|redundancy_confirmed_by_client_uuid|string|API client||
-|redundancy_confirmed_at|datetime|||
-|redundancy_confirmed_as|number|||
+|description|text|||
+|portable_data_hash|string|||
|manifest_text|text|||
+|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@|
+|replication_confirmed|number|Replication level most recently confirmed by the storage system. This field is null when a collection is first created, and is reset to null when the manifest_text changes in a way that introduces a new data block. An integer value indicates the replication level of the _least replicated_ data block in the collection.|@2@, null|
+|replication_confirmed_at|datetime|When replication_confirmed was confirmed. If replication_confirmed is null, this field is also null.||
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 4383514..4e7acca 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -408,10 +408,14 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
print >>stderr, error
sys.exit(1)
- # Apply default replication, if none specified. TODO (#3410): Use
- # default replication given by discovery document.
- if args.replication <= 0:
- args.replication = 2
+ # write_copies is how many copies of each data block we write to
+ # Keep. args.replication is how many copies we instruct Arvados to
+ # maintain (by passing it in collections().create() after all data
+ # is written). If args.replication is given as None, it should
+ # stay None, but we still need to write a suitable number of
+ # copies to Keep.
+ write_copies = (args.replication or
+ api_client._rootDesc.get('defaultCollectionReplication', 2))
if args.progress:
reporter = progress_writer(human_progress)
@@ -437,12 +441,12 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
writer = ArvPutCollectionWriter(
resume_cache, reporter, bytes_expected,
num_retries=args.retries,
- replication=args.replication)
+ replication=write_copies)
else:
writer = ArvPutCollectionWriter.from_cache(
resume_cache, reporter, bytes_expected,
num_retries=args.retries,
- replication=args.replication)
+ replication=write_copies)
# Install our signal handler for each code in CAUGHT_SIGNALS, and save
# the originals.
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 196264f..4e4e096 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -539,7 +539,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
def test_put_collection_with_default_redundancy(self):
collection = self.run_and_find_collection("")
- self.assertEqual(2, collection['replication_desired'])
+ self.assertEqual(None, collection['replication_desired'])
def test_put_collection_with_unnamed_project_link(self):
link = self.run_and_find_collection(
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 99d1a66..3ab97bf 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -9,6 +9,7 @@ class Collection < ArvadosModel
before_validation :check_signatures
before_validation :strip_manifest_text
before_validation :set_portable_data_hash
+ before_validation :maybe_clear_redundancy_confirmed
validate :ensure_hash_matches_manifest_text
before_save :set_file_names
@@ -22,6 +23,8 @@ class Collection < ArvadosModel
t.add :portable_data_hash
t.add :signed_manifest_text, as: :manifest_text
t.add :replication_desired
+ t.add :replication_confirmed
+ t.add :replication_confirmed_at
end
def self.attributes_required_columns
@@ -179,27 +182,6 @@ class Collection < ArvadosModel
end
end
- def replication_desired
- # Shim until database columns get fixed up in #3410.
- redundancy or 2
- end
-
- def redundancy_status
- if redundancy_confirmed_as.nil?
- 'unconfirmed'
- elsif redundancy_confirmed_as < redundancy
- 'degraded'
- else
- if redundancy_confirmed_at.nil?
- 'unconfirmed'
- elsif Time.now - redundancy_confirmed_at < 7.days
- 'OK'
- else
- 'stale'
- end
- end
- end
-
def signed_manifest_text
if has_attribute? :manifest_text
token = current_api_client_authorization.andand.api_token
@@ -232,6 +214,15 @@ class Collection < ArvadosModel
end
end
+ def self.each_manifest_locator manifest
+ # Given a manifest text and a block, yield each locator.
+ manifest.andand.scan(/ ([[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+))/) do |word|
+ if loc = Keep::Locator.parse(word[1])
+ yield loc
+ end
+ end
+ end
+
def self.normalize_uuid uuid
hash_part = nil
size_part = nil
@@ -328,4 +319,32 @@ class Collection < ArvadosModel
'+' +
portable_manifest.bytesize.to_s)
end
+
+ def maybe_clear_redundancy_confirmed
+ if manifest_text_changed?
+ # If the new manifest_text contains locators whose hashes
+ # weren't in the old manifest_text, storage replication is no
+ # longer confirmed.
+ in_old_manifest = {}
+ self.class.each_manifest_locator(manifest_text_was) do |loc|
+ in_old_manifest[loc.hash] = true
+ end
+ self.class.each_manifest_locator(manifest_text) do |loc|
+ if not in_old_manifest[loc.hash]
+ replication_confirmed_at = nil
+ replication_confirmed = nil
+ break
+ end
+ end
+ end
+ end
+
+ def ensure_permission_to_save
+ if (not current_user.andand.is_admin and
+ (replication_confirmed_at_changed? or replication_confirmed_changed?) and
+ not (replication_confirmed_at.nil? and replication_confirmed.nil?))
+ raise ArvadosModel::PermissionDeniedError.new("replication_confirmed and replication_confirmed_at attributes cannot be changed, except by setting both to nil")
+ end
+ super
+ end
end
commit ec0c0f54da513b2b8221d65d9a2c621a7d95d79e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 6 18:49:59 2015 -0500
3410: Add default_collection_replication to config and discovery doc.
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index bc5a20f..9e694fb 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -20,6 +20,7 @@ class Arvados::V1::SchemaController < ApplicationController
title: "Arvados API",
description: "The API to interact with Arvados.",
documentationLink: "http://doc.arvados.org/api/index.html",
+ defaultCollectionReplication: Rails.configuration.default_collection_replication,
protocol: "rest",
baseUrl: root_url + "arvados/v1/",
basePath: "/arvados/v1/",
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 2d62e40..f40508e 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -244,4 +244,8 @@ common:
# Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
# On (sso) server. Should only be enabled during development when the SSO
# server is using a self-signed cert.
- sso_insecure: false
\ No newline at end of file
+ sso_insecure: false
+
+ # Set replication level for collections whose replication_desired
+ # attribute is nil.
+ default_collection_replication: 2
commit 353a72e637532f2641e55c79edc0de52e2dd3508
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 6 18:49:33 2015 -0500
3410: Rename redundancy -> replication columns.
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 334f3c6..99d1a66 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -32,10 +32,6 @@ class Collection < ArvadosModel
# API response, and never let clients select the
# manifest_text column.
'manifest_text' => ['manifest_text'],
-
- # This is a shim until the database column gets
- # renamed to replication_desired in #3410.
- 'replication_desired' => ['redundancy'],
)
end
diff --git a/services/api/db/migrate/20150206230342_rename_replication_attributes.rb b/services/api/db/migrate/20150206230342_rename_replication_attributes.rb
new file mode 100644
index 0000000..e1519a3
--- /dev/null
+++ b/services/api/db/migrate/20150206230342_rename_replication_attributes.rb
@@ -0,0 +1,30 @@
+class RenameReplicationAttributes < ActiveRecord::Migration
+ RENAME = [[:redundancy, :replication_desired],
+ [:redundancy_confirmed_as, :replication_confirmed],
+ [:redundancy_confirmed_at, :replication_confirmed_at]]
+
+ def up
+ RENAME.each do |oldname, newname|
+ rename_column :collections, oldname, newname
+ end
+ remove_column :collections, :redundancy_confirmed_by_client_uuid
+ Collection.reset_column_information
+
+ # Removing that column dropped some indexes. Let's put them back.
+ add_index :collections, ["owner_uuid", "modified_by_client_uuid", "modified_by_user_uuid", "portable_data_hash", "uuid", "name", "file_names"], name: 'collections_search_index'
+ execute "CREATE INDEX collections_full_text_search_idx ON collections USING gin(#{Collection.full_text_tsvector});"
+ end
+
+ def down
+ remove_index :collections, name: 'collections_search_index'
+ add_column :collections, :redundancy_confirmed_by_client_uuid, :string
+ RENAME.reverse.each do |oldname, newname|
+ rename_column :collections, newname, oldname
+ end
+ remove_index :collections, :name => 'collections_full_text_search_idx'
+ Collection.reset_column_information
+
+ execute "CREATE INDEX collections_full_text_search_idx ON collections USING gin(#{Collection.full_text_tsvector});"
+ add_index :collections, ["owner_uuid", "modified_by_client_uuid", "modified_by_user_uuid", "portable_data_hash", "redundancy_confirmed_by_client_uuid", "uuid", "name", "file_names"], name: 'collections_search_index'
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index e2c6b66..756dfc9 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -159,10 +159,9 @@ CREATE TABLE collections (
modified_by_user_uuid character varying(255),
modified_at timestamp without time zone,
portable_data_hash character varying(255),
- redundancy integer,
- redundancy_confirmed_by_client_uuid character varying(255),
- redundancy_confirmed_at timestamp without time zone,
- redundancy_confirmed_as integer,
+ replication_desired integer,
+ replication_confirmed_at timestamp without time zone,
+ replication_confirmed integer,
updated_at timestamp without time zone NOT NULL,
uuid character varying(255),
manifest_text text,
@@ -1311,14 +1310,14 @@ CREATE UNIQUE INDEX collection_owner_uuid_name_unique ON collections USING btree
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(redundancy_confirmed_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
-- Name: collections_search_index; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
-CREATE INDEX collections_search_index ON collections USING btree (owner_uuid, modified_by_client_uuid, modified_by_user_uuid, portable_data_hash, redundancy_confirmed_by_client_uuid, uuid, name, file_names);
+CREATE INDEX collections_search_index ON collections USING btree (owner_uuid, modified_by_client_uuid, modified_by_user_uuid, portable_data_hash, uuid, name, file_names);
--
@@ -2357,4 +2356,6 @@ INSERT INTO schema_migrations (version) VALUES ('20150123142953');
INSERT INTO schema_migrations (version) VALUES ('20150203180223');
-INSERT INTO schema_migrations (version) VALUES ('20150206210804');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20150206210804');
+
+INSERT INTO schema_migrations (version) VALUES ('20150206230342');
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list