[ARVADOS] updated: 46460c9615f07184ecb9917e4c948635712859da
git at public.curoverse.com
git at public.curoverse.com
Wed May 28 18:13:09 EDT 2014
Summary of changes:
apps/admin/setup-new-user.rb | 168 --------
apps/workbench/app/assets/javascripts/event_log.js | 45 +++
.../app/assets/javascripts/pipeline_instances.js | 26 ++
.../app/assets/javascripts/select_modal.js | 39 ++
.../app/assets/stylesheets/folders.css.scss | 16 +-
.../app/assets/stylesheets/select_modal.css.scss | 3 +
.../app/controllers/application_controller.rb | 24 +-
.../app/controllers/folders_controller.rb | 65 ++--
.../app/helpers/pipeline_instances_helper.rb | 20 +
apps/workbench/app/models/arvados_base.rb | 17 +-
apps/workbench/app/models/collection.rb | 4 +
apps/workbench/app/models/group.rb | 4 +
apps/workbench/app/models/human.rb | 3 +
apps/workbench/app/models/job.rb | 4 +
apps/workbench/app/models/pipeline_instance.rb | 4 +
apps/workbench/app/models/pipeline_template.rb | 4 +
apps/workbench/app/models/specimen.rb | 3 +
apps/workbench/app/models/trait.rb | 3 +
.../views/application/_selection_checkbox.html.erb | 2 +-
.../app/views/application/_show_recent.html.erb | 12 +-
apps/workbench/app/views/folders/_choose.html.erb | 42 ++
apps/workbench/app/views/folders/_choose.js.erb | 8 +
.../app/views/folders/_show_folders.html.erb | 29 ++
.../app/views/folders/_show_my_folders.html.erb | 2 -
.../views/folders/_show_shared_with_me.html.erb | 2 -
apps/workbench/app/views/folders/show.html.erb | 35 +-
.../app/views/layouts/application.html.erb | 18 +-
.../pipeline_instances/_show_components.html.erb | 50 ++-
.../_show_components_editable.html.erb | 2 +-
.../views/pipeline_templates/_show_recent.html.erb | 10 +-
.../app/views/user_agreements/index.html.erb | 2 +-
apps/workbench/app/views/users/_tables.html.erb | 6 +-
apps/workbench/config/routes.rb | 3 +-
apps/workbench/test/integration/folders_test.rb | 41 +-
.../tutorial-firstscript.html.textile.liquid | 10 +-
sdk/cli/bin/arv-run-pipeline-instance | 5 +-
sdk/go/build.sh | 37 --
sdk/go/go.sh | 17 +
sdk/go/src/arvados.org/keepclient/keepclient.go | 111 +++++-
.../src/arvados.org/keepclient/keepclient_test.go | 109 +++---
sdk/go/src/arvados.org/keepclient/support.go | 67 ++--
.../main/java/org/arvados/sdk/java/Arvados.java | 111 ++++--
.../java/org/arvados/sdk/java/ArvadosTest.java | 72 +++-
sdk/python/arvados/api.py | 5 +-
sdk/python/arvados/collection.py | 27 +-
sdk/python/arvados/errors.py | 2 +
sdk/python/arvados/keep.py | 76 +++-
sdk/python/run_test_server.py | 47 ++-
sdk/python/test_cmdline.py | 65 ++++
sdk/python/test_keep_client.py | 81 ++++
.../controllers/arvados/v1/groups_controller.rb | 44 +--
services/api/app/models/group.rb | 16 +
services/api/app/models/pipeline_template.rb | 1 +
services/api/app/models/user.rb | 4 +-
services/api/config/application.yml.example | 10 +-
...152921_add_description_to_pipeline_templates.rb | 5 +
services/api/db/schema.rb | 3 +-
services/api/script/crunch-dispatch.rb | 50 ++-
services/api/test/integration/websocket_test.rb | 2 +-
services/keep/build.sh | 36 --
services/keep/go.sh | 17 +
.../keep/src/arvados.org/keepproxy/keepproxy.go | 423 +++++++++++++++++++++
.../src/arvados.org/keepproxy/keepproxy_test.go | 276 ++++++++++++++
services/keep/src/keep/handler_test.go | 11 +
services/keep/src/keep/keep.go | 36 ++
65 files changed, 1946 insertions(+), 546 deletions(-)
delete mode 100755 apps/admin/setup-new-user.rb
create mode 100644 apps/workbench/app/assets/javascripts/event_log.js
create mode 100644 apps/workbench/app/assets/javascripts/select_modal.js
create mode 100644 apps/workbench/app/assets/stylesheets/select_modal.css.scss
create mode 100644 apps/workbench/app/views/folders/_choose.html.erb
create mode 100644 apps/workbench/app/views/folders/_choose.js.erb
create mode 100644 apps/workbench/app/views/folders/_show_folders.html.erb
delete mode 100644 apps/workbench/app/views/folders/_show_my_folders.html.erb
delete mode 100644 apps/workbench/app/views/folders/_show_shared_with_me.html.erb
delete mode 100755 sdk/go/build.sh
create mode 100755 sdk/go/go.sh
create mode 100644 sdk/python/test_cmdline.py
create mode 100644 services/api/db/migrate/20140527152921_add_description_to_pipeline_templates.rb
delete mode 100755 services/keep/build.sh
create mode 100755 services/keep/go.sh
create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy.go
create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy_test.go
via 46460c9615f07184ecb9917e4c948635712859da (commit)
via 40b6a69496aa50349d07daa6b28ae36bddd8a6a4 (commit)
via e289f5b586c24baf0bbd74e206d906297751db56 (commit)
via a01f0a9b9860ca574e0078a0e95afb783e711dc1 (commit)
via 5805fc322c7fd0c83eb9805a915aecd693cef5ee (commit)
via f049df0d8d69b8f4e235869103be2c814382bcce (commit)
via 122e392ac6d9b36e376937f4dc2f910118b112ac (commit)
via 9e4c1f166766dcbd0cfc8bc17ff13f7a2dc8508d (commit)
via b80867f0ae046c907cbd208facf7f98bee5a4f85 (commit)
via 68f03bd3db859e60237507ec08c789a2c3a89ca9 (commit)
via fbb501e3c0efb10851c00a91c0538bedb8c5e454 (commit)
via ab5ff91383be492d5a5bcbfdcc008cb32f37a03d (commit)
via 16413f6ebe1bba3a070cdff3d7436ad508db8514 (commit)
via 319cc7b3862ff831798f92bcb2f0c921d208147d (commit)
via 7199f034c15930a3a733e526e12fa93df7f9d4fb (commit)
via 4d50cea3609918775bf2a07981f1ef719067502b (commit)
via fd7ac9bf21002cc8a3cdb9a5e16c588ff734dfab (commit)
via a5f005b1b1175771ea708daee632a76f502eb9e6 (commit)
via fba94dcc012a22c48b2b3b3bc9b0e134d605e725 (commit)
via 7c810a0a2ad922e1bf966c026a96327c03ded77f (commit)
via 30228998e995720e4c2e1a3f0b52be231d9fbb9a (commit)
via 7ca1a382f7c59e29a03c765a6542f95a31ecdd07 (commit)
via 93901e794e11dbb5717288b006714fac7d13cb21 (commit)
via fe84c84827af6092784408629e4cde0663fe6af8 (commit)
via 1113b89204db5201806d03ecedaa99f177a91a2d (commit)
via a8da71bb7b0c4f319afd50ee6cd5d593f1237d33 (commit)
via facb17a2885e6b6d3e998e15862a3c6e970e8cf1 (commit)
via f60b9d7057fef32a7b61ea4de55b3d95b5b28f6c (commit)
via 037ec427df8bb32953f5158d0cd9d074ace776e8 (commit)
via 52bf3c0924da239488fff3ed9907c3478ee44e6b (commit)
via 010a56c1ac501c8ae18b71198d53c17e60f9d9e3 (commit)
via 8e60aa5c834a1a2a741a87a8155b8db9b4c46da3 (commit)
via a1e0f75938c65b827396e6796e9ccb99eecfe608 (commit)
via c61379a97dd46273da01792a83f4cae9736d6adf (commit)
via 2b2adb421b9b82b75fe8a635442dfe8e1fab775a (commit)
via c3b30a4104264f78088133b7d758e1920b1525af (commit)
via 69a1f282ad2983d36bd8fb7c41c681e1baa5dfa1 (commit)
via a8749c6b2c7d339c3355dd130ca37c9c876f72a5 (commit)
via 86680eab88b02d7f8bfe2f76308668957c825c36 (commit)
via f7a3e11f6017b329bb381f7ef10f9bae9dcece8f (commit)
via 9c4c0fd4015865e8f2f9f9c9867c7d847cd34aeb (commit)
via f4b7ce1ad093bd4c131ee603d053a329f003b754 (commit)
via 6df740c374ba869fd2aba618ab6d26ef641b6efd (commit)
via 6abd2d88cce48e0c5f908ed584cd5f575a82de90 (commit)
via 8633979010e2f15b3b88cab2a610019a336745c7 (commit)
via 31acc5982c7f002944e1c8cecf83ca98cc3c3e76 (commit)
via 1e0a693f16e599bb05a11933154da0f73030ecf7 (commit)
via e96a00962a6f5fcf65529630e4f2850fbfde2b73 (commit)
via f36fe36bd5e64b55b8699fae187c6accb678db2b (commit)
via 5cdf402cfc23ad287f1570b9db19a72a64b91fdc (commit)
via 39e38f830e8a12c93d5a788d15a5834b6f911217 (commit)
via b25c3bd978ccc68b42b3b68392863456fe4441fa (commit)
via 7c8e4a8d357ad8140eab83687c36cafd4e110e71 (commit)
via 56d21dc3d5823117729df3db6186da96f277b328 (commit)
via 7a82503a45c632918f8f032913dbbf6711a02e2e (commit)
via c0c0d769cd812e87efa392649380dc5ba8a25cd4 (commit)
via 293b52241445edf9eb86e0a0f9b63fccc3aa7fbd (commit)
via b06affa05ffbb121512ae179feb06c0bf90541bd (commit)
via 8570c81e79a9c00a314e2cf28a008b9aeae65037 (commit)
via 8815915612d1d8fef2001083bec2a24b3ed2eb3f (commit)
via 7331dca3c9f4a75e291b9b975f7cd570ad2b3f96 (commit)
via b668cb88f24085f92858b268120650c28d3a79af (commit)
via ec4e4a338f45ce39b4081f67c2b991c15fdf0fcd (commit)
via 14eddaec927be590682e226b9998c3e616c0b72e (commit)
via 216733543e5348ef8ef25ee5edbe6ed2dd35f5d3 (commit)
via 34ab7b208c7eb14ccae1e31831286acf9ea29487 (commit)
via 45d037864f655373dacd681d753ea3270f4ba997 (commit)
via 744ca00ccd1e7fda9d4210b18c1f4a734f3b2261 (commit)
via ed101805b04c70bcbf1b070dceba2aee72170377 (commit)
via 27ffca811e6f43225fc82d582d1962eebbd1ab6e (commit)
via b646cec74484bf07a54f4be2de712f50dc387aa0 (commit)
via 0d35501f448a8e7ca44152429d3d38edaa9bb30c (commit)
via f3dd3608f2c32fd4dcd9551bfa1b871d816954d5 (commit)
via 8efba7f42f882b2126a15e1bbcf385a5ab207b11 (commit)
via 8e5db2221d848c35e751b932da8126b559d292f0 (commit)
via 43a2aef67a96033f54e9f19cd9788e059e72115a (commit)
via c95e2762749ac2eb98fd7aec45134549174213d8 (commit)
via 12f8c88325daf4c6af8cbf091ea64cc5d64566c0 (commit)
via 575457ac8645c61ca71e94ce074291ec002b4c24 (commit)
via be92cf19aa7b61576ddcee0fd360103ca43ef5ea (commit)
via 67492abc51acc09abec9e754fc1d845679db3e87 (commit)
via 49a6ced3c7a540a7da7155ab1c3120a5227c620c (commit)
via 86dffe03ec74387d14da9ceb17934bbdab1239b9 (commit)
via 178f2834f7c9585485df30b1fe0e53b27e4edac5 (commit)
via fff404a0e2ebe2582f616526c486ad0dbecce3c8 (commit)
via 66f1be77149feb033e3adc17c37a09a391eec6e9 (commit)
via 2d4553d3d8e6f4d2e8f17e21664d46607a11dfb8 (commit)
via 2328a5363e90f2588113cc6ea76329ef93197ca0 (commit)
via c66b82f901113491f8a16a046ee52fb40d8d2939 (commit)
via 84decc8949fc51ec2eaf906fc2c4cbbb3df807dd (commit)
via 5aac4c685edaa685c255dcf79aefde46c267e6d7 (commit)
via 8ab98660dfc1973cced8b1deed355336cf4c5e64 (commit)
via 540dbe405430bc8d58a07cfff141b4b4bc9811c1 (commit)
via ea17ec3d053bd2f792730c5ea74d03109ef942ce (commit)
via a46ebd3da9f41522abde23685bba92f538f76f03 (commit)
via 8f20c1723c0614fabedb2c2468932d9245a1fc86 (commit)
via 77504f6d369bd7bd323748d5347d6d99ed9c75ca (commit)
via f8743ac8fa1ed9dc8c7c4f7a23803ffe8721cfa6 (commit)
via b64b0a4082eeba5d17705dca3bedf53b99ded617 (commit)
via 77d3ed33f525bd6859580954c6d7b9b36b394b98 (commit)
via 4d1e1b22d470d2fec9078c64c058c69850e440d6 (commit)
via bbaa4e880fd232bd270644bff24e33eb06784f85 (commit)
via e5b58426fb9064da76b207cd6d8aaaff05331d64 (commit)
via b0d3b048f6f16dd80e119623e0d4cb558a6537f7 (commit)
via 7d2ce5fa717f587963f64e466a9c3dd33faef27b (commit)
via 0c5de14523d1e90297c091675ee621ce6664f65b (commit)
via cdec3d68520096ca95a66f73ecf22bb0b22363a1 (commit)
via 86a02fd0842381002c46b590d774c561dbbaf19a (commit)
via aa7ec423d92b977320cc5708bd9556021d50d77a (commit)
via a84be631958ac0bb45b5e06efdb2782f02fdb2af (commit)
via c08254d5d6afcb5104cd65109d16479bf44ebb85 (commit)
via 8a4b7ab667a30f883dfeb0892f9fe03defc47a8f (commit)
via d07841a4b10d5677a6e4f80df1ba4f1a8bd13be9 (commit)
via 94abf66ab822425b025b8c88ebf9a2754b776ac3 (commit)
via 1b6c6a52d0bba5b0a03524c7477ee9e8836c5f5b (commit)
via 5cf70c36dd291410cdf177786a1a0ea0c25c06cb (commit)
via e6805dea66e86c6cdad036407f16fe17f45b8f94 (commit)
via 6ea7ba8654f7620a5e0ba4f66d653026b3f002fe (commit)
via e35566eddb0c1e383e437e95b228f7898e7a1c16 (commit)
via cd85fc2c07a0e9d8c34b81e4523ee3d1ebd696e9 (commit)
via f2574ec2e4ddef03c9504cd58741e73426ea1b17 (commit)
via e2eca205db4e40de582117790b1e48dbf3e5bf3a (commit)
via 48ffdd5ac196771381c8dc9ab47cfad5f1929720 (commit)
via 548f183c23606b9febcadbbfd658aa921c3baaf5 (commit)
via f5339841222a2ac17d3c04d4e9912c0608041b2f (commit)
via eb8eaafa2351d8cb8d4ba7408e0052d47fd56a2f (commit)
via 675118822f0a9968672dd99320d9b3494bc2b148 (commit)
via 243a6df519ee539ad9e5476cd6fcb51e4b0a48fe (commit)
via b1c303302dbac2ea6e9ab6775aecaf661b5da9c8 (commit)
via 5c2372f1e53184a60d9e1366d26ce41bf405a0a9 (commit)
via 465b105b0282efcdd63d88e1fb4e2eae11a195bc (commit)
via 9e0031c0bf48acdb76118ee643290d97a94926de (commit)
via a8ebe9f478e897334af0fcda79704de77c4dcea6 (commit)
via 264b0a629a67888a9d982b3a4398c0e0e4119089 (commit)
from d269c98c322e0a65ef36730e6a57c98a1ed754fa (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 46460c9615f07184ecb9917e4c948635712859da
Author: Tim Pierce <twp at curoverse.com>
Date: Wed May 28 17:58:09 2014 -0400
2755: add support to arv-put for signed manifests.
When arv-put finishes a stream, the manifest it stores in Keep now has
been stripped of signatures and other variable hints.
test_cmdline.py tests arv-put's handling of the manifest to make sure
that, when permissions are enabled, the manifest in Keep lacks
signatures, and the same manifest returned from the API server includes
signatures.
Refs #2755.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index fb3dea4..ab0b028 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -156,6 +156,7 @@ class CollectionWriter(object):
self._current_stream_name = '.'
self._current_file_name = None
self._current_file_pos = 0
+ self._manifest_text = ''
self._finished_streams = []
def __enter__(self):
@@ -272,9 +273,31 @@ class CollectionWriter(object):
self._current_file_name = None
def finish(self):
- return Keep.put(self.manifest_text())
-
+ # Send the stripped manifest to Keep, to ensure that we use the
+ # same UUID regardless of what hints are used on the collection.
+ return Keep.put(self.stripped_manifest())
+
+ def stripped_manifest(self):
+ """
+ Return the manifest for the current collection with all hints
+ (other than size) removed from the locators in the manifest.
+ """
+ raw = self.manifest_text()
+ clean = ''
+ for line in raw.split("\n"):
+ fields = line.split()
+ if len(fields) > 0:
+ locators = [ re.sub(r'\+[A-Z][a-z0-9 at _-]+', '', x)
+ for x in fields[1:-1] ]
+ clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
+ return clean
+
def manifest_text(self):
+ if self._manifest_text == '':
+ self._manifest_text = self.generate_manifest()
+ return self._manifest_text
+
+ def generate_manifest(self):
self.finish_current_stream()
manifest = ''
diff --git a/sdk/python/test_cmdline.py b/sdk/python/test_cmdline.py
new file mode 100644
index 0000000..a7029e2
--- /dev/null
+++ b/sdk/python/test_cmdline.py
@@ -0,0 +1,65 @@
+import os
+import re
+import subprocess
+import unittest
+import tempfile
+import yaml
+
+import arvados
+import run_test_server
+
+class ArvPutTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ try:
+ del os.environ['KEEP_LOCAL_STORE']
+ except KeyError:
+ pass
+
+ # Use the blob_signing_key from the Rails "test" configuration
+ # to provision the Keep server.
+ with open(os.path.join(os.path.dirname(__file__),
+ run_test_server.ARV_API_SERVER_DIR,
+ "config",
+ "application.yml")) as f:
+ rails_config = yaml.load(f.read())
+ config_blob_signing_key = rails_config["test"]["blob_signing_key"]
+ run_test_server.run()
+ run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
+ enforce_permissions=True)
+
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop()
+ run_test_server.stop_keep()
+
+ def test_ArvPutSignedManifest(self):
+ run_test_server.authorize_with('active')
+ for v in ["ARVADOS_API_HOST",
+ "ARVADOS_API_HOST_INSECURE",
+ "ARVADOS_API_TOKEN"]:
+ os.environ[v] = arvados.config.settings()[v]
+
+ datadir = tempfile.mkdtemp()
+ with open(os.path.join(datadir, "foo"), "w") as f:
+ f.write("The quick brown fox jumped over the lazy dog")
+ p = subprocess.Popen(["arv-put", datadir],
+ stdout=subprocess.PIPE)
+ (arvout, arverr) = p.communicate()
+ self.assertEqual(arverr, None)
+
+ # The manifest UUID returned by arv-put must be signed.
+ manifest_uuid = arvout.strip()
+ self.assertRegexpMatches(manifest_uuid, r'\+A[0-9a-f]+@[0-9a-f]{8}')
+
+ # The manifest text stored in Keep must contain unsigned locators.
+ m = arvados.Keep.get(manifest_uuid)
+ self.assertEqual(m, ". 08a008a01d498c404b0c30852b39d3b8+44 0:44:foo\n")
+
+ # The manifest text stored in the API server under the same
+ # manifest UUID must use signed locators.
+ api = arvados.api('v1', cache=False)
+ c = api.collections().get(uuid=manifest_uuid).execute()
+ self.assertRegexpMatches(
+ c['manifest_text'],
+ r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 030e238..d6db3a4 100644
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -38,11 +38,11 @@ production:
# The version below is suitable for AWS.
# Uncomment and change <%# to <%= to use it.
# compute_node_nameservers: <%#
- require 'net/http'
- ['local', 'public'].collect do |iface|
- Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- end << '172.16.0.23'
- %>
+ # require 'net/http'
+ # ['local', 'public'].collect do |iface|
+ # Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+ # end << '172.16.0.23'
+ # %>
test:
uuid_prefix: zzzzz
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list