[ARVADOS] updated: 46460c9615f07184ecb9917e4c948635712859da

git at public.curoverse.com git at public.curoverse.com
Wed May 28 18:13:09 EDT 2014


Summary of changes:
 apps/admin/setup-new-user.rb                       | 168 --------
 apps/workbench/app/assets/javascripts/event_log.js |  45 +++
 .../app/assets/javascripts/pipeline_instances.js   |  26 ++
 .../app/assets/javascripts/select_modal.js         |  39 ++
 .../app/assets/stylesheets/folders.css.scss        |  16 +-
 .../app/assets/stylesheets/select_modal.css.scss   |   3 +
 .../app/controllers/application_controller.rb      |  24 +-
 .../app/controllers/folders_controller.rb          |  65 ++--
 .../app/helpers/pipeline_instances_helper.rb       |  20 +
 apps/workbench/app/models/arvados_base.rb          |  17 +-
 apps/workbench/app/models/collection.rb            |   4 +
 apps/workbench/app/models/group.rb                 |   4 +
 apps/workbench/app/models/human.rb                 |   3 +
 apps/workbench/app/models/job.rb                   |   4 +
 apps/workbench/app/models/pipeline_instance.rb     |   4 +
 apps/workbench/app/models/pipeline_template.rb     |   4 +
 apps/workbench/app/models/specimen.rb              |   3 +
 apps/workbench/app/models/trait.rb                 |   3 +
 .../views/application/_selection_checkbox.html.erb |   2 +-
 .../app/views/application/_show_recent.html.erb    |  12 +-
 apps/workbench/app/views/folders/_choose.html.erb  |  42 ++
 apps/workbench/app/views/folders/_choose.js.erb    |   8 +
 .../app/views/folders/_show_folders.html.erb       |  29 ++
 .../app/views/folders/_show_my_folders.html.erb    |   2 -
 .../views/folders/_show_shared_with_me.html.erb    |   2 -
 apps/workbench/app/views/folders/show.html.erb     |  35 +-
 .../app/views/layouts/application.html.erb         |  18 +-
 .../pipeline_instances/_show_components.html.erb   |  50 ++-
 .../_show_components_editable.html.erb             |   2 +-
 .../views/pipeline_templates/_show_recent.html.erb |  10 +-
 .../app/views/user_agreements/index.html.erb       |   2 +-
 apps/workbench/app/views/users/_tables.html.erb    |   6 +-
 apps/workbench/config/routes.rb                    |   3 +-
 apps/workbench/test/integration/folders_test.rb    |  41 +-
 .../tutorial-firstscript.html.textile.liquid       |  10 +-
 sdk/cli/bin/arv-run-pipeline-instance              |   5 +-
 sdk/go/build.sh                                    |  37 --
 sdk/go/go.sh                                       |  17 +
 sdk/go/src/arvados.org/keepclient/keepclient.go    | 111 +++++-
 .../src/arvados.org/keepclient/keepclient_test.go  | 109 +++---
 sdk/go/src/arvados.org/keepclient/support.go       |  67 ++--
 .../main/java/org/arvados/sdk/java/Arvados.java    | 111 ++++--
 .../java/org/arvados/sdk/java/ArvadosTest.java     |  72 +++-
 sdk/python/arvados/api.py                          |   5 +-
 sdk/python/arvados/collection.py                   |  27 +-
 sdk/python/arvados/errors.py                       |   2 +
 sdk/python/arvados/keep.py                         |  76 +++-
 sdk/python/run_test_server.py                      |  47 ++-
 sdk/python/test_cmdline.py                         |  65 ++++
 sdk/python/test_keep_client.py                     |  81 ++++
 .../controllers/arvados/v1/groups_controller.rb    |  44 +--
 services/api/app/models/group.rb                   |  16 +
 services/api/app/models/pipeline_template.rb       |   1 +
 services/api/app/models/user.rb                    |   4 +-
 services/api/config/application.yml.example        |  10 +-
 ...152921_add_description_to_pipeline_templates.rb |   5 +
 services/api/db/schema.rb                          |   3 +-
 services/api/script/crunch-dispatch.rb             |  50 ++-
 services/api/test/integration/websocket_test.rb    |   2 +-
 services/keep/build.sh                             |  36 --
 services/keep/go.sh                                |  17 +
 .../keep/src/arvados.org/keepproxy/keepproxy.go    | 423 +++++++++++++++++++++
 .../src/arvados.org/keepproxy/keepproxy_test.go    | 276 ++++++++++++++
 services/keep/src/keep/handler_test.go             |  11 +
 services/keep/src/keep/keep.go                     |  36 ++
 65 files changed, 1946 insertions(+), 546 deletions(-)
 delete mode 100755 apps/admin/setup-new-user.rb
 create mode 100644 apps/workbench/app/assets/javascripts/event_log.js
 create mode 100644 apps/workbench/app/assets/javascripts/select_modal.js
 create mode 100644 apps/workbench/app/assets/stylesheets/select_modal.css.scss
 create mode 100644 apps/workbench/app/views/folders/_choose.html.erb
 create mode 100644 apps/workbench/app/views/folders/_choose.js.erb
 create mode 100644 apps/workbench/app/views/folders/_show_folders.html.erb
 delete mode 100644 apps/workbench/app/views/folders/_show_my_folders.html.erb
 delete mode 100644 apps/workbench/app/views/folders/_show_shared_with_me.html.erb
 delete mode 100755 sdk/go/build.sh
 create mode 100755 sdk/go/go.sh
 create mode 100644 sdk/python/test_cmdline.py
 create mode 100644 services/api/db/migrate/20140527152921_add_description_to_pipeline_templates.rb
 delete mode 100755 services/keep/build.sh
 create mode 100755 services/keep/go.sh
 create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy.go
 create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy_test.go

       via  46460c9615f07184ecb9917e4c948635712859da (commit)
       via  40b6a69496aa50349d07daa6b28ae36bddd8a6a4 (commit)
       via  e289f5b586c24baf0bbd74e206d906297751db56 (commit)
       via  a01f0a9b9860ca574e0078a0e95afb783e711dc1 (commit)
       via  5805fc322c7fd0c83eb9805a915aecd693cef5ee (commit)
       via  f049df0d8d69b8f4e235869103be2c814382bcce (commit)
       via  122e392ac6d9b36e376937f4dc2f910118b112ac (commit)
       via  9e4c1f166766dcbd0cfc8bc17ff13f7a2dc8508d (commit)
       via  b80867f0ae046c907cbd208facf7f98bee5a4f85 (commit)
       via  68f03bd3db859e60237507ec08c789a2c3a89ca9 (commit)
       via  fbb501e3c0efb10851c00a91c0538bedb8c5e454 (commit)
       via  ab5ff91383be492d5a5bcbfdcc008cb32f37a03d (commit)
       via  16413f6ebe1bba3a070cdff3d7436ad508db8514 (commit)
       via  319cc7b3862ff831798f92bcb2f0c921d208147d (commit)
       via  7199f034c15930a3a733e526e12fa93df7f9d4fb (commit)
       via  4d50cea3609918775bf2a07981f1ef719067502b (commit)
       via  fd7ac9bf21002cc8a3cdb9a5e16c588ff734dfab (commit)
       via  a5f005b1b1175771ea708daee632a76f502eb9e6 (commit)
       via  fba94dcc012a22c48b2b3b3bc9b0e134d605e725 (commit)
       via  7c810a0a2ad922e1bf966c026a96327c03ded77f (commit)
       via  30228998e995720e4c2e1a3f0b52be231d9fbb9a (commit)
       via  7ca1a382f7c59e29a03c765a6542f95a31ecdd07 (commit)
       via  93901e794e11dbb5717288b006714fac7d13cb21 (commit)
       via  fe84c84827af6092784408629e4cde0663fe6af8 (commit)
       via  1113b89204db5201806d03ecedaa99f177a91a2d (commit)
       via  a8da71bb7b0c4f319afd50ee6cd5d593f1237d33 (commit)
       via  facb17a2885e6b6d3e998e15862a3c6e970e8cf1 (commit)
       via  f60b9d7057fef32a7b61ea4de55b3d95b5b28f6c (commit)
       via  037ec427df8bb32953f5158d0cd9d074ace776e8 (commit)
       via  52bf3c0924da239488fff3ed9907c3478ee44e6b (commit)
       via  010a56c1ac501c8ae18b71198d53c17e60f9d9e3 (commit)
       via  8e60aa5c834a1a2a741a87a8155b8db9b4c46da3 (commit)
       via  a1e0f75938c65b827396e6796e9ccb99eecfe608 (commit)
       via  c61379a97dd46273da01792a83f4cae9736d6adf (commit)
       via  2b2adb421b9b82b75fe8a635442dfe8e1fab775a (commit)
       via  c3b30a4104264f78088133b7d758e1920b1525af (commit)
       via  69a1f282ad2983d36bd8fb7c41c681e1baa5dfa1 (commit)
       via  a8749c6b2c7d339c3355dd130ca37c9c876f72a5 (commit)
       via  86680eab88b02d7f8bfe2f76308668957c825c36 (commit)
       via  f7a3e11f6017b329bb381f7ef10f9bae9dcece8f (commit)
       via  9c4c0fd4015865e8f2f9f9c9867c7d847cd34aeb (commit)
       via  f4b7ce1ad093bd4c131ee603d053a329f003b754 (commit)
       via  6df740c374ba869fd2aba618ab6d26ef641b6efd (commit)
       via  6abd2d88cce48e0c5f908ed584cd5f575a82de90 (commit)
       via  8633979010e2f15b3b88cab2a610019a336745c7 (commit)
       via  31acc5982c7f002944e1c8cecf83ca98cc3c3e76 (commit)
       via  1e0a693f16e599bb05a11933154da0f73030ecf7 (commit)
       via  e96a00962a6f5fcf65529630e4f2850fbfde2b73 (commit)
       via  f36fe36bd5e64b55b8699fae187c6accb678db2b (commit)
       via  5cdf402cfc23ad287f1570b9db19a72a64b91fdc (commit)
       via  39e38f830e8a12c93d5a788d15a5834b6f911217 (commit)
       via  b25c3bd978ccc68b42b3b68392863456fe4441fa (commit)
       via  7c8e4a8d357ad8140eab83687c36cafd4e110e71 (commit)
       via  56d21dc3d5823117729df3db6186da96f277b328 (commit)
       via  7a82503a45c632918f8f032913dbbf6711a02e2e (commit)
       via  c0c0d769cd812e87efa392649380dc5ba8a25cd4 (commit)
       via  293b52241445edf9eb86e0a0f9b63fccc3aa7fbd (commit)
       via  b06affa05ffbb121512ae179feb06c0bf90541bd (commit)
       via  8570c81e79a9c00a314e2cf28a008b9aeae65037 (commit)
       via  8815915612d1d8fef2001083bec2a24b3ed2eb3f (commit)
       via  7331dca3c9f4a75e291b9b975f7cd570ad2b3f96 (commit)
       via  b668cb88f24085f92858b268120650c28d3a79af (commit)
       via  ec4e4a338f45ce39b4081f67c2b991c15fdf0fcd (commit)
       via  14eddaec927be590682e226b9998c3e616c0b72e (commit)
       via  216733543e5348ef8ef25ee5edbe6ed2dd35f5d3 (commit)
       via  34ab7b208c7eb14ccae1e31831286acf9ea29487 (commit)
       via  45d037864f655373dacd681d753ea3270f4ba997 (commit)
       via  744ca00ccd1e7fda9d4210b18c1f4a734f3b2261 (commit)
       via  ed101805b04c70bcbf1b070dceba2aee72170377 (commit)
       via  27ffca811e6f43225fc82d582d1962eebbd1ab6e (commit)
       via  b646cec74484bf07a54f4be2de712f50dc387aa0 (commit)
       via  0d35501f448a8e7ca44152429d3d38edaa9bb30c (commit)
       via  f3dd3608f2c32fd4dcd9551bfa1b871d816954d5 (commit)
       via  8efba7f42f882b2126a15e1bbcf385a5ab207b11 (commit)
       via  8e5db2221d848c35e751b932da8126b559d292f0 (commit)
       via  43a2aef67a96033f54e9f19cd9788e059e72115a (commit)
       via  c95e2762749ac2eb98fd7aec45134549174213d8 (commit)
       via  12f8c88325daf4c6af8cbf091ea64cc5d64566c0 (commit)
       via  575457ac8645c61ca71e94ce074291ec002b4c24 (commit)
       via  be92cf19aa7b61576ddcee0fd360103ca43ef5ea (commit)
       via  67492abc51acc09abec9e754fc1d845679db3e87 (commit)
       via  49a6ced3c7a540a7da7155ab1c3120a5227c620c (commit)
       via  86dffe03ec74387d14da9ceb17934bbdab1239b9 (commit)
       via  178f2834f7c9585485df30b1fe0e53b27e4edac5 (commit)
       via  fff404a0e2ebe2582f616526c486ad0dbecce3c8 (commit)
       via  66f1be77149feb033e3adc17c37a09a391eec6e9 (commit)
       via  2d4553d3d8e6f4d2e8f17e21664d46607a11dfb8 (commit)
       via  2328a5363e90f2588113cc6ea76329ef93197ca0 (commit)
       via  c66b82f901113491f8a16a046ee52fb40d8d2939 (commit)
       via  84decc8949fc51ec2eaf906fc2c4cbbb3df807dd (commit)
       via  5aac4c685edaa685c255dcf79aefde46c267e6d7 (commit)
       via  8ab98660dfc1973cced8b1deed355336cf4c5e64 (commit)
       via  540dbe405430bc8d58a07cfff141b4b4bc9811c1 (commit)
       via  ea17ec3d053bd2f792730c5ea74d03109ef942ce (commit)
       via  a46ebd3da9f41522abde23685bba92f538f76f03 (commit)
       via  8f20c1723c0614fabedb2c2468932d9245a1fc86 (commit)
       via  77504f6d369bd7bd323748d5347d6d99ed9c75ca (commit)
       via  f8743ac8fa1ed9dc8c7c4f7a23803ffe8721cfa6 (commit)
       via  b64b0a4082eeba5d17705dca3bedf53b99ded617 (commit)
       via  77d3ed33f525bd6859580954c6d7b9b36b394b98 (commit)
       via  4d1e1b22d470d2fec9078c64c058c69850e440d6 (commit)
       via  bbaa4e880fd232bd270644bff24e33eb06784f85 (commit)
       via  e5b58426fb9064da76b207cd6d8aaaff05331d64 (commit)
       via  b0d3b048f6f16dd80e119623e0d4cb558a6537f7 (commit)
       via  7d2ce5fa717f587963f64e466a9c3dd33faef27b (commit)
       via  0c5de14523d1e90297c091675ee621ce6664f65b (commit)
       via  cdec3d68520096ca95a66f73ecf22bb0b22363a1 (commit)
       via  86a02fd0842381002c46b590d774c561dbbaf19a (commit)
       via  aa7ec423d92b977320cc5708bd9556021d50d77a (commit)
       via  a84be631958ac0bb45b5e06efdb2782f02fdb2af (commit)
       via  c08254d5d6afcb5104cd65109d16479bf44ebb85 (commit)
       via  8a4b7ab667a30f883dfeb0892f9fe03defc47a8f (commit)
       via  d07841a4b10d5677a6e4f80df1ba4f1a8bd13be9 (commit)
       via  94abf66ab822425b025b8c88ebf9a2754b776ac3 (commit)
       via  1b6c6a52d0bba5b0a03524c7477ee9e8836c5f5b (commit)
       via  5cf70c36dd291410cdf177786a1a0ea0c25c06cb (commit)
       via  e6805dea66e86c6cdad036407f16fe17f45b8f94 (commit)
       via  6ea7ba8654f7620a5e0ba4f66d653026b3f002fe (commit)
       via  e35566eddb0c1e383e437e95b228f7898e7a1c16 (commit)
       via  cd85fc2c07a0e9d8c34b81e4523ee3d1ebd696e9 (commit)
       via  f2574ec2e4ddef03c9504cd58741e73426ea1b17 (commit)
       via  e2eca205db4e40de582117790b1e48dbf3e5bf3a (commit)
       via  48ffdd5ac196771381c8dc9ab47cfad5f1929720 (commit)
       via  548f183c23606b9febcadbbfd658aa921c3baaf5 (commit)
       via  f5339841222a2ac17d3c04d4e9912c0608041b2f (commit)
       via  eb8eaafa2351d8cb8d4ba7408e0052d47fd56a2f (commit)
       via  675118822f0a9968672dd99320d9b3494bc2b148 (commit)
       via  243a6df519ee539ad9e5476cd6fcb51e4b0a48fe (commit)
       via  b1c303302dbac2ea6e9ab6775aecaf661b5da9c8 (commit)
       via  5c2372f1e53184a60d9e1366d26ce41bf405a0a9 (commit)
       via  465b105b0282efcdd63d88e1fb4e2eae11a195bc (commit)
       via  9e0031c0bf48acdb76118ee643290d97a94926de (commit)
       via  a8ebe9f478e897334af0fcda79704de77c4dcea6 (commit)
       via  264b0a629a67888a9d982b3a4398c0e0e4119089 (commit)
      from  d269c98c322e0a65ef36730e6a57c98a1ed754fa (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 46460c9615f07184ecb9917e4c948635712859da
Author: Tim Pierce <twp at curoverse.com>
Date:   Wed May 28 17:58:09 2014 -0400

    2755: add support to arv-put for signed manifests.
    
    When arv-put finishes a stream, the manifest it stores in Keep now has
    been stripped of signatures and other variable hints.
    
    test_cmdline.py tests arv-put's handling of the manifest to make sure
    that, when permissions are enabled, the manifest in Keep lacks
    signatures, and the same manifest returned from the API server includes
    signatures.
    
    Refs #2755.

diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index fb3dea4..ab0b028 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -156,6 +156,7 @@ class CollectionWriter(object):
         self._current_stream_name = '.'
         self._current_file_name = None
         self._current_file_pos = 0
+        self._manifest_text = ''
         self._finished_streams = []
 
     def __enter__(self):
@@ -272,9 +273,31 @@ class CollectionWriter(object):
         self._current_file_name = None
 
     def finish(self):
-        return Keep.put(self.manifest_text())
-
+        # Send the stripped manifest to Keep, to ensure that we use the
+        # same UUID regardless of what hints are used on the collection.
+        return Keep.put(self.stripped_manifest())
+
+    def stripped_manifest(self):
+        """
+        Return the manifest for the current collection with all hints
+        (other than size) removed from the locators in the manifest.
+        """
+        raw = self.manifest_text()
+        clean = ''
+        for line in raw.split("\n"):
+            fields = line.split()
+            if len(fields) > 0:
+                locators = [ re.sub(r'\+[A-Z][a-z0-9 at _-]+', '', x)
+                             for x in fields[1:-1] ]
+                clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
+        return clean
+        
     def manifest_text(self):
+        if self._manifest_text == '':
+            self._manifest_text = self.generate_manifest()
+        return self._manifest_text
+
+    def generate_manifest(self):
         self.finish_current_stream()
         manifest = ''
 
diff --git a/sdk/python/test_cmdline.py b/sdk/python/test_cmdline.py
new file mode 100644
index 0000000..a7029e2
--- /dev/null
+++ b/sdk/python/test_cmdline.py
@@ -0,0 +1,65 @@
+import os
+import re
+import subprocess
+import unittest
+import tempfile
+import yaml
+
+import arvados
+import run_test_server
+
+class ArvPutTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        try:
+            del os.environ['KEEP_LOCAL_STORE']
+        except KeyError:
+            pass
+
+        # Use the blob_signing_key from the Rails "test" configuration
+        # to provision the Keep server.
+        with open(os.path.join(os.path.dirname(__file__),
+                               run_test_server.ARV_API_SERVER_DIR,
+                               "config",
+                               "application.yml")) as f:
+            rails_config = yaml.load(f.read())
+        config_blob_signing_key = rails_config["test"]["blob_signing_key"]
+        run_test_server.run()
+        run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
+                                 enforce_permissions=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        run_test_server.stop()
+        run_test_server.stop_keep()
+
+    def test_ArvPutSignedManifest(self):
+        run_test_server.authorize_with('active')
+        for v in ["ARVADOS_API_HOST",
+                  "ARVADOS_API_HOST_INSECURE",
+                  "ARVADOS_API_TOKEN"]:
+            os.environ[v] = arvados.config.settings()[v]
+
+        datadir = tempfile.mkdtemp()
+        with open(os.path.join(datadir, "foo"), "w") as f:
+            f.write("The quick brown fox jumped over the lazy dog")
+        p = subprocess.Popen(["arv-put", datadir],
+                             stdout=subprocess.PIPE)
+        (arvout, arverr) = p.communicate()
+        self.assertEqual(arverr, None)
+
+        # The manifest UUID returned by arv-put must be signed.
+        manifest_uuid = arvout.strip()
+        self.assertRegexpMatches(manifest_uuid, r'\+A[0-9a-f]+@[0-9a-f]{8}')
+
+        # The manifest text stored in Keep must contain unsigned locators.
+        m = arvados.Keep.get(manifest_uuid)
+        self.assertEqual(m, ". 08a008a01d498c404b0c30852b39d3b8+44 0:44:foo\n")
+
+        # The manifest text stored in the API server under the same
+        # manifest UUID must use signed locators.
+        api = arvados.api('v1', cache=False)
+        c = api.collections().get(uuid=manifest_uuid).execute()
+        self.assertRegexpMatches(
+            c['manifest_text'],
+            r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 030e238..d6db3a4 100644
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -38,11 +38,11 @@ production:
   # The version below is suitable for AWS.
   # Uncomment and change <%# to <%= to use it.
   # compute_node_nameservers: <%#
-    require 'net/http'
-    ['local', 'public'].collect do |iface|
-      Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
-    end << '172.16.0.23'
-  %>
+  #   require 'net/http'
+  #   ['local', 'public'].collect do |iface|
+  #     Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+  #   end << '172.16.0.23'
+  # %>
 
 test:
   uuid_prefix: zzzzz

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list