[ARVADOS] updated: 4e5e843708a52e8587344be28269e14d7ad41409

git at public.curoverse.com git at public.curoverse.com
Thu Aug 7 19:47:58 EDT 2014


Summary of changes:
 sdk/python/arvados/commands/put.py          | 32 ++++++------
 sdk/python/tests/test_arv_put.py            | 81 +++++++++++++++++------------
 sdk/python/tests/test_keep_client.py        | 45 ++++++++--------
 services/api/config/application.default.yml |  2 +-
 4 files changed, 89 insertions(+), 71 deletions(-)

       via  4e5e843708a52e8587344be28269e14d7ad41409 (commit)
       via  21be87a976a7234e1a69e7f2a61c43c80e90b804 (commit)
       via  61e0cea2c42441dcdedd3776ca4520c8079eaa4b (commit)
       via  7f15a960d533ae692112d90e79bfa1a69f4dd27f (commit)
      from  8e76d0a54757172747f93eaa797e6d1746f8cd32 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 4e5e843708a52e8587344be28269e14d7ad41409
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Aug 7 19:47:53 2014 -0400

    3499: Reconcile Python tests to new arv-put "home project" behavior.

diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index c9c63e9..7a82768 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -346,6 +346,7 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
 
     def check_link(self, link, project_uuid, link_name=None):
         self.assertEqual(project_uuid, link.get('tail_uuid'))
+        self.assertEqual(project_uuid, link.get('owner_uuid'))
         self.assertEqual('name', link.get('link_class'))
         if link_name is None:
             self.assertNotIn('name', link)
@@ -376,6 +377,7 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
         else:
             self.fail("no warning emitted about the lack of collection name")
 
+    @unittest.skip("prep_project_link needs an API lookup for this case")
     def test_collection_without_project_defaults_to_home(self):
         link = self.prep_link_from_arguments(['--name', 'test link BBB'])
         self.check_link(link, self.Z_UUID)
@@ -443,10 +445,6 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
             arv_put.ResumeCache.CACHE_DIR = orig_cachedir
             os.chmod(cachedir, 0o700)
 
-    def test_link_without_project_uuid_aborts(self):
-        self.assertRaises(SystemExit, self.call_main_with_args,
-                          ['--name', 'test without project UUID', '/dev/null'])
-
     def test_link_without_collection_aborts(self):
         self.assertRaises(SystemExit, self.call_main_with_args,
                           ['--name', 'test without Collection',
@@ -497,6 +495,9 @@ class ArvPutIntegrationTest(unittest.TestCase):
                   "ARVADOS_API_TOKEN"]:
             os.environ[v] = arvados.config.settings()[v]
 
+    def current_user(self):
+        return arvados.api('v1').users().current().execute()
+
     def test_check_real_project_found(self):
         self.assertTrue(arv_put.check_project_exists(self.PROJECT_UUID),
                         "did not correctly find test fixture project")
@@ -569,29 +570,38 @@ class ArvPutIntegrationTest(unittest.TestCase):
     def run_and_find_link(self, text, extra_args=[]):
         self.authorize_with('active')
         pipe = subprocess.Popen(
-            [sys.executable, arv_put.__file__,
-             '--project-uuid', self.PROJECT_UUID] + extra_args,
+            [sys.executable, arv_put.__file__] + extra_args,
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE, env=self.ENVIRON)
         stdout, stderr = pipe.communicate(text)
         link_list = arvados.api('v1', cache=False).links().list(
             filters=[['head_uuid', '=', stdout.strip()],
-                     ['tail_uuid', '=', self.PROJECT_UUID],
                      ['link_class', '=', 'name']]).execute().get('items', [])
         self.assertEqual(1, len(link_list))
         return link_list[0]
 
     def test_put_collection_with_unnamed_project_link(self):
-        link = self.run_and_find_link("Test unnamed collection")
+        link = self.run_and_find_link("Test unnamed collection",
+                                      ['--project-uuid', self.PROJECT_UUID])
         username = pwd.getpwuid(os.getuid()).pw_name
         self.assertRegexpMatches(
             link['name'],
             r'^Saved at .* by {}@'.format(re.escape(username)))
 
+    def test_put_collection_with_name_and_no_project(self):
+        link_name = 'Test Collection Link in home project'
+        link = self.run_and_find_link("Test named collection in home project",
+                                      ['--name', link_name])
+        self.assertEqual(link_name, link['name'])
+        my_user_uuid = self.current_user()['uuid']
+        self.assertEqual(my_user_uuid, link['tail_uuid'])
+        self.assertEqual(my_user_uuid, link['owner_uuid'])
+
     def test_put_collection_with_named_project_link(self):
         link_name = 'Test auto Collection Link'
         link = self.run_and_find_link("Test named collection",
-                                      ['--name', link_name])
+                                      ['--name', link_name,
+                                       '--project-uuid', self.PROJECT_UUID])
         self.assertEqual(link_name, link['name'])
 
 

commit 21be87a976a7234e1a69e7f2a61c43c80e90b804
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Aug 7 19:46:36 2014 -0400

    3499: Fix Python SDK tests to accept signed blob locators.

diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index f5ab614..8706e21 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -36,9 +36,10 @@ class KeepTestCase(unittest.TestCase):
 
     def test_KeepBasicRWTest(self):
         foo_locator = arvados.Keep.put('foo')
-        self.assertEqual(foo_locator,
-                         'acbd18db4cc2f85cedef654fccc4a4d8+3',
-                         'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
+        self.assertRegexpMatches(
+            foo_locator,
+            '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
+            'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
         self.assertEqual(arvados.Keep.get(foo_locator),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
@@ -46,10 +47,10 @@ class KeepTestCase(unittest.TestCase):
     def test_KeepBinaryRWTest(self):
         blob_str = '\xff\xfe\xf7\x00\x01\x02'
         blob_locator = arvados.Keep.put(blob_str)
-        self.assertEqual(blob_locator,
-                         '7fc7c53b45e53926ba52821140fef396+6',
-                         ('wrong locator from Keep.put(<binarydata>):' +
-                          blob_locator))
+        self.assertRegexpMatches(
+            blob_locator,
+            '^7fc7c53b45e53926ba52821140fef396\+6',
+            ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
         self.assertEqual(arvados.Keep.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
@@ -59,10 +60,10 @@ class KeepTestCase(unittest.TestCase):
         for i in range(0,23):
             blob_str = blob_str + blob_str
         blob_locator = arvados.Keep.put(blob_str)
-        self.assertEqual(blob_locator,
-                         '84d90fc0d8175dd5dcfab04b999bc956+67108864',
-                         ('wrong locator from Keep.put(<binarydata>): ' +
-                          blob_locator))
+        self.assertRegexpMatches(
+            blob_locator,
+            '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
+            ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
         self.assertEqual(arvados.Keep.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
@@ -70,10 +71,10 @@ class KeepTestCase(unittest.TestCase):
     def test_KeepSingleCopyRWTest(self):
         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
         blob_locator = arvados.Keep.put(blob_str, copies=1)
-        self.assertEqual(blob_locator,
-                         'c902006bc98a3eb4a3663b65ab4a6fab+8',
-                         ('wrong locator from Keep.put(<binarydata>): ' +
-                          blob_locator))
+        self.assertRegexpMatches(
+            blob_locator,
+            '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
+            ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
         self.assertEqual(arvados.Keep.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
@@ -249,9 +250,10 @@ class KeepProxyTestCase(unittest.TestCase):
         arvados.config._settings = None
 
         baz_locator = arvados.Keep.put('baz')
-        self.assertEqual(baz_locator,
-                         '73feffa4b7f6bb68e44cf984c85f6e88+3',
-                         'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
+        self.assertRegexpMatches(
+            baz_locator,
+            '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
+            'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
         self.assertEqual(arvados.Keep.get(baz_locator),
                          'baz',
                          'wrong content from Keep.get(md5("baz"))')
@@ -271,9 +273,10 @@ class KeepProxyTestCase(unittest.TestCase):
         # keep_services/accessible
 
         baz_locator = arvados.Keep.put('baz2')
-        self.assertEqual(baz_locator,
-                         '91f372a266fe2bf2823cb8ec7fda31ce+4',
-                         'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
+        self.assertRegexpMatches(
+            baz_locator,
+            '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
+            'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
         self.assertEqual(arvados.Keep.get(baz_locator),
                          'baz2',
                          'wrong content from Keep.get(md5("baz2"))')

commit 61e0cea2c42441dcdedd3776ca4520c8079eaa4b
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Aug 7 19:45:12 2014 -0400

    3499: Improve Python SDK's ability to read apiserver's config files.
    
    Use a hardcoded secret in the default "test" configuration, rather
    than generating a random string with erb. Otherwise, the Python test
    suite can't figure out what blob signing key the test server actually
    ends up using.

diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 59d3a8e..c9c63e9 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -466,18 +466,24 @@ class ArvPutIntegrationTest(unittest.TestCase):
 
         # Use the blob_signing_key from the Rails "test" configuration
         # to provision the Keep server.
-        with open(os.path.join(os.path.dirname(__file__),
-                               run_test_server.ARV_API_SERVER_DIR,
-                               "config",
-                               "application.yml")) as f:
-            rails_config = yaml.load(f.read())
-        try:
-            config_blob_signing_key = rails_config["test"]["blob_signing_key"]
-        except KeyError:
-            config_blob_signing_key = rails_config["common"]["blob_signing_key"]
+        config_blob_signing_key = None
+        for config_file in ['application.yml', 'application.default.yml']:
+            with open(os.path.join(os.path.dirname(__file__),
+                                   run_test_server.ARV_API_SERVER_DIR,
+                                   "config",
+                                   config_file)) as f:
+                rails_config = yaml.load(f.read())
+                for config_section in ['test', 'common']:
+                    try:
+                        config_blob_signing_key = rails_config[config_section]["blob_signing_key"]
+                        break
+                    except KeyError, AttributeError:
+                        pass
+            if config_blob_signing_key != None:
+                break
         run_test_server.run()
         run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
-                                 enforce_permissions=True)
+                                 enforce_permissions=(config_blob_signing_key != None))
 
     @classmethod
     def tearDownClass(cls):
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 5436f02..c32900c 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -41,7 +41,7 @@ test:
   active_record.mass_assignment_sanitizer: :strict
   uuid_prefix: zzzzz
   secret_token: <%= rand(2**512).to_s(36) %>
-  blob_signing_key: <%= rand(2**512).to_s(36) %>
+  blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
 
 common:
   uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>

commit 7f15a960d533ae692112d90e79bfa1a69f4dd27f
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Aug 7 10:58:31 2014 -0400

    3499: arv-put collections in home project if no --project-uuid specified.

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index fa648da..342915e 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -352,31 +352,29 @@ def prep_project_link(args, stderr, project_exists=check_project_exists):
     # to create the desired project link for this Collection, or None.
     # Raises ValueError if the arguments request something impossible.
     making_collection = not (args.raw or args.stream)
-    any_link_spec = args.project_uuid or args.name
     if not making_collection:
-        if any_link_spec:
+        if args.name or args.project_uuid:
             raise ValueError("Requested a Link without creating a Collection")
         return None
-    elif not any_link_spec:
-        stderr.write(
-            "arv-put: No --project-uuid or --name specified.  This data will be cached\n"
-            "in Keep.  You will need to find this upload by its locator(s) later.\n")
-        return None
-    elif not args.project_uuid:
-        raise ValueError("--name requires --project-uuid")
-    elif not project_exists(args.project_uuid):
+    link = {'tail_uuid': args.project_uuid,
+            'link_class': 'name',
+            'name': args.name}
+    if not link['tail_uuid']:
+        link['tail_uuid'] = arvados.api('v1').users().current().execute()['uuid']
+    elif not project_exists(link['tail_uuid']):
         raise ValueError("Project {} not found".format(args.project_uuid))
-    link = {'tail_uuid': args.project_uuid, 'link_class': 'name'}
-    if args.name:
-        link['name'] = args.name
+    if not link['name']:
+        link['name'] = "Saved at {} by {}@{}".format(
+            datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"),
+            pwd.getpwuid(os.getuid()).pw_name,
+            socket.gethostname())
+        stderr.write(
+            "arv-put: No --name specified. Saving as \"%s\"\n" % link['name'])
+    link['owner_uuid'] = link['tail_uuid']
     return link
 
 def create_project_link(locator, link):
     link['head_uuid'] = locator
-    link.setdefault('name', "Collection saved by {}@{} at {}".format(
-            pwd.getpwuid(os.getuid()).pw_name,
-            socket.gethostname(),
-            datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")))
     return arvados.api('v1').links().create(body=link).execute()
 
 def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 1a107a5..59d3a8e 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -363,17 +363,23 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
         self.check_stderr_empty()
 
     def test_project_link_without_name(self):
+        username = pwd.getpwuid(os.getuid()).pw_name
         link = self.prep_link_from_arguments(['--project-uuid', self.Z_UUID])
-        self.check_link(link, self.Z_UUID)
-        self.check_stderr_empty()
-
-    def test_collection_without_project_warned(self):
-        self.assertIsNone(self.prep_link_from_arguments([]))
+        self.assertIsNotNone(link.get('name', None))
+        self.assertRegexpMatches(
+            link['name'],
+            r'^Saved at .* by {}@'.format(re.escape(username)))
+        self.check_link(link, self.Z_UUID, link.get('name', None))
         for line in self.stderr:
-            if "--project-uuid or --name" in line:
+            if "No --name specified" in line:
                 break
         else:
-            self.fail("no warning emitted about the lack of project name")
+            self.fail("no warning emitted about the lack of collection name")
+
+    def test_collection_without_project_defaults_to_home(self):
+        link = self.prep_link_from_arguments(['--name', 'test link BBB'])
+        self.check_link(link, self.Z_UUID)
+        self.check_stderr_empty()
 
     def test_no_link_or_warning_with_no_collection(self):
         self.assertIsNone(self.prep_link_from_arguments(['--raw']))
@@ -384,11 +390,6 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
                           self.prep_link_from_arguments,
                           ['--project-uuid', self.Z_UUID], False)
 
-    def test_name_without_project_is_error(self):
-        self.assertRaises(ValueError,
-                          self.prep_link_from_arguments,
-                          ['--name', 'test'])
-
     def test_link_without_collection_is_error(self):
         self.assertRaises(ValueError,
                           self.prep_link_from_arguments,
@@ -579,7 +580,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
         username = pwd.getpwuid(os.getuid()).pw_name
         self.assertRegexpMatches(
             link['name'],
-            r'^Collection saved by {}@'.format(re.escape(username)))
+            r'^Saved at .* by {}@'.format(re.escape(username)))
 
     def test_put_collection_with_named_project_link(self):
         link_name = 'Test auto Collection Link'

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list