[ARVADOS] updated: b6017f4edc19188fa26f74ff0da2bedb94841dcf

git at public.curoverse.com git at public.curoverse.com
Mon Sep 8 10:56:23 EDT 2014


Summary of changes:
 apps/workbench/Gemfile                             |   2 +
 apps/workbench/Gemfile.lock                        |  36 +++++
 .../app/assets/javascripts/select_modal.js         |  16 ++-
 apps/workbench/app/assets/javascripts/selection.js |  26 ++--
 .../app/controllers/actions_controller.rb          |  38 ++---
 .../app/controllers/collections_controller.rb      |  15 +-
 .../app/controllers/projects_controller.rb         |  29 +---
 apps/workbench/app/helpers/collections_helper.rb   |  22 ++-
 apps/workbench/app/models/collection.rb            |  32 +++--
 .../app/views/application/_choose.html.erb         |  24 ++--
 .../workbench/app/views/application/_choose.js.erb |  23 ++-
 .../app/views/collections/_show_files.html.erb     |   2 +-
 .../app/views/collections/show_file_links.html.erb |   7 +-
 .../app/views/projects/_show_tab_contents.html.erb |  45 ++++--
 apps/workbench/app/views/projects/show.html.erb    |   2 +-
 apps/workbench/config/routes.rb                    |   1 -
 .../test/integration/pipeline_instances_test.rb    |  45 +++++-
 apps/workbench/test/integration/projects_test.rb   | 157 +++++++++++++++++++++
 sdk/cli/Gemfile                                    |   2 +-
 sdk/cli/Gemfile.lock                               |  42 ++++--
 sdk/cli/bin/arv                                    |   2 +-
 sdk/cli/bin/arv-run-pipeline-instance              |  36 +++--
 sdk/cli/bin/crunch-job                             |  32 +++--
 sdk/python/arvados/commands/keepdocker.py          | 104 ++++++++++----
 sdk/python/arvados/commands/put.py                 |  21 +--
 sdk/ruby/arvados.gemspec                           |   2 +-
 sdk/ruby/lib/arvados/keep.rb                       | 134 ++++++++++++++++++
 sdk/ruby/test/test_big_request.rb                  |  21 +--
 sdk/ruby/test/test_keep_manifest.rb                |  87 ++++++++++++
 services/api/.gitignore                            |   1 -
 services/api/Gemfile                               |   3 +-
 services/api/Gemfile.lock                          |   7 +-
 .../api/app/assets/stylesheets/application.css     |   2 +-
 .../api/app/controllers/application_controller.rb  |  42 +++++-
 .../arvados/v1/collections_controller.rb           |  14 +-
 .../controllers/arvados/v1/schema_controller.rb    |  28 ++--
 services/api/app/models/collection.rb              | 100 ++-----------
 services/api/app/models/locator.rb                 | 104 --------------
 .../api/app/views/layouts/application.html.erb     |   2 +-
 services/api/app/views/static/intro.html.erb       |  10 +-
 .../api/app/views/static/login_failure.html.erb    |   2 +-
 services/api/db/structure.sql                      |  20 +--
 services/api/test/fixtures/collections.yml         |  12 ++
 services/api/test/fixtures/pipeline_templates.yml  |  21 +++
 .../arvados/v1/collections_controller_test.rb      |  70 +++------
 45 files changed, 950 insertions(+), 493 deletions(-)
 create mode 100644 sdk/ruby/lib/arvados/keep.rb
 create mode 100644 sdk/ruby/test/test_keep_manifest.rb
 delete mode 100644 services/api/app/models/locator.rb

       via  b6017f4edc19188fa26f74ff0da2bedb94841dcf (commit)
       via  9e9506f9acce0d11a82f7abcea31680761d6427b (commit)
       via  d43fb929ea2fdf984a86bc00c257e989e4b7a88c (commit)
       via  c6537733d0bd44f0319dec074492e7e943549145 (commit)
       via  0ac9eaceae35e2a466b0ad09c903384d9bd21b9e (commit)
       via  b60e52e031eeba96ef1a301660de2f8cec4518b6 (commit)
       via  15f512c3d8bc8bf090972653b3e53742197316e8 (commit)
       via  1c679ba62c8d7b0d962532bdb5fa2de4c685d286 (commit)
       via  3b7e5549bf4a17907f8e3236592bcbd5cf6ad623 (commit)
       via  33a6cc0d0b748b4ec5a01db6218b8fa21afb11ed (commit)
       via  f94f97fe63d3d860d373a25d3539d559b53542c5 (commit)
       via  41c7f2010ebdbb76fada25a21f184e2d1f4049b3 (commit)
       via  0c3c1a495e8048195151d30d831208a5c75e36f0 (commit)
       via  d13ad5bbb7f5e74cb43d0570d9d1a8159237f5ff (commit)
       via  d95e85ac808f5fedfaf573d3b5fcc0bb02605f38 (commit)
       via  ac80f69f2a4469891f70b86c9c196e3ebd256081 (commit)
       via  80df142a2b7a53d420c837acc467fc39ac9ae312 (commit)
       via  35e39f925dae467785ed1b1fa752dd4f76133414 (commit)
       via  5184743db419caa1dca30b2d15e3b9d1cdb0c51c (commit)
       via  5828355672e3fc7926125dcc35882a642a4cde1a (commit)
       via  812f2714c2908e5b427a8e37032ddcd46fb08297 (commit)
       via  2449ae26b0d8be41aa5411e62032c9fb1ea9af71 (commit)
       via  0eb5711ade0f74e556b0a1c10909dbf0bdecb63f (commit)
       via  0fdafcb846ec6bb34e4b8ae91ad62c1ccd328355 (commit)
       via  682a02f163af60655ec30391689cbb84392e9a19 (commit)
       via  9ec9d36e9f236613fc67256ebdb75b3d1e2bec35 (commit)
       via  44bcabe94af2a0b34edd242e82fd529d1d56524a (commit)
       via  9c81bfe7c85658ddab49a985b923a248824a0f99 (commit)
       via  81ead9fd78dafa906a1a92a47fcec170a31f4d15 (commit)
       via  94982cd84ed3b39dca8d47a7786133eac3885a92 (commit)
       via  4c517008ac098b8bd107ed22edce0c50ba20f648 (commit)
       via  0e0279e2566ff5ee19a741961dd21b3e2ab310db (commit)
       via  c23cb2dd9b373a8e391e810a9406114749cc2c59 (commit)
       via  73988a7629d8ce151983997aac45c04bb150f8f1 (commit)
       via  5b953c11c79e63cbbe0a08037c429f3f7ed0ff13 (commit)
       via  0f30cd32ae07e1d514fa5b9fff7fd796a95a6d84 (commit)
       via  a16c764ae43c82913b69be61b526c6eb04ab20e6 (commit)
       via  4938505bc99e6b0c01b2c8dbc6a54814e93d5a30 (commit)
       via  4917bb2c0b7194af05a6e41c461f8a0acce1540d (commit)
       via  2679a9f68531ee7c29ee4e1d68faef408de92d48 (commit)
       via  0b0070702a59ca73d8bdd7bc117c63efff932ede (commit)
       via  a3be17c575845ad0d84969128850cf87a0b71110 (commit)
       via  419fb180375ea9087df1418f245581a5c4a7ba74 (commit)
      from  c8de202a8ba2b7e128f9698c4b57f0d7d9e89c1d (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b6017f4edc19188fa26f74ff0da2bedb94841dcf
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 8 10:56:19 2014 -0400

    3453: Add ensure_unique_name=True to collection create.

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 5be4ef4..df13a94 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -257,7 +257,8 @@ def main(arguments=None):
                     # create new collection owned by the project
                     coll_uuid = api.collections().create(body={"manifest_text": collections[0]['manifest_text'], 
                                                                "name": collection_name, 
-                                                               "owner_uuid": parent_project_uuid}).execute()['uuid']
+                                                               "owner_uuid": parent_project_uuid}, 
+                                                         ensure_unique_name=True).execute()['uuid']
 
                 link_base = {'owner_uuid': parent_project_uuid, 
                              'head_uuid':  coll_uuid }
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index bf51ccd..ed405dc 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -461,6 +461,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
                 'manifest_text': writer.manifest_text(),
                 'owner_uuid': project_link['tail_uuid']
                 },
+            ensure_unique_name=True
             ).execute()
 
         if args.portable_data_hash and 'portable_data_hash' in collection and collection['portable_data_hash']:

commit 9e9506f9acce0d11a82f7abcea31680761d6427b
Merge: c653773 d43fb92
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 8 10:53:52 2014 -0400

    Merge branch 'master' into 3453-arv-list-docker-images


commit c6537733d0bd44f0319dec074492e7e943549145
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 8 10:43:49 2014 -0400

    3453: Create links and collection objects in the specified project, even if
    the docker image has already been uploaded to some other project.

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index fa4e441..5be4ef4 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -9,6 +9,7 @@ import subprocess
 import sys
 import tarfile
 import tempfile
+import textwrap
 
 from collections import namedtuple
 from stat import *
@@ -22,19 +23,12 @@ STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
 DockerImage = namedtuple('DockerImage',
                          ['repo', 'tag', 'hash', 'created', 'vsize'])
 
-opt_parser = argparse.ArgumentParser(add_help=False)
-opt_parser.add_argument(
+keepdocker_parser = argparse.ArgumentParser(add_help=False)
+keepdocker_parser.add_argument(
     '-f', '--force', action='store_true', default=False,
     help="Re-upload the image even if it already exists on the server")
 
-opt_parser.add_argument(
-    '--project-uuid',
-    help="Add the Docker image and metadata to the specified project.  Goes into user 'home' project by default.")
-opt_parser.add_argument(
-    '--name',
-    help="Name to use for the collection that will contain the docker image.")
-
-_group = opt_parser.add_mutually_exclusive_group()
+_group = keepdocker_parser.add_mutually_exclusive_group()
 _group.add_argument(
     '--pull', action='store_true', default=False,
     help="Try to pull the latest image from Docker registry")
@@ -42,16 +36,19 @@ _group.add_argument(
     '--no-pull', action='store_false', dest='pull',
     help="Use locally installed image only, don't pull image from Docker registry (default)")
 
-opt_parser.add_argument(
+keepdocker_parser.add_argument(
     'image', nargs='?',
     help="Docker image to upload, as a repository name or hash")
-opt_parser.add_argument(
+keepdocker_parser.add_argument(
     'tag', nargs='?', default='latest',
     help="Tag of the Docker image to upload (default 'latest')")
 
+# Combine keepdocker options listed above with run_opts options of arv-put.
+# The options inherited from arv-put include --name, --project-uuid,
+# --progress/--no-progress/--batch-progress and --resume/--no-resume.
 arg_parser = argparse.ArgumentParser(
         description="Upload or list Docker images in Arvados",
-        parents=[opt_parser, arv_put.run_opts])
+        parents=[keepdocker_parser, arv_put.run_opts])
 
 class DockerError(Exception):
     pass
@@ -193,8 +190,8 @@ def list_images_in_arv():
 
     st = sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"]))
 
-    fmt = "{:30}  {:10}  {:12}  {:38}  {:20}"
-    print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "KEEP LOCATOR", "CREATED")
+    fmt = "{:30}  {:10}  {:12}  {:29}  {:20}"
+    print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
     for i, j in st:
         print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:11], i, j["timestamp"].strftime("%c")))
 
@@ -215,18 +212,67 @@ def main(arguments=None):
     except DockerError as error:
         print >>sys.stderr, "arv-keepdocker:", error.message
         sys.exit(1)
+
+    image_repo_tag = '{}:{}'.format(args.image, args.tag)
+
+    if args.name is None:
+        collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:11])
+    else:
+        collection_name = args.name
+
+    api = arvados.api('v1')
+
     if not args.force:
-        # Abort if this image is already in Arvados.
-        existing_links = arvados.api('v1').links().list(
+        # Check if this image is already in Arvados.
+
+        # Project where everything should be owned
+        parent_project_uuid = args.project_uuid if args.project_uuid else api.users().current().execute()['uuid']
+
+        # Find image hash tags
+        existing_links = api.links().list(
             filters=[['link_class', '=', 'docker_image_hash'],
                      ['name', '=', image_hash]]).execute()['items']
         if existing_links:
-            message = [
-                "arv-keepdocker: Image {} already stored in collection(s):".
-                format(image_hash)]
-            message.extend(link['head_uuid'] for link in existing_links)
-            print >>sys.stderr, "\n".join(message)
-            sys.exit(0)
+            # get readable collections
+            collections = api.collections().list(
+                filters=[['uuid', 'in', [link['head_uuid'] for link in existing_links]]], 
+                select=["uuid", "owner_uuid", "name", "manifest_text"]).execute()['items']
+
+            if collections:
+                # check for repo+tag links on these collections
+                existing_repo_tag = api.links().list(
+                    filters=[['link_class', '=', 'docker_image_repo+tag'],
+                             ['name', '=', image_repo_tag],
+                             ['head_uuid', 'in', collections]]).execute()['items']
+
+                # Filter on elements owned by the parent project
+                owned_col = [c for c in collections if c['owner_uuid'] == parent_project_uuid]
+                owned_img = [c for c in existing_links if c['owner_uuid'] == parent_project_uuid] 
+                owned_rep = [c for c in existing_repo_tag if c['owner_uuid'] == parent_project_uuid] 
+
+                if owned_col:
+                    # already have a collection owned by this project
+                    coll_uuid = owned_col[0]['uuid']
+                else:
+                    # create new collection owned by the project
+                    coll_uuid = api.collections().create(body={"manifest_text": collections[0]['manifest_text'], 
+                                                               "name": collection_name, 
+                                                               "owner_uuid": parent_project_uuid}).execute()['uuid']
+
+                link_base = {'owner_uuid': parent_project_uuid, 
+                             'head_uuid':  coll_uuid }
+
+                if not owned_img:
+                    # create image link owned by the project
+                    make_link('docker_image_hash', image_hash, **link_base)
+
+                if not owned_rep:
+                    # create repo+tag link owned by the project
+                    make_link('docker_image_repo+tag', image_repo_tag, **link_base)
+
+                print(coll_uuid)
+
+                sys.exit(0)                
 
     # Open a file for the saved image, and write it if needed.
     outfile_name = '{}.tar'.format(image_hash)
@@ -236,15 +282,10 @@ def main(arguments=None):
 
     # Call arv-put with switches we inherited from it
     # (a.k.a., switches that aren't our own).
-    put_args = opt_parser.parse_known_args(arguments)[1]
+    put_args = keepdocker_parser.parse_known_args(arguments)[1]
 
     if args.name is None:
-        put_args += ['--name', 'Docker image {}:{} {}'.format(args.image, args.tag, image_hash[0:11])]
-    else:
-        put_args += ['--name', args.name]
-
-    if args.project_uuid is not None:
-        put_args += ['--project-uuid', args.project_uuid]
+        put_args += ['--name', collection_name]
 
     coll_uuid = arv_put.main(
         put_args + ['--filename', outfile_name, image_file.name]).strip()
@@ -264,7 +305,7 @@ def main(arguments=None):
 
     make_link('docker_image_hash', image_hash, **link_base)
     if not image_hash.startswith(args.image.lower()):
-        make_link('docker_image_repo+tag', '{}:{}'.format(args.image, args.tag),
+        make_link('docker_image_repo+tag', image_repo_tag,
                   **link_base)
 
     # Clean up.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 6abaf77..bf51ccd 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -40,16 +40,6 @@ a single stream. Default: -1 (unlimited), i.e., exactly one manifest
 stream per filesystem directory that contains files.
 """)
 
-upload_opts.add_argument('--project-uuid', metavar='UUID', help="""
-Store the collection in the specified project, instead of your Home
-project.
-""")
-
-upload_opts.add_argument('--name', help="""
-Save the collection with the specified name, rather than the default
-generic name "Saved at {time} by {username}@{host}".
-""")
-
 _group = upload_opts.add_mutually_exclusive_group()
 
 _group.add_argument('--as-stream', action='store_true', dest='stream',
@@ -113,6 +103,16 @@ created by the upload.
 """)
 
 run_opts = argparse.ArgumentParser(add_help=False)
+
+run_opts.add_argument('--project-uuid', metavar='UUID', help="""
+Store the collection in the specified project, instead of your Home
+project.
+""")
+
+run_opts.add_argument('--name', help="""
+Save the collection with the specified name.
+""")
+
 _group = run_opts.add_mutually_exclusive_group()
 _group.add_argument('--progress', action='store_true',
                    help="""

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list