[ARVADOS] updated: e8451457c28477a5c3716e878d09d0be147186d6
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 23 13:37:06 EDT 2014
Summary of changes:
sdk/python/arvados/commands/copy.py | 39 +++++++++++++++----------------
sdk/python/arvados/commands/keepdocker.py | 31 +++++++++++++++++-------
2 files changed, 42 insertions(+), 28 deletions(-)
via e8451457c28477a5c3716e878d09d0be147186d6 (commit)
from d3dbc2c0557801f0e269a035e257beb09fd53618 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e8451457c28477a5c3716e878d09d0be147186d6
Author: Tim Pierce <twp at curoverse.com>
Date: Thu Oct 23 13:36:10 2014 -0400
3699: figure out correct docker image to fetch
Use arvados.commands.keepdocker.list_images_in_arv to figure out which
Docker image is represented by a given image name+tag.
diff --git a/sdk/python/arvados/commands/copy.py b/sdk/python/arvados/commands/copy.py
index 16bf137..7841f90 100755
--- a/sdk/python/arvados/commands/copy.py
+++ b/sdk/python/arvados/commands/copy.py
@@ -30,6 +30,7 @@ import arvados.config
import arvados.keep
import arvados.util
import arvados.commands._util as arv_cmd
+import arvados.commands.keepdocker
logger = logging.getLogger('arvados.arv-copy')
@@ -535,12 +536,13 @@ def copy_docker_images(pipeline, src, dst, args):
"""Copy any docker images named in the pipeline components'
runtime_constraints field from src to dst."""
- for c in pipeline['components']:
- if ('runtime_constraints' in c and
- 'docker_image' in c['runtime_constraints']):
+ logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
+ for c_name, c_info in pipeline['components'].iteritems():
+ if ('runtime_constraints' in c_info and
+ 'docker_image' in c_info['runtime_constraints']):
copy_docker_image(
- c['runtime_constraints']['docker_image'],
- c['runtime_constraints'].get('docker_image_tag'),
+ c_info['runtime_constraints']['docker_image'],
+ c_info['runtime_constraints'].get('docker_image_tag', 'latest'),
src, dst, args)
@@ -551,39 +553,36 @@ def copy_docker_image(docker_image, docker_image_tag, src, dst, args):
"""
+ logger.debug('copying docker image {}:{}'.format(docker_image, docker_image_tag))
+
# Find the link identifying this docker image.
- docker_link_name = "{}:{}".format(docker_image, docker_image_tag or "latest")
- links = src.links().list(
- filters=[
- ['link_class', '=', 'docker_image_repo+tag'],
- ['name', '=', docker_link_name],
- ]
- ).execute(num_retries=args.retries)
- if links['items_available'] == 0:
- raise ValueError("no docker image {} at src".format(docker_link_name))
- docker_link = links['items'][0]
- docker_image_uuid = docker_link['head_uuid']
+ docker_image_list = arvados.commands.keepdocker.list_images_in_arv(
+ src, args.retries, docker_image, docker_image_tag)
+ image_uuid, image_info = docker_image_list[0]
+ logger.debug('copying collection {} {}'.format(image_uuid, image_info))
# Copy the collection it refers to.
- dst_image_col = copy_collection(docker_image_uuid, src, dst, args)
+ dst_image_col = copy_collection(image_uuid, src, dst, args)
# Create docker_image_repo+tag and docker_image_hash links
# at the destination.
- dst.links().create(
+ lk = dst.links().create(
body={
'head_uuid': dst_image_col['uuid'],
'link_class': 'docker_image_repo+tag',
- 'name': docker_link_name,
+ 'name': "{}:{}".format(docker_image, docker_image_tag),
}
).execute(num_retries=args.retries)
+ logger.debug('created dst link {}'.format(lk))
- dst.links().create(
+ lk = dst.links().create(
body={
'head_uuid': dst_image_col['uuid'],
'link_class': 'docker_image_hash',
'name': dst_image_col['portable_data_hash'],
}
).execute(num_retries=args.retries)
+ logger.debug('created dst link {}'.format(lk))
# git_rev_parse(rev, repo)
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 6fa1745..536889d 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -164,9 +164,25 @@ def ptimestamp(t):
t = s[0] + s[1][-1:]
return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%SZ")
-def list_images_in_arv(api_client, num_retries):
+def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None):
+ """List all Docker images known to the api_client with image_name and
+ image_tag. If no image_name is given, defaults to listing all
+ Docker images.
+
+ Returns a list of tuples representing matching Docker images,
+ sorted in preference order (i.e. the first collection in the list
+ is the one that the API server would use). Each tuple is a
+ (collection_uuid, collection_info) pair, where collection_info is
+ a dict with fields "dockerhash", "repo", "tag", and "timestamp".
+
+ """
+ docker_image_filters = [['link_class', 'in', ['docker_image_hash', 'docker_image_repo+tag']]]
+ if image_name:
+ image_link_name = "{}:{}".format(image_name, image_tag or 'latest')
+ docker_image_filters.append(['name', '=', image_link_name])
+
existing_links = api_client.links().list(
- filters=[['link_class', 'in', ['docker_image_hash', 'docker_image_repo+tag']]]
+ filters=docker_image_filters
).execute(num_retries=num_retries)['items']
images = {}
for link in existing_links:
@@ -191,19 +207,18 @@ def list_images_in_arv(api_client, num_retries):
else:
images[collection_uuid]["timestamp"] = ptimestamp(link["created_at"])
- st = sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"]))
+ return sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"]))
- fmt = "{:30} {:10} {:12} {:29} {:20}"
- print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
- for i, j in st:
- print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
def main(arguments=None):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
- list_images_in_arv(api, args.retries)
+ fmt = "{:30} {:10} {:12} {:29} {:20}"
+ print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
+ for i, j in list_images_in_arv(api, args.retries):
+ print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list