[ARVADOS] updated: 5fcf5ebe7b89517edcb610bf1f1df8abf99df9ae
git at public.curoverse.com
git at public.curoverse.com
Wed Sep 3 13:48:35 EDT 2014
Summary of changes:
sdk/cli/bin/arv-copy | 116 +--------------------
.../arv-copy => python/arvados/commands/copy.py} | 39 +++++--
sdk/python/bin/arv-copy | 4 +
3 files changed, 35 insertions(+), 124 deletions(-)
mode change 100755 => 120000 sdk/cli/bin/arv-copy
copy sdk/{cli/bin/arv-copy => python/arvados/commands/copy.py} (75%)
create mode 100755 sdk/python/bin/arv-copy
via 5fcf5ebe7b89517edcb610bf1f1df8abf99df9ae (commit)
from 1f3035cfe645741753e45ff8d3cf43a3fc5b2385 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5fcf5ebe7b89517edcb610bf1f1df8abf99df9ae
Author: Tim Pierce <twp at curoverse.com>
Date: Wed Sep 3 13:47:33 2014 -0400
3699: arv-copy supports collections
arv-copy can be passed a collection UUID. copy_collection(uuid) copies
each block in the collection from the source Arvados to the destination,
copies the manifest and then writes a collection record.
diff --git a/sdk/cli/bin/arv-copy b/sdk/cli/bin/arv-copy
deleted file mode 100755
index 25a8b50..0000000
--- a/sdk/cli/bin/arv-copy
+++ /dev/null
@@ -1,115 +0,0 @@
-#! /usr/bin/env python
-
-# arv-copy [--recursive] [--no-recursive] object-uuid src dst
-#
-# Copies an object from Arvados instance src to instance dst.
-#
-# By default, arv-copy recursively copies any dependent objects
-# necessary to make the object functional in the new instance
-# (e.g. for a pipeline instance, arv-copy copies the pipeline
-# template, input collection, docker images, git repositories). If
-# --no-recursive is given, arv-copy copies only the single record
-# identified by object-uuid.
-#
-# The user must have files $HOME/.config/arvados/{src}.conf and
-# $HOME/.config/arvados/{dst}.conf with valid login credentials for
-# instances src and dst. If either of these files is not found,
-# arv-copy will issue an error.
-
-import argparse
-import hashlib
-import os
-import re
-import string
-import sys
-import logging
-
-import arvados
-import arvados.config
-
-def main():
- logger = logging.getLogger('arvados.arv-copy')
-
- parser = argparse.ArgumentParser(
- description='Copy a pipeline instance from one Arvados instance to another.')
-
- parser.add_argument('--recursive', dest='recursive', action='store_true')
- parser.add_argument('--no-recursive', dest='recursive', action='store_false')
- parser.add_argument('object_uuid')
- parser.add_argument('source_arvados')
- parser.add_argument('destination_arvados')
- parser.set_defaults(recursive=True)
-
- args = parser.parse_args()
-
- # Create API clients for the source and destination instances
- src_arv = api_for_instance(args.source_arvados)
- dst_arv = api_for_instance(args.destination_arvados)
-
- # And now for the copying.
- t = uuid_type(args.object_uuid)
- if t == 'collection':
- copy_collection(args.object_uuid, src=src_arv, dst=dst_arv)
- elif t == 'pipeline_instance':
- copy_pipeline_instance(args.object_uuid, src=src_arv, dst=dst_arv)
- elif t == 'pipeline_template':
- new_pt = copy_pipeline_template(args.object_uuid, src=src_arv, dst=dst_arv)
- print new_pt
- else:
- abort("cannot copy object {} of type {}".format(args.object_uuid, t))
-
- exit(0)
-
-# Creates an API client for the Arvados instance identified by
-# instance_name. Looks in $HOME/.config/arvados/instance_name.conf
-# for credentials.
-#
-def api_for_instance(instance_name):
- if '/' in instance_name:
- abort('illegal instance name {}'.format(instance_name))
- config_file = os.path.join(os.environ['HOME'], '.config', 'arvados', "{}.conf".format(instance_name))
- cfg = arvados.config.load(config_file)
-
- if 'ARVADOS_API_HOST' in cfg and 'ARVADOS_API_TOKEN' in cfg:
- api_is_insecure = (
- cfg.get('ARVADOS_API_HOST_INSECURE', '').lower() in set(
- ['1', 't', 'true', 'y', 'yes']))
- client = arvados.api('v1',
- host=cfg['ARVADOS_API_HOST'],
- token=cfg['ARVADOS_API_TOKEN'],
- insecure=api_is_insecure,
- cache=False)
- else:
- abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name))
- return client
-
-def copy_collection(obj_uuid, src=None, dst=None):
- raise NotImplementedError
-
-def copy_pipeline_instance(obj_uuid, src=None, dst=None):
- raise NotImplementedError
-
-def copy_pipeline_template(obj_uuid, src=None, dst=None):
- # fetch the pipeline template from the source instance
- old_pt = src.pipeline_templates().get(uuid=obj_uuid).execute()
- old_pt['name'] = old_pt['name'] + ' copy'
- del old_pt['uuid']
- del old_pt['owner_uuid']
- return dst.pipeline_templates().create(body=old_pt).execute()
-
-uuid_type_map = {
- "4zz18": "collection",
- "d1hrv": "pipeline_instance",
- "p5p6p": "pipeline_template",
-}
-
-def uuid_type(object_uuid):
- type_str = object_uuid.split('-')[1]
- return uuid_type_map.get(type_str, None)
-
-def abort(msg, code=1):
- print >>sys.stderr, "arv-copy:", msg
- exit(code)
-
-if __name__ == '__main__':
- main()
diff --git a/sdk/cli/bin/arv-copy b/sdk/cli/bin/arv-copy
new file mode 120000
index 0000000..1ad64f4
--- /dev/null
+++ b/sdk/cli/bin/arv-copy
@@ -0,0 +1 @@
+../../python/bin/arv-copy
\ No newline at end of file
diff --git a/sdk/cli/bin/arv-copy b/sdk/python/arvados/commands/copy.py
similarity index 75%
copy from sdk/cli/bin/arv-copy
copy to sdk/python/arvados/commands/copy.py
index 25a8b50..554b00f 100755
--- a/sdk/cli/bin/arv-copy
+++ b/sdk/python/arvados/commands/copy.py
@@ -17,15 +17,14 @@
# arv-copy will issue an error.
import argparse
-import hashlib
import os
-import re
-import string
+import sets
import sys
import logging
import arvados
import arvados.config
+import arvados.keep
def main():
logger = logging.getLogger('arvados.arv-copy')
@@ -46,18 +45,18 @@ def main():
src_arv = api_for_instance(args.source_arvados)
dst_arv = api_for_instance(args.destination_arvados)
- # And now for the copying.
+ # Identify the kind of object we have been given, and begin copying.
t = uuid_type(args.object_uuid)
if t == 'collection':
- copy_collection(args.object_uuid, src=src_arv, dst=dst_arv)
+ result = copy_collection(args.object_uuid, src=src_arv, dst=dst_arv)
elif t == 'pipeline_instance':
- copy_pipeline_instance(args.object_uuid, src=src_arv, dst=dst_arv)
+ result = copy_pipeline_instance(args.object_uuid, src=src_arv, dst=dst_arv)
elif t == 'pipeline_template':
- new_pt = copy_pipeline_template(args.object_uuid, src=src_arv, dst=dst_arv)
- print new_pt
+ result = copy_pipeline_template(args.object_uuid, src=src_arv, dst=dst_arv)
else:
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
+ print result
exit(0)
# Creates an API client for the Arvados instance identified by
@@ -84,7 +83,29 @@ def api_for_instance(instance_name):
return client
def copy_collection(obj_uuid, src=None, dst=None):
- raise NotImplementedError
+ # Fetch the collection's manifest.
+ c = src.collections().get(uuid=obj_uuid).execute()
+ manifest = c['manifest_text']
+
+ # Enumerate the block locators found in the manifest.
+ collection_blocks = sets.Set()
+ src_keep = arvados.keep.KeepClient(src)
+ for line in manifest.splitlines():
+ try:
+ block_hash = line.split()[1]
+ collection_blocks.add(block_hash)
+ except ValueError:
+ abort('bad manifest line in collection {}: {}'.format(obj_uuid, f))
+
+ # Copy each block from src_keep to dst_keep.
+ for locator in collection_blocks:
+ data = src_keep.get(locator)
+ logger.info("Retrieved %d bytes", len(data))
+ dst_keep.put(data)
+
+ # Copy the manifest and save the collection.
+ dst_keep.put(manifest)
+ return dst_keep.collections().create(manifest_text=manifest).execute()
def copy_pipeline_instance(obj_uuid, src=None, dst=None):
raise NotImplementedError
diff --git a/sdk/python/bin/arv-copy b/sdk/python/bin/arv-copy
new file mode 100755
index 0000000..2de0f0c
--- /dev/null
+++ b/sdk/python/bin/arv-copy
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from arvados.commands.copy import main
+main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list