[ARVADOS] updated: 565e1cc82ed0af5a4d58ea78d6e0294afdd1cba8
Git user
git at public.curoverse.com
Fri Jan 13 14:24:39 EST 2017
Summary of changes:
doc/user/topics/arv-copy.html.textile.liquid | 23 ++++++++++
sdk/python/arvados/commands/arv_copy.py | 64 +++++++++++++++++++++++++++-
sdk/python/setup.py | 3 +-
3 files changed, 88 insertions(+), 2 deletions(-)
via 565e1cc82ed0af5a4d58ea78d6e0294afdd1cba8 (commit)
via d28ddb0442c1d46fe443521a212faf9ccbbe4066 (commit)
via b8b7bec03f7e13f5703d37560d1019fbd3112bac (commit)
via 2375573bd5a7f909433435c607803df8d2b7bb38 (commit)
from 554fe927169e928d91c2d8c4bed158aef4d4d746 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 565e1cc82ed0af5a4d58ea78d6e0294afdd1cba8
Merge: 554fe92 d28ddb0
Author: radhika <radhika at curoverse.com>
Date: Thu Jan 12 20:59:34 2017 -0500
closes #10110
Merge branch '10110-arv-copy-workflow'
commit d28ddb0442c1d46fe443521a212faf9ccbbe4066
Author: radhika <radhika at curoverse.com>
Date: Thu Jan 12 20:33:06 2017 -0500
10110: add ruamel.yaml to setup.py
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 76ff1c1..abb1a6b 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -88,7 +88,7 @@ We will use the uuid @zzzzz-7fd4e-sampleworkflow1@ as an example workflow.
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial zzzzz-7fd4e-sampleworkflow1</span>
-zzzzz-4zz18-jidprdejysravcr: 1143M / 1143M 100.0%
+zzzzz-4zz18-jidprdejysravcr: 1143M / 1143M 100.0%
2017-01-04 04:11:58 arvados.arv-copy[5906] INFO:
2017-01-04 04:11:58 arvados.arv-copy[5906] INFO: Success: created copy with uuid dst_cluster-7fd4e-ojtgpne594ubkt7
</code></pre>
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 1af6aa7..5c51928 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -113,7 +113,7 @@ def main():
copy_opts.set_defaults(recursive=True)
parser = argparse.ArgumentParser(
- description='Copy a pipeline instance, template or collection from one Arvados instance to another.',
+ description='Copy a pipeline instance, template, workflow, or collection from one Arvados instance to another.',
parents=[copy_opts, arv_cmd.retry_opt])
args = parser.parse_args()
@@ -435,7 +435,7 @@ def copy_workflow(wf_uuid, src, dst, args):
if graph is not None:
workflow_collections(graph, locations, docker_images)
else:
- workflow_collections(graph, locations, docker_images)
+ workflow_collections(wf_def, locations, docker_images)
if locations:
copy_collections(locations, src, dst, args)
@@ -455,9 +455,7 @@ def workflow_collections(obj, locations, docker_images):
if loc.startswith("keep:"):
locations.append(loc[5:])
- docker_image = obj.get('dockerImageId', None)
- if docker_image is None:
- docker_image = obj.get('dockerPull', None)
+ docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None)
if docker_image is not None:
ds = docker_image.split(":", 1)
tag = ds[1] if len(ds)==2 else 'latest'
@@ -465,8 +463,7 @@ def workflow_collections(obj, locations, docker_images):
for x in obj:
workflow_collections(obj[x], locations, docker_images)
-
- if isinstance(obj, list):
+ elif isinstance(obj, list):
for x in obj:
workflow_collections(x, locations, docker_images)
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index d470ab4..0dc59e5 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -51,7 +51,8 @@ setup(name='arvados-python-client',
'pycurl >=7.19.5.1, <7.21.5',
'python-gflags<3.0',
'setuptools',
- 'ws4py'
+ 'ws4py',
+ 'ruamel.yaml==0.13.7'
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
commit b8b7bec03f7e13f5703d37560d1019fbd3112bac
Author: radhika <radhika at curoverse.com>
Date: Wed Jan 4 07:50:20 2017 -0500
10110: also copy any docker images during arv-copy of a workflow and update documentation.
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 223f2fe..76ff1c1 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -81,3 +81,26 @@ For example, we can copy the same object using this tag.
<pre><code>~$ <span class="userinput">arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive qr1hi-p5p6p-9pkaxt6qjnkxhhu</span>
</code></pre>
</notextile>
+
+h3. How to copy a workflow
+
+We will use the uuid @zzzzz-7fd4e-sampleworkflow1@ as an example workflow.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial zzzzz-7fd4e-sampleworkflow1</span>
+zzzzz-4zz18-jidprdejysravcr: 1143M / 1143M 100.0%
+2017-01-04 04:11:58 arvados.arv-copy[5906] INFO:
+2017-01-04 04:11:58 arvados.arv-copy[5906] INFO: Success: created copy with uuid dst_cluster-7fd4e-ojtgpne594ubkt7
+</code></pre>
+</notextile>
+
+The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *locations* and *docker images* found in the src workflow definition will also be copied to the destination recursively.
+
+If you would like to copy the object without dependencies, you can use the @--no-recursive@ flag.
+
+For example, we can copy the same object non-recursively using the following:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive zzzzz-7fd4e-sampleworkflow1</span>
+</code></pre>
+</notextile>
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 71820b3..1af6aa7 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -425,33 +425,50 @@ def copy_workflow(wf_uuid, src, dst, args):
# fetch the workflow from the source instance
wf = src.workflows().get(uuid=wf_uuid).execute(num_retries=args.retries)
+ # copy collections and docker images
if args.recursive:
wf_def = yaml.safe_load(wf["definition"])
if wf_def is not None:
- colls = []
+ locations = []
+ docker_images = {}
graph = wf_def.get('$graph', None)
if graph is not None:
- workflow_collections(graph, colls)
+ workflow_collections(graph, locations, docker_images)
else:
- workflow_collections(wf_def, colls)
- copy_collections(colls, src, dst, args)
+ workflow_collections(graph, locations, docker_images)
+ if locations:
+ copy_collections(locations, src, dst, args)
+
+ for image in docker_images:
+ copy_docker_image(image, docker_images[image], src, dst, args)
+
+ # copy the workflow itself
del wf['uuid']
wf['owner_uuid'] = args.project_uuid
-
return dst.workflows().create(body=wf).execute(num_retries=args.retries)
-def workflow_collections(obj, colls):
+def workflow_collections(obj, locations, docker_images):
if isinstance(obj, dict):
loc = obj.get('location', None)
if loc is not None:
if loc.startswith("keep:"):
- colls.append(loc[5:])
+ locations.append(loc[5:])
+
+ docker_image = obj.get('dockerImageId', None)
+ if docker_image is None:
+ docker_image = obj.get('dockerPull', None)
+ if docker_image is not None:
+ ds = docker_image.split(":", 1)
+ tag = ds[1] if len(ds)==2 else 'latest'
+ docker_images[ds[0]] = tag
+
for x in obj:
- workflow_collections(obj[x], colls)
+ workflow_collections(obj[x], locations, docker_images)
+
if isinstance(obj, list):
for x in obj:
- workflow_collections(x, colls)
+ workflow_collections(x, locations, docker_images)
# copy_collections(obj, src, dst, args)
#
commit 2375573bd5a7f909433435c607803df8d2b7bb38
Author: radhika <radhika at curoverse.com>
Date: Wed Jan 4 04:20:01 2017 -0500
10110: arv-copy workflows
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 1f72635..71820b3 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -33,6 +33,7 @@ import arvados.keep
import arvados.util
import arvados.commands._util as arv_cmd
import arvados.commands.keepdocker
+import ruamel.yaml as yaml
from arvados.api import OrderedJsonModel
from arvados._version import __version__
@@ -144,6 +145,9 @@ def main():
set_src_owner_uuid(src_arv.pipeline_templates(), args.object_uuid, args)
result = copy_pipeline_template(args.object_uuid,
src_arv, dst_arv, args)
+ elif t == 'Workflow':
+ set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args)
+ result = copy_workflow(args.object_uuid, src_arv, dst_arv, args)
else:
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
@@ -405,6 +409,50 @@ def copy_pipeline_template(pt_uuid, src, dst, args):
return dst.pipeline_templates().create(body=pt, ensure_unique_name=True).execute(num_retries=args.retries)
+# copy_workflow(wf_uuid, src, dst, args)
+#
+# Copies a workflow identified by wf_uuid from src to dst.
+#
+# If args.recursive is True, also copy any collections
+# referenced in the workflow definition yaml.
+#
+# The owner_uuid of the new workflow is set to any given
+# project_uuid or the user who copied the template.
+#
+# Returns the copied workflow object.
+#
+def copy_workflow(wf_uuid, src, dst, args):
+ # fetch the workflow from the source instance
+ wf = src.workflows().get(uuid=wf_uuid).execute(num_retries=args.retries)
+
+ if args.recursive:
+ wf_def = yaml.safe_load(wf["definition"])
+ if wf_def is not None:
+ colls = []
+ graph = wf_def.get('$graph', None)
+ if graph is not None:
+ workflow_collections(graph, colls)
+ else:
+ workflow_collections(wf_def, colls)
+ copy_collections(colls, src, dst, args)
+
+ del wf['uuid']
+ wf['owner_uuid'] = args.project_uuid
+
+ return dst.workflows().create(body=wf).execute(num_retries=args.retries)
+
+def workflow_collections(obj, colls):
+ if isinstance(obj, dict):
+ loc = obj.get('location', None)
+ if loc is not None:
+ if loc.startswith("keep:"):
+ colls.append(loc[5:])
+ for x in obj:
+ workflow_collections(obj[x], colls)
+ if isinstance(obj, list):
+ for x in obj:
+ workflow_collections(x, colls)
+
# copy_collections(obj, src, dst, args)
#
# Recursively copies all collections referenced by 'obj' from src
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list