[ARVADOS] updated: 8d83cc78c3d4ae5163d0b69c6169a6782344079f
Git user
git at public.curoverse.com
Wed Oct 12 14:04:16 EDT 2016
Summary of changes:
crunch_scripts/cwl-runner | 96 +--------------------------------
sdk/cwl/arvados_cwl/crunch_script.py | 100 +++++++++++++++++++++++++++++++++++
sdk/cwl/arvados_cwl/pathmapper.py | 8 +--
sdk/cwl/tests/test_pathmapper.py | 5 +-
4 files changed, 107 insertions(+), 102 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/crunch_script.py
via 8d83cc78c3d4ae5163d0b69c6169a6782344079f (commit)
via 0f9a4795fdcd60ea4f02e335618627e4240b6b92 (commit)
from 79fb879f870585c394e15ea8f4d470f7f8665f3f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8d83cc78c3d4ae5163d0b69c6169a6782344079f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Oct 12 12:00:11 2016 -0400
10221: Move contents of cwl-runner crunch script into arvados_cwl package to
avoid version skew.
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 5d7f553..a313805 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -2,98 +2,6 @@
# Crunch script integration for running arvados-cwl-runner (importing
# arvados_cwl module) inside a crunch job.
-#
-# This gets the job record, transforms the script parameters into a valid CWL
-# input object, then executes the CWL runner to run the underlying workflow or
-# tool. When the workflow completes, record the output object in an output
-# collection for this runner job.
-import arvados
-import arvados_cwl
-import arvados.collection
-import arvados.util
-import cwltool.main
-import logging
-import os
-import json
-import argparse
-import re
-import functools
-
-from arvados.api import OrderedJsonModel
-from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
-from cwltool.load_tool import load_tool
-
-# Print package versions
-logging.info(cwltool.main.versionstring())
-
-api = arvados.api("v1")
-
-try:
- job_order_object = arvados.current_job()['script_parameters']
-
- pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
-
- def keeppath(v):
- if pdh_path.match(v):
- return "keep:%s" % v
- else:
- return v
-
- def keeppathObj(v):
- v["location"] = keeppath(v["location"])
-
- job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
-
- for k,v in job_order_object.items():
- if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
- job_order_object[k] = {
- "class": "File",
- "location": "keep:%s" % v
- }
-
- adjustFileObjs(job_order_object, keeppathObj)
- adjustDirObjs(job_order_object, keeppathObj)
- normalizeFilesDirs(job_order_object)
- adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
-
- output_name = None
- if "arv:output_name" in job_order_object:
- output_name = job_order_object["arv:output_name"]
- del job_order_object["arv:output_name"]
-
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
- output_name=output_name)
-
- t = load_tool(job_order_object, runner.arv_make_tool)
-
- args = argparse.Namespace()
- args.project_uuid = arvados.current_job()["owner_uuid"]
- args.enable_reuse = True
- args.submit = False
- args.debug = True
- args.quiet = False
- args.ignore_docker_for_reuse = False
- args.basedir = os.getcwd()
- args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
- outputObj = runner.arv_executor(t, job_order_object, **vars(args))
-
- if runner.final_output_collection:
- outputCollection = runner.final_output_collection.portable_data_hash()
- else:
- outputCollection = None
-
- api.job_tasks().update(uuid=arvados.current_task()['uuid'],
- body={
- 'output': outputCollection,
- 'success': True,
- 'progress':1.0
- }).execute()
-except Exception as e:
- logging.exception("Unhandled exception")
- api.job_tasks().update(uuid=arvados.current_task()['uuid'],
- body={
- 'output': None,
- 'success': False,
- 'progress':1.0
- }).execute()
+import arvados_cwl.crunch_script
+arvados_cwl.crunch_script.run()
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
new file mode 100644
index 0000000..d800bc7
--- /dev/null
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -0,0 +1,100 @@
+# Crunch script integration for running arvados-cwl-runner (importing
+# arvados_cwl module) inside a crunch job.
+#
+# This gets the job record, transforms the script parameters into a valid CWL
+# input object, then executes the CWL runner to run the underlying workflow or
+# tool. When the workflow completes, record the output object in an output
+# collection for this runner job.
+
+import arvados
+import arvados_cwl
+import arvados.collection
+import arvados.util
+import cwltool.main
+import logging
+import os
+import json
+import argparse
+import re
+import functools
+
+from arvados.api import OrderedJsonModel
+from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
+from cwltool.load_tool import load_tool
+
+def run():
+ # Print package versions
+ logging.info(cwltool.main.versionstring())
+
+ api = arvados.api("v1")
+
+ arvados_cwl.add_arv_hints()
+
+ try:
+ job_order_object = arvados.current_job()['script_parameters']
+
+ pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
+ def keeppath(v):
+ if pdh_path.match(v):
+ return "keep:%s" % v
+ else:
+ return v
+
+ def keeppathObj(v):
+ v["location"] = keeppath(v["location"])
+
+ job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
+
+ for k,v in job_order_object.items():
+ if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
+ job_order_object[k] = {
+ "class": "File",
+ "location": "keep:%s" % v
+ }
+
+ adjustFileObjs(job_order_object, keeppathObj)
+ adjustDirObjs(job_order_object, keeppathObj)
+ normalizeFilesDirs(job_order_object)
+ adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
+
+ output_name = None
+ if "arv:output_name" in job_order_object:
+ output_name = job_order_object["arv:output_name"]
+ del job_order_object["arv:output_name"]
+
+ runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
+ output_name=output_name)
+
+ t = load_tool(job_order_object, runner.arv_make_tool)
+
+ args = argparse.Namespace()
+ args.project_uuid = arvados.current_job()["owner_uuid"]
+ args.enable_reuse = True
+ args.submit = False
+ args.debug = True
+ args.quiet = False
+ args.ignore_docker_for_reuse = False
+ args.basedir = os.getcwd()
+ args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
+ outputObj = runner.arv_executor(t, job_order_object, **vars(args))
+
+ if runner.final_output_collection:
+ outputCollection = runner.final_output_collection.portable_data_hash()
+ else:
+ outputCollection = None
+
+ api.job_tasks().update(uuid=arvados.current_task()['uuid'],
+ body={
+ 'output': outputCollection,
+ 'success': True,
+ 'progress':1.0
+ }).execute()
+ except Exception as e:
+ logging.exception("Unhandled exception")
+ api.job_tasks().update(uuid=arvados.current_task()['uuid'],
+ body={
+ 'output': None,
+ 'success': False,
+ 'progress':1.0
+ }).execute()
commit 0f9a4795fdcd60ea4f02e335618627e4240b6b92
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Oct 12 11:51:03 2016 -0400
10221: Don't depend on st.keepref from arvados.commands.run.statfile and uploadfile, use file_pattern to construct predictable keep reference.
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 288397a..73c81ce 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -37,11 +37,11 @@ class ArvPathMapper(PathMapper):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
- st = arvados.commands.run.statfile("", ab, fnPattern=self.file_pattern)
+ st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s")
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt("keep:"+st.keepref, st.fn, "File")
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
elif src.startswith("_:"):
if "contents" in srcobj:
pass
@@ -91,12 +91,12 @@ class ArvPathMapper(PathMapper):
self.arvrunner.api,
dry_run=False,
num_retries=self.arvrunner.num_retries,
- fnPattern=self.file_pattern,
+ fnPattern="keep:%s/%s",
name=self.name,
project=self.arvrunner.project_uuid)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt("keep:" + st.keepref, st.fn, "File")
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
self.arvrunner.add_uploaded(src, self._pathmap[src])
for srcobj in referenced_files:
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index 065904d..7e13066 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -18,9 +18,7 @@ from arvados_cwl.pathmapper import ArvPathMapper
def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
pdh = "99999999999999999999999999999991+99"
for c in files:
- c.fn = os.path.basename(c.fn)
- c.keepref = "%s/%s" % (pdh, c.fn)
- c.fn = fnPattern % (pdh, c.fn)
+ c.fn = fnPattern % (pdh, os.path.basename(c.fn))
class TestPathmap(unittest.TestCase):
def test_keepref(self):
@@ -79,7 +77,6 @@ class TestPathmap(unittest.TestCase):
# keep mount, so we can construct a direct reference directly without upload.
def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
st = arvados.commands.run.ArvFile("", fnPattern % ("99999999999999999999999999999991+99", "hw.py"))
- st.keepref = "99999999999999999999999999999991+99/hw.py"
return st
upl.side_effect = upload_mock
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list