[ARVADOS] updated: 84d86c1721bf549c2dc38df95f29f3579b36a5ae
Git user
git at public.curoverse.com
Tue Oct 4 16:44:08 EDT 2016
Summary of changes:
crunch_scripts/cwl-runner | 8 +++++++-
sdk/cwl/arvados_cwl/__init__.py | 22 ++++++++++++++--------
sdk/cwl/arvados_cwl/arvcontainer.py | 7 ++++++-
sdk/cwl/arvados_cwl/arvjob.py | 5 ++++-
sdk/cwl/arvados_cwl/runner.py | 3 ++-
5 files changed, 33 insertions(+), 12 deletions(-)
via 84d86c1721bf549c2dc38df95f29f3579b36a5ae (commit)
from c7a461e97640697e390aff96bf25126972396b19 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 84d86c1721bf549c2dc38df95f29f3579b36a5ae
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Oct 4 16:24:46 2016 -0400
10165: Add --output-name option to set name of output collection.
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 3f2d53d..5d7f553 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -57,7 +57,13 @@ try:
normalizeFilesDirs(job_order_object)
adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
+ output_name = None
+ if "arv:output_name" in job_order_object:
+ output_name = job_order_object["arv:output_name"]
+ del job_order_object["arv:output_name"]
+
+ runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
+ output_name=output_name)
t = load_tool(job_order_object, runner.arv_make_tool)
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7d5590b..46850b3 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -24,6 +24,7 @@ import arvados.config
from .arvcontainer import ArvadosContainer, RunnerContainer
from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
+from. runner import Runner
from .arvtool import ArvadosCommandTool
from .arvworkflow import ArvadosWorkflow, upload_workflow
from .fsaccess import CollectionFsAccess
@@ -47,7 +48,7 @@ class ArvCwlRunner(object):
"""
- def __init__(self, api_client, work_api=None, keep_client=None):
+ def __init__(self, api_client, work_api=None, keep_client=None, output_name=None):
self.api = api_client
self.processes = {}
self.lock = threading.Lock()
@@ -62,6 +63,7 @@ class ArvCwlRunner(object):
self.poll_api = None
self.pipeline = None
self.final_output_collection = None
+ self.output_name = output_name
if keep_client is not None:
self.keep_client = keep_client
else:
@@ -217,7 +219,9 @@ class ArvCwlRunner(object):
final.save_new(name=name, owner_uuid=self.project_uuid, ensure_unique_name=True)
- logger.info("Final output collection %s (%s)", final.portable_data_hash(), final.manifest_locator())
+ logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
+ final.api_response()["name"],
+ final.manifest_locator())
self.final_output_collection = final
@@ -270,9 +274,9 @@ class ArvCwlRunner(object):
self.output_callback,
**kwargs).next()
else:
- runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"))
+ runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name)
else:
- runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"))
+ runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name)
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
# Create pipeline for local run
@@ -353,11 +357,12 @@ class ArvCwlRunner(object):
if self.final_output is None:
raise WorkflowException("Workflow did not return a result.")
- if kwargs.get("submit"):
+ if kwargs.get("submit") and isinstance(runnerjob, Runner):
logger.info("Final output collection %s", runnerjob.final_output)
else:
- self.make_output_collection("Output of %s" % (shortname(tool.tool["id"])),
- self.final_output)
+ if self.output_name is None:
+ self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
+ self.make_output_collection(self.output_name, self.final_output)
if kwargs.get("compute_checksum"):
adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
@@ -409,6 +414,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="")
parser.add_argument("--project-uuid", type=str, metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
+ parser.add_argument("--output-name", type=str, help="Name to use for collection that stores the final output.", default=None)
parser.add_argument("--ignore-docker-for-reuse", action="store_true",
help="Ignore Docker image version when deciding whether to reuse past jobs.",
default=False)
@@ -466,7 +472,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
try:
if api_client is None:
api_client=arvados.api('v1', model=OrderedJsonModel())
- runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client)
+ runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client, output_name=arvargs.output_name)
except Exception as e:
logger.error(e)
return 1
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index cd8f84e..fa32456 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -179,8 +179,13 @@ class RunnerContainer(Runner):
pull_image,
self.arvrunner.project_uuid)
+ command = ["arvados-cwl-runner", "--local", "--api=containers"]
+ if self.output_name:
+ command.append("--output-name=" + self.output_name)
+ command.extend([workflowpath, jobpath])
+
return {
- "command": ["arvados-cwl-runner", "--local", "--api=containers", workflowpath, jobpath],
+ "command": command,
"owner_uuid": self.arvrunner.project_uuid,
"name": self.name,
"output_path": "/var/spool/cwl",
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index d881082..c7345b0 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -228,6 +228,8 @@ class RunnerJob(Runner):
workflowmapper = super(RunnerJob, self).arvados_job_spec(dry_run=dry_run, pull_image=pull_image, **kwargs)
self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:]
+ if self.output_name:
+ self.job_order["arv:output_name"] = self.output_name
return {
"script": "cwl-runner",
"script_version": "master",
@@ -283,7 +285,8 @@ class RunnerTemplate(object):
runner=runner,
tool=tool,
job_order=job_order,
- enable_reuse=enable_reuse)
+ enable_reuse=enable_reuse,
+ output_name=None)
def pipeline_component_spec(self):
"""Return a component that Workbench and a-r-p-i will understand.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 08e7d02..c52c952 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -114,7 +114,7 @@ def upload_docker(arvrunner, tool):
class Runner(object):
- def __init__(self, runner, tool, job_order, enable_reuse):
+ def __init__(self, runner, tool, job_order, enable_reuse, output_name):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -122,6 +122,7 @@ class Runner(object):
self.enable_reuse = enable_reuse
self.uuid = None
self.final_output = None
+ self.output_name = output_name
def update_pipeline_component(self, record):
pass
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list