[ARVADOS] updated: 8ffcc8cc6847919afbb16944012e4ada3eff5419
Git user
git at public.curoverse.com
Tue Jun 28 13:25:19 EDT 2016
Summary of changes:
crunch_scripts/cwl-runner | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 10 +++++-----
sdk/cwl/arvados_cwl/arvtool.py | 5 ++++-
sdk/cwl/tests/test_container.py | 7 +++++--
sdk/cwl/tests/test_job.py | 5 +++--
5 files changed, 18 insertions(+), 11 deletions(-)
via 8ffcc8cc6847919afbb16944012e4ada3eff5419 (commit)
via 2f1d6ce89425424a57e6ed041b9a7d50a826eba3 (commit)
via a9f1adf0a3e2df296ce0a8c0a1d735b7e5044baa (commit)
from 966acb6ce28cdb4b0d30bdcd238cbbce73deaa87 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8ffcc8cc6847919afbb16944012e4ada3eff5419
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 28 13:25:13 2016 -0400
9442: Incorporate check for correct $(runtime.outdir) into tests.
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index df7d142..a65c02e 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -29,7 +29,8 @@ class TestContainer(unittest.TestCase):
tool = {
"inputs": [],
"outputs": [],
- "baseCommand": "ls"
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}]
}
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, basedir="")
arvtool.formatgraph = None
@@ -52,7 +53,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'container_image': '99999999999999999999999999999993+99',
- 'command': ['ls'],
+ 'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl'
})
@@ -126,6 +127,7 @@ class TestContainer(unittest.TestCase):
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.successCodes = [0]
+ arvjob.outdir = "/var/spool/cwl"
arvjob.done({
"state": "Complete",
@@ -170,6 +172,7 @@ class TestContainer(unittest.TestCase):
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.successCodes = [0]
+ arvjob.outdir = "/var/spool/cwl"
arvjob.done({
"state": "Complete",
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index abaf161..0d3a494 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -23,7 +23,8 @@ class TestJob(unittest.TestCase):
tool = {
"inputs": [],
"outputs": [],
- "baseCommand": "ls"
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}]
}
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="")
arvtool.formatgraph = None
@@ -36,7 +37,7 @@ class TestJob(unittest.TestCase):
'script_parameters': {
'tasks': [{
'task.env': {'TMPDIR': '$(task.tmpdir)'},
- 'command': ['ls']
+ 'command': ['ls', '$(task.outdir)']
}],
},
'script_version': 'master',
commit 2f1d6ce89425424a57e6ed041b9a7d50a826eba3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 28 13:15:20 2016 -0400
9442: Make cwl-runner crunch script backwards compatible (note-6)
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 71ba68d..c786fc1 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -47,7 +47,7 @@ try:
adjustFiles(job_order_object, keeppath)
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()), work_api="jobs")
+ runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
t = load_tool(job_order_object, runner.arvMakeTool)
commit a9f1adf0a3e2df296ce0a8c0a1d735b7e5044baa
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 28 13:15:02 2016 -0400
9442: Fix runtime.outdir for containers.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 94a7579..9bf93e7 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -29,14 +29,14 @@ class ArvadosContainer(object):
"command": self.command_line,
"owner_uuid": self.arvrunner.project_uuid,
"name": self.name,
- "output_path": "/var/spool/cwl",
- "cwd": "/var/spool/cwl",
+ "output_path": self.outdir,
+ "cwd": self.outdir,
"priority": 1,
"state": "Committed"
}
runtime_constraints = {}
mounts = {
- "/var/spool/cwl": {
+ self.outdir: {
"kind": "tmp"
}
}
@@ -60,7 +60,7 @@ class ArvadosContainer(object):
if self.stdout:
mounts["stdout"] = {"kind": "file",
- "path": "/var/spool/cwl/%s" % (self.stdout)}
+ "path": "%s/%s" % (self.outdir, self.stdout)}
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if not docker_req:
@@ -114,7 +114,7 @@ class ArvadosContainer(object):
try:
outputs = {}
if record["output"]:
- outputs = done.done(self, record, "/tmp", "/var/spool/cwl", "/keep")
+ outputs = done.done(self, record, "/tmp", self.outdir, "/keep")
except WorkflowException as e:
logger.error("Error while collecting container outputs:\n%s", e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 7347eac..a2dffa6 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -30,5 +30,8 @@ class ArvadosCommandTool(CommandLineTool):
**kwargs)
def job(self, joborder, output_callback, **kwargs):
- kwargs["outdir"] = "$(task.outdir)"
+ if self.work_api == "containers":
+ kwargs["outdir"] = "/var/spool/cwl"
+ elif self.work_api == "jobs":
+ kwargs["outdir"] = "$(task.outdir)"
return super(ArvadosCommandTool, self).job(joborder, output_callback, **kwargs)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list