[ARVADOS] updated: ae61fc4e1e7c1cbd4695bb3cc715eba5492c2b20
Git user
git at public.curoverse.com
Fri Dec 9 12:02:14 EST 2016
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 13 ++++++++-----
sdk/cwl/arvados_cwl/runner.py | 2 +-
sdk/cwl/tests/test_submit.py | 37 +++++++++++++++++++++----------------
3 files changed, 30 insertions(+), 22 deletions(-)
via ae61fc4e1e7c1cbd4695bb3cc715eba5492c2b20 (commit)
from 0479c53c53a4a458b2759b067ae2cb928cbbf051 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ae61fc4e1e7c1cbd4695bb3cc715eba5492c2b20
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Dec 9 12:01:52 2016 -0500
10576: Set template_uuid when creating container request from workflow. Set
default container request name from workflow name.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index eb0afc1..2f31602 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -199,7 +199,8 @@ class RunnerContainer(Runner):
"vcpus": 1,
"ram": 1024*1024 * self.submit_runner_ram,
"API": True
- }
+ },
+ "properties": {}
}
workflowcollection = workflowmapper.mapper(self.tool.tool["id"])[1]
@@ -213,14 +214,16 @@ class RunnerContainer(Runner):
}
elif workflowcollection.startswith("arvwf:"):
workflowpath = "/var/lib/cwl/workflow.json#main"
- fetcher = CollectionFetcher({}, None,
- api_client=self.arvrunner.api,
- keep_client=self.arvrunner.keep_client)
- wfobj = yaml.safe_load(fetcher.fetch_text(workflowcollection))
+ wfuuid = workflowcollection[6:workflowcollection.index("#")]
+ wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute()
+ wfobj = yaml.safe_load(wfrecord["definition"])
+ if container_req["name"].startswith("arvwf:"):
+ container_req["name"] = wfrecord["name"]
container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
"kind": "json",
"json": wfobj
}
+ container_req["properties"]["template_uuid"] = wfuuid
command = ["arvados-cwl-runner", "--local", "--api=containers"]
if self.output_name:
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 490695a..1152b7a 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -184,7 +184,7 @@ class Runner(object):
pass
def arvados_job_spec(self, *args, **kwargs):
- self.name = os.path.basename(self.tool.tool["id"])
+ self.name = self.tool.tool.get("label") or os.path.basename(self.tool.tool["id"])
# Need to filter this out, gets added by cwltool when providing
# parameters on the command line.
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8381548..f082c71 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -202,7 +202,8 @@ def stubs(func):
'API': True,
'vcpus': 1,
'ram': 1024*1024*1024
- }
+ },
+ "properties": {}
}
stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
@@ -278,7 +279,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
- body=expect_pipeline)
+ body=JsonDiffMatcher(expect_pipeline))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_pipeline_uuid + '\n')
@@ -297,7 +298,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
- body=expect_pipeline)
+ body=JsonDiffMatcher(expect_pipeline))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_pipeline_uuid + '\n')
@@ -328,7 +329,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
- body=expect_pipeline)
+ body=JsonDiffMatcher(expect_pipeline))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_pipeline_uuid + '\n')
@@ -348,7 +349,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
- body=expect_pipeline)
+ body=JsonDiffMatcher(expect_pipeline))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_pipeline_uuid + '\n')
@@ -367,7 +368,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
expect_pipeline["owner_uuid"] = project_uuid
stubs.api.pipeline_instances().create.assert_called_with(
- body=expect_pipeline)
+ body=JsonDiffMatcher(expect_pipeline))
@stubs
def test_submit_container(self, stubs):
@@ -408,7 +409,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -428,7 +429,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -450,7 +451,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -472,7 +473,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -492,7 +493,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -555,11 +556,12 @@ class TestSubmit(unittest.TestCase):
'API': True,
'vcpus': 1,
'ram': 1073741824
- }
+ },
+ "properties": {}
}
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
@@ -570,7 +572,7 @@ class TestSubmit(unittest.TestCase):
capture_stdout = cStringIO.StringIO()
with open("tests/wf/expect_arvworkflow.cwl") as f:
- stubs.api.workflows().get().execute.return_value = {"definition": f.read()}
+ stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -634,7 +636,7 @@ class TestSubmit(unittest.TestCase):
}, 'state': 'Committed',
'owner_uuid': None,
'output_path': '/var/spool/cwl',
- 'name': 'arvwf:962eh-7fd4e-gkbzl62qqtfig37#main',
+ 'name': 'a test workflow',
'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
@@ -642,11 +644,14 @@ class TestSubmit(unittest.TestCase):
'API': True,
'vcpus': 1,
'ram': 1073741824
+ },
+ "properties": {
+ "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
}
}
stubs.api.container_requests().create.assert_called_with(
- body=expect_container)
+ body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list