[ARVADOS] created: 960e626794f8ea94759e5ef4d040df4f7eb68e4b
Git user
git at public.curoverse.com
Fri Apr 22 11:20:54 EDT 2016
at 960e626794f8ea94759e5ef4d040df4f7eb68e4b (commit)
commit 960e626794f8ea94759e5ef4d040df4f7eb68e4b
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 15:08:01 2016 -0400
8653: Add arvados-cwl-runner --create-template flag
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8f2102c..d714a69 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -322,7 +322,13 @@ class RunnerJob(object):
for s in tool.steps:
self.upload_docker(s.embedded_tool)
- def run(self, dry_run=False, pull_image=True, **kwargs):
+ def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs):
+ """Create an Arvados job specification for this workflow.
+
+ The returned dict can be used to create a job (i.e., passed as
+ the +body+ argument to jobs().create()), or as a component in
+ a pipeline template or pipeline instance.
+ """
self.upload_docker(self.tool)
workflowfiles = set()
@@ -364,9 +370,7 @@ class RunnerJob(object):
del self.job_order["id"]
self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"])[1]
-
- response = self.arvrunner.api.jobs().create(body={
- "owner_uuid": self.arvrunner.project_uuid,
+ return {
"script": "cwl-runner",
"script_version": "master",
"repository": "arvados",
@@ -374,7 +378,16 @@ class RunnerJob(object):
"runtime_constraints": {
"docker_image": "arvados/jobs"
}
- }, find_or_create=self.enable_reuse).execute(num_retries=self.arvrunner.num_retries)
+ }
+
+ def run(self, *args, **kwargs):
+ job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+ response = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
self.arvrunner.jobs[response["uuid"]] = self
@@ -401,6 +414,31 @@ class RunnerJob(object):
finally:
del self.arvrunner.jobs[record["uuid"]]
+
+class RunnerTemplate(object):
+ """An Arvados pipeline template that invokes a CWL workflow."""
+
+ def __init__(self, runner, tool, job_order, enable_reuse):
+ self.runner = runner
+ self.job = RunnerJob(
+ runner=runner,
+ tool=tool,
+ job_order=job_order,
+ enable_reuse=enable_reuse)
+
+ def save(self):
+ job_spec = self.job.arvados_job_spec()
+ response = self.runner.api.pipeline_templates().create(body={
+ "components": {
+ self.job.name: job_spec,
+ },
+ "name": self.job.name,
+ "owner_uuid": self.runner.project_uuid,
+ }).execute(num_retries=self.runner.num_retries)
+ self.uuid = response["uuid"]
+ logger.info("Created template %s", self.uuid)
+
+
class ArvPathMapper(cwltool.pathmapper.PathMapper):
"""Convert container-local paths to and from Keep collection ids."""
@@ -502,7 +540,6 @@ class ArvCwlRunner(object):
body={"state": "Failed"}).execute(num_retries=self.num_retries)
self.final_output = out
-
def on_message(self, event):
if "object_uuid" in event:
if event["object_uuid"] in self.jobs and event["event_type"] == "update":
@@ -541,6 +578,13 @@ class ArvCwlRunner(object):
self.project_uuid = args.project_uuid if args.project_uuid else useruuid
self.pipeline = None
+ if args.create_template:
+ tmpl = RunnerTemplate(self, tool, job_order, args.enable_reuse)
+ tmpl.save()
+ # cwltool.main will print our return value (quoted as
+ # JSON, without a newline) on stdout.
+ return tmpl.uuid
+
if args.submit:
runnerjob = RunnerJob(self, tool, job_order, args.enable_reuse)
if not args.wait:
@@ -655,6 +699,7 @@ def main(args, stdout, stderr, api_client=None):
default=True, dest="submit")
exgroup.add_argument("--local", action="store_false", help="Run workflow on local host (submits jobs to Arvados).",
default=True, dest="submit")
+ exgroup.add_argument("--create-template", action="store_true", help="Create an Arvados pipeline template.")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--wait", action="store_true", help="After submitting workflow runner job, wait for completion.",
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 5734821..750eeae 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -2,12 +2,16 @@ import arvados
import arvados.keep
import arvados.collection
import arvados_cwl
+import copy
+import cStringIO
import functools
+import json
import hashlib
import mock
import sys
import unittest
+
def stubs(func):
@functools.wraps(func)
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@@ -29,7 +33,9 @@ def stubs(func):
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
stubs.api = mock.MagicMock()
- stubs.api.users().current().execute.return_value = {"uuid": stubs.fake_user_uuid}
+ stubs.api.users().current().execute.return_value = {
+ "uuid": stubs.fake_user_uuid,
+ }
stubs.api.collections().list().execute.return_value = {"items": []}
stubs.api.collections().create().execute.side_effect = ({
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
@@ -42,8 +48,11 @@ def stubs(func):
"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
"state": "Queued",
}
+ stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
+ stubs.api.pipeline_templates().create().execute.return_value = {
+ "uuid": stubs.expect_pipeline_template_uuid,
+ }
stubs.expect_job_spec = {
- 'owner_uuid': stubs.fake_user_uuid,
'runtime_constraints': {
'docker_image': 'arvados/jobs'
},
@@ -52,7 +61,8 @@ def stubs(func):
'path': '99999999999999999999999999999992+99/blorp.txt',
'class': 'File'
},
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ 'cwl:tool':
+ '99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
'script_version': 'master',
@@ -66,27 +76,33 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit(self, stubs):
arvados_cwl.main(
- ["--debug", "--submit", "--no-wait",
+ ["--submit", "--no-wait",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
sys.stdout, sys.stderr, api_client=stubs.api)
stubs.api.collections().create.assert_has_calls([
mock.call(),
mock.call(body={
- 'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
+ 'manifest_text':
+ './tool 84ec4df683711de31b782505389a8843+429 '
+ '0:16:blub.txt 16:413:submit_tool.cwl\n./wf '
+ '81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
mock.call().execute(),
mock.call(body={
- 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
+ 'manifest_text':
+ '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': '#',
}, ensure_unique_name=True),
mock.call().execute()])
+ expect_job = copy.deepcopy(stubs.expect_job_spec)
+ expect_job["owner_uuid"] = stubs.fake_user_uuid
stubs.api.jobs().create.assert_called_with(
- body=stubs.expect_job_spec,
+ body=expect_job,
find_or_create=True)
@stubs
@@ -94,13 +110,45 @@ class TestSubmit(unittest.TestCase):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
arvados_cwl.main(
- ["--debug", "--submit", "--no-wait",
+ ["--submit", "--no-wait",
"--project-uuid", project_uuid,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
sys.stdout, sys.stderr, api_client=stubs.api)
- expect_body = stubs.expect_job_spec.copy()
+ expect_body = copy.deepcopy(stubs.expect_job_spec)
expect_body["owner_uuid"] = project_uuid
stubs.api.jobs().create.assert_called_with(
body=expect_body,
find_or_create=True)
+
+
+class TestCreateTemplate(unittest.TestCase):
+ @stubs
+ def test_create_pipeline_template(self, stubs):
+ project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+
+ capture_stdout = cStringIO.StringIO()
+
+ exited = arvados_cwl.main(
+ ["--create-template", "--no-wait",
+ "--project-uuid", project_uuid,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ stubs.api.pipeline_instances().create.refute_called()
+ stubs.api.jobs().create.refute_called()
+
+ expect_component = stubs.expect_job_spec
+ expect_template = {
+ "components": {
+ "submit_wf.cwl": expect_component,
+ },
+ "name": "submit_wf.cwl",
+ "owner_uuid": project_uuid,
+ }
+ stubs.api.pipeline_templates().create.assert_called_with(
+ body=expect_template)
+
+ self.assertEqual(capture_stdout.getvalue(),
+ json.dumps(stubs.expect_pipeline_template_uuid))
commit aa63face41bcc2028350249a3083eb2bc10896c1
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 15:15:56 2016 -0400
8653: DRY testing code.
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 19745a0..5734821 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1,103 +1,106 @@
-import unittest
-import mock
-import arvados_cwl
-import sys
import arvados
import arvados.keep
import arvados.collection
+import arvados_cwl
+import functools
import hashlib
+import mock
+import sys
+import unittest
-class TestSubmit(unittest.TestCase):
+def stubs(func):
+ @functools.wraps(func)
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@mock.patch("arvados.collection.KeepClient")
@mock.patch("arvados.events.subscribe")
- def test_submit(self, events, keep, keepdocker):
- api = mock.MagicMock()
+ def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
+ class Stubs:
+ pass
+ stubs = Stubs()
+ stubs.events = events
+ stubs.KeepClient = KeepClient
+ stubs.keepdocker = keepdocker
+
def putstub(p, **kwargs):
return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- keep().put.side_effect = putstub
- keepdocker.return_value = True
- user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
- api.users().current().execute.return_value = {"uuid": user_uuid}
- api.collections().list().execute.return_value = {"items": []}
- api.collections().create().execute.side_effect = ({"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
- "portable_data_hash": "99999999999999999999999999999991+99"},
- {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
- "portable_data_hash": "99999999999999999999999999999992+99"})
- api.jobs().create().execute.return_value = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "state": "Queued"}
+ stubs.KeepClient().put.side_effect = putstub
+
+ stubs.keepdocker.return_value = True
+ stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+
+ stubs.api = mock.MagicMock()
+ stubs.api.users().current().execute.return_value = {"uuid": stubs.fake_user_uuid}
+ stubs.api.collections().list().execute.return_value = {"items": []}
+ stubs.api.collections().create().execute.side_effect = ({
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+ "portable_data_hash": "99999999999999999999999999999991+99",
+ }, {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "portable_data_hash": "99999999999999999999999999999992+99",
+ })
+ stubs.api.jobs().create().execute.return_value = {
+ "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "state": "Queued",
+ }
+ stubs.expect_job_spec = {
+ 'owner_uuid': stubs.fake_user_uuid,
+ 'runtime_constraints': {
+ 'docker_image': 'arvados/jobs'
+ },
+ 'script_parameters': {
+ 'x': {
+ 'path': '99999999999999999999999999999992+99/blorp.txt',
+ 'class': 'File'
+ },
+ 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ },
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'cwl-runner'
+ }
+ return func(self, stubs, *args, **kwargs)
+ return wrapped
+
- arvados_cwl.main(["--debug", "--submit", "--no-wait", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdout, sys.stderr, api_client=api)
+class TestSubmit(unittest.TestCase):
+ @stubs
+ def test_submit(self, stubs):
+ arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdout, sys.stderr, api_client=stubs.api)
- api.collections().create.assert_has_calls([
+ stubs.api.collections().create.assert_has_calls([
mock.call(),
- mock.call(body={'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
- 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'name': 'submit_wf.cwl'
- }, ensure_unique_name=True),
+ mock.call(body={
+ 'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'submit_wf.cwl',
+ }, ensure_unique_name=True),
mock.call().execute(),
- mock.call(body={'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'name': '#'
- }, ensure_unique_name=True),
+ mock.call(body={
+ 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': '#',
+ }, ensure_unique_name=True),
mock.call().execute()])
- api.jobs().create.assert_called_with(
- body={
- 'owner_uuid': user_uuid,
- 'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
- },
- 'script_parameters': {
- 'x': {
- 'path': '99999999999999999999999999999992+99/blorp.txt',
- 'class': 'File'
- },
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
- },
+ stubs.api.jobs().create.assert_called_with(
+ body=stubs.expect_job_spec,
find_or_create=True)
- @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
- @mock.patch("arvados.collection.KeepClient")
- @mock.patch("arvados.events.subscribe")
- def test_submit_with_project_uuid(self, events, keep, keepdocker):
- api = mock.MagicMock()
- def putstub(p, **kwargs):
- return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- keep().put.side_effect = putstub
- keepdocker.return_value = True
- api.users().current().execute.return_value = {"uuid": "zzzzz-tpzed-zzzzzzzzzzzzzzz"}
- api.collections().list().execute.return_value = {"items": []}
- api.collections().create().execute.side_effect = ({"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
- "portable_data_hash": "99999999999999999999999999999991+99"},
- {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
- "portable_data_hash": "99999999999999999999999999999992+99"})
- api.jobs().create().execute.return_value = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "state": "Queued"}
+ @stubs
+ def test_submit_with_project_uuid(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- arvados_cwl.main(["--debug", "--submit", "--project-uuid", project_uuid,
- "--no-wait", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdout, sys.stderr, api_client=api)
+ arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait",
+ "--project-uuid", project_uuid,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdout, sys.stderr, api_client=stubs.api)
- api.jobs().create.assert_called_with(
- body={
- 'owner_uuid': project_uuid,
- 'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
- },
- 'script_parameters': {
- 'x': {
- 'path': '99999999999999999999999999999992+99/blorp.txt',
- 'class': 'File'
- },
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
- },
+ expect_body = stubs.expect_job_spec.copy()
+ expect_body["owner_uuid"] = project_uuid
+ stubs.api.jobs().create.assert_called_with(
+ body=expect_body,
find_or_create=True)
commit 91ac0186b3f1c1bcebf5d9b4123fe6683ef7dc43
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 10:55:18 2016 -0400
8653: Turn off debug messages / verbose logging in test suite.
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index e1ef605..e74e000 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -1,6 +1,13 @@
-import unittest
-import mock
import arvados_cwl
+import logging
+import mock
+import unittest
+import os
+
+if not os.getenv('ARVADOS_DEBUG'):
+ logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
+ logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
+
class TestJob(unittest.TestCase):
commit 3d43e0fbd074089f78568d65e332f75bfecdca81
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 10:54:11 2016 -0400
8653: Fix whitespace.
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index bb15e60..e1ef605 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -28,9 +28,9 @@ class TestJob(unittest.TestCase):
'tasks': [{
'task.env': {'TMPDIR': '$(task.tmpdir)'},
'command': ['ls']
- }]
- },
- 'script_version': 'master',
+ }],
+ },
+ 'script_version': 'master',
'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
'repository': 'arvados',
'script': 'crunchrunner',
@@ -40,8 +40,8 @@ class TestJob(unittest.TestCase):
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
}
- },
- find_or_create=True,
+ },
+ find_or_create=True,
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
@@ -90,7 +90,7 @@ class TestJob(unittest.TestCase):
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
}
- },
+ },
find_or_create=True,
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list