[ARVADOS] created: de4ecbc700759ff22e76948a58f7d70e5d3c1464
Git user
git at public.curoverse.com
Fri Jan 20 09:48:05 EST 2017
at de4ecbc700759ff22e76948a58f7d70e5d3c1464 (commit)
commit de4ecbc700759ff22e76948a58f7d70e5d3c1464
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Jan 20 09:46:50 2017 -0500
10812: Add --submit-runner-image to override default Docker image when submitting workflow runner.
Conflicts:
sdk/cwl/arvados_cwl/__init__.py
sdk/cwl/arvados_cwl/runner.py
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3ffc4c7..5d83300 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -384,13 +384,21 @@ class ArvCwlRunner(object):
self.output_callback,
**kwargs).next()
else:
- runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
- self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs.get("name"), on_error=kwargs.get("on_error"))
- else:
- runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
- self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs.get("name"), on_error=kwargs.get("on_error"))
+ runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"),
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=kwargs.get("submit_runner_ram"),
+ name=kwargs["name"],
+ on_error=kwargs.get("on_error"),
+ submit_runner_image=kwargs.get("submit_runner_image"))
+ elif self.work_api == "jobs":
+ runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=kwargs.get("submit_runner_ram"),
+ name=kwargs["name"],
+ on_error=kwargs.get("on_error"),
+ submit_runner_image=kwargs.get("submit_runner_image"))
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
# Create pipeline for local run
@@ -573,6 +581,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="RAM (in MiB) required for the workflow runner job (default 1024)",
default=1024)
+ parser.add_argument("--submit-runner-image", type=str,
+ help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
+ default=None)
+
parser.add_argument("--name", type=str,
help="Name to use for workflow execution instance.",
default=None)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 987b0d6..a3a26aa 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -84,7 +84,7 @@ class ArvadosContainer(object):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if not docker_req:
- docker_req = {"dockerImageId": arvados_jobs_image(self.arvrunner)}
+ docker_req = {"dockerImageId": "arvados/jobs"}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
@@ -199,7 +199,7 @@ class RunnerContainer(Runner):
"cwd": "/var/spool/cwl",
"priority": 1,
"state": "Committed",
- "container_image": arvados_jobs_image(self.arvrunner),
+ "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
"mounts": {
"/var/lib/cwl/cwl.input.json": {
"kind": "json",
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 11ef653..4bbbda2 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -92,7 +92,7 @@ class ArvadosJob(object):
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
- runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
+ runtime_constraints["docker_image"] = "arvados/jobs"
resources = self.builder.resources
if resources is not None:
@@ -261,7 +261,7 @@ class RunnerJob(Runner):
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": arvados_jobs_image(self.arvrunner),
+ "docker_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
"min_ram_mb_per_node": self.submit_runner_ram
}
}
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 1c3625e..09ca63b 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -156,8 +156,9 @@ def upload_instance(arvrunner, name, tool, job_order):
return workflowmapper
-def arvados_jobs_image(arvrunner):
- img = "arvados/jobs:"+__version__
+def arvados_jobs_image(arvrunner, img):
+ """Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
+
try:
arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
@@ -167,7 +168,7 @@ def arvados_jobs_image(arvrunner):
class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
- name=None, on_error=None):
+ name=None, on_error=None, submit_runner_image=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -179,6 +180,7 @@ class Runner(object):
self.output_tags = output_tags
self.name = name
self.on_error = on_error
+ self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 7675e3d..8aafb4a 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -62,7 +62,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_cores_per_node': 1,
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
@@ -72,7 +72,7 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
+ ['docker_image_locator', 'in docker', 'arvados/jobs']]
)
# The test passes some fields in builder.resources
@@ -130,7 +130,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_cores_per_node': 3,
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
@@ -141,7 +141,7 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
+ ['docker_image_locator', 'in docker', 'arvados/jobs']])
@mock.patch("arvados.collection.CollectionReader")
def test_done(self, reader):
@@ -295,14 +295,14 @@ class TestWorkflow(unittest.TestCase):
'runtime_constraints': {
'min_scratch_mb_per_node': 2048,
'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_ram_mb_per_node': 1024
},
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+ ['docker_image_locator', 'in docker', 'arvados/jobs']],
find_or_create=True)
mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0a01fb4..13cc483 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -793,7 +793,7 @@ class TestSubmit(unittest.TestCase):
}], "items_available": 1, "offset": 0},
{"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
+ self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list