[ARVADOS] updated: 93442a7ff68174ac5f02a2d7455e1572dc40dcb2
Git user
git at public.curoverse.com
Thu Jan 19 18:42:49 EST 2017
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 12 ++++++++++--
sdk/cwl/arvados_cwl/arvcontainer.py | 4 ++--
sdk/cwl/arvados_cwl/arvjob.py | 4 ++--
sdk/cwl/arvados_cwl/runner.py | 6 +++---
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_job.py | 12 ++++++------
sdk/cwl/tests/test_submit.py | 2 +-
7 files changed, 25 insertions(+), 17 deletions(-)
via 93442a7ff68174ac5f02a2d7455e1572dc40dcb2 (commit)
via 235383b85d1ae657bafbc1a65a78598467ac87ba (commit)
from d961917a27d29ef177770ec757c243431065ad0b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 93442a7ff68174ac5f02a2d7455e1572dc40dcb2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 19 18:42:44 2017 -0500
10812: Bump cwltool version
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 39ff152..c3a5bcd 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -48,7 +48,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170119182607',
+ 'cwltool==1.0.20170119234115',
'schema-salad==2.2.20170119151016',
'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20170112173420',
commit 235383b85d1ae657bafbc1a65a78598467ac87ba
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 19 17:36:08 2017 -0500
10812: Add --submit-runner-image to override default Docker image when submitting workflow runner.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index a0c9d57..93a899c 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -416,13 +416,17 @@ class ArvCwlRunner(object):
self.output_name,
self.output_tags,
submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs["name"], on_error=kwargs.get("on_error"))
+ name=kwargs["name"],
+ on_error=kwargs.get("on_error"),
+ submit_runner_image=kwargs.get("submit_runner_image"))
elif self.work_api == "jobs":
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
self.output_name,
self.output_tags,
submit_runner_ram=kwargs.get("submit_runner_ram"),
- name=kwargs["name"], on_error=kwargs.get("on_error"))
+ name=kwargs["name"],
+ on_error=kwargs.get("on_error"),
+ submit_runner_image=kwargs.get("submit_runner_image"))
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and self.work_api == "jobs":
# Create pipeline for local run
@@ -605,6 +609,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="RAM (in MiB) required for the workflow runner job (default 1024)",
default=1024)
+ parser.add_argument("--submit-runner-image", type=str,
+ help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
+ default=None)
+
parser.add_argument("--name", type=str,
help="Name to use for workflow execution instance.",
default=None)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 6015c1d..235e9b8 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -84,7 +84,7 @@ class ArvadosContainer(object):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if not docker_req:
- docker_req = {"dockerImageId": arvados_jobs_image(self.arvrunner)}
+ docker_req = {"dockerImageId": "arvados/jobs"}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
@@ -199,7 +199,7 @@ class RunnerContainer(Runner):
"cwd": "/var/spool/cwl",
"priority": 1,
"state": "Committed",
- "container_image": arvados_jobs_image(self.arvrunner),
+ "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
"mounts": {
"/var/lib/cwl/cwl.input.json": {
"kind": "json",
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index bf1052c..7b31802 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -97,7 +97,7 @@ class ArvadosJob(object):
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
- runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
+ runtime_constraints["docker_image"] = "arvados/jobs"
resources = self.builder.resources
if resources is not None:
@@ -295,7 +295,7 @@ class RunnerJob(Runner):
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": arvados_jobs_image(self.arvrunner),
+ "docker_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
"min_ram_mb_per_node": self.submit_runner_ram
}
}
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 53a4a6c..d3e0a0e 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -206,10 +206,9 @@ def upload_workflow_deps(arvrunner, tool):
tool.visit(upload_tool_deps)
-def arvados_jobs_image(arvrunner):
+def arvados_jobs_image(arvrunner, img):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
- img = "arvados/jobs:"+__version__
try:
arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
@@ -222,7 +221,7 @@ class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
- name=None, on_error=None):
+ name=None, on_error=None, submit_runner_image=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -234,6 +233,7 @@ class Runner(object):
self.output_tags = output_tags
self.name = name
self.on_error = on_error
+ self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 7675e3d..8aafb4a 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -62,7 +62,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_cores_per_node': 1,
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
@@ -72,7 +72,7 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
+ ['docker_image_locator', 'in docker', 'arvados/jobs']]
)
# The test passes some fields in builder.resources
@@ -130,7 +130,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_cores_per_node': 3,
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
@@ -141,7 +141,7 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
+ ['docker_image_locator', 'in docker', 'arvados/jobs']])
@mock.patch("arvados.collection.CollectionReader")
def test_done(self, reader):
@@ -295,14 +295,14 @@ class TestWorkflow(unittest.TestCase):
'runtime_constraints': {
'min_scratch_mb_per_node': 2048,
'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': 'arvados/jobs',
'min_ram_mb_per_node': 1024
},
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+ ['docker_image_locator', 'in docker', 'arvados/jobs']],
find_or_create=True)
mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8b3db8f..f1dd119 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -808,7 +808,7 @@ class TestSubmit(unittest.TestCase):
}], "items_available": 1, "offset": 0},
{"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
+ self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list