[ARVADOS] created: 62469691c9818f668e1341258b9fc355ec25c08e
Git user
git at public.curoverse.com
Wed Sep 27 22:27:50 EDT 2017
at 62469691c9818f668e1341258b9fc355ec25c08e (commit)
commit 62469691c9818f668e1341258b9fc355ec25c08e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Sep 27 22:24:06 2017 -0400
12347: Fix --disable-reuse for jobs API.
The --disable-reuse flag was creating a pipeline instance containing a
job that ran cwl with reuse disabled, but it didn't set
properties.run_options.enable_job_reuse on the pipeline instance
itself. If you had run before with --disable-reuse, that previous run
would be eligible for reuse.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index c2973e0..737c958 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -327,12 +327,20 @@ class RunnerJob(Runner):
del job_spec["owner_uuid"]
job_spec["job"] = job
+
+ instance_spec = {
+ "owner_uuid": self.arvrunner.project_uuid,
+ "name": self.name,
+ "components": {
+ "cwl-runner": job_spec,
+ },
+ "state": "RunningOnServer",
+ }
+ if not self.enable_reuse:
+ instance_spec["properties"] = {"run_options": {"enable_job_reuse": False}}
+
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
- body={
- "owner_uuid": self.arvrunner.project_uuid,
- "name": self.name,
- "components": {"cwl-runner": job_spec },
- "state": "RunningOnServer"}).execute(num_retries=self.arvrunner.num_retries)
+ body=instance_spec).execute(num_retries=self.arvrunner.num_retries)
logger.info("Created pipeline %s", self.arvrunner.pipeline["uuid"])
if kwargs.get("wait") is False:
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 92933d8..6735cec 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -321,6 +321,7 @@ class TestSubmit(unittest.TestCase):
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
+ expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
stubs.api.pipeline_instances().create.assert_called_with(
body=JsonDiffMatcher(expect_pipeline))
commit b0f335abada22c2f9f3abf4c2b8e389f4d9ec52f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Sep 27 22:16:32 2017 -0400
12347: Fix formatting.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 794c994..c2973e0 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -184,17 +184,19 @@ class ArvadosJob(object):
if self.arvrunner.pipeline:
self.arvrunner.pipeline["components"][self.name] = {"job": record}
with Perf(metrics, "update_pipeline_component %s" % self.name):
- self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update(uuid=self.arvrunner.pipeline["uuid"],
- body={
- "components": self.arvrunner.pipeline["components"]
- }).execute(num_retries=self.arvrunner.num_retries)
+ self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update(
+ uuid=self.arvrunner.pipeline["uuid"],
+ body={
+ "components": self.arvrunner.pipeline["components"]
+ }).execute(num_retries=self.arvrunner.num_retries)
if self.arvrunner.uuid:
try:
job = self.arvrunner.api.jobs().get(uuid=self.arvrunner.uuid).execute()
if job:
components = job["components"]
components[self.name] = record["uuid"]
- self.arvrunner.api.jobs().update(uuid=self.arvrunner.uuid,
+ self.arvrunner.api.jobs().update(
+ uuid=self.arvrunner.uuid,
body={
"components": components
}).execute(num_retries=self.arvrunner.num_retries)
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d7ec1ac..92933d8 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -496,9 +496,10 @@ class TestSubmit(unittest.TestCase):
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
- '--disable-reuse', '--on-error=continue',
- '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["command"] = [
+ 'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ '--disable-reuse', '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
stubs.api.container_requests().create.assert_called_with(
commit 61b3e0de6b5a5be764c5de73c55184f50a82fc02
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Sep 27 22:11:11 2017 -0400
12347: Fix --disable-reuse for containers API.
The --disable-reuse flag was requesting a container to run
"arvados-cwl-runner --disable-reuse", but it didn't set
use_existing=False on the container request itself. If you had run
before with --disable-reuse, that previous run would be eligible for
reuse.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 78086ed..15e02f3 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -351,8 +351,10 @@ class RunnerContainer(Runner):
if self.enable_reuse:
command.append("--enable-reuse")
+ container_req["use_existing"] = True
else:
command.append("--disable-reuse")
+ container_req["use_existing"] = False
if self.on_error:
command.append("--on-error=" + self.on_error)
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0314720..d7ec1ac 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -245,7 +245,8 @@ def stubs(func):
'vcpus': 1,
'ram': 1024*1024*1024
},
- "properties": {}
+ 'use_existing': True,
+ 'properties': {}
}
stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
@@ -498,6 +499,7 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
'--disable-reuse', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["use_existing"] = False
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -705,7 +707,8 @@ class TestSubmit(unittest.TestCase):
'vcpus': 1,
'ram': 1073741824
},
- "properties": {}
+ 'use_existing': True,
+ 'properties': {}
}
stubs.api.container_requests().create.assert_called_with(
@@ -820,7 +823,8 @@ class TestSubmit(unittest.TestCase):
'vcpus': 1,
'ram': 1073741824
},
- "properties": {
+ 'use_existing': True,
+ 'properties': {
"template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
}
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list