[ARVADOS] created: 1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7
Git user
git at public.curoverse.com
Wed Nov 30 16:45:06 EST 2016
at 1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7 (commit)
commit 1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 16:39:14 2016 -0500
10293: Improve error messages for --api and unsupported InitialWorkDirRequirement.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index d394949..bd65252 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -567,6 +567,10 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
return 1
arvargs.work_api = want_api
+ if arvargs.work_api not in ("jobs", "containers"):
+ logger.error("Unknown --api '%s' expected one of 'jobs' or 'containers'", arvargs.work_api)
+ return 1
+
if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order:
job_order_object = ({}, "")
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 0c65df4..4dd79a1 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -62,7 +62,7 @@ class ArvadosContainer(object):
}
if self.generatefiles["listing"]:
- raise UnsupportedRequirement("Generate files not supported")
+ raise UnsupportedRequirement("InitialWorkDirRequirement not supported with --api=containers")
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
if self.environment:
commit e64e03d71d1ad9c4ed9354fce55be7af17b4a56e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 16:04:00 2016 -0500
10293: Don't try to json decode empty output file to reduce spurious errors.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 6b5cd71..178285d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -204,7 +204,8 @@ class Runner(object):
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
with outc.open("cwl.output.json") as f:
- outputs = json.load(f)
+ if f.size() > 0:
+ outputs = json.load(f)
def keepify(fileobj):
path = fileobj["location"]
if not path.startswith("keep:"):
commit 2b3af1807efa6bc4b3aaa298c1e24998d8ddd1c5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 15:52:38 2016 -0500
10293: Override workflow runner done() to get container from container request.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 9da5fcb..0c65df4 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -247,3 +247,17 @@ class RunnerContainer(Runner):
if response["state"] == "Final":
self.done(response)
+
+ def done(self, record):
+ try:
+ container = self.arvrunner.api.containers().get(
+ uuid=record["container_uuid"]
+ ).execute(num_retries=self.arvrunner.num_retries)
+ except Exception as e:
+ logger.exception("While getting runner container: %s", e)
+ self.arvrunner.output_callback({}, "permanentFail")
+ del self.arvrunner.processes[record["uuid"]]
+ else:
+ super(RunnerContainer, self).done(container)
+ finally:
+ del self.arvrunner.processes[record["uuid"]]
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3bbcb8b..6b5cd71 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -215,4 +215,5 @@ class Runner(object):
logger.exception("While getting final output object: %s", e)
self.arvrunner.output_callback(outputs, processStatus)
finally:
- del self.arvrunner.processes[record["uuid"]]
+ if record["uuid"] in self.arvrunner.processes:
+ del self.arvrunner.processes[record["uuid"]]
commit 79e63a733c48b04d7c9b6bcc6120af72b3f14641
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 15:26:30 2016 -0500
10293: Bug fixes
* Now polling state on self.poll_api.container_requests() table
* Runner object puts "self" into processes table
* Don't report spurious ApiError when not running in container
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8fab946..d394949 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -21,6 +21,7 @@ import schema_salad
import arvados
import arvados.config
+from arvados.errors import ApiError
from .arvcontainer import ArvadosContainer, RunnerContainer
from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
@@ -150,7 +151,7 @@ class ArvCwlRunner(object):
continue
if self.work_api == "containers":
- table = self.poll_api.containers()
+ table = self.poll_api.container_requests()
elif self.work_api == "jobs":
table = self.poll_api.jobs()
@@ -277,6 +278,12 @@ class ArvCwlRunner(object):
if self.work_api == "containers":
try:
current = self.api.containers().current().execute(num_retries=self.num_retries)
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
+ logger.info("Getting current container: %s", e)
+ return
+ try:
self.api.containers().update(uuid=current['uuid'],
body={
'output': self.final_output_collection.portable_data_hash(),
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4c1915f..9da5fcb 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -241,7 +241,7 @@ class RunnerContainer(Runner):
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
- self.arvrunner.processes[response["uuid"]] = response["uuid"]
+ self.arvrunner.processes[response["uuid"]] = self
logger.info("Submitted container %s", response["uuid"])
commit f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a
Merge: 15635ff 1334193
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 28 08:12:48 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit 15635ff9fa2c964fb45467c9846ed92f7f6388b9
Merge: e259833 0e1522f
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 22 16:31:28 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit e2598332f9ecd40ab0dfd025bd1e5eba02ad8673
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 22 16:30:45 2016 -0500
10293: done_outputs in a try/except block
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index c349839..4c1915f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -148,12 +148,12 @@ class ArvadosContainer(object):
outputs = {}
if container["output"]:
- outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
- try:
- self.output_callback(outputs, processStatus)
- except Exception as e:
- logger.error("Got error %s" % str(e))
- self.output_callback({}, "permanentFail")
+ try:
+ outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
+ except Exception as e:
+ logger.error("Got error %s" % str(e))
+ self.output_callback({}, "permanentFail")
+ self.output_callback(outputs, processStatus)
finally:
del self.arvrunner.processes[record["uuid"]]
commit e4a51416586f73593ac68bf0d2a74c53a4875f7e
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 22 14:06:59 2016 -0500
10293: invoke output_callback in a try/except block
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b127f68..c349839 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -115,9 +115,9 @@ class ArvadosContainer(object):
body=container_request
).execute(num_retries=self.arvrunner.num_retries)
- self.arvrunner.processes[response["uuid"]] = response["uuid"]
+ self.arvrunner.processes[response["uuid"]] = self
- logger.info("Container request %s (%s) state is %s with container %s %s", self.name, response["uuid"], response["state"])
+ logger.info("Container request %s (%s) state is %s", self.name, response["uuid"], response["state"])
if response["state"] == "Final":
self.done(response)
@@ -146,9 +146,14 @@ class ArvadosContainer(object):
processStatus = "permanentFail"
outputs = {}
+
if container["output"]:
outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
- self.output_callback(outputs, processStatus)
+ try:
+ self.output_callback(outputs, processStatus)
+ except Exception as e:
+ logger.error("Got error %s" % str(e))
+ self.output_callback({}, "permanentFail")
finally:
del self.arvrunner.processes[record["uuid"]]
commit 81ccd9264d0742ba1bf0b9dfbc31ff4f15929f3e
Merge: 9f53e08 6f461f4
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 22 13:38:27 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit 9f53e085c98249ba79d85ba59e6f1ca624fede10
Merge: c40265a a07c8bf
Author: radhika <radhika at curoverse.com>
Date: Thu Nov 17 14:21:16 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
Conflicts:
sdk/cwl/arvados_cwl/__init__.py
sdk/cwl/arvados_cwl/arvcontainer.py
diff --cc doc/api/methods/container_requests.html.textile.liquid
index 1d7a244,304226d..05a8cf5
--- a/doc/api/methods/container_requests.html.textile.liquid
+++ b/doc/api/methods/container_requests.html.textile.liquid
@@@ -43,11 -40,9 +43,11 @@@ table(table table-bordered table-conden
|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.|
|command|array of strings|Command to execute in the container.|Required. e.g., @["echo","hello"]@|
|output_path|string|Path to a directory or file inside the container that should be preserved as container's output when it finishes. This path must be, or be inside, one of the mount targets. For best performance, point output_path to a writable collection mount.|Required.|
--|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit container requests with zero priority in order to prevew the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".|
++|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit container requests with zero priority in order to preview the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".|
|expires_at|datetime|After this time, priority is considered to be zero.|Not yet implemented.|
|use_existing|boolean|If possible, use an existing (non-failed) container to satisfy the request instead of creating a new one.|Default is true|
+|log_uuid|string|Log collection containing log messages provided by the scheduler and crunch processes.|Null if the container has not yet completed.|
+|output_uuid|string|Output collection created when the container finished successfully.|Null if the container has failed or not yet completed.|
|filters|string|Additional constraints for satisfying the container_request, given in the same form as the filters parameter accepted by the container_requests.list API.|
h2(#mount_types). {% include 'mount_types' %}
diff --cc sdk/cwl/arvados_cwl/arvcontainer.py
index 178aeef,1fda412..b127f68
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@@ -145,7 -146,24 +145,9 @@@ class ArvadosContainer(object)
else:
processStatus = "permanentFail"
- outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
- try:
- outputs = {}
- if record["output"]:
- outputs = done.done(self, record, "/tmp", self.outdir, "/keep")
- except WorkflowException as e:
- logger.error("Error while collecting output for container %s:\n%s", self.name, e, exc_info=(e if self.arvrunner.debug else False))
- processStatus = "permanentFail"
- except Exception as e:
- logger.exception("Got unknown exception while collecting output for container %s:", self.name)
- processStatus = "permanentFail"
-
- # Note: Currently, on error output_callback is expecting an empty dict,
- # anything else will fail.
- if not isinstance(outputs, dict):
- logger.error("Unexpected output type %s '%s'", type(outputs), outputs)
- outputs = {}
- processStatus = "permanentFail"
-
++ outputs = {}
++ if container["output"]:
++ outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
self.output_callback(outputs, processStatus)
finally:
del self.arvrunner.processes[record["uuid"]]
commit c40265a873d73f03a9ca077f18fe305d883fb4a5
Author: radhika <radhika at curoverse.com>
Date: Thu Nov 17 13:11:08 2016 -0500
10293: update documentation with scheduling_parameters, log_uuid and output_uuid.
diff --git a/doc/_includes/_container_runtime_constraints.liquid b/doc/_includes/_container_runtime_constraints.liquid
index 849db42..d505bfd 100644
--- a/doc/_includes/_container_runtime_constraints.liquid
+++ b/doc/_includes/_container_runtime_constraints.liquid
@@ -8,4 +8,3 @@ table(table table-bordered table-condensed).
|vcpus|integer|Number of cores to be used to run this process.|Optional. However, a ContainerRequest that is in "Committed" state must provide this.|
|keep_cache_ram|integer|Number of keep cache bytes to be used to run this process.|Optional.|
|API|boolean|When set, ARVADOS_API_HOST and ARVADOS_API_TOKEN will be set, and container will have networking enabled to access the Arvados API server.|Optional.|
-|partition|array of strings|Specify the names of one or more compute partitions that may run this container. If not provided, the system chooses where to run the container.|Optional.|
diff --git a/doc/_includes/_container_scheduling_parameters.liquid b/doc/_includes/_container_scheduling_parameters.liquid
new file mode 100644
index 0000000..ee2ca07
--- /dev/null
+++ b/doc/_includes/_container_scheduling_parameters.liquid
@@ -0,0 +1,7 @@
+Scheduling parameters
+
+Parameters to be passed to the container scheduler (e.g., SLURM) when running a container.
+
+table(table table-bordered table-condensed).
+|_. Key|_. Type|_. Description|_. Notes|
+|partitions|array of strings|The names of one or more compute partitions that may run this container. If not provided, the system will choose where to run the container.|Optional.|
diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid
index 304226d..1d7a244 100644
--- a/doc/api/methods/container_requests.html.textile.liquid
+++ b/doc/api/methods/container_requests.html.textile.liquid
@@ -35,6 +35,9 @@ table(table table-bordered table-condensed).
"vcpus":2,
"API":true
}</code></pre>See "Runtime constraints":#runtime_constraints for more details.|
+|scheduling_parameters|hash|Parameters to be passed to the container scheduler when running this container.|e.g.,<pre><code>{
+"partitions":["fastcpu","vfastcpu"]
+}</code></pre>See "Scheduling parameters":#scheduling_parameters for more details.|
|container_image|string|Portable data hash of a collection containing the docker image to run the container.|Required.|
|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.||
|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.|
@@ -43,12 +46,16 @@ table(table table-bordered table-condensed).
|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit container requests with zero priority in order to prevew the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".|
|expires_at|datetime|After this time, priority is considered to be zero.|Not yet implemented.|
|use_existing|boolean|If possible, use an existing (non-failed) container to satisfy the request instead of creating a new one.|Default is true|
+|log_uuid|string|Log collection containing log messages provided by the scheduler and crunch processes.|Null if the container has not yet completed.|
+|output_uuid|string|Output collection created when the container finished successfully.|Null if the container has failed or not yet completed.|
|filters|string|Additional constraints for satisfying the container_request, given in the same form as the filters parameter accepted by the container_requests.list API.|
h2(#mount_types). {% include 'mount_types' %}
h2(#runtime_constraints). {% include 'container_runtime_constraints' %}
+h2(#scheduling_parameters). {% include 'container_scheduling_parameters' %}
+
h2(#container_reuse). Container reuse
When a container request is "Committed", the system will try to find and reuse any preexisting Container with the same exact command, cwd, environment, output_path, container_image, mounts, and runtime_constraints as this container request. The serialized fields environment, mounts and runtime_constraints are sorted to facilitate comparison.
diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid
index 221141c..7eed8b0 100644
--- a/doc/api/methods/containers.html.textile.liquid
+++ b/doc/api/methods/containers.html.textile.liquid
@@ -36,6 +36,9 @@ Generally this will contain additional keys that are not present in any correspo
"vcpus":2,
"API":true
}</code></pre>See "Runtime constraints":#runtime_constraints for more details.|
+|scheduling_parameters|hash|Parameters to be passed to the container scheduler when running this container.|e.g.,<pre><code>{
+"partitions":["fastcpu","vfastcpu"]
+}</code></pre>See "Scheduling parameters":#scheduling_parameters for more details.|
|output|string|Portable data hash of the output collection.|Null if the container is not yet finished.|
|container_image|string|Portable data hash of a collection containing the docker image used to run the container.||
|progress|number|A number between 0.0 and 1.0 describing the fraction of work done.||
@@ -58,6 +61,8 @@ h2(#mount_types). {% include 'mount_types' %}
h2(#runtime_constraints). {% include 'container_runtime_constraints' %}
+h2(#scheduling_parameters). {% include 'container_scheduling_parameters' %}
+
h2. Methods
See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update at .
commit d53262e8e07785a9d2566966ee41ab8949ef1962
Merge: a4762b5 ce924ad
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 16 17:04:23 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit a4762b52738e12c39d93c14501a2f62463b05d07
Merge: 5fe0d5e 4bccbae
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 16 15:58:45 2016 -0500
Merge branch '10293-container-request-output-uuid' into 10293-cwl-cr-output
commit 5fe0d5e7496fad7bd1c4bab0e5ca5f348c0eec63
Merge: a9a677e b6bc9bb
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 16 15:58:35 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit a9a677e1655c461e742e46cc3c239f8605f4fc6b
Merge: 8668135 d2f9e78
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 16 15:33:47 2016 -0500
Merge branch 'master' into 10293-cwl-cr-output
commit 8668135855b400e7f6047ecacd9cfa27fed723f7
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 16 15:25:01 2016 -0500
10293: update cwl runner
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 92be92d..f3e1a26 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -119,7 +119,7 @@ class ArvCwlRunner(object):
logger.info("Job %s (%s) is Running", j.name, uuid)
j.running = True
j.update_pipeline_component(event["properties"]["new_attributes"])
- elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"):
+ elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled", "Final"):
uuid = event["object_uuid"]
try:
self.cond.acquire()
@@ -148,7 +148,7 @@ class ArvCwlRunner(object):
continue
if self.work_api == "containers":
- table = self.poll_api.containers()
+ table = self.poll_api.container_requests()
elif self.work_api == "jobs":
table = self.poll_api.jobs()
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index aa088c5..e9e6b32 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -115,24 +115,23 @@ class ArvadosContainer(object):
body=container_request
).execute(num_retries=self.arvrunner.num_retries)
- self.arvrunner.processes[response["container_uuid"]] = self
+ self.arvrunner.processes[response["uuid"]] = response["uuid"]
- container = self.arvrunner.api.containers().get(
- uuid=response["container_uuid"]
- ).execute(num_retries=self.arvrunner.num_retries)
+ logger.info("Container request %s (%s) state is %s with container %s %s", self.name, response["uuid"], response["state"])
- logger.info("Container request %s (%s) state is %s with container %s %s", self.name, response["uuid"], response["state"], container["uuid"], container["state"])
-
- if container["state"] in ("Complete", "Cancelled"):
- self.done(container)
+ if response["state"] == "Final":
+ self.done(response)
except Exception as e:
logger.error("Got error %s" % str(e))
self.output_callback({}, "permanentFail")
def done(self, record):
try:
- if record["state"] == "Complete":
- rcode = record["exit_code"]
+ container = self.arvrunner.api.containers().get(
+ uuid=record["container_uuid"]
+ ).execute(num_retries=self.arvrunner.num_retries)
+ if container["state"] == "Complete":
+ rcode = container["exit_code"]
if self.successCodes and rcode in self.successCodes:
processStatus = "success"
elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
@@ -146,17 +145,7 @@ class ArvadosContainer(object):
else:
processStatus = "permanentFail"
- try:
- outputs = {}
- if record["output"]:
- outputs = done.done(self, record, "/tmp", self.outdir, "/keep")
- except WorkflowException as e:
- logger.error("Error while collecting container outputs:\n%s", e, exc_info=(e if self.arvrunner.debug else False))
- processStatus = "permanentFail"
- except Exception as e:
- logger.exception("Got unknown exception while collecting job outputs:")
- processStatus = "permanentFail"
-
+ outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
self.output_callback(outputs, processStatus)
finally:
del self.arvrunner.processes[record["uuid"]]
@@ -242,9 +231,9 @@ class RunnerContainer(Runner):
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
- self.arvrunner.processes[response["container_uuid"]] = self
+ self.arvrunner.processes[response["uuid"]] = response["uuid"]
logger.info("Submitted container %s", response["uuid"])
- if response["state"] in ("Complete", "Failed", "Cancelled"):
+ if response["state"] == "Final":
self.done(response)
diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 31f353e..7bdae5a 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -35,6 +35,9 @@ def done(self, record, tmpdir, outdir, keepdir):
}, ensure_unique_name=True).execute(
num_retries=self.arvrunner.num_retries)
+ done_outputs(self, record, tmpdir, outdir, keepdir)
+
+def done_outputs(self, record, tmpdir, outdir, keepdir):
self.builder.outdir = outdir
self.builder.pathmapper.keepdir = keepdir
return self.collect_outputs("keep:" + record["output"])
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index bb4bac3..b13bcde 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -172,29 +172,14 @@ class TestContainer(unittest.TestCase):
arvjob.outdir = "/var/spool/cwl"
arvjob.done({
- "state": "Complete",
- "output": "99999999999999999999999999999993+99",
- "log": "99999999999999999999999999999994+99",
- "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
- "exit_code": 0
+ "state": "Final",
+ "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+ "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
+ "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
- api.collections().list.assert_has_calls([
- mock.call(),
- mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
- ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
- ['name', '=', 'Output 9999999 of testjob']]),
- mock.call().execute(num_retries=0),
- mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
- select=['manifest_text']),
- mock.call().execute(num_retries=0)])
-
- api.collections().create.assert_called_with(
- ensure_unique_name=True,
- body={'portable_data_hash': '99999999999999999999999999999993+99',
- 'manifest_text': 'XYZ',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'name': 'Output 9999999 of testjob'})
+ self.assertFalse(api.collections().create.called)
@mock.patch("arvados.collection.Collection")
def test_done_use_existing_collection(self, col):
@@ -217,18 +202,12 @@ class TestContainer(unittest.TestCase):
arvjob.outdir = "/var/spool/cwl"
arvjob.done({
- "state": "Complete",
- "output": "99999999999999999999999999999993+99",
- "log": "99999999999999999999999999999994+99",
- "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
- "exit_code": 0
+ "state": "Final",
+ "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+ "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
+ "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
- api.collections().list.assert_has_calls([
- mock.call(),
- mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
- ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
- ['name', '=', 'Output 9999999 of testjob']]),
- mock.call().execute(num_retries=0)])
-
self.assertFalse(api.collections().create.called)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list