[ARVADOS] updated: 4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759
Git user
git at public.curoverse.com
Wed Nov 30 16:26:41 EST 2016
Summary of changes:
build/README | 6 +-
build/build-dev-docker-jobs-image.sh | 63 ++++++++++
build/run-build-packages-one-target.sh | 11 +-
build/run-build-packages.sh | 21 +++-
build/run-library.sh | 17 +++
doc/images/upload-using-workbench.png | Bin 49690 -> 23979 bytes
doc/images/workbench-dashboard.png | Bin 94257 -> 57930 bytes
doc/images/workbench-move-selected.png | Bin 47833 -> 15450 bytes
doc/user/cwl/cwl-runner.html.textile.liquid | 2 +-
.../getting_started/workbench.html.textile.liquid | 2 +-
...tutorial-pipeline-workbench.html.textile.liquid | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 9 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 16 ++-
sdk/cwl/arvados_cwl/runner.py | 6 +-
sdk/cwl/setup.py | 2 +-
sdk/dev-jobs.dockerfile | 38 ++++++
sdk/python/arvados/keep.py | 134 ++++++++++++---------
sdk/python/tests/test_keep_client.py | 76 +++++++-----
.../controllers/arvados/v1/groups_controller.rb | 9 +-
.../arvados/v1/groups_controller_test.rb | 19 ++-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +
21 files changed, 330 insertions(+), 105 deletions(-)
create mode 100755 build/build-dev-docker-jobs-image.sh
create mode 100644 sdk/dev-jobs.dockerfile
via 4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759 (commit)
via 04722e5aafb50a01f064925afd7165fe62a1fdf5 (commit)
via 2bc63b75485240e8c8c682ffa57c0ef4a979b17b (commit)
via 6eb6c5932cf9a6927dfa3541faf786b8104039ee (commit)
via 277ad30ef4824ab6363ba24f10f62d9fb6544ad7 (commit)
via 515e3f5c3dac3076217bd59545d01604008c01e6 (commit)
via 5646a899f667ee14efc32e9db84c72c0938ac6dc (commit)
via bde488e7eff0e9a94cbf9709d28de13603857a3d (commit)
via dfd9492ee371ce86f7b4543106eb5060bc98928c (commit)
via dcc80dc5b02e46170400d42ca72672cceff03ba2 (commit)
via b5d3273b5aa503b22fdbeab7f8979e720e30c119 (commit)
via 7bf6588ce6589a194df2c7f45b9a443025c2ff67 (commit)
via d71a3ed8a91b7491df1f675389e1038d1a781e8e (commit)
via b738c7e7357a143dade94dc5a1bad2b69bac2b27 (commit)
via e4664336c420836bf26f423faf2af9316302da93 (commit)
via 16942303133cb4e6be76d4d33a211d1e0ce2ea38 (commit)
via 52f9f7b3cf35efb7e8d1189b80f9b0f7afe0e111 (commit)
via 92fbda579013213b6f0d101f4fee71d149f3cb02 (commit)
via 86f04235021d84afa0d28d105111422e0dd15738 (commit)
via 7c2b38632ae2af8a2f0f5c8fa2e55523220a3335 (commit)
from f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 16:04:00 2016 -0500
10293: Don't try to json decode empty output file to reduce spurious errors.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 6b5cd71..178285d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -204,7 +204,8 @@ class Runner(object):
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
with outc.open("cwl.output.json") as f:
- outputs = json.load(f)
+ if f.size() > 0:
+ outputs = json.load(f)
def keepify(fileobj):
path = fileobj["location"]
if not path.startswith("keep:"):
commit 04722e5aafb50a01f064925afd7165fe62a1fdf5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 15:52:38 2016 -0500
10293: Override workflow runner done() to get container from container request.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 9da5fcb..0c65df4 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -247,3 +247,17 @@ class RunnerContainer(Runner):
if response["state"] == "Final":
self.done(response)
+
+ def done(self, record):
+ try:
+ container = self.arvrunner.api.containers().get(
+ uuid=record["container_uuid"]
+ ).execute(num_retries=self.arvrunner.num_retries)
+ except Exception as e:
+ logger.exception("While getting runner container: %s", e)
+ self.arvrunner.output_callback({}, "permanentFail")
+ del self.arvrunner.processes[record["uuid"]]
+ else:
+ super(RunnerContainer, self).done(container)
+ finally:
+ del self.arvrunner.processes[record["uuid"]]
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3bbcb8b..6b5cd71 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -215,4 +215,5 @@ class Runner(object):
logger.exception("While getting final output object: %s", e)
self.arvrunner.output_callback(outputs, processStatus)
finally:
- del self.arvrunner.processes[record["uuid"]]
+ if record["uuid"] in self.arvrunner.processes:
+ del self.arvrunner.processes[record["uuid"]]
commit 2bc63b75485240e8c8c682ffa57c0ef4a979b17b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 15:26:30 2016 -0500
10293: Bug fixes
* Now polling state on self.poll_api.container_requests() table
* Runner object puts "self" into processes table
* Don't report spurious ApiError when not running in container
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8fab946..d394949 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -21,6 +21,7 @@ import schema_salad
import arvados
import arvados.config
+from arvados.errors import ApiError
from .arvcontainer import ArvadosContainer, RunnerContainer
from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
@@ -150,7 +151,7 @@ class ArvCwlRunner(object):
continue
if self.work_api == "containers":
- table = self.poll_api.containers()
+ table = self.poll_api.container_requests()
elif self.work_api == "jobs":
table = self.poll_api.jobs()
@@ -277,6 +278,12 @@ class ArvCwlRunner(object):
if self.work_api == "containers":
try:
current = self.api.containers().current().execute(num_retries=self.num_retries)
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
+ logger.info("Getting current container: %s", e)
+ return
+ try:
self.api.containers().update(uuid=current['uuid'],
body={
'output': self.final_output_collection.portable_data_hash(),
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4c1915f..9da5fcb 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -241,7 +241,7 @@ class RunnerContainer(Runner):
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
- self.arvrunner.processes[response["uuid"]] = response["uuid"]
+ self.arvrunner.processes[response["uuid"]] = self
logger.info("Submitted container %s", response["uuid"])
commit 6eb6c5932cf9a6927dfa3541faf786b8104039ee
Merge: f17da3a 277ad30
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 30 14:41:50 2016 -0500
Merge branch '10647-dev-jobs-image' into origin-10293-cwl-cr-output
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list