[ARVADOS] updated: 4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759

Git user git at public.curoverse.com
Wed Nov 30 16:26:41 EST 2016


Summary of changes:
 build/README                                       |   6 +-
 build/build-dev-docker-jobs-image.sh               |  63 ++++++++++
 build/run-build-packages-one-target.sh             |  11 +-
 build/run-build-packages.sh                        |  21 +++-
 build/run-library.sh                               |  17 +++
 doc/images/upload-using-workbench.png              | Bin 49690 -> 23979 bytes
 doc/images/workbench-dashboard.png                 | Bin 94257 -> 57930 bytes
 doc/images/workbench-move-selected.png             | Bin 47833 -> 15450 bytes
 doc/user/cwl/cwl-runner.html.textile.liquid        |   2 +-
 .../getting_started/workbench.html.textile.liquid  |   2 +-
 ...tutorial-pipeline-workbench.html.textile.liquid |   2 +-
 sdk/cwl/arvados_cwl/__init__.py                    |   9 +-
 sdk/cwl/arvados_cwl/arvcontainer.py                |  16 ++-
 sdk/cwl/arvados_cwl/runner.py                      |   6 +-
 sdk/cwl/setup.py                                   |   2 +-
 sdk/dev-jobs.dockerfile                            |  38 ++++++
 sdk/python/arvados/keep.py                         | 134 ++++++++++++---------
 sdk/python/tests/test_keep_client.py               |  76 +++++++-----
 .../controllers/arvados/v1/groups_controller.rb    |   9 +-
 .../arvados/v1/groups_controller_test.rb           |  19 ++-
 tools/arvbox/lib/arvbox/docker/Dockerfile.base     |   2 +
 21 files changed, 330 insertions(+), 105 deletions(-)
 create mode 100755 build/build-dev-docker-jobs-image.sh
 create mode 100644 sdk/dev-jobs.dockerfile

       via  4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759 (commit)
       via  04722e5aafb50a01f064925afd7165fe62a1fdf5 (commit)
       via  2bc63b75485240e8c8c682ffa57c0ef4a979b17b (commit)
       via  6eb6c5932cf9a6927dfa3541faf786b8104039ee (commit)
       via  277ad30ef4824ab6363ba24f10f62d9fb6544ad7 (commit)
       via  515e3f5c3dac3076217bd59545d01604008c01e6 (commit)
       via  5646a899f667ee14efc32e9db84c72c0938ac6dc (commit)
       via  bde488e7eff0e9a94cbf9709d28de13603857a3d (commit)
       via  dfd9492ee371ce86f7b4543106eb5060bc98928c (commit)
       via  dcc80dc5b02e46170400d42ca72672cceff03ba2 (commit)
       via  b5d3273b5aa503b22fdbeab7f8979e720e30c119 (commit)
       via  7bf6588ce6589a194df2c7f45b9a443025c2ff67 (commit)
       via  d71a3ed8a91b7491df1f675389e1038d1a781e8e (commit)
       via  b738c7e7357a143dade94dc5a1bad2b69bac2b27 (commit)
       via  e4664336c420836bf26f423faf2af9316302da93 (commit)
       via  16942303133cb4e6be76d4d33a211d1e0ce2ea38 (commit)
       via  52f9f7b3cf35efb7e8d1189b80f9b0f7afe0e111 (commit)
       via  92fbda579013213b6f0d101f4fee71d149f3cb02 (commit)
       via  86f04235021d84afa0d28d105111422e0dd15738 (commit)
       via  7c2b38632ae2af8a2f0f5c8fa2e55523220a3335 (commit)
      from  f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 4553a0c5e4fdc7628c4ed76dd8cce52b8d24e759
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Nov 30 16:04:00 2016 -0500

    10293: Don't try to json decode empty output file to reduce spurious errors.

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 6b5cd71..178285d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -204,7 +204,8 @@ class Runner(object):
                                                            keep_client=self.arvrunner.keep_client,
                                                            num_retries=self.arvrunner.num_retries)
                 with outc.open("cwl.output.json") as f:
-                    outputs = json.load(f)
+                    if f.size() > 0:
+                        outputs = json.load(f)
                 def keepify(fileobj):
                     path = fileobj["location"]
                     if not path.startswith("keep:"):

commit 04722e5aafb50a01f064925afd7165fe62a1fdf5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Nov 30 15:52:38 2016 -0500

    10293: Override workflow runner done() to get container from container request.

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 9da5fcb..0c65df4 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -247,3 +247,17 @@ class RunnerContainer(Runner):
 
         if response["state"] == "Final":
             self.done(response)
+
+    def done(self, record):
+        try:
+            container = self.arvrunner.api.containers().get(
+                uuid=record["container_uuid"]
+            ).execute(num_retries=self.arvrunner.num_retries)
+        except Exception as e:
+            logger.exception("While getting runner container: %s", e)
+            self.arvrunner.output_callback({}, "permanentFail")
+            del self.arvrunner.processes[record["uuid"]]
+        else:
+            super(RunnerContainer, self).done(container)
+        finally:
+            del self.arvrunner.processes[record["uuid"]]
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3bbcb8b..6b5cd71 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -215,4 +215,5 @@ class Runner(object):
                 logger.exception("While getting final output object: %s", e)
             self.arvrunner.output_callback(outputs, processStatus)
         finally:
-            del self.arvrunner.processes[record["uuid"]]
+            if record["uuid"] in self.arvrunner.processes:
+                del self.arvrunner.processes[record["uuid"]]

commit 2bc63b75485240e8c8c682ffa57c0ef4a979b17b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Nov 30 15:26:30 2016 -0500

    10293: Bug fixes
    
    * Now polling state on self.poll_api.container_requests() table
    * Runner object puts "self" into processes table
    * Don't report spurious ApiError when not running in container

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8fab946..d394949 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -21,6 +21,7 @@ import schema_salad
 
 import arvados
 import arvados.config
+from arvados.errors import ApiError
 
 from .arvcontainer import ArvadosContainer, RunnerContainer
 from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
@@ -150,7 +151,7 @@ class ArvCwlRunner(object):
                     continue
 
                 if self.work_api == "containers":
-                    table = self.poll_api.containers()
+                    table = self.poll_api.container_requests()
                 elif self.work_api == "jobs":
                     table = self.poll_api.jobs()
 
@@ -277,6 +278,12 @@ class ArvCwlRunner(object):
         if self.work_api == "containers":
             try:
                 current = self.api.containers().current().execute(num_retries=self.num_retries)
+            except ApiError as e:
+                # Status code 404 just means we're not running in a container.
+                if e.resp.status != 404:
+                    logger.info("Getting current container: %s", e)
+                return
+            try:
                 self.api.containers().update(uuid=current['uuid'],
                                              body={
                                                  'output': self.final_output_collection.portable_data_hash(),
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4c1915f..9da5fcb 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -241,7 +241,7 @@ class RunnerContainer(Runner):
         ).execute(num_retries=self.arvrunner.num_retries)
 
         self.uuid = response["uuid"]
-        self.arvrunner.processes[response["uuid"]] = response["uuid"]
+        self.arvrunner.processes[response["uuid"]] = self
 
         logger.info("Submitted container %s", response["uuid"])
 

commit 6eb6c5932cf9a6927dfa3541faf786b8104039ee
Merge: f17da3a 277ad30
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Nov 30 14:41:50 2016 -0500

    Merge branch '10647-dev-jobs-image' into origin-10293-cwl-cr-output


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list