[ARVADOS] updated: 617ef14a3f2fac4dad5f699af7000187143ebbe3
Git user
git at public.curoverse.com
Thu Oct 20 16:09:02 EDT 2016
Summary of changes:
apps/workbench/test/diagnostics/pipeline_test.rb | 2 +-
apps/workbench/test/diagnostics_test_helper.rb | 7 +-
crunch_scripts/cwl-runner | 23 +-
.../install-arv-git-httpd.html.textile.liquid | 52 ++--
sdk/cli/bin/arv-run-pipeline-instance | 31 +--
sdk/cwl/arvados_cwl/__init__.py | 46 +++-
sdk/cwl/arvados_cwl/arvjob.py | 6 +
sdk/cwl/arvados_cwl/crunch_script.py | 99 ++++++++
sdk/cwl/arvados_cwl/runner.py | 6 +-
sdk/cwl/tests/test_job.py | 121 +++++----
sdk/cwl/tests/test_make_output.py | 14 +-
sdk/cwl/tests/test_pathmapper.py | 12 +-
sdk/cwl/tests/test_submit.py | 22 +-
sdk/go/arvados/duration.go | 5 +
sdk/go/streamer/streamer.go | 30 ++-
sdk/go/streamer/streamer_test.go | 10 +
sdk/python/tests/run_test_server.py | 3 +-
services/api/Gemfile | 2 +-
services/api/Gemfile.lock | 6 +-
.../api/app/controllers/application_controller.rb | 8 +
.../controllers/arvados/v1/schema_controller.rb | 4 +
.../app/controllers/user_sessions_controller.rb | 3 +
services/api/app/models/collection.rb | 2 +-
services/api/app/models/container_request.rb | 6 +-
services/api/app/models/user.rb | 36 +--
services/api/config/application.default.yml | 5 +
...71346_add_use_existing_to_container_requests.rb | 9 +
services/api/db/structure.sql | 7 +-
services/api/lib/josh_id.rb | 1 +
services/api/test/fixtures/collections.yml | 17 +-
.../functional/arvados/v1/jobs_controller_test.rb | 13 +
.../arvados/v1/schema_controller_test.rb | 19 ++
.../api/test/integration/user_sessions_test.rb | 21 +-
services/api/test/unit/container_request_test.rb | 43 ++--
services/arv-git-httpd/arvados-git-httpd.service | 2 +-
services/arv-git-httpd/doc.go | 35 ---
services/arv-git-httpd/git_handler.go | 31 ++-
services/arv-git-httpd/git_handler_test.go | 10 +-
services/arv-git-httpd/gitolite_test.go | 10 +-
services/arv-git-httpd/integration_test.go | 2 +
services/arv-git-httpd/main.go | 21 +-
services/arv-git-httpd/usage.go | 39 ++-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 7 +-
services/crunch-dispatch-slurm/squeue.go | 6 +-
services/keepstore/azure_blob_volume.go | 161 ++++++------
services/keepstore/azure_blob_volume_test.go | 12 +-
services/keepstore/bufferpool_test.go | 4 +-
services/keepstore/config.go | 179 +++++++++++++
services/keepstore/deprecated.go | 43 ++++
services/keepstore/handler_test.go | 110 ++++----
services/keepstore/handlers.go | 26 +-
services/keepstore/keepstore.go | 276 ++++-----------------
.../keepstore.service} | 6 +-
services/keepstore/keepstore_test.go | 28 +--
services/keepstore/perms.go | 9 +-
services/keepstore/perms_test.go | 28 ++-
services/keepstore/pull_worker_test.go | 34 +--
services/keepstore/s3_volume.go | 254 +++++++++++--------
services/keepstore/s3_volume_test.go | 110 ++++----
services/keepstore/trash_worker.go | 12 +-
services/keepstore/trash_worker_test.go | 22 +-
services/keepstore/usage.go | 124 +++++++++
services/keepstore/volume.go | 22 +-
services/keepstore/volume_generic_test.go | 51 ++--
services/keepstore/volume_test.go | 11 +-
services/keepstore/volume_unix.go | 138 +++++++----
services/keepstore/volume_unix_test.go | 37 ++-
services/login-sync/bin/arvados-login-sync | 41 ++-
.../docker/service/arv-git-httpd/run-service | 9 +-
69 files changed, 1620 insertions(+), 981 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/crunch_script.py
create mode 100644 services/api/db/migrate/20161019171346_add_use_existing_to_container_requests.rb
delete mode 100644 services/arv-git-httpd/doc.go
create mode 100644 services/keepstore/config.go
create mode 100644 services/keepstore/deprecated.go
copy services/{keepproxy/keepproxy.service => keepstore/keepstore.service} (53%)
create mode 100644 services/keepstore/usage.go
via 617ef14a3f2fac4dad5f699af7000187143ebbe3 (commit)
via 0e5198142fdba0ce2af6eb2852d45dff46ffb2e2 (commit)
via 3d984f3ee6182391094ece7e32d61c40be7ba903 (commit)
via d5524135b1495b919de332df4f952926664961f5 (commit)
via cfa2c529c7c8105e4d424b404db2c48ff2066e27 (commit)
via f84926fc3f934ea1958c810637851ec91994fc16 (commit)
via 7e54f217895d2f401563c54704c73a196e2d236d (commit)
via 81d1a618e724c6e9f20f4efa47acbcef4bba5f15 (commit)
via 8ffceeb0bddd457cee62586d405afd8e082e1d6f (commit)
via 360d5f885a6613a506e283ff74b18382249cf929 (commit)
via e9b2c5fc743957598d839e072c49e027752b3287 (commit)
via 6ebfa334f7965f2c4dbe1f52676993dcbd745f65 (commit)
via 5c5e51e7fd601d4e760c76099e3225845d91bb5f (commit)
via b1578a5e964e1a04d8c96c541017c44498e966cb (commit)
via 6bf3b279cd2d5ea5c1d6b849265b7206461467b8 (commit)
via 3ec9630b7d8b4be9d9ca149e45fa5d9cc4ea0f42 (commit)
via 787856704fb3e4421f0367eaca17c7743e844962 (commit)
via cfa2a5ac51c55fa317e1d4a725ebf4fd630b9322 (commit)
via ce5e483d141207a9adddfb82bc8073caaf555907 (commit)
via 99256e7b8649d6be2f380ef9cd1cf506e2bbe8d7 (commit)
via 94588fc14c200e4163144cd53bfa548a27fff7e0 (commit)
via d26136521a20387b12a3a632dac18a210fe88430 (commit)
via 54a8c91cda42a5be9fa0f532605e3bc3262b240a (commit)
via 2dd5e12d5cd92fbc746bb1d3e30dfba3e871fd72 (commit)
via 3891d484a96b50a6c2561ec0f296bf4f0399ad03 (commit)
via 921fef9e349cada916934ae0131067adac8d0335 (commit)
via fa57e1ba30bbf2ab2956d5a7ae3da14380da8669 (commit)
via a85e7c7526d985f66f073320155ae5d70590be60 (commit)
via 570509ab4d2ef93d870fd2b1f2eab178afb1bad9 (commit)
via f156def25b0720dac1e102727f3a3821b98fb551 (commit)
via 687906a92dc61fb9824e5963de675ed3f907e6b0 (commit)
via 6c2276acb0a049d0defc71dbb2578cf172717520 (commit)
via 5a0d9381edaa6c7eede45c9171ac089d1a4debf8 (commit)
via 794e704d8945dd6ce0e8523e4e209b86e0dd5523 (commit)
via b8d98eba35c5a9d902eddbf42fd53a37bb6b30c4 (commit)
via 5fd9b5a5fb88f40b999ab6c9fa9435ad01f595ff (commit)
via 160644c09224113debdbae7c0be0c6e661dfebd8 (commit)
via 50cd2aaee0e70d35ac4d90e3c29304fa48dd568a (commit)
via 0cb1e7c1e0d6eda274b175f395a2739806f2cfea (commit)
via 0f89acd51a22391e00b5cb270ca93e48357fd618 (commit)
via e1902b8a0490aa6b7ffc544c1609d4d57a5110ce (commit)
via c62e4ae2026770d983ef5b7ffda0cf6d0f48ffc2 (commit)
via b307cc4cf2f31eccd1ae5ef3310cfee309f6688e (commit)
via df2a5463aa313dde8e0dba33c357e3eb4272ba35 (commit)
via a7d5b045f79eecc54949929abfdf13689626fcc3 (commit)
via cd6a8c82f19a83dcd76ef86bd16af49b71a639f4 (commit)
via b9d2799dfebae724dda3b3e28641116ca5daf5c7 (commit)
via 82cc494a64aaac576dfefe0487faafdda82482bc (commit)
via f935965259f9c0476a9c5ffa79e5c27ce9da4800 (commit)
via e5b9158fc357495fba2d110bf6eafa72b1ebe145 (commit)
via d3b56f18947c3ccb42fee52ff33b3b00db0dffd9 (commit)
via 8b03f0b0e66190b35c55ce1f0917dd2104b8a0b9 (commit)
via a278be11225d5f4d77eaa332854421995ce20224 (commit)
via 6a7beeebde1fed4ab925704238d1fb980e9fe238 (commit)
via f0769a2c41c1ae03d2180e9d25e1738b88650064 (commit)
via 1825696494802d264c74ed2af58cc39b6ff8c7f8 (commit)
via 70f1b69fe1f0567483afc182e47fe717756b8bd1 (commit)
via 4ae3e06d829da2018e952eab6f544f3bed754d67 (commit)
via f78ee6387642d4dd0faf9071e44319a2ee42ce2f (commit)
via 20a717071130eee4afb4704617c51ff165d735d9 (commit)
via 51eaad58e0a135bdf88d4bdcd819d219d75de0df (commit)
via ea0de4d588c6eac28458fb20d70e008081f9ecbb (commit)
via 59acf0718d442c05c64ba012b4f48478a4087b7b (commit)
via 8baaf20400a376977b277ac7675ebfd0c4afe917 (commit)
via c978ad2d773b1ffc1ec6608123e27b89e65c428e (commit)
via 195b3f3cd0870195c546fec3e05522c0d1c89b72 (commit)
via 1d3d5386e0586f91984fcfb9a117ffcc38b817b4 (commit)
via 64efd1030538d59821ce288a7674e29d49c35744 (commit)
from 754913fc3e5037903048863f5545c9512fbf7bfc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 617ef14a3f2fac4dad5f699af7000187143ebbe3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 20 16:08:52 2016 -0400
10172: crunch_script only updates task record on error, will normally be
updated in set_crunch_output().
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index e5c6d67..9b0680b 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -21,6 +21,7 @@ import functools
from arvados.api import OrderedJsonModel
from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
from cwltool.load_tool import load_tool
+from cwltool.errors import WorkflowException
logger = logging.getLogger('arvados.cwl-runner')
@@ -32,6 +33,7 @@ def run():
arvados_cwl.add_arv_hints()
+ runner = None
try:
job_order_object = arvados.current_job()['script_parameters']
@@ -80,23 +82,18 @@ def run():
args.basedir = os.getcwd()
args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
outputObj = runner.arv_executor(t, job_order_object, **vars(args))
-
- if runner.final_output_collection:
+ except Exception as e:
+ if isinstance(e, WorkflowException):
+ logging.info("Workflow error %s", e)
+ else:
+ logging.exception("Unhandled exception")
+ if runner and runner.final_output_collection:
outputCollection = runner.final_output_collection.portable_data_hash()
else:
outputCollection = None
-
api.job_tasks().update(uuid=arvados.current_task()['uuid'],
body={
'output': outputCollection,
- 'success': True,
- 'progress':1.0
- }).execute()
- except Exception as e:
- logging.exception("Unhandled exception")
- api.job_tasks().update(uuid=arvados.current_task()['uuid'],
- body={
- 'output': None,
'success': False,
'progress':1.0
}).execute()
commit 0e5198142fdba0ce2af6eb2852d45dff46ffb2e2
Merge: 3d984f3 d552413
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 20 15:47:59 2016 -0400
Merge branch 'master' into 10172-crunch2-container-output
commit 3d984f3ee6182391094ece7e32d61c40be7ba903
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 20 15:42:28 2016 -0400
10172: Detect when running as crunch container or job and set output.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7ebb13f..5ba6412 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -227,6 +227,24 @@ class ArvCwlRunner(object):
self.final_output_collection = final
+ def set_crunch_output(self):
+ if self.work_api == "containers":
+ try:
+ current = self.api.containers().current().execute(num_retries=self.num_retries)
+ self.api.containers().update(uuid=current['uuid'],
+ body={
+ 'output': self.final_output_collection.portable_data_hash(),
+ }).execute(num_retries=self.num_retries)
+ except Exception as e:
+ logger.info("Setting container output: %s", e)
+ elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
+ self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
+ body={
+ 'output': self.final_output_collection.portable_data_hash(),
+ 'success': self.final_status == "success",
+ 'progress':1.0
+ }).execute(num_retries=self.num_retries)
+
def arv_executor(self, tool, job_order, **kwargs):
self.debug = kwargs.get("debug")
@@ -355,9 +373,6 @@ class ArvCwlRunner(object):
if self.final_status == "UnsupportedRequirement":
raise UnsupportedRequirement("Check log for details.")
- if self.final_status != "success":
- raise WorkflowException("Workflow failed.")
-
if self.final_output is None:
raise WorkflowException("Workflow did not return a result.")
@@ -367,6 +382,10 @@ class ArvCwlRunner(object):
if self.output_name is None:
self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
self.make_output_collection(self.output_name, self.final_output)
+ self.set_crunch_output()
+
+ if self.final_status != "success":
+ raise WorkflowException("Workflow failed.")
if kwargs.get("compute_checksum"):
adjustDirObjs(self.final_output, partial(getListing, self.fs_access))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list