[ARVADOS] updated: a83e4de23b64a24e89321714511f6d9ef4ce4831
Git user
git at public.curoverse.com
Tue Feb 21 12:48:11 EST 2017
Summary of changes:
.../controllers/pipeline_instances_controller.rb | 9 +
.../app/controllers/projects_controller.rb | 7 +-
apps/workbench/app/models/job.rb | 2 +-
apps/workbench/app/models/job_work_unit.rb | 4 +
apps/workbench/app/models/pipeline_instance.rb | 4 +
apps/workbench/app/models/work_unit.rb | 4 +
.../workbench/app/views/jobs/_show_status.html.erb | 32 +-
.../pipeline_instances/_show_tab_buttons.html.erb | 5 +-
.../app/views/work_units/_show_child.html.erb | 4 +-
.../app/views/work_units/_show_component.html.erb | 7 +-
apps/workbench/config/routes.rb | 1 +
.../test/controllers/jobs_controller_test.rb | 15 +
apps/workbench/test/integration/work_units_test.rb | 32 +-
build/package-build-dockerfiles/Makefile | 2 +-
build/package-build-dockerfiles/centos7/Dockerfile | 6 +-
build/package-build-dockerfiles/debian8/Dockerfile | 6 +-
.../ubuntu1204/Dockerfile | 6 +-
.../ubuntu1404/Dockerfile | 6 +-
.../ubuntu1604/Dockerfile | 6 +-
doc/_includes/_install_ruby_and_bundler.liquid | 8 +-
doc/_includes/_mount_types.liquid | 55 ++++
.../methods/container_requests.html.textile.liquid | 2 +-
sdk/cli/arvados-cli.gemspec | 4 +-
sdk/cli/bin/crunch-job | 5 +-
sdk/cli/test/binstub_output_coll_owner/python | 4 +
sdk/cli/test/test_crunch-job.rb | 9 +
sdk/cwl/arvados_cwl/__init__.py | 7 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 63 +++-
sdk/cwl/arvados_cwl/arvjob.py | 11 +-
sdk/cwl/arvados_cwl/done.py | 2 +-
sdk/cwl/arvados_cwl/fsaccess.py | 9 +-
sdk/cwl/arvados_cwl/pathmapper.py | 4 +-
sdk/cwl/setup.py | 4 +-
sdk/cwl/test_with_arvbox.sh | 2 +
sdk/cwl/tests/test_container.py | 125 +++++++-
sdk/cwl/tests/test_submit.py | 9 +-
sdk/go/arvados/collection.go | 5 +-
sdk/go/arvados/container.go | 15 +-
sdk/go/dispatch/dispatch.go | 340 ++++++++++----------
sdk/go/keepclient/collectionreader.go | 6 +-
sdk/go/keepclient/collectionreader_test.go | 10 +-
sdk/go/manifest/manifest.go | 341 ++++++++++++++++++---
sdk/go/manifest/manifest_test.go | 126 +++++++-
sdk/go/manifest/testdata/long_manifest | 8 +-
sdk/go/manifest/testdata/short_manifest | 7 +-
sdk/python/arvados/collection.py | 12 +-
sdk/python/arvados/commands/keepdocker.py | 63 ++++
sdk/python/arvados/commands/put.py | 19 +-
sdk/python/bin/arv-migrate-docker19 | 4 +
sdk/python/setup.py | 1 +
sdk/python/tests/test_arv_keepdocker.py | 17 +
sdk/python/tests/test_arv_put.py | 4 +-
sdk/python/tests/test_arvfile.py | 24 +-
sdk/ruby/arvados.gemspec | 4 +-
.../app/controllers/arvados/v1/jobs_controller.rb | 2 +-
.../arvados/v1/pipeline_instances_controller.rb | 6 +
services/api/app/models/arvados_model.rb | 6 +-
services/api/app/models/collection.rb | 18 ++
services/api/app/models/container_request.rb | 8 +-
services/api/app/models/job.rb | 51 ++-
services/api/app/models/node.rb | 4 +-
services/api/app/models/pipeline_instance.rb | 27 ++
services/api/app/models/workflow.rb | 4 +
services/api/config/routes.rb | 4 +-
..._and_workflow_def_in_full_text_search_index.rb} | 17 +-
services/api/db/structure.sql | 138 +++++----
services/api/lib/crunch_dispatch.rb | 2 +-
services/api/lib/has_uuid.rb | 4 +-
services/api/lib/sweep_trashed_collections.rb | 3 +
services/api/test/fixtures/collections.yml | 2 +-
services/api/test/fixtures/jobs.yml | 175 +++++++++++
services/api/test/fixtures/links.yml | 56 ++++
services/api/test/fixtures/pipeline_instances.yml | 39 +++
.../v1/pipeline_instances_controller_test.rb | 20 ++
.../api/test/helpers/docker_migration_helper.rb | 13 +
services/api/test/unit/collection_test.rb | 16 +
services/api/test/unit/container_request_test.rb | 19 ++
services/api/test/unit/job_test.rb | 103 +++++++
services/api/test/unit/link_test.rb | 24 +-
.../crunch-dispatch-local/crunch-dispatch-local.go | 18 +-
.../crunch-dispatch-local_test.go | 42 +--
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 170 +++++-----
.../crunch-dispatch-slurm_test.go | 77 +++--
services/crunch-run/crunchrun.go | 124 +++++++-
services/crunch-run/crunchrun_test.go | 269 ++++++++++++++--
.../arvnodeman/computenode/dispatch/__init__.py | 42 +--
.../arvnodeman/computenode/dispatch/slurm.py | 26 +-
services/nodemanager/arvnodeman/daemon.py | 52 ++--
services/nodemanager/arvnodeman/jobqueue.py | 20 +-
services/nodemanager/arvnodeman/launcher.py | 2 +-
services/nodemanager/arvnodeman/nodelist.py | 42 ++-
.../nodemanager/tests/test_computenode_dispatch.py | 37 ++-
.../tests/test_computenode_dispatch_slurm.py | 25 +-
.../tests/test_computenode_driver_azure.py | 4 +-
services/nodemanager/tests/test_daemon.py | 57 ++--
services/nodemanager/tests/test_failure.py | 4 +-
services/nodemanager/tests/test_jobqueue.py | 19 +-
services/nodemanager/tests/test_nodelist.py | 45 ++-
services/nodemanager/tests/testutil.py | 4 +-
services/ws/event_source.go | 94 ++++--
services/ws/main.go | 8 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 23 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 4 +-
tools/arvbox/lib/arvbox/docker/createusers.sh | 15 +-
104 files changed, 2674 insertions(+), 797 deletions(-)
create mode 100755 sdk/cli/test/binstub_output_coll_owner/python
create mode 100755 sdk/python/bin/arv-migrate-docker19
copy services/api/db/migrate/{20161213172944_full_text_search_indexes.rb => 20170216170823_no_cr_mounts_and_workflow_def_in_full_text_search_index.rb} (55%)
create mode 100644 services/api/test/helpers/docker_migration_helper.rb
via a83e4de23b64a24e89321714511f6d9ef4ce4831 (commit)
via 8ed521f7fd1e48e1e415125745ed8c6627a62c91 (commit)
via 8b75947ee3f99b87eec443763653ca6ae3eb21e1 (commit)
via 4bd54c4c42f13434b84410b7917a8cd208d613d7 (commit)
via d069de03a99dc58fd38f241435fcbaac84e9f63a (commit)
via d0ba6eaac1387ad817de1c2df2d7f4f00800aaa5 (commit)
via b1fffbeb4e06d0ec36c41c2fd9a0f23871f081c5 (commit)
via 484370cbfe47b04e1d4222dd4a7606171c87a324 (commit)
via 85887cd7fed798345e340480062b8ffcf3cf053a (commit)
via 3fa4a2b6138e3e9e468dd885a743ca38f08f0755 (commit)
via ed1acd6fd780467ba69998e76e28fda61beedc0f (commit)
via 24b1aecb485b170c6e127251201a6bac87d7860a (commit)
via d5ddfd9d876a75327795793544d105051f2a306e (commit)
via 3a6966e5997ed5de342947759042ec5584f770c6 (commit)
via 21598295f38998d8028aaa117f192de6b5758808 (commit)
via 425f9a753e4390215b95b794785efd38bbc5f42d (commit)
via be33e85dd22b898e3a8f27ba8b42d9faef6e8516 (commit)
via 73c2bd55dc6e13891c6d1be63fc0613728bd929a (commit)
via 72873affa7f249faa16d5d21200e935d27aea911 (commit)
via dc086bed8661d0e7579df7a240f7e67c86946a59 (commit)
via 197dd5583c13fb6cddba8d74848df20ba57ed924 (commit)
via 2d80bb7d83bda777181542afcb0b7293cb53eefd (commit)
via 1aa3f43606ba5e31633c063851d25b8fab4b93e2 (commit)
via 0403a0c816df1edea311b9197147fd254d131712 (commit)
via 5624fec61db977d386ce03ca333241c74ca251b5 (commit)
via 4edf67aee4e8d4b351ee08bcb19c051a9bddeacd (commit)
via 31d76600cdb691251d0823cc6be601d958b4e1a4 (commit)
via 03306e210c6835c6de1c908d5afe02ba964e7dbc (commit)
via 6873eca0bf64a2a94ea7536bbc9a5a3788695136 (commit)
via 6b775ec45db0143c0d476cf2f0fcfb8bdd39a845 (commit)
via fb60dbc9b79cfa8f34a10909ea0fd2a51b600ce6 (commit)
via 314f3c34442e0daffbea4ed62adea9e722673810 (commit)
via 15c1329834e02814fb5b510bf13f1ea2a97ee028 (commit)
via 0e6c1a287933d8e55508c2457b7ce31a2bb5a965 (commit)
via e86743ca1a4e1431d6d8417083ef1a56461b45e9 (commit)
via 908959eafe8a9925cb2c204d6511095f702c1667 (commit)
via f14ce11321e919cc39b878fe9f7847e1a9bb0de3 (commit)
via bf9af31f49954d949317475bdcdc0694d247f82d (commit)
via 4440f049a94bd3570271f8f0b2461d7f3f3c4582 (commit)
via f782a2505422ad9c853c4c416640c41f3b1e7e79 (commit)
via 3f30342290d901408a3b9a640a21e41f011041cd (commit)
via c90f6525e949c2314ccc9c2cb980bccbc23e2434 (commit)
via 6e8c06b81867a3d54e89694f1301ab037d3f5d22 (commit)
via 72b6a2992c69f557f3b6b850914c5ee914cf862b (commit)
via 38bf86b7d9e256aaae50870174ae2cf6a84e8656 (commit)
via c86cbaa6f286e50900dae3203a42044449e042f7 (commit)
via 0030aceea911108449d67a785d802e894943b340 (commit)
via 70f47bfdccdbdcc773225a94015bba2d2975d7f7 (commit)
via 7c667d8963c7a3cf9acd04c1d938b5273b761228 (commit)
via 5763c67176e8e34656cd96881074777b14b2dc4a (commit)
via 53b24cf4fafa5c8ce76eb4abf5e61de944bb4a6a (commit)
via 232a69cedf3024380546b9e67ad10d6beb357649 (commit)
via 10b3fe2ae3a37ee473684177aa6e4e9f090a230e (commit)
via 90982bd47200f6a555074842c817ac2893c2e391 (commit)
via 01779cbbd49870dbe12478713bf6cd3332d08144 (commit)
via 2a469c4874895b05ee137e2382fd882680b3feb2 (commit)
via 091c92aef16f9657cf7b9eb8f8778aafa33f12c1 (commit)
via e2f3f8e7347a8ded697d3c0127f82325d672e7ab (commit)
via 08c06bc813d96634b362e8c5736341fb2d874f59 (commit)
via d6aab18f9688d46bc1f86f021d68e02e5601cfe7 (commit)
via 5dfc48c1a1697d9c29eac0061a491e96c9b72441 (commit)
via dc82c1c16a9b18e094e60af2c3973e034d5e2068 (commit)
via d3719c6ba29daec66eb1dd4003f2580f97da8e41 (commit)
via 35ae0e8d87dde84dd3a0ab41cef9567ab93b042a (commit)
via 2e9cb970e94c1ba259c6365ea4b456ca87e85cee (commit)
via fbc6c71ef67461cde5ba914e61f3f3b8740a4045 (commit)
via 397a747639fe360c18f3aa79846fef530b263184 (commit)
via a2fe6f9367de3ee93064fbee3f2df78ce84aa318 (commit)
via 9333c9f65503d86c12776e0bc8bfcb6fc07dc79c (commit)
via cff6cd57cbbaa032602ca2fb930c78812c5824dc (commit)
via 6eded76b722df150ce59a1bddd2fb7e5175a596d (commit)
via d6cc6f748c79e9bbd0ba18dd8d32ae210a7b897b (commit)
via ef9bbef07f63dd2207e1336e85884aa90434ad29 (commit)
via ad464d416de6996a41d0c752124e0d201de0d3fc (commit)
via 80689aac71f6e3e9b103f0f6b668bd173a76554f (commit)
via 4ee34d218487d8b330147b185a2c9bcea8d68b09 (commit)
via 6b1bf77727379c7ffaf1620399c37ea0106a0909 (commit)
via ddee3839f8a82b889f84171e2354108cb20f93e0 (commit)
via 78b94fd03e3c0a2d225d0ab8882ee62dee9f8a11 (commit)
via d95e6df5184ab2ec137c8098b47caf9ebcf4e7d9 (commit)
via 7b4c1b70b7d1ef4c977c57c89c4dc0f479e7e5ae (commit)
via 4fa9baf6590e63975b6053a049543590db0e2527 (commit)
via 4c081dd00f65f1e5a8e0cea34276d60ecbb49f40 (commit)
via 2b297df85b61ac7f2ded512eca7c307d75b1cd8e (commit)
via a0195a506af77771434302b96a5e05ecfd3814d8 (commit)
via 85c684122fd678dc24035d58236dd5734aed50e5 (commit)
via 1c434a9f9ec70d8f23583bc737a516c3ef0eb91d (commit)
via 310f7b0266f012355e38f3296bc6defe14a3d25f (commit)
via 15b49783aa0ac76508986e772b98ffa9d187c57f (commit)
via 5bf9312a5174f97f00db383836eb7666dc500293 (commit)
via 2b7834020290b28d797333f90fcb87e5da67d616 (commit)
via 4fd89ed7b10f0860a6030c25e44d4df45a087b2e (commit)
via 8ee2c7c4f231a55601fdc90b087e42985b52fb20 (commit)
via d5effac093938be2b57d579cc1fecfa90310af02 (commit)
via 0fc6eaead0bf7691e99d19e74ec33636909001a7 (commit)
via 210af1cc7ede88914026fde078e45ef84c187a0c (commit)
via bf0945d1136d2578e40909d54e8614085d6f9c34 (commit)
via ca56623679bcf5733a3266711f513f8c23f8b0df (commit)
via 4dbaf8c355f3743bd42ff5b917eda57d3e90abf8 (commit)
via 108467e2b85c9f3c44b483bc79f68acf6ae3b963 (commit)
via ab6a70e86dd041f3b4da167c59e3e91309f14365 (commit)
via 3f8336875d5938afb6b00289e9d6c9941456b57f (commit)
via ef934e66fee76dcf760dd5fc835f27127e4cc791 (commit)
via 8734a7391a5672eebcdf572d93bae1b3ed1179c9 (commit)
via 8ee173e80b4954159490c8f9327dab07a4bffcf3 (commit)
via 837c509a55f56fa653454e7b99e293f8f87ebef2 (commit)
via 8adad21150d3678aad0f88e5fb30a088145478ee (commit)
via 925a1526383299a1ade38a18e616564ab8c38da4 (commit)
via 785c967e74a7dab0b29b276162d1d7513ce1cf6b (commit)
via 471270bc6ec9453fdd4c1faf97b65a8291543c6f (commit)
via 6a7d7a2fa8e217e1ff9440769f39a2095d5bb837 (commit)
via 04a8e2c7d8a8dd3afd891292415cdfaea25fd481 (commit)
via e6986ed88126f61ecad0f557c78981961f901044 (commit)
via 5caa73a3bf5a17241bf908c75aa2c3aa427289d1 (commit)
via da42861619eb478cd1f01d58a1ebe59f1a25002e (commit)
via dc7d01f4d4031962ffd5734ca0c64146a7217e4a (commit)
via a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a (commit)
via 280621308e41e9cc8b8557d27738f186e795780b (commit)
via 8e9a3e39375b4dd689cd85a92e77a5eee03b4908 (commit)
via 6b91e8201f24d2b4126532d809abba42a5ab959c (commit)
via fbacdaec3dbba425155ab6348c7e6b80ff4e710b (commit)
via edd77406c5a428af8780ed4dffd512adcc70bc0f (commit)
via 49d7a1fbd42359aa0948efd0968349eaf854d6cb (commit)
via 9cc8eb72a4d8dddd2ffe2014d085517ecd0f6a67 (commit)
via 89ce36332958f698bc571eca770fb94e579957cd (commit)
via d2942208ef4bc31bc965d5f72f254b8d39a1bc9b (commit)
from e2ea7599812d12dd027930069ee92e8816ee9de8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a83e4de23b64a24e89321714511f6d9ef4ce4831
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Feb 21 14:41:58 2017 -0300
11002: Do not try to save internal state when receiving a KeyboardInterrupt exception.
Updated test accordingly.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 30ba882..5598e54 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -391,6 +391,7 @@ class ArvPutUploadJob(object):
self._upload_started = False
self.logger = logger
self.dry_run = dry_run
+ self._checkpoint_before_quit = True
if not self.use_cache and self.resume:
raise ArvPutArgumentConflict('resume cannot be True when use_cache is False')
@@ -447,20 +448,21 @@ class ArvPutUploadJob(object):
# Actual file upload
self._upload_started = True # Used by the update thread to start checkpointing
self._upload_files()
+ except KeyboardInterrupt:
+ self.logger.warning("User interrupt request, cleaning up before exiting.")
+ self._checkpoint_before_quit = False
+ raise
finally:
if not self.dry_run:
# Stop the thread before doing anything else
self._stop_checkpointer.set()
self._checkpointer.join()
- try:
+ if self._checkpoint_before_quit:
# Commit all pending blocks & one last _update()
self._local_collection.manifest_text()
self._update(final=True)
if save_collection:
self.save_collection()
- except AttributeError:
- # Exception caught in inconsistent state, finish as is.
- self.logger.warning("Couldn't save last checkpoint while exiting.")
if self.use_cache:
self._cache_file.close()
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index f9af01d..f0c76c2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -347,7 +347,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
# Test for bug #11002
def test_graceful_exit_while_repacking_small_blocks(self):
def wrapped_commit(*args, **kwargs):
- raise SystemExit("Simulated error")
+ raise KeyboardInterrupt("Simulated error")
with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock',
autospec=True) as mocked_commit:
@@ -359,7 +359,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
writer = arv_put.ArvPutUploadJob([self.small_files_dir],
replication_desired=1)
try:
- with self.assertRaises(SystemExit):
+ with self.assertRaises(KeyboardInterrupt):
writer.start(save_collection=False)
except AttributeError:
self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002")
commit 8ed521f7fd1e48e1e415125745ed8c6627a62c91
Merge: e2ea759 8b75947
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Feb 21 13:22:07 2017 -0300
11002: Merge branch 'master' into 11002-arvput-crash-fix
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list