[ARVADOS] updated: a83e4de23b64a24e89321714511f6d9ef4ce4831

Git user git at public.curoverse.com
Tue Feb 21 12:48:11 EST 2017


Summary of changes:
 .../controllers/pipeline_instances_controller.rb   |   9 +
 .../app/controllers/projects_controller.rb         |   7 +-
 apps/workbench/app/models/job.rb                   |   2 +-
 apps/workbench/app/models/job_work_unit.rb         |   4 +
 apps/workbench/app/models/pipeline_instance.rb     |   4 +
 apps/workbench/app/models/work_unit.rb             |   4 +
 .../workbench/app/views/jobs/_show_status.html.erb |  32 +-
 .../pipeline_instances/_show_tab_buttons.html.erb  |   5 +-
 .../app/views/work_units/_show_child.html.erb      |   4 +-
 .../app/views/work_units/_show_component.html.erb  |   7 +-
 apps/workbench/config/routes.rb                    |   1 +
 .../test/controllers/jobs_controller_test.rb       |  15 +
 apps/workbench/test/integration/work_units_test.rb |  32 +-
 build/package-build-dockerfiles/Makefile           |   2 +-
 build/package-build-dockerfiles/centos7/Dockerfile |   6 +-
 build/package-build-dockerfiles/debian8/Dockerfile |   6 +-
 .../ubuntu1204/Dockerfile                          |   6 +-
 .../ubuntu1404/Dockerfile                          |   6 +-
 .../ubuntu1604/Dockerfile                          |   6 +-
 doc/_includes/_install_ruby_and_bundler.liquid     |   8 +-
 doc/_includes/_mount_types.liquid                  |  55 ++++
 .../methods/container_requests.html.textile.liquid |   2 +-
 sdk/cli/arvados-cli.gemspec                        |   4 +-
 sdk/cli/bin/crunch-job                             |   5 +-
 sdk/cli/test/binstub_output_coll_owner/python      |   4 +
 sdk/cli/test/test_crunch-job.rb                    |   9 +
 sdk/cwl/arvados_cwl/__init__.py                    |   7 +-
 sdk/cwl/arvados_cwl/arvcontainer.py                |  63 +++-
 sdk/cwl/arvados_cwl/arvjob.py                      |  11 +-
 sdk/cwl/arvados_cwl/done.py                        |   2 +-
 sdk/cwl/arvados_cwl/fsaccess.py                    |   9 +-
 sdk/cwl/arvados_cwl/pathmapper.py                  |   4 +-
 sdk/cwl/setup.py                                   |   4 +-
 sdk/cwl/test_with_arvbox.sh                        |   2 +
 sdk/cwl/tests/test_container.py                    | 125 +++++++-
 sdk/cwl/tests/test_submit.py                       |   9 +-
 sdk/go/arvados/collection.go                       |   5 +-
 sdk/go/arvados/container.go                        |  15 +-
 sdk/go/dispatch/dispatch.go                        | 340 ++++++++++----------
 sdk/go/keepclient/collectionreader.go              |   6 +-
 sdk/go/keepclient/collectionreader_test.go         |  10 +-
 sdk/go/manifest/manifest.go                        | 341 ++++++++++++++++++---
 sdk/go/manifest/manifest_test.go                   | 126 +++++++-
 sdk/go/manifest/testdata/long_manifest             |   8 +-
 sdk/go/manifest/testdata/short_manifest            |   7 +-
 sdk/python/arvados/collection.py                   |  12 +-
 sdk/python/arvados/commands/keepdocker.py          |  63 ++++
 sdk/python/arvados/commands/put.py                 |  19 +-
 sdk/python/bin/arv-migrate-docker19                |   4 +
 sdk/python/setup.py                                |   1 +
 sdk/python/tests/test_arv_keepdocker.py            |  17 +
 sdk/python/tests/test_arv_put.py                   |   4 +-
 sdk/python/tests/test_arvfile.py                   |  24 +-
 sdk/ruby/arvados.gemspec                           |   4 +-
 .../app/controllers/arvados/v1/jobs_controller.rb  |   2 +-
 .../arvados/v1/pipeline_instances_controller.rb    |   6 +
 services/api/app/models/arvados_model.rb           |   6 +-
 services/api/app/models/collection.rb              |  18 ++
 services/api/app/models/container_request.rb       |   8 +-
 services/api/app/models/job.rb                     |  51 ++-
 services/api/app/models/node.rb                    |   4 +-
 services/api/app/models/pipeline_instance.rb       |  27 ++
 services/api/app/models/workflow.rb                |   4 +
 services/api/config/routes.rb                      |   4 +-
 ..._and_workflow_def_in_full_text_search_index.rb} |  17 +-
 services/api/db/structure.sql                      | 138 +++++----
 services/api/lib/crunch_dispatch.rb                |   2 +-
 services/api/lib/has_uuid.rb                       |   4 +-
 services/api/lib/sweep_trashed_collections.rb      |   3 +
 services/api/test/fixtures/collections.yml         |   2 +-
 services/api/test/fixtures/jobs.yml                | 175 +++++++++++
 services/api/test/fixtures/links.yml               |  56 ++++
 services/api/test/fixtures/pipeline_instances.yml  |  39 +++
 .../v1/pipeline_instances_controller_test.rb       |  20 ++
 .../api/test/helpers/docker_migration_helper.rb    |  13 +
 services/api/test/unit/collection_test.rb          |  16 +
 services/api/test/unit/container_request_test.rb   |  19 ++
 services/api/test/unit/job_test.rb                 | 103 +++++++
 services/api/test/unit/link_test.rb                |  24 +-
 .../crunch-dispatch-local/crunch-dispatch-local.go |  18 +-
 .../crunch-dispatch-local_test.go                  |  42 +--
 .../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 170 +++++-----
 .../crunch-dispatch-slurm_test.go                  |  77 +++--
 services/crunch-run/crunchrun.go                   | 124 +++++++-
 services/crunch-run/crunchrun_test.go              | 269 ++++++++++++++--
 .../arvnodeman/computenode/dispatch/__init__.py    |  42 +--
 .../arvnodeman/computenode/dispatch/slurm.py       |  26 +-
 services/nodemanager/arvnodeman/daemon.py          |  52 ++--
 services/nodemanager/arvnodeman/jobqueue.py        |  20 +-
 services/nodemanager/arvnodeman/launcher.py        |   2 +-
 services/nodemanager/arvnodeman/nodelist.py        |  42 ++-
 .../nodemanager/tests/test_computenode_dispatch.py |  37 ++-
 .../tests/test_computenode_dispatch_slurm.py       |  25 +-
 .../tests/test_computenode_driver_azure.py         |   4 +-
 services/nodemanager/tests/test_daemon.py          |  57 ++--
 services/nodemanager/tests/test_failure.py         |   4 +-
 services/nodemanager/tests/test_jobqueue.py        |  19 +-
 services/nodemanager/tests/test_nodelist.py        |  45 ++-
 services/nodemanager/tests/testutil.py             |   4 +-
 services/ws/event_source.go                        |  94 ++++--
 services/ws/main.go                                |   8 +-
 tools/arvbox/lib/arvbox/docker/Dockerfile.base     |  23 +-
 tools/arvbox/lib/arvbox/docker/api-setup.sh        |   4 +-
 tools/arvbox/lib/arvbox/docker/createusers.sh      |  15 +-
 104 files changed, 2674 insertions(+), 797 deletions(-)
 create mode 100755 sdk/cli/test/binstub_output_coll_owner/python
 create mode 100755 sdk/python/bin/arv-migrate-docker19
 copy services/api/db/migrate/{20161213172944_full_text_search_indexes.rb => 20170216170823_no_cr_mounts_and_workflow_def_in_full_text_search_index.rb} (55%)
 create mode 100644 services/api/test/helpers/docker_migration_helper.rb

       via  a83e4de23b64a24e89321714511f6d9ef4ce4831 (commit)
       via  8ed521f7fd1e48e1e415125745ed8c6627a62c91 (commit)
       via  8b75947ee3f99b87eec443763653ca6ae3eb21e1 (commit)
       via  4bd54c4c42f13434b84410b7917a8cd208d613d7 (commit)
       via  d069de03a99dc58fd38f241435fcbaac84e9f63a (commit)
       via  d0ba6eaac1387ad817de1c2df2d7f4f00800aaa5 (commit)
       via  b1fffbeb4e06d0ec36c41c2fd9a0f23871f081c5 (commit)
       via  484370cbfe47b04e1d4222dd4a7606171c87a324 (commit)
       via  85887cd7fed798345e340480062b8ffcf3cf053a (commit)
       via  3fa4a2b6138e3e9e468dd885a743ca38f08f0755 (commit)
       via  ed1acd6fd780467ba69998e76e28fda61beedc0f (commit)
       via  24b1aecb485b170c6e127251201a6bac87d7860a (commit)
       via  d5ddfd9d876a75327795793544d105051f2a306e (commit)
       via  3a6966e5997ed5de342947759042ec5584f770c6 (commit)
       via  21598295f38998d8028aaa117f192de6b5758808 (commit)
       via  425f9a753e4390215b95b794785efd38bbc5f42d (commit)
       via  be33e85dd22b898e3a8f27ba8b42d9faef6e8516 (commit)
       via  73c2bd55dc6e13891c6d1be63fc0613728bd929a (commit)
       via  72873affa7f249faa16d5d21200e935d27aea911 (commit)
       via  dc086bed8661d0e7579df7a240f7e67c86946a59 (commit)
       via  197dd5583c13fb6cddba8d74848df20ba57ed924 (commit)
       via  2d80bb7d83bda777181542afcb0b7293cb53eefd (commit)
       via  1aa3f43606ba5e31633c063851d25b8fab4b93e2 (commit)
       via  0403a0c816df1edea311b9197147fd254d131712 (commit)
       via  5624fec61db977d386ce03ca333241c74ca251b5 (commit)
       via  4edf67aee4e8d4b351ee08bcb19c051a9bddeacd (commit)
       via  31d76600cdb691251d0823cc6be601d958b4e1a4 (commit)
       via  03306e210c6835c6de1c908d5afe02ba964e7dbc (commit)
       via  6873eca0bf64a2a94ea7536bbc9a5a3788695136 (commit)
       via  6b775ec45db0143c0d476cf2f0fcfb8bdd39a845 (commit)
       via  fb60dbc9b79cfa8f34a10909ea0fd2a51b600ce6 (commit)
       via  314f3c34442e0daffbea4ed62adea9e722673810 (commit)
       via  15c1329834e02814fb5b510bf13f1ea2a97ee028 (commit)
       via  0e6c1a287933d8e55508c2457b7ce31a2bb5a965 (commit)
       via  e86743ca1a4e1431d6d8417083ef1a56461b45e9 (commit)
       via  908959eafe8a9925cb2c204d6511095f702c1667 (commit)
       via  f14ce11321e919cc39b878fe9f7847e1a9bb0de3 (commit)
       via  bf9af31f49954d949317475bdcdc0694d247f82d (commit)
       via  4440f049a94bd3570271f8f0b2461d7f3f3c4582 (commit)
       via  f782a2505422ad9c853c4c416640c41f3b1e7e79 (commit)
       via  3f30342290d901408a3b9a640a21e41f011041cd (commit)
       via  c90f6525e949c2314ccc9c2cb980bccbc23e2434 (commit)
       via  6e8c06b81867a3d54e89694f1301ab037d3f5d22 (commit)
       via  72b6a2992c69f557f3b6b850914c5ee914cf862b (commit)
       via  38bf86b7d9e256aaae50870174ae2cf6a84e8656 (commit)
       via  c86cbaa6f286e50900dae3203a42044449e042f7 (commit)
       via  0030aceea911108449d67a785d802e894943b340 (commit)
       via  70f47bfdccdbdcc773225a94015bba2d2975d7f7 (commit)
       via  7c667d8963c7a3cf9acd04c1d938b5273b761228 (commit)
       via  5763c67176e8e34656cd96881074777b14b2dc4a (commit)
       via  53b24cf4fafa5c8ce76eb4abf5e61de944bb4a6a (commit)
       via  232a69cedf3024380546b9e67ad10d6beb357649 (commit)
       via  10b3fe2ae3a37ee473684177aa6e4e9f090a230e (commit)
       via  90982bd47200f6a555074842c817ac2893c2e391 (commit)
       via  01779cbbd49870dbe12478713bf6cd3332d08144 (commit)
       via  2a469c4874895b05ee137e2382fd882680b3feb2 (commit)
       via  091c92aef16f9657cf7b9eb8f8778aafa33f12c1 (commit)
       via  e2f3f8e7347a8ded697d3c0127f82325d672e7ab (commit)
       via  08c06bc813d96634b362e8c5736341fb2d874f59 (commit)
       via  d6aab18f9688d46bc1f86f021d68e02e5601cfe7 (commit)
       via  5dfc48c1a1697d9c29eac0061a491e96c9b72441 (commit)
       via  dc82c1c16a9b18e094e60af2c3973e034d5e2068 (commit)
       via  d3719c6ba29daec66eb1dd4003f2580f97da8e41 (commit)
       via  35ae0e8d87dde84dd3a0ab41cef9567ab93b042a (commit)
       via  2e9cb970e94c1ba259c6365ea4b456ca87e85cee (commit)
       via  fbc6c71ef67461cde5ba914e61f3f3b8740a4045 (commit)
       via  397a747639fe360c18f3aa79846fef530b263184 (commit)
       via  a2fe6f9367de3ee93064fbee3f2df78ce84aa318 (commit)
       via  9333c9f65503d86c12776e0bc8bfcb6fc07dc79c (commit)
       via  cff6cd57cbbaa032602ca2fb930c78812c5824dc (commit)
       via  6eded76b722df150ce59a1bddd2fb7e5175a596d (commit)
       via  d6cc6f748c79e9bbd0ba18dd8d32ae210a7b897b (commit)
       via  ef9bbef07f63dd2207e1336e85884aa90434ad29 (commit)
       via  ad464d416de6996a41d0c752124e0d201de0d3fc (commit)
       via  80689aac71f6e3e9b103f0f6b668bd173a76554f (commit)
       via  4ee34d218487d8b330147b185a2c9bcea8d68b09 (commit)
       via  6b1bf77727379c7ffaf1620399c37ea0106a0909 (commit)
       via  ddee3839f8a82b889f84171e2354108cb20f93e0 (commit)
       via  78b94fd03e3c0a2d225d0ab8882ee62dee9f8a11 (commit)
       via  d95e6df5184ab2ec137c8098b47caf9ebcf4e7d9 (commit)
       via  7b4c1b70b7d1ef4c977c57c89c4dc0f479e7e5ae (commit)
       via  4fa9baf6590e63975b6053a049543590db0e2527 (commit)
       via  4c081dd00f65f1e5a8e0cea34276d60ecbb49f40 (commit)
       via  2b297df85b61ac7f2ded512eca7c307d75b1cd8e (commit)
       via  a0195a506af77771434302b96a5e05ecfd3814d8 (commit)
       via  85c684122fd678dc24035d58236dd5734aed50e5 (commit)
       via  1c434a9f9ec70d8f23583bc737a516c3ef0eb91d (commit)
       via  310f7b0266f012355e38f3296bc6defe14a3d25f (commit)
       via  15b49783aa0ac76508986e772b98ffa9d187c57f (commit)
       via  5bf9312a5174f97f00db383836eb7666dc500293 (commit)
       via  2b7834020290b28d797333f90fcb87e5da67d616 (commit)
       via  4fd89ed7b10f0860a6030c25e44d4df45a087b2e (commit)
       via  8ee2c7c4f231a55601fdc90b087e42985b52fb20 (commit)
       via  d5effac093938be2b57d579cc1fecfa90310af02 (commit)
       via  0fc6eaead0bf7691e99d19e74ec33636909001a7 (commit)
       via  210af1cc7ede88914026fde078e45ef84c187a0c (commit)
       via  bf0945d1136d2578e40909d54e8614085d6f9c34 (commit)
       via  ca56623679bcf5733a3266711f513f8c23f8b0df (commit)
       via  4dbaf8c355f3743bd42ff5b917eda57d3e90abf8 (commit)
       via  108467e2b85c9f3c44b483bc79f68acf6ae3b963 (commit)
       via  ab6a70e86dd041f3b4da167c59e3e91309f14365 (commit)
       via  3f8336875d5938afb6b00289e9d6c9941456b57f (commit)
       via  ef934e66fee76dcf760dd5fc835f27127e4cc791 (commit)
       via  8734a7391a5672eebcdf572d93bae1b3ed1179c9 (commit)
       via  8ee173e80b4954159490c8f9327dab07a4bffcf3 (commit)
       via  837c509a55f56fa653454e7b99e293f8f87ebef2 (commit)
       via  8adad21150d3678aad0f88e5fb30a088145478ee (commit)
       via  925a1526383299a1ade38a18e616564ab8c38da4 (commit)
       via  785c967e74a7dab0b29b276162d1d7513ce1cf6b (commit)
       via  471270bc6ec9453fdd4c1faf97b65a8291543c6f (commit)
       via  6a7d7a2fa8e217e1ff9440769f39a2095d5bb837 (commit)
       via  04a8e2c7d8a8dd3afd891292415cdfaea25fd481 (commit)
       via  e6986ed88126f61ecad0f557c78981961f901044 (commit)
       via  5caa73a3bf5a17241bf908c75aa2c3aa427289d1 (commit)
       via  da42861619eb478cd1f01d58a1ebe59f1a25002e (commit)
       via  dc7d01f4d4031962ffd5734ca0c64146a7217e4a (commit)
       via  a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a (commit)
       via  280621308e41e9cc8b8557d27738f186e795780b (commit)
       via  8e9a3e39375b4dd689cd85a92e77a5eee03b4908 (commit)
       via  6b91e8201f24d2b4126532d809abba42a5ab959c (commit)
       via  fbacdaec3dbba425155ab6348c7e6b80ff4e710b (commit)
       via  edd77406c5a428af8780ed4dffd512adcc70bc0f (commit)
       via  49d7a1fbd42359aa0948efd0968349eaf854d6cb (commit)
       via  9cc8eb72a4d8dddd2ffe2014d085517ecd0f6a67 (commit)
       via  89ce36332958f698bc571eca770fb94e579957cd (commit)
       via  d2942208ef4bc31bc965d5f72f254b8d39a1bc9b (commit)
      from  e2ea7599812d12dd027930069ee92e8816ee9de8 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit a83e4de23b64a24e89321714511f6d9ef4ce4831
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Tue Feb 21 14:41:58 2017 -0300

    11002: Do not try to save internal state when receiving a KeyboardInterrupt exception.
    Updated test accordingly.

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 30ba882..5598e54 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -391,6 +391,7 @@ class ArvPutUploadJob(object):
         self._upload_started = False
         self.logger = logger
         self.dry_run = dry_run
+        self._checkpoint_before_quit = True
 
         if not self.use_cache and self.resume:
             raise ArvPutArgumentConflict('resume cannot be True when use_cache is False')
@@ -447,20 +448,21 @@ class ArvPutUploadJob(object):
             # Actual file upload
             self._upload_started = True # Used by the update thread to start checkpointing
             self._upload_files()
+        except KeyboardInterrupt:
+            self.logger.warning("User interrupt request, cleaning up before exiting.")
+            self._checkpoint_before_quit = False
+            raise
         finally:
             if not self.dry_run:
                 # Stop the thread before doing anything else
                 self._stop_checkpointer.set()
                 self._checkpointer.join()
-                try:
+                if self._checkpoint_before_quit:
                     # Commit all pending blocks & one last _update()
                     self._local_collection.manifest_text()
                     self._update(final=True)
                     if save_collection:
                         self.save_collection()
-                except AttributeError:
-                    # Exception caught in inconsistent state, finish as is.
-                    self.logger.warning("Couldn't save last checkpoint while exiting.")
             if self.use_cache:
                 self._cache_file.close()
 
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index f9af01d..f0c76c2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -347,7 +347,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
     # Test for bug #11002
     def test_graceful_exit_while_repacking_small_blocks(self):
         def wrapped_commit(*args, **kwargs):
-            raise SystemExit("Simulated error")
+            raise KeyboardInterrupt("Simulated error")
 
         with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock',
                         autospec=True) as mocked_commit:
@@ -359,7 +359,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
             writer = arv_put.ArvPutUploadJob([self.small_files_dir],
                                              replication_desired=1)
             try:
-                with self.assertRaises(SystemExit):
+                with self.assertRaises(KeyboardInterrupt):
                     writer.start(save_collection=False)
             except AttributeError:
                 self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002")

commit 8ed521f7fd1e48e1e415125745ed8c6627a62c91
Merge: e2ea759 8b75947
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Tue Feb 21 13:22:07 2017 -0300

    11002: Merge branch 'master' into 11002-arvput-crash-fix


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list