[ARVADOS] updated: 1.3.0-760-g454ee2b8f
Git user
git at public.curoverse.com
Fri Apr 19 14:59:44 UTC 2019
Summary of changes:
.licenseignore | 2 +
build/run-library.sh | 14 +
build/run-tests.sh | 4 +-
doc/_config.yml | 5 +
doc/_includes/_config_default_yml.liquid | 1 +
doc/admin/config-migration.html.textile.liquid | 50 +++
doc/admin/config.html.textile.liquid | 19 +
...controlling-container-reuse.html.textile.liquid | 21 +
.../logs-table-management.html.textile.liquid | 55 +++
doc/admin/upgrading.html.textile.liquid | 4 +
doc/api/methods/containers.html.textile.liquid | 4 +-
doc/install/arvbox.html.textile.liquid | 55 ++-
.../install-nodemanager.html.textile.liquid | 2 +-
lib/config/config.default.yml | 455 +++++++++++++++++++
lib/dispatchcloud/container/queue_test.go | 4 +-
sdk/cwl/arvados_cwl/executor.py | 3 +-
sdk/cwl/arvados_cwl/pathmapper.py | 13 +-
.../api/app/controllers/application_controller.rb | 20 +-
.../controllers/arvados/v1/groups_controller.rb | 2 +-
.../arvados/v1/healthcheck_controller.rb | 2 +-
.../controllers/arvados/v1/schema_controller.rb | 55 ++-
services/api/app/controllers/static_controller.rb | 4 +-
.../app/controllers/user_sessions_controller.rb | 4 +-
services/api/app/mailers/admin_notifier.rb | 16 +-
services/api/app/mailers/profile_notifier.rb | 2 +-
services/api/app/mailers/user_notifier.rb | 2 +-
.../api/app/models/api_client_authorization.rb | 16 +-
services/api/app/models/arvados_model.rb | 8 +-
services/api/app/models/blob.rb | 10 +-
services/api/app/models/collection.rb | 28 +-
services/api/app/models/commit.rb | 2 +-
services/api/app/models/commit_ancestor.rb | 2 +-
services/api/app/models/container.rb | 11 +-
services/api/app/models/container_request.rb | 6 +-
services/api/app/models/job.rb | 8 +-
services/api/app/models/node.rb | 43 +-
services/api/app/models/repository.rb | 25 +-
services/api/app/models/user.rb | 20 +-
.../admin_notifier/new_inactive_user.text.erb | 4 +-
.../api/app/views/admin_notifier/new_user.text.erb | 7 +-
.../views/user_notifier/account_is_setup.text.erb | 4 +-
services/api/config/application.default.yml | 495 ---------------------
services/api/config/application.rb | 2 +
services/api/config/arvados_config.rb | 260 +++++++++++
services/api/config/config.default.yml | 1 +
.../api/config/initializers/legacy_jobs_api.rb | 1 -
services/api/config/initializers/load_config.rb | 77 ----
services/api/config/initializers/lograge.rb | 4 +-
services/api/config/initializers/omniauth_init.rb | 12 +-
.../api/config/initializers/preload_all_models.rb | 1 -
services/api/db/structure.sql | 1 +
services/api/lib/audit_logs.rb | 4 +-
services/api/lib/config_loader.rb | 205 +++++++++
services/api/lib/crunch_dispatch.rb | 48 +-
services/api/lib/current_api_client.rb | 10 +-
services/api/lib/enable_jobs_api.rb | 6 +-
services/api/lib/has_uuid.rb | 2 +-
services/api/lib/josh_id.rb | 2 +-
services/api/lib/load_param.rb | 2 +-
services/api/lib/log_reuse_info.rb | 2 +-
services/api/lib/refresh_permission_view.rb | 4 +-
services/api/lib/sweep_trashed_objects.rb | 4 +-
services/api/lib/tasks/config.rake | 51 +++
services/api/lib/tasks/config_check.rake | 28 --
services/api/lib/tasks/config_dump.rake | 10 -
.../api/lib/tasks/delete_old_container_logs.rake | 2 +-
services/api/lib/tasks/delete_old_job_logs.rake | 2 +-
services/api/lib/trashable.rb | 6 +-
.../arvados/v1/collections_controller_test.rb | 30 +-
.../api/test/functional/arvados/v1/filters_test.rb | 4 +-
.../arvados/v1/groups_controller_test.rb | 4 +-
.../functional/arvados/v1/jobs_controller_test.rb | 10 +-
.../functional/arvados/v1/nodes_controller_test.rb | 2 +-
.../arvados/v1/repositories_controller_test.rb | 14 +-
.../arvados/v1/schema_controller_test.rb | 8 +-
.../functional/arvados/v1/users_controller_test.rb | 4 +-
services/api/test/helpers/git_test_helper.rb | 8 +-
.../api/test/integration/collections_api_test.rb | 6 +-
services/api/test/integration/groups_test.rb | 2 +-
services/api/test/integration/remote_user_test.rb | 6 +-
.../api/test/integration/user_sessions_test.rb | 8 +-
.../test/tasks/delete_old_container_logs_test.rb | 2 +-
.../api/test/tasks/delete_old_job_logs_test.rb | 2 +-
services/api/test/test_helper.rb | 7 +-
services/api/test/unit/blob_test.rb | 6 +-
services/api/test/unit/collection_test.rb | 46 +-
services/api/test/unit/commit_test.rb | 8 +-
services/api/test/unit/container_request_test.rb | 26 +-
services/api/test/unit/container_test.rb | 57 ++-
services/api/test/unit/crunch_dispatch_test.rb | 6 +-
services/api/test/unit/fail_jobs_test.rb | 6 +-
services/api/test/unit/job_test.rb | 37 +-
services/api/test/unit/log_test.rb | 12 +-
services/api/test/unit/node_test.rb | 38 +-
services/api/test/unit/repository_test.rb | 6 +-
services/api/test/unit/user_notifier_test.rb | 4 +-
services/api/test/unit/user_test.rb | 48 +-
tools/arvbox/bin/arvbox | 163 ++++---
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 8 +
tools/arvbox/lib/arvbox/docker/createusers.sh | 10 +-
tools/arvbox/lib/arvbox/docker/devenv.sh | 2 +-
101 files changed, 1743 insertions(+), 1111 deletions(-)
create mode 120000 doc/_includes/_config_default_yml.liquid
create mode 100644 doc/admin/config-migration.html.textile.liquid
create mode 100644 doc/admin/config.html.textile.liquid
create mode 100644 doc/admin/controlling-container-reuse.html.textile.liquid
create mode 100644 doc/admin/logs-table-management.html.textile.liquid
create mode 100644 lib/config/config.default.yml
create mode 100644 services/api/config/arvados_config.rb
create mode 120000 services/api/config/config.default.yml
delete mode 100644 services/api/config/initializers/load_config.rb
create mode 100644 services/api/lib/config_loader.rb
create mode 100644 services/api/lib/tasks/config.rake
delete mode 100644 services/api/lib/tasks/config_check.rake
delete mode 100644 services/api/lib/tasks/config_dump.rake
via 454ee2b8f0385c542b6f1165a3baf2820425e1a3 (commit)
via 25107815e9da9483bb7e6055379420113bfc3640 (commit)
via 3c87946740b83f612561f998f7d83586593be830 (commit)
via aec31c21df670646bdb6dedd3fd2fa690b8ac7e4 (commit)
via 30fc42873b2c82e72a95393eb053abf1f7052618 (commit)
via 0ab9897ec19e24b7964b2467984065327010d400 (commit)
via 5756eb031199c2ab015463292e7d24f3357c1852 (commit)
via 0e13b30f173cf03c856376bb534a20b9d07d643b (commit)
via 375ddb8a0f46854963d30792072e2bcf06648d17 (commit)
via 1476c801c430df6693d3e2e456ac5718d20f31cf (commit)
via 5e95c9b723e36cf80e0b9c1bf02206520503d4f1 (commit)
via eb6982790aca2975f9d6c01e40b9b9c5aab879ea (commit)
via 79c34c27034d447d3c7141b4f357a1e76e36f831 (commit)
via 9f9e263d16dbd934754fa42375084478f5c74464 (commit)
via aeff8498a0e64e2cbf842e09ac1ad26ca9b8e76c (commit)
via 085146e2717c4a2a3cca1081d924c34cf0b52c45 (commit)
via 8b5dff30937a7c12dcf7cd71153770393266500d (commit)
via a7c0781642028fd0ce66208133a83bb85c5c6b3e (commit)
via b05be6e2d1db4e63074fc28e978c40a271376f1f (commit)
via 2ea729f97166cb5fda4a6ca0818f0a0ab15388ab (commit)
via dee79c4cf4eb34d80b925168e76d1114dfc02c2a (commit)
via 0e5e80d054cc19c145f5bc02dfc3bd6271fde316 (commit)
via 1346cc5072843797a8091cd67a0eb75499a97642 (commit)
via 648a56262d7b5b1c550077341f81cc98e1850305 (commit)
via 4475be5167afe63bef3ffe2b8145d35aaebe4761 (commit)
via f195fde23e4b21d99519906be14aa21ae4ef0834 (commit)
via 63287a6a6153e9e8196922f01ded0321cfc762dd (commit)
via 39172f2972a6c700792d4b071072a0adc2f6f739 (commit)
via 023015bacfded08503d0240d3f71838204d8dcb7 (commit)
via 234878d253910806a12f599cef8484c8890e7ef9 (commit)
via 6e9fcde0422b33d081e2985975e3104eb2434957 (commit)
via eb0bb0118051b0acbff09cf87287ad83a48ee337 (commit)
via 90944740f40ab0dbfc4bdfc1b16accfbf6559e4f (commit)
via ff043c7f69a599f35c47a283cc7f78addffe5fdc (commit)
via 7112c01c75ad8be748b33b86a033e668e607eb6c (commit)
via f98a4eb7fcd4f762c8dc707ec1661ca5a5802f6e (commit)
via 17028c6f18a3e83b39de1655b6bfeafe2938220d (commit)
via a3a0940ccf0aeacb45aa7216035a2d815ddf7bcf (commit)
via 1a613db714afd5f1c5a14a807fea02e125a4a75b (commit)
via f9ae5b90a5e04477133ca7a7d34bd3eebf862474 (commit)
via 8a0e9c549595e114a0eadc9d6792a17fb59d0f3e (commit)
via 450b51a554050504d6b510c2c9c1adf463b937a0 (commit)
via 09a1ecf507df8ca110e6620efeb3593bc0d90192 (commit)
via 47abf19591c9816f88f83db9fd2cbe93e2262e79 (commit)
via 7baff32e82b0bfc961dc9a285da8ce187d4fe0b6 (commit)
via 0195e1bbf4c1e5810f637212e9605d2d2dc03e7e (commit)
via d7ef7ddbfac8fe5e0641101f3be58f7e0b35dad0 (commit)
via 91545d0a1fb40c79eef0f3013f832734f7a9695e (commit)
from 3332afccd61fd457514eb5682f20de23870d40b1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 454ee2b8f0385c542b6f1165a3baf2820425e1a3
Merge: 25107815e 3c8794674
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Fri Apr 19 10:59:24 2019 -0400
Merge branch 'master' into 14723-cwl-multiple-file-targets
refs #14723
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
commit 25107815e9da9483bb7e6055379420113bfc3640
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Fri Apr 19 10:59:06 2019 -0400
14723: Adds comments for the changes to StagingPathMapper
Also improves executor exception handling mapping logic.
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 0e46c5360..eeb44dbd7 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -461,7 +461,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
raise
except IOError as e:
- logger.warning("While preparing output collection: %s", e)
+ logger.error("While preparing output collection: %s", e)
+ raise
def rewrite(fileobj):
fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index f08d7d7b2..56c15a4a4 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -265,6 +265,13 @@ class ArvPathMapper(PathMapper):
class StagingPathMapper(PathMapper):
+ # Note that StagingPathMapper internally maps files from target to source.
+ # Specifically, the 'self._pathmap' dict keys are the target location and the
+ # values are 'MapperEnt' named tuples from which we use the 'resolved' attribute
+ # as the file identifier. This makes it possible to map an input file to multiple
+ # target directories. The exception is for file literals, which store the contents of
+ # the file in 'MapperEnt.resolved' and are therefore still mapped from source to target.
+
_follow_dirs = True
def __init__(self, referenced_files, basedir, stagedir, separateDirs=True):
@@ -307,10 +314,12 @@ class StagingPathMapper(PathMapper):
self._pathmap[tgt] = MapperEnt(loc, tgt, "File", staged)
self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
- def mapper(self, src): # type: (Text) -> MapperEnt
+ def mapper(self, src): # type: (Text) -> MapperEnt.
+ # Overridden to maintain the use case of mapping by source (identifier) to
+ # target regardless of how the map is structured interally.
def getMapperEnt(src):
for k,v in viewitems(self._pathmap):
- if v.resolved == src or (v.type == "CreateFile" and k == src):
+ if (v.type != "CreateFile" and v.resolved == src) or (v.type == "CreateFile" and k == src):
return v
if u"#" in src:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list