[ARVADOS] updated: 1.2.0-59-g95e5ccacf
Git user
git at public.curoverse.com
Fri Aug 24 15:28:30 EDT 2018
Summary of changes:
README.md | 4 +-
apps/workbench/Gemfile.lock | 6 +-
.../app/assets/javascripts/models/session_db.js | 11 +-
.../app/controllers/work_units_controller.rb | 29 +-
apps/workbench/app/helpers/application_helper.rb | 27 +-
apps/workbench/app/helpers/version_helper.rb | 6 +
.../views/application/_report_issue_popup.html.erb | 6 +-
.../application/_show_text_with_locators.html.erb | 49 +-
apps/workbench/config/application.default.yml | 6 +
apps/workbench/lib/app_version.rb | 15 +
.../container_requests_controller_test.rb | 2 +
.../test/controllers/projects_controller_test.rb | 14 +-
apps/workbench/test/test_helper.rb | 30 +-
build/build.list | 49 +-
build/libcloud-pin.sh | 4 +-
build/package-build-dockerfiles/Makefile | 10 +-
build/package-build-dockerfiles/centos7/Dockerfile | 4 +-
build/package-build-dockerfiles/debian8/Dockerfile | 4 +-
build/package-build-dockerfiles/debian9/Dockerfile | 2 +-
.../ubuntu1404/Dockerfile | 4 +-
.../ubuntu1604/Dockerfile | 4 +-
.../{ubuntu1404 => ubuntu1804}/Dockerfile | 10 +-
build/package-test-dockerfiles/centos7/Dockerfile | 2 +-
build/package-test-dockerfiles/debian8/Dockerfile | 2 +-
.../package-test-dockerfiles/ubuntu1404/Dockerfile | 2 +-
.../package-test-dockerfiles/ubuntu1604/Dockerfile | 2 +-
.../{ubuntu1604 => ubuntu1804}/Dockerfile | 10 +-
.../etc-apt-preferences.d-arvados | 0
build/package-testing/deb-common-test-packages.sh | 8 +-
...ages-debian8.sh => test-packages-ubuntu1804.sh} | 0
build/run-build-packages-sso.sh | 15 +-
build/run-build-packages.sh | 47 +-
build/run-library.sh | 35 +-
build/run-tests.sh | 36 +-
cmd/arvados-server/arvados-controller.service | 28 +
doc/_config.yml | 40 +-
.../_container_scheduling_parameters.liquid | 2 +
doc/_includes/_install_compute_docker.liquid | 27 +
doc/_includes/_skip_sso_server_install.liquid | 12 -
doc/admin/activation.html.textile.liquid | 229 +++++
doc/admin/health-checks.html.textile.liquid | 70 ++
doc/admin/management-token.html.textile.liquid | 56 ++
doc/admin/metrics.html.textile.liquid | 189 ++++
doc/admin/spot-instances.html.textile.liquid | 78 ++
doc/admin/storage-classes.html.textile.liquid | 47 +
doc/admin/upgrading.html.textile.liquid | 46 +-
doc/api/execution.html.textile.liquid | 28 +
doc/api/methods.html.textile.liquid | 2 +-
doc/api/methods/collections.html.textile.liquid | 1 +
.../methods/container_requests.html.textile.liquid | 2 +-
doc/api/methods/groups.html.textile.liquid | 24 +-
doc/api/methods/links.html.textile.liquid | 2 +-
doc/api/methods/nodes.html.textile.liquid | 3 +-
doc/api/tokens.html.textile.liquid | 4 +
doc/css/code.css | 9 +
doc/css/images.css | 8 +
.../arvados-on-kubernetes-GKE.html.textile.liquid | 62 ++
...ados-on-kubernetes-minikube.html.textile.liquid | 34 +
.../arvados-on-kubernetes.html.textile.liquid | 133 +++
doc/install/cheat_sheet.html.textile.liquid | 2 +-
...onfigure-azure-blob-storage.html.textile.liquid | 86 +-
.../configure-fs-storage.html.textile.liquid | 56 ++
...configure-s3-object-storage.html.textile.liquid | 112 +++
...opy_pipeline_from_curoverse.html.textile.liquid | 20 +-
.../create-standard-objects.html.textile.liquid | 84 --
.../install-dispatch.html.textile.liquid | 31 +-
.../install-slurm.html.textile.liquid | 2 -
doc/install/index.html.textile.liquid | 22 +-
doc/install/install-api-server.html.textile.liquid | 68 +-
.../install-arv-git-httpd.html.textile.liquid | 45 +
doc/install/install-components.html.textile.liquid | 28 +
doc/install/install-composer.html.textile.liquid | 59 ++
doc/install/install-controller.html.textile.liquid | 180 ++++
.../install-keep-balance.html.textile.liquid | 7 +-
doc/install/install-keepproxy.html.textile.liquid | 23 +-
doc/install/install-keepstore.html.textile.liquid | 289 ++++---
...nstall-manual-prerequisites.html.textile.liquid | 8 +-
.../install-nodemanager.html.textile.liquid | 29 +-
.../firstpipeline.html.textile.liquid | 2 +-
doc/user/composer/c1.png | Bin 0 -> 17420 bytes
doc/user/composer/c10.png | Bin 0 -> 33020 bytes
doc/user/composer/c11.png | Bin 0 -> 26144 bytes
doc/user/composer/c12.png | Bin 0 -> 175462 bytes
doc/user/composer/c13.png | Bin 0 -> 11645 bytes
doc/user/composer/c14.png | Bin 0 -> 13116 bytes
doc/user/composer/c15.png | Bin 0 -> 50066 bytes
doc/user/composer/c16.png | Bin 0 -> 32472 bytes
doc/user/composer/c17.png | Bin 0 -> 34259 bytes
doc/user/composer/c18.png | Bin 0 -> 59780 bytes
doc/user/composer/c19.png | Bin 0 -> 53580 bytes
doc/user/composer/c2.png | Bin 0 -> 41693 bytes
doc/user/composer/c20.png | Bin 0 -> 23625 bytes
doc/user/composer/c21.png | Bin 0 -> 11379 bytes
doc/user/composer/c22.png | Bin 0 -> 13713 bytes
doc/user/composer/c23.png | Bin 0 -> 12253 bytes
doc/user/composer/c24.png | Bin 0 -> 19931 bytes
doc/user/composer/c2b.png | Bin 0 -> 14285 bytes
doc/user/composer/c2c.png | Bin 0 -> 9123 bytes
doc/user/composer/c3.png | Bin 0 -> 38471 bytes
doc/user/composer/c4.png | Bin 0 -> 8811 bytes
doc/user/composer/c5.png | Bin 0 -> 24020 bytes
doc/user/composer/c6.png | Bin 0 -> 41978 bytes
doc/user/composer/c7.png | Bin 0 -> 26904 bytes
doc/user/composer/c8.png | Bin 0 -> 44358 bytes
doc/user/composer/c9.png | Bin 0 -> 18920 bytes
doc/user/composer/composer.html.textile.liquid | 119 +++
doc/user/cwl/cwl-extensions.html.textile.liquid | 23 +
doc/user/cwl/cwl-runner.html.textile.liquid | 10 +-
doc/user/cwl/cwl-style.html.textile.liquid | 6 +-
doc/user/topics/arv-copy.html.textile.liquid | 4 +-
doc/user/topics/arv-docker.html.textile.liquid | 10 +-
.../topics/storage-classes.html.textile.liquid | 53 ++
.../running-external-program.html.textile.liquid | 4 +-
...tutorial-workflow-workbench.html.textile.liquid | 4 +
.../writing-cwl-workflow.html.textile.liquid | 6 +-
lib/cmd/cmd.go | 11 +-
lib/controller/cmd.go | 70 +-
lib/controller/federation.go | 117 +++
lib/controller/federation_test.go | 301 +++++++
lib/controller/handler.go | 147 +++-
lib/controller/handler_test.go | 132 +++
lib/controller/proxy.go | 83 ++
lib/controller/server_test.go | 68 ++
lib/crunchstat/crunchstat.go | 64 +-
lib/dispatchcloud/node_size.go | 34 +-
lib/dispatchcloud/node_size_test.go | 76 +-
lib/service/cmd.go | 120 +++
sdk/R/DESCRIPTION | 2 +-
sdk/R/R/Arvados.R | 950 +++++++++++----------
sdk/R/R/ArvadosFile.R | 113 ++-
sdk/R/R/Collection.R | 133 ++-
sdk/R/R/CollectionTree.R | 92 +-
sdk/R/R/HttpParser.R | 6 +-
sdk/R/R/HttpRequest.R | 17 +-
sdk/R/R/RESTService.R | 37 +-
sdk/R/R/Subcollection.R | 102 ++-
sdk/R/R/autoGenAPI.R | 16 +-
sdk/R/R/zzz.R | 14 +
sdk/R/README.Rmd | 55 +-
sdk/R/man/Arvados.Rd | 1 +
sdk/R/man/ArvadosFile.Rd | 6 +-
sdk/R/man/Collection.Rd | 8 +-
sdk/R/man/Subcollection.Rd | 6 +-
sdk/R/man/users.merge.Rd | 22 +
sdk/R/tests/testthat/fakes/FakeHttpParser.R | 6 +-
sdk/R/tests/testthat/fakes/FakeHttpRequest.R | 33 +-
sdk/R/tests/testthat/fakes/FakeRESTService.R | 16 +-
sdk/R/tests/testthat/test-ArvadosFile.R | 136 ++-
sdk/R/tests/testthat/test-Collection.R | 87 +-
sdk/R/tests/testthat/test-CollectionTree.R | 20 +-
sdk/R/tests/testthat/test-HttpParser.R | 10 +-
sdk/R/tests/testthat/test-HttpRequest.R | 85 +-
sdk/R/tests/testthat/test-RESTService.R | 133 ++-
sdk/R/tests/testthat/test-Subcollection.R | 148 +++-
sdk/R/tests/testthat/test-util.R | 20 +-
sdk/cli/arvados-cli.gemspec | 4 +-
sdk/cli/bin/crunch-job | 10 +
sdk/cwl/arvados_cwl/__init__.py | 260 +++---
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 44 +
sdk/cwl/arvados_cwl/arvcontainer.py | 81 +-
sdk/cwl/arvados_cwl/arvjob.py | 54 +-
sdk/cwl/arvados_cwl/arvtool.py | 73 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 69 +-
sdk/cwl/arvados_cwl/context.py | 33 +
sdk/cwl/arvados_cwl/crunch_script.py | 19 +-
sdk/cwl/arvados_cwl/done.py | 10 +-
sdk/cwl/arvados_cwl/fsaccess.py | 13 +-
sdk/cwl/arvados_cwl/http.py | 16 +-
sdk/cwl/arvados_cwl/pathmapper.py | 48 +-
sdk/cwl/arvados_cwl/runner.py | 52 +-
sdk/cwl/arvados_cwl/util.py | 31 +
sdk/cwl/arvados_version.py | 17 +-
sdk/cwl/setup.py | 8 +-
sdk/cwl/tests/13931-size-job.yml | 3 +
sdk/cwl/tests/13931-size.cwl | 10 +
sdk/cwl/tests/13976-keepref-wf.cwl | 17 +
sdk/cwl/tests/arvados-tests.sh | 4 +
sdk/cwl/tests/arvados-tests.yml | 17 +
.../collection_per_tool_packed.cwl | 126 ++-
sdk/cwl/tests/makes_intermediates/echo.cwl | 14 +
.../cwl/tests/makes_intermediates/hello1.txt | 0
.../tests/makes_intermediates/run_in_single.cwl | 38 +
sdk/cwl/tests/makes_intermediates/subwf.cwl | 15 +
sdk/cwl/tests/test_container.py | 182 ++--
sdk/cwl/tests/test_job.py | 147 ++--
sdk/cwl/tests/test_make_output.py | 4 +-
sdk/cwl/tests/test_pathmapper.py | 19 +-
sdk/cwl/tests/test_submit.py | 121 ++-
sdk/cwl/tests/test_util.py | 45 +
sdk/cwl/tests/wf/expect_packed.cwl | 129 ++-
sdk/cwl/tests/wf/submit_wf_runner_resources.cwl | 31 +
sdk/dev-jobs.dockerfile | 2 +-
sdk/go/arvados/api_client_authorization.go | 4 +
sdk/go/arvados/byte_size.go | 91 ++
sdk/go/arvados/byte_size_test.go | 70 ++
sdk/go/arvados/config.go | 138 ++-
sdk/go/arvados/config_test.go | 37 +
sdk/go/arvados/container.go | 4 +-
sdk/go/arvados/postgresql.go | 20 +
sdk/go/arvadostest/fixtures.go | 2 +
sdk/go/auth/auth.go | 24 +-
sdk/go/auth/salt.go | 48 ++
sdk/go/dispatch/dispatch.go | 39 +-
sdk/go/health/aggregator.go | 16 +-
sdk/go/health/aggregator_test.go | 14 +-
sdk/go/httpserver/error.go | 21 +
sdk/go/httpserver/logger.go | 30 +-
sdk/go/httpserver/logger_test.go | 14 +-
sdk/go/httpserver/metrics.go | 135 +++
sdk/python/arvados/__init__.py | 2 -
sdk/python/arvados/api.py | 9 +-
sdk/python/arvados/collection.py | 90 +-
sdk/python/arvados/commands/keepdocker.py | 2 +-
sdk/python/arvados/commands/run.py | 11 +-
sdk/python/arvados/keep.py | 19 +-
sdk/python/arvados/safeapi.py | 6 +
sdk/python/setup.py | 5 +-
sdk/python/tests/nginx.conf | 91 +-
sdk/python/tests/run_test_server.py | 121 ++-
sdk/python/tests/test_collections.py | 38 +-
sdk/python/tests/test_keep_client.py | 25 +
sdk/ruby/lib/arvados/google_api_client.rb | 5 +
services/api/Gemfile | 5 +-
services/api/Gemfile.lock | 16 +-
.../api/app/controllers/application_controller.rb | 8 +-
.../arvados/v1/containers_controller.rb | 6 +-
.../controllers/arvados/v1/groups_controller.rb | 63 +-
.../controllers/arvados/v1/schema_controller.rb | 2 +
services/api/app/controllers/static_controller.rb | 2 +-
.../api/app/models/api_client_authorization.rb | 3 +-
services/api/app/models/collection.rb | 20 +-
services/api/app/models/container_request.rb | 77 +-
services/api/app/models/group.rb | 3 +
services/api/app/models/user.rb | 3 +-
services/api/config/application.default.yml | 26 +-
services/api/config/application.rb | 3 +
services/api/config/initializers/oj_mimic_json.rb | 11 +
services/api/config/initializers/time_format.rb | 2 +
services/api/config/routes.rb | 1 +
.../migrate/20180607175050_properties_to_jsonb.rb | 32 +
.../20180608123145_add_properties_to_groups.rb | 18 +
.../migrate/20180806133039_index_all_filenames.rb | 12 +
...30357_add_pdh_and_trash_index_to_collections.rb | 9 +
...180820135808_drop_pdh_index_from_collections.rb | 9 +
services/api/db/structure.sql | 669 ++++++++-------
services/api/lib/app_version.rb | 15 +
services/api/lib/crunch_dispatch.rb | 13 +-
services/api/lib/load_param.rb | 14 +-
services/api/lib/safe_json.rb | 6 +
services/api/lib/update_priority.rb | 25 +-
services/api/lib/whitelist_update.rb | 2 +-
.../test/fixtures/api_client_authorizations.yml | 7 +
services/api/test/fixtures/container_requests.yml | 2 +-
services/api/test/fixtures/containers.yml | 2 +-
services/api/test/fixtures/groups.yml | 8 +
.../v1/container_requests_controller_test.rb | 6 +-
.../arvados/v1/groups_controller_test.rb | 131 ++-
.../arvados/v1/schema_controller_test.rb | 14 +-
services/api/test/integration/cross_origin_test.rb | 4 +-
services/api/test/integration/remote_user_test.rb | 33 +
services/api/test/test_helper.rb | 2 +-
services/api/test/unit/arvados_model_test.rb | 2 +-
services/api/test/unit/container_request_test.rb | 147 +++-
services/api/test/unit/container_test.rb | 8 +-
services/api/test/unit/update_priority_test.rb | 30 +
services/api/test/unit/user_test.rb | 4 +-
.../crunch-dispatch-local_test.go | 2 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 28 +-
.../crunch-dispatch-slurm_test.go | 40 +-
services/crunch-dispatch-slurm/squeue.go | 98 ++-
services/crunch-dispatch-slurm/squeue_test.go | 44 +
services/crunch-dispatch-slurm/usage.go | 1 +
services/crunch-run/crunchrun.go | 124 ++-
services/crunch-run/crunchrun_test.go | 68 +-
services/crunch-run/logging_test.go | 2 +-
services/crunchstat/crunchstat.go | 4 +-
services/fuse/arvados_fuse/fusedir.py | 74 +-
services/health/main.go | 2 +-
services/keep-balance/balance.go | 162 +++-
services/keep-balance/balance_test.go | 206 ++++-
services/keep-web/cache.go | 136 ++-
services/keep-web/cache_test.go | 87 +-
services/keep-web/cadaver_test.go | 2 +-
services/keep-web/doc.go | 6 +
services/keep-web/handler.go | 24 +-
services/keep-web/handler_test.go | 9 +-
services/keep-web/server.go | 10 +-
services/keep-web/server_test.go | 97 +++
services/keep-web/status_test.go | 1 -
services/keepproxy/keepproxy.go | 2 +-
services/keepproxy/keepproxy_test.go | 2 +-
services/keepstore/config.go | 67 +-
services/keepstore/handlers.go | 12 +-
services/login-sync/arvados-login-sync.gemspec | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 31 +-
.../arvnodeman/computenode/dispatch/slurm.py | 2 +-
.../arvnodeman/computenode/driver/__init__.py | 10 +-
.../arvnodeman/computenode/driver/azure.py | 7 +-
.../arvnodeman/computenode/driver/dummy.py | 2 +-
.../arvnodeman/computenode/driver/ec2.py | 11 +-
.../arvnodeman/computenode/driver/gce.py | 25 +-
services/nodemanager/arvnodeman/config.py | 50 +-
services/nodemanager/arvnodeman/daemon.py | 73 +-
services/nodemanager/arvnodeman/jobqueue.py | 39 +-
services/nodemanager/arvnodeman/launcher.py | 15 +-
services/nodemanager/arvnodeman/nodelist.py | 6 +-
.../nodemanager/arvnodeman/test/fake_driver.py | 15 +-
services/nodemanager/doc/azure.example.cfg | 10 +
services/nodemanager/doc/ec2.example.cfg | 21 +
services/nodemanager/doc/gce.example.cfg | 9 +
services/nodemanager/setup.py | 11 +-
services/nodemanager/tests/fake_azure.cfg.template | 6 +-
services/nodemanager/tests/integration_test.py | 7 +-
.../nodemanager/tests/test_computenode_dispatch.py | 23 +-
.../tests/test_computenode_dispatch_slurm.py | 14 +-
.../nodemanager/tests/test_computenode_driver.py | 2 +-
.../tests/test_computenode_driver_azure.py | 23 +-
.../tests/test_computenode_driver_ec2.py | 27 +-
.../tests/test_computenode_driver_gce.py | 11 +
services/nodemanager/tests/test_config.py | 24 +-
services/nodemanager/tests/test_daemon.py | 16 +
services/nodemanager/tests/test_jobqueue.py | 12 +-
services/nodemanager/tests/test_nodelist.py | 5 +-
services/nodemanager/tests/testutil.py | 4 +-
services/ws/config.go | 4 +-
services/ws/event_source.go | 16 -
services/ws/event_source_test.go | 30 +-
services/ws/server.go | 2 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 8 +-
tools/arvbox/lib/arvbox/docker/common.sh | 4 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 2 +-
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 23 +-
.../lib/arvbox/docker/service/api/run-service | 4 +-
.../docker/service/arv-git-httpd/run-service | 2 +-
.../lib/arvbox/docker/service/composer/run-service | 2 +-
.../service/{api => controller}/log/main/.gitstub | 0
.../docker/service/{api => controller}/log/run | 0
.../lib/arvbox/docker/service/controller/run | 50 ++
.../service/crunch-dispatch-local/run-service | 2 +-
tools/arvbox/lib/arvbox/docker/service/doc/run | 10 +-
.../lib/arvbox/docker/service/doc/run-service | 31 +-
.../lib/arvbox/docker/service/gitolite/run-service | 4 +-
.../lib/arvbox/docker/service/keep-web/run-service | 2 +-
.../arvbox/docker/service/keepproxy/run-service | 2 +-
.../service/{api => nginx}/log/main/.gitstub | 0
.../arvbox/docker/service/{api => nginx}/log/run | 0
.../lib/arvbox/docker/service/{api => nginx}/run | 0
.../lib/arvbox/docker/service/nginx/run-service | 54 ++
tools/arvbox/lib/arvbox/docker/service/ready/run | 10 +-
.../lib/arvbox/docker/service/ready/run-service | 4 +-
.../lib/arvbox/docker/service/sso/run-service | 2 +-
tools/arvbox/lib/arvbox/docker/service/vm/run | 2 +-
.../lib/arvbox/docker/service/vm/run-service | 2 +-
.../arvbox/docker/service/websockets/run-service | 2 +-
.../arvbox/docker/service/workbench/run-service | 6 +-
...application_yml_override.py => yml_override.py} | 9 +-
vendor/vendor.json | 86 +-
358 files changed, 9870 insertions(+), 3182 deletions(-)
copy build/package-build-dockerfiles/{ubuntu1404 => ubuntu1804}/Dockerfile (76%)
copy build/package-test-dockerfiles/{ubuntu1604 => ubuntu1804}/Dockerfile (73%)
copy build/package-test-dockerfiles/{ubuntu1604 => ubuntu1804}/etc-apt-preferences.d-arvados (100%)
copy build/package-testing/{test-packages-debian8.sh => test-packages-ubuntu1804.sh} (100%)
create mode 100644 cmd/arvados-server/arvados-controller.service
delete mode 100644 doc/_includes/_skip_sso_server_install.liquid
create mode 100644 doc/admin/activation.html.textile.liquid
create mode 100644 doc/admin/health-checks.html.textile.liquid
create mode 100644 doc/admin/management-token.html.textile.liquid
create mode 100644 doc/admin/metrics.html.textile.liquid
create mode 100644 doc/admin/spot-instances.html.textile.liquid
create mode 100644 doc/admin/storage-classes.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes.html.textile.liquid
create mode 100644 doc/install/configure-fs-storage.html.textile.liquid
create mode 100644 doc/install/configure-s3-object-storage.html.textile.liquid
delete mode 100644 doc/install/create-standard-objects.html.textile.liquid
create mode 100644 doc/install/install-components.html.textile.liquid
create mode 100644 doc/install/install-composer.html.textile.liquid
create mode 100644 doc/install/install-controller.html.textile.liquid
create mode 100644 doc/user/composer/c1.png
create mode 100644 doc/user/composer/c10.png
create mode 100644 doc/user/composer/c11.png
create mode 100644 doc/user/composer/c12.png
create mode 100644 doc/user/composer/c13.png
create mode 100644 doc/user/composer/c14.png
create mode 100644 doc/user/composer/c15.png
create mode 100644 doc/user/composer/c16.png
create mode 100644 doc/user/composer/c17.png
create mode 100644 doc/user/composer/c18.png
create mode 100644 doc/user/composer/c19.png
create mode 100644 doc/user/composer/c2.png
create mode 100644 doc/user/composer/c20.png
create mode 100644 doc/user/composer/c21.png
create mode 100644 doc/user/composer/c22.png
create mode 100644 doc/user/composer/c23.png
create mode 100644 doc/user/composer/c24.png
create mode 100644 doc/user/composer/c2b.png
create mode 100644 doc/user/composer/c2c.png
create mode 100644 doc/user/composer/c3.png
create mode 100644 doc/user/composer/c4.png
create mode 100644 doc/user/composer/c5.png
create mode 100644 doc/user/composer/c6.png
create mode 100644 doc/user/composer/c7.png
create mode 100644 doc/user/composer/c8.png
create mode 100644 doc/user/composer/c9.png
create mode 100644 doc/user/composer/composer.html.textile.liquid
create mode 100644 doc/user/topics/storage-classes.html.textile.liquid
create mode 100644 lib/controller/federation.go
create mode 100644 lib/controller/federation_test.go
create mode 100644 lib/controller/handler_test.go
create mode 100644 lib/controller/proxy.go
create mode 100644 lib/controller/server_test.go
create mode 100644 lib/service/cmd.go
create mode 100644 sdk/R/R/zzz.R
create mode 100644 sdk/R/man/users.merge.Rd
create mode 100644 sdk/cwl/arvados_cwl/context.py
create mode 100644 sdk/cwl/arvados_cwl/util.py
create mode 100644 sdk/cwl/tests/13931-size-job.yml
create mode 100644 sdk/cwl/tests/13931-size.cwl
create mode 100644 sdk/cwl/tests/13976-keepref-wf.cwl
create mode 100644 sdk/cwl/tests/makes_intermediates/echo.cwl
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/makes_intermediates/hello1.txt (100%)
create mode 100644 sdk/cwl/tests/makes_intermediates/run_in_single.cwl
create mode 100644 sdk/cwl/tests/makes_intermediates/subwf.cwl
create mode 100644 sdk/cwl/tests/test_util.py
create mode 100644 sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
create mode 100644 sdk/go/arvados/byte_size.go
create mode 100644 sdk/go/arvados/byte_size_test.go
create mode 100644 sdk/go/arvados/config_test.go
create mode 100644 sdk/go/arvados/postgresql.go
create mode 100644 sdk/go/auth/salt.go
create mode 100644 sdk/go/httpserver/error.go
create mode 100644 sdk/go/httpserver/metrics.go
create mode 100644 services/api/config/initializers/oj_mimic_json.rb
create mode 100644 services/api/db/migrate/20180607175050_properties_to_jsonb.rb
create mode 100644 services/api/db/migrate/20180608123145_add_properties_to_groups.rb
create mode 100644 services/api/db/migrate/20180806133039_index_all_filenames.rb
create mode 100644 services/api/db/migrate/20180820130357_add_pdh_and_trash_index_to_collections.rb
create mode 100644 services/api/db/migrate/20180820135808_drop_pdh_index_from_collections.rb
create mode 100644 services/api/test/unit/update_priority_test.rb
copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/controller/run
mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/doc/run
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/run (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/nginx/run-service
mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/ready/run
rename tools/arvbox/lib/arvbox/docker/{application_yml_override.py => yml_override.py} (79%)
via 95e5ccacf6c1193b313fa90a6d39baafa2ba67d8 (commit)
via e73008e45f0c67d26bd4e05c932de4c9d4431575 (commit)
via 27f2739825c3d543301a82871ab6f07d69785998 (commit)
via 43227b3a265d1529efe3e5b12964fe3a622f139b (commit)
via 9af9cecdc25dc15243e6ca54895620ab472780df (commit)
via c11f9cac7535f27aff9575002bc31ba0149ac111 (commit)
via ac2cc876733c6137d525d12780275f2c02d84383 (commit)
via bc97d806fa1fb15e49cefe819f00089ebc33d376 (commit)
via bf742fa9565561e8862bbc86a38a411accbcb8c3 (commit)
via 301e4dc1069fd7f97d977d9cbeb682e697660e49 (commit)
via e4fca76d5db9bd844530454894d07ddc729b4a9a (commit)
via 3df02aa618ee1f9c125d9f2253d95a474d3d8e57 (commit)
via 067a7263a97e1b4d96ee6572e79884bdcd937bbf (commit)
via 0d93140ca76f38f05f1b689bdf31168efbc6984f (commit)
via 5c1d46ab958097eb0da1ca692af23345e1faf66c (commit)
via 22488d2c44b03bd255a9d223b8f8498d56534bdf (commit)
via b7d199af0a1844a4b6db38d315c26365617bfe41 (commit)
via a0446e23b6795c8c90c5e66438a3a1bb82f9368a (commit)
via 662a70957f0652acd6579efc635eed8f708ab48c (commit)
via 1a373b5f2c37cead0fe41482805fdb93ca871e37 (commit)
via 6d6a0b0f7d5780c92c865ba2e6979195510c27b2 (commit)
via 489aed58cb0d8bd816e07128cfcb9f5a06224083 (commit)
via 24972a99eb2cb4d914b687ccd4050f2da0333214 (commit)
via d7b63b6901f31b8b9de89762cdf76aa9ae5b00b7 (commit)
via 99263be28ab2ec28db721ced52357c05d11947d7 (commit)
via 70e5c7a3c6a5860d702d5e5c219dc0f3a3696d35 (commit)
via a96cca47cc60c482316d3b1a25cfe1cb4b838f41 (commit)
via fd4d26e448b1f9f45f84c0ff9ec10db54471d880 (commit)
via a41c0f6aa41b658c8f2947c46cb90778894f5cf3 (commit)
via 5d9908601b8c16c556d0153640f67aa3b26c2f57 (commit)
via 89698c15b13dcb151ebb673a2d73b1df7339c607 (commit)
via eb1c9afa9a92c1506e5d4d1161b6e74d919e8f00 (commit)
via 34d989929d8ecf3620640b5b7a5e89b89da70a89 (commit)
via 37e385b7aaf52ac391c6964557e30f662cb1de36 (commit)
via f0d67ab6f71cef3e7ce0fa850f7309a94e21671b (commit)
via 9c93e1f79c385269435299d90d00e1b36c09cf3c (commit)
via 0ba8a53ba145475153b01dc498f85dbd2f03228b (commit)
via bb31e272ab27d14a5d7dce7e4270c10b99035f45 (commit)
via 61ad64acbca1b8cc990ecf0e9ca2c54ad9eb8fe6 (commit)
via 789e15f578cf4464834ecae347fdfe0d337b7464 (commit)
via 6d3d48d6706c6aa32b5d1bf1977ea172b8cde7a4 (commit)
via 4d00ed0a34cae5a5da3992ca9adac46d3682fee1 (commit)
via bc49a7967325c523a7c25d48df30ae718a66aa63 (commit)
via 22a42fb90bb5e878b75ffffe54bd76d4d31cacf9 (commit)
via 17cf67c1a04b3ffbccdf08dd9b3aa5f25ef84a1b (commit)
via 982498669e3a703e8aad888056faedadf340e21f (commit)
via a9a202895a63f29e8bc045c50f5518800b008275 (commit)
via d06843b42863fb2db47d11f18ce60b80cc5a8c55 (commit)
via 420a8e7fb7b159452da834062cc3e040dd1b411b (commit)
via 14c7497f4d0e4749ae3240cf7d3ae5cbce391da1 (commit)
via fc582bfe0561092165f45b7b6f4d0025892aebfd (commit)
via a362958e10d84233ccc4cc8d05a9183a171618a3 (commit)
via 57fd7c42e7e447bcd9174b6b0fbde011d2db37bd (commit)
via a2c87cc8989f8076367a8245c810560ece55f7e7 (commit)
via 4019ca8eab40af7ba3e2bf465695b720f907abc8 (commit)
via 1ea2dbfc7fd6466a98cf8f239a6bea9b4947ef57 (commit)
via df2f836528899ae50ee4a07b10c47d5d08f59366 (commit)
via 32d403dd4d791e88ee93ddda7865a1566e3da116 (commit)
via 23574f4d4039641f9a9f624d3b16b64148de3826 (commit)
via 63def826c674fe3d277bff61084e6da22688e951 (commit)
via 39c17737ac69d7693684fe2f95bef0ec235a28bf (commit)
via 2a298d6cca655c16847733c5ddfc3cc02e71ff8c (commit)
via 0befd8da0101dbc7845c8216637c8b621519bc68 (commit)
via 363094904b570d223d59eb911150279308e98011 (commit)
via fdf14c1b9b4866c371fce9791c09b939194f3b6c (commit)
via eeef002557bc361a10483d1924c49e35e8a23fde (commit)
via 21bf21abe900918a7882f7e43c102418e04159be (commit)
via 38c4ce8bd5aefed6784d457ed3caf28f279d6de4 (commit)
via b0ab93cc292146f1e38c061ab04df3b9a114232f (commit)
via b57ca9691db0527a086c6ac781abdb956a1b7406 (commit)
via f1697a4720541a84e581fee2513c84a8dc26ff93 (commit)
via 00e5b1e6e2946edc49dc2371a3b3d70c4dcbfe5d (commit)
via 4c5600a83bd887894d135474fe3a7e0200dfb01b (commit)
via 6db4c94527903f403e77ba4fce2d1d5fe4e29b03 (commit)
via 9f9cf2e1da7233d6b341602faeaa69454b7e201f (commit)
via 61611952a2ac9a8f40fddd1a259c3eae5cd8d6b0 (commit)
via f8f3944e6ba1f470eedf48aa609a50c780fe4b6d (commit)
via 94d277295c4a8d331622e4077ace1eb4570d54e5 (commit)
via ce3f2d624d8600a27030994990b8893fcb7df24f (commit)
via 71df6eb49c3c3ef4a9ef463f9193764149cfd5d2 (commit)
via 809a07ce8704dcb579fc653664bd365f1a6e9ef7 (commit)
via 7828c8a729955de05135c2a53172b29c515a03ec (commit)
via 63e4678b027feb29ee7c1fc5e344b6d58be8abfa (commit)
via d94ed391d56f17f23e98a8e116849a44cac27e3b (commit)
via fb95e36d2f0525fc7e024854b1bb3705027ca779 (commit)
via d22b0f215c0c6c94277d7114756f497cd7a3f07b (commit)
via 9a752eae723dfe23d5b98db8ba7ca600ab9f9573 (commit)
via d49a9ee4cd176e35f1929c1bb60caa751b53903d (commit)
via 0fe48142de81e20967eaac3972e35d115aaaa497 (commit)
via 2b0c4469c697b3ee752c690bab57b421a70e3853 (commit)
via df29ab509d3a9f6275fade0b7b8c9b5cd8d6f74d (commit)
via 718e3b0c18916a2a48d16cb5cb67d972ea5be6b8 (commit)
via 4c393c35f171c5b9e76972223a6c2739c02e0072 (commit)
via a464c2a216412032b58688194b4ff730fbe3a28f (commit)
via da14703fb4e1a249f47685b29310c4c69441ff08 (commit)
via 562130229d7e85cbf813179a315772594e298127 (commit)
via 7c7514411d3ed3b4d4695f55eb1a02f452a894a4 (commit)
via 336638f432e46f61cc87b958580e098e81cea921 (commit)
via e43159541b9da21730d3da27568530f47f7194f4 (commit)
via 73167d511b4ed1a0441bafa0f7bac26263b20231 (commit)
via 353cfc20742aaa069eecba2b291cb56a78e4d497 (commit)
via 3847e0a3ecba006108130fa890e55c020a6dc3e9 (commit)
via f4faf6f01860cb4683536a2c5e793a93b4193a42 (commit)
via b479bd43dfcca5ccc4bebdb62d8453a63f51527e (commit)
via c435ef61a97a115797c3075a6e5870b9a9587348 (commit)
via c4e13fdbb3f7de9d587f9c07b4711c08c7eae09e (commit)
via ccfad8f850f6d3edc43e7757fbeed112864efd18 (commit)
via 0be45d32255d83c410aad4e34560c478917ee6c4 (commit)
via b727cb3c8c32389f03e6ae71ad128c518a2c2a8c (commit)
via e8ebb2101fa89294c91b404953cc2fbabb796e40 (commit)
via de76be07dcbb36d38f6714480570cddaeabe2194 (commit)
via c8a4dee5e52feed137ca3cb4c4a4e224efbb694f (commit)
via d80674f09b714ee2ced537f18d73d325672db3c2 (commit)
via db33fa2b2d1f09a186bc07a2956cb0edae380d2c (commit)
via 413db07b4c81ea08663f90f31ee03227349d2be4 (commit)
via 996776292b5bfaeadb89457943ea1fba0700d030 (commit)
via cd5cd1855c02319ca31410943c4d944133294b17 (commit)
via 676d3522be04f774ba3a7db38d88b284252e0408 (commit)
via 43175f034f8015c6ca89a04fa8d9a322cf156265 (commit)
via c0b02133b2988b05772de5f2716d152aec4864ca (commit)
via 904e0b03fd558491d9dee39d7679a6c77eb6abd5 (commit)
via d6c7e2952413c33a4af9b1599e4000b9df622661 (commit)
via 35ac725af39a6326323cac66b28c532e49e27510 (commit)
via c9bfa91e3c3ec90778795969fdd3787578a76ed4 (commit)
via 342c15f625fda8746e7d74510a298286b3a52196 (commit)
via 8fdac0b6f954b6265798390b95e61f8192b85630 (commit)
via c7cbb121dec3c356bf7d72087d601c7593410b8c (commit)
via f5d7521ca506d63f631c603938cac5f40663bcca (commit)
via 22f96cd164a9a86e577b1ca4de24a281e7fbfa17 (commit)
via 641ef213571f793bb290a182dee3c4325bc85096 (commit)
via cdc146d447ae1ba42275d4ce7f34bd0b04158306 (commit)
via 92fba2405fc9cd6bbb6b1c4f5a7774f15b242696 (commit)
via 74cffe96768727e8b727cdb54358729c91bec130 (commit)
via c775680aefe1b6813b50f0a97dc073caeb4befbc (commit)
via 85f6919fae720c41474e60f5c21710f70e446070 (commit)
via c41b3893a26d542142de86a641e43077df3bda16 (commit)
via 3ae140fa072b2f2fbc8576c20ffd81fe463e78a5 (commit)
via b211e857d304f7fbe8787d2b65a307da841d047b (commit)
via 57153b5a518646399e9b62ad693ce213dae21972 (commit)
via 0853b16ad82a6b4411f04910d0035a2c653cfc29 (commit)
via b9a322e1aba0030f2dc13c2cf71fb581876c68cc (commit)
via d63984fdc1cf7c48cab9a57640415e85c8bfc48f (commit)
via 0a3d7a02236cbec448203a1b2218b5e0630d1c00 (commit)
via a77e606772cd4909f2401f55cd7c3c08f8325fb8 (commit)
via cec8e2705d260c9df1042858941419a3b9160c0e (commit)
via 9cc572d6a44262e21251372e28b549cfc09e681a (commit)
via 115e837fd16eecd80868267b0a6f99a531d4cc93 (commit)
via febdebbb58592be73dcf7d4bd4b2c7ff96657741 (commit)
via bef56b9a22efac9ce73006623080e84a0b57f243 (commit)
via b3b3d214f54d7909f12e60a911b02676818400b1 (commit)
via 711711827bb0c3564836707bb7d4453c60c6a98c (commit)
via 3271e860e871b19a2f3be68376c57c2ce88993ac (commit)
via 8df5b745955fafef08056a742680979b81faeb1e (commit)
via d6e1bfee59569d79f0f3e24620280e70aa161403 (commit)
via 0985649e03f3112be396fb67ee7507ff47a37571 (commit)
via d91eb0fae8d2ef4c2348769694573bf725ef16d4 (commit)
via 6c66177816fe128595454e9890d2c82a984cbc90 (commit)
via 2f4a5bef7433097558b89844c84d4cc40a447c53 (commit)
via 8ab6b482342b95ad35775867bcdb8fd691b78fb7 (commit)
via 58a026e09bda4c1e2374347615c325007c64fac4 (commit)
via e2cc6c1935565ee965f38501a4974e9d0e6b29e5 (commit)
via d2bb622dce5e765e55942e4da6348f949883d9fc (commit)
via c09663a2b1342e755daafcf8e4a4c2b121e4a253 (commit)
via 649d52a70fd75e255437deb1798fa9f09697b78c (commit)
via fba33040ea730ccc30035557226fc1a1de32ba6e (commit)
via 7f0f3ab4e13dd808b7b761039c857d55c7843175 (commit)
via 9a80d15b7cab21efe16ec2b543dfb566bea9def4 (commit)
via 297c4aaf43858eff5022a1e72eb8e09660bde4b0 (commit)
via 080c940d7a8134a6e277a53b7e45eb27e2b2c87f (commit)
via eef816fa8450fd3ce5388f4701ed888691ba897a (commit)
via 55137e6828bf11f76c3f9ec61e4a76954f5d6fa1 (commit)
via cb64653912a604809c3429f7dbda781741acd18a (commit)
via 0011b5236fc9a562bc13f943f9a431c496b2b7cd (commit)
via 3c23a0a57bde2978402e5d46ca2b003becee6d58 (commit)
via 42012c4746d6dd81eb3c72fddb2fdd36cff381e7 (commit)
via bc66b072e80a69cd9cb4f0c4bf746995305eaf8d (commit)
via 6fe8e52020d421797306e5c6536afbcee761510a (commit)
via a4166f402b34e018940ae1df726351e8c52ac1c1 (commit)
via 355173ba2e8c42b29011493d1d8c7cc4d69295c6 (commit)
via ab3afbb684bc1b32577c2696e13882123bfff7d2 (commit)
via 321e025cde18b8069dfc8977754af39f2efe505a (commit)
via 265fe64e5b7a931736f156c3cb446fbbbc27f018 (commit)
via 84f21d5634e17be62748f29f4303a86e0be6715b (commit)
via 83d08d7ccbc622ec97948929c83fb91f96743ca2 (commit)
via 051b9e763b0a14f9039f536859bd2c1580faa31c (commit)
via 66fc74001e777ac7ceff2b02cfc459b1368f42f3 (commit)
via d27d94532c173335cae9dddc30f1cc9a2e372bd7 (commit)
via b3c1156e1d4ed9f15a36bc51f2cd125a65aab3d3 (commit)
via 843b4e5cf7727528fb5b45629030fd0d6a364ab7 (commit)
via a93ef946eb1e73ee190ea4ff19c4f9278235530c (commit)
via db5107dca09b786374f06a35abb51ffc3f032abd (commit)
via 50cbdcbd67f8b0da06d3b188e7bfdea3963661a7 (commit)
via c9653ef070009e03061e59f2e13ea06cf7c16b01 (commit)
via 0279d2f8a27170c151833c470b88cf361aa3b91e (commit)
via f9a05f61abdf33891b09d62205d009d1cae73d1b (commit)
via ef7718f230f076d369f8c66064e7b542d24e57ed (commit)
via 224ac505b1162837f2f84fe2735a959d71bd5ce9 (commit)
via f0d62ea064d32b980f723e37972788cbd693c2ff (commit)
via e7870334a9f2c44fcd8580d3e75074df216c647a (commit)
via f62cff30c9948815d3cf2b00294da01c146cb8f2 (commit)
via 9ba887f887aa3736ab50728ccd2c0ae20e45d2a4 (commit)
via ec9ecf93b25f849e8480ece08b330c22b13bfe5f (commit)
via e0033099ba12ae2c42d41d3d049b9d759663d35d (commit)
via d4df74752927fbd464567f13723ba0dc5d9e5ff0 (commit)
via cd5141d405db649f0b99ca55b46ff68f16dc13b9 (commit)
via 336095d3ba51bf434c11a6d9d67e510bad165a0a (commit)
via 8d75c13b5601cb3450ad253cdddfef41499ba493 (commit)
via 5c2e355aca8dd456a646e743652a1be87113113f (commit)
via b5e8bde902d5b6d806c087fb30aea568da01a81c (commit)
via 0c27a910dc594c3d9eee5a69866cac220c0b18ba (commit)
via 115d8b18837272ee53144586454a684541318b84 (commit)
via 1980d00a9afeaea0bc6f266892e43de14ccd297e (commit)
via 751cd3892cbcf7ecb65ec416dba52ee84a2fee2a (commit)
via 7c893856e22073b10dc147b887b0fd64b11a1f81 (commit)
via d640a546f2cd55cfbd0a959bcfb84be1eecdd6a6 (commit)
via 09afabfb3de546d55c3a33ae0b68291edf414fcb (commit)
via 29776dabd84b8070b647ee38289214b7a56b2c1c (commit)
via 146f7179c88c0d25acc91ff2b5faff8e7f98d1be (commit)
via 3712b5943d9d32346552cb91ec5c4690e14485b9 (commit)
via 2ac43c8c20d917376b2b92c09c46fa5641d054f8 (commit)
via 02d7f4e820b65d83a5e3709dc14b6d72b9f5ab49 (commit)
via 8e693a9981f03d229ff2bff7dd6e5d06e0790c19 (commit)
via ac6a1a232a670d10ce12d19e6d2a7900d353943a (commit)
via bc248010427eded4610d7f337d3ee01d4d06538a (commit)
via e0784f339e794a64108f7c18e478d44e5751cc19 (commit)
via 4369714821950366db98a54e4b62fdb5d09951a6 (commit)
via be0cdc7814a49fa093b86b698a9756971ba80fcf (commit)
via ce85cd0b2605a6cbfb96bf18c1f2913d33b60f44 (commit)
via 8ae6680ebae91af9b0aafd6c9cfe5e5fb97f8b12 (commit)
via 5eda2f0ee6fcab67d3a74517a85946435931c67b (commit)
via 7c1e0140fb0d059299af08a102d30d65dfb60990 (commit)
via f4fd17ccc26166a195b7344b84d95e9ff2315a36 (commit)
via 5b0cb2c6292e8784b7a54b75b444e47a09c10a05 (commit)
via cf24a6137c3b70060fc5a9197f751595b1033131 (commit)
via 688319dca9cbbb27452a0062943bb38b6935254e (commit)
via eead78e2eb11528af8cb862dcb6f9a41737a8a14 (commit)
via cda50fb57f9b18cb769432876433b52f484692bd (commit)
via 64937458c5bc2918fa679c97478b2931bd26cfc1 (commit)
via 8d3ecc67ad5d208e5327b81dffda0c10ef0a8378 (commit)
via 3934348cf5166ed53248bed7f2aa3b98d067cbc2 (commit)
via acf545571425b2a2d8a26cb703f75926b6b2a987 (commit)
via 08540a58a67f2d575b1e15338c6d49b8b74e2c38 (commit)
via 2c87b580a87a55010da626d352307343f75d6d3a (commit)
via e2f50c08c4fc6da7065be222c2620264b521897f (commit)
via 6b495b26e49bb32eccaddffea36d91f34d3ba6f8 (commit)
via 0a2cc29e409f85381a8ee1ae1f0097937eb11804 (commit)
via fe8e22166ce16ec8ee78c53e49d0ab23f767d9a6 (commit)
via b478b8a0b4263b0e54060e3fecaab9b6ecdcb085 (commit)
via 33c10053b22fd5065516eb7df4c58b55a70d490c (commit)
via c405f9adab08daa3d4edef9cddd5453a79446c7b (commit)
via 6dd60d7c70a63ac884515387cd8c92bb97433e1b (commit)
via 710399e0355ff8ed016a9d3c830730af2e228ad5 (commit)
via 81fa59a1e7797da8d420a94216bd28031178ae51 (commit)
via f339946832e0bb7ad175acaf59733445e6915f7a (commit)
via e5a3fb0a69df6c27dc567949a2e64d0e7da65384 (commit)
via 04c664d6150683e3258168eecc4440a635ef8b16 (commit)
via 951c8a79bca7224fdf8c50463d0c9a60b43ce930 (commit)
via b29c13b372bd39075eabda49ea57b0cb6d5a8452 (commit)
via 9b16a843a641159867ea390c4c2384b320183b15 (commit)
via 77cfd1ee241b4a2d408bd3ec5ebc69f34c7496f5 (commit)
via d3e2fb85a39562e38ea67f84e9f065815450294a (commit)
via b2a50a8539be09b73c5d65719e8eae4b2a15273b (commit)
via bd55c808de1d4d25fea7cc047956c45ec9752fa5 (commit)
via a8d2a183ac7246bf368d40dba1af226a4968a9c9 (commit)
via 69138b061aca4e0cbbb7dad080158205f6734e77 (commit)
via 535856c28a12bb07dc986b980b0f4ccfdfd25640 (commit)
via 335ee76030e85fa6ac4da79b598cf4c4a212443d (commit)
via 81d5bd428962c1819c0c60de5bc5d93710798424 (commit)
via cd6d6f1f15bc1452a09c16b1a3524b5b289c6100 (commit)
via d22f92638664d9744a38e60dce9f0d95a2d393f0 (commit)
via 6b8a3d3d912573ab432c319121eaffd0bbe2216b (commit)
via c584cb69cdcfb377deed94745785330562a54ae3 (commit)
via 3f278cea46034660149403d68e05f0f450330854 (commit)
via 3a7481703542b0c95d5a23cb45b0d2e7c3ac4c9e (commit)
via 66c644efeabea073428288378a39a8e988e5b69b (commit)
via 9b6abcd0448567146b471ad02162d33fd4b1d5a8 (commit)
via 5c4d9d38dcee73a7ffb6221c80f707c3924da64f (commit)
via ee4c01999aff1ebc1e2dc338a70d1d7b812c633c (commit)
via 380e4da5aab5d24d0e90ea27880974c232538fbf (commit)
via 424298694945c9f66676bd401753be44f6fc5fa8 (commit)
via 64a357242cc7ba9fb877a9b3d622160318098c5c (commit)
via 57074c15587686e066d7e44adec7b6c7154d6ac4 (commit)
via a12e4d36901ab3b3027f1168f1ea06b26122829e (commit)
via c1ab715afeb9c095efed7e6064ce10021c067c35 (commit)
via 0695b41ae52b87adcdf49c6411f0da353d7213a9 (commit)
via 816764a283c2cbf2d41b4582113065922b99bd52 (commit)
via 8f0815985dfb74fc1c7468426359ac3fbbd8c12d (commit)
via 1f9519fba9a34f2a596c683ed6395b2e291935b7 (commit)
via 4a78e8e91fdad38e567fef0cd43aa8cb6bd33580 (commit)
via c425f01b6dd63dfe5d99a19f57c8155c2f0fb195 (commit)
via 3813e892be7f3136e5b14580e7ee64d6c786bf8e (commit)
via 4a2dc82a1acce855151928abe0030e1dd7dbf728 (commit)
via 9a9930cebb7f36131490d0f144e23f5066e7f01d (commit)
via d128fe6bc8da01fe8e7829db988819dd1c159298 (commit)
via 76a2c8a1eb1771b5255e8582b7e9823d072e8ef4 (commit)
via 62f6a306eb965cee6bd814b194bf5ce3bb910116 (commit)
via 11ab2e23511c7e8962e0110c3aad44b74fea2dbd (commit)
via 7d4123813b42d8f6478e239895e62f6f600a1c6f (commit)
via 596884ea30c551d7c2699b2cf3fdea2cede3101a (commit)
via bd059862d1dd7e68642dd365f0a536621ff6735f (commit)
via 95de13bdab14eb803a4c9e2243df50e1eb0df69a (commit)
via e1633d0a8365f63db56777524b1281d90bd9ee46 (commit)
via 810f2d8a0e6720568a963462e538daa5c692ace3 (commit)
via 890195dc78d25d9a8f1e513197e4adc71be4f146 (commit)
via 0765fb8b19dd7b76ec5d2d05edda3f4fd4347194 (commit)
via 967821ec0ac00aef677f943e709ea079ebc30f78 (commit)
via 286f7fb2dbdcd860275a24f3480e938b5bd414ec (commit)
via aeaee020e585b69f3fea9381deb0c595eebf7c9a (commit)
via 32d954c32f20a982bec4b3615bfd7cfca449c280 (commit)
via 57fd9fa6bf0ee3062d7d38aceb7e97543791d241 (commit)
via 116f934f0359b27ae3bd3047860fc5e7f27bdcf5 (commit)
via 9df6d2c2152b5b1968649c970664c4f69d9e92e8 (commit)
via edf5021cddc7646c920d3c938a2a3b34d6d8c0a5 (commit)
via b004059a3372af7aa347ef7b8b73522d489ffec2 (commit)
via c90adfb66544a4e6ff34b13268fe3e05847144c2 (commit)
via e9e440847c574d1152b128cfd508cf473f49121b (commit)
via a4ea40603c5f9c71ce7051fe13e3b493e0829128 (commit)
via eed6ea7af40d0d7ee0a11bcb484113bd606f51ed (commit)
via ab7bb79f2da9f44eae4b1fd910680ca472b9c5a7 (commit)
via ba908bbe90213f1d50422611052c92280eae0dcd (commit)
via 945258f22c3c02d12e0dada049b8c37fa5139af2 (commit)
via a31c1accfc353ed6bb3c9982ee694f98f6c965ec (commit)
via 42d62e3d140360a179293b8995aaf535e8c4c30c (commit)
via 059d04053d1a7ac62c796ad5757191b9c5dd5aae (commit)
via 93661ec76c6c1affcde86563dccda5843a879239 (commit)
via 7f9465d37fcc3277128d3f4a611b778e24e530a5 (commit)
via 59cf374273bfae0baec8370526a747bbb8e5edb7 (commit)
via ebb7681e5cf4bc2825e8786ecda895e219158703 (commit)
via 6df0096f2b246959bb9449a0940e9c539bcc9100 (commit)
via a6ab70e907a47e8e28aae2dd3eb357aa72c8b673 (commit)
via 4fdb030e1ad940215b75b4b8a8f7aca5635211e3 (commit)
via 78a9021dc8baa5603feb512c45c1fbca5f3accfc (commit)
via b7acf6db5beab0b082c863fe5a9692d68152a132 (commit)
via 0bcbbb13f9e278347e500fa344ee600891a9bcb8 (commit)
via 0e94528878341faa941247eb3d990230c941868c (commit)
via 645e9829bec9147f52141b17b439f9b561ed3445 (commit)
via 5d3a682b0fc41f4152051b43b8e5311b9a130f13 (commit)
via ec88f99b1eeb811159b3aea9a6f810dc06e9417b (commit)
via bd14cd71274109789ea076b1b4219bdf3beef657 (commit)
via 75f835af6ac337c490a76ac60453242719f6ceca (commit)
via 905a7305413f57e3188a66fe55d35cecc484b1f2 (commit)
via c7414891876cd6d0d02abe71c7376b8104bfbc32 (commit)
via a200bee21cc9632428c6ed9e37dd3a4f90682f9d (commit)
via 731ff645f6de779a324999f3b78bf90469dbb4fb (commit)
via 9fa635c9dc288317f19471291accecf8690f5718 (commit)
via f9e94997cb5c2166d8b71874f263544cfc2fe5ba (commit)
via e677e6b27fa35ba2f81a8a35b00fe860926a69ad (commit)
via a7cd9cfe43468acd6cbc1b674241fb3c7ffcf0cf (commit)
via fa4fc4f452c5b29427512758f4bd14df2fcc3c42 (commit)
via cc556a86ae6a83ba4cf5b7125fbabe704207ae60 (commit)
via 911abf131639401b2ccfd23a6bc3ec983f7ed8f4 (commit)
via 41e15e626cc301d867c48ebb1f945bbec6b7d555 (commit)
via af1978730ccd9bd3584fc1cdace37809eff63a13 (commit)
via 43e349a8916b4e952dfafee2f88d4d2952be12c1 (commit)
via a8bfbac314335eb0bff3f4ff5e856d3c327de31d (commit)
via 73ad2ee9af3b97c46293bdfc9e2925a67726b786 (commit)
via af3bfbb6126ebce381c52ced87fc8af372d84118 (commit)
via 6f1a5b9cb0bc761115ef4bc31472ee47db83a906 (commit)
via 2538d2a88afa75f4c23961d77fe9ffccb048368a (commit)
via 5f478e932583f04fc1bd067e437e2d929060dd18 (commit)
via 519850a6a801553c4cfbc7384236536b5a9cedf2 (commit)
via 6ac7e6d3d1f75cc6d6fe625d2ceaaf00b86a720d (commit)
via bb5b4b99bd9957fd1b4713c5168b0d7e148899ee (commit)
via cb97316068f201ffd03f54d67074a83601c2bb45 (commit)
via aace3aa52c1b9fe3b2e907f43e8d6801ceeb28f9 (commit)
via 75adb33e1b2038691e1f62078ee530d1ba01c873 (commit)
via 18e820bdfe2598ec1901fd52fd80c8dae4f55c32 (commit)
via 3fbabc1b5236b6667f453a2949849d96f6e683df (commit)
via 187d6318298adca84193d8c78952e023f303bc2d (commit)
via 02763280f54fd0c2a499285f8ce6afcbd8b9e082 (commit)
via 87c2b7450a906fbfc87c90b1e30cff11ac5e2aca (commit)
via 988c59f51aa579ce8bf0eab1cc729e05a5ee5631 (commit)
via c4b2d43234205a0d9b1310e55b1f00dee9c20625 (commit)
via 3ac93fa684e01ee2e8ababa6d9cc617cdbd3cbea (commit)
via fe561d69b42d55191d6d1f01d4f3cdcf1fc9faaa (commit)
via 94f2b439783a8e63d6d7b9ba2760f54fc642a8fb (commit)
via 23d57ba45b348b580fc584bbd77fe3960796622d (commit)
via f777c74882e6b0f52b15f62d1d6251cd180979e4 (commit)
via 1b5156270c5cb8d7a4a1b095d981f1a84a98554f (commit)
via b58c06e93fc1392aea0347ea099376b41ec4b7c3 (commit)
via a1f0e517f6b37ea987c0146a4ca93f50715f00f2 (commit)
via d9c1f53cde2e3a09c92e7e6f009f35237b2272a9 (commit)
via 21c5372c6b670820e842e01336eb6b191d6e10b7 (commit)
via 4a98eba9ae08ffccb822842f74b1b805302a1ad1 (commit)
via e26dc8ebc182bec997624213c771f06e9b0179e8 (commit)
via 46434cfe5a053097440bcccc35d0ce7d00bbcfee (commit)
via 679d5507d6c390fd03e5c71300001f2696b1f5b1 (commit)
via 18d6239d25924545ba91825011d467861cd5513c (commit)
via 48fd863c654325eefceb8dfd182c88a8149ca309 (commit)
via 8666f138c10e2a201ee288770f29c5a20b9fc706 (commit)
via 488bc59b2d90e0a9a23801b034c8a54525d83da4 (commit)
via e2f03263c7c2496ff3ee84e43eb133fe171905f9 (commit)
via db7330822cb7dbdd1b61a34737d1b24158d8068d (commit)
via 2e0b7fcafcccc50602f8fd4df11b6312467e95fa (commit)
via fa8fd28e3ca22518a147cf34bf7146ef2a173257 (commit)
via 36f8e449321e4fa02d88fee1fded14aa8ff81723 (commit)
via 8f76037ba8a37c488612285ffe70d26d0d038124 (commit)
via 0557b1dbb08d5328e29eac805824a8e21040f582 (commit)
via 4f1e8828caf40af5a80ca8189c46a6cdfe8d541f (commit)
via 4d82174d21dd5cdf0668a10ca016e72338fd97e6 (commit)
via 1a37db7b0a70b5672cd0f7c86b19b85ba824378f (commit)
via 0275a62826a50d84f8f633f475c930fde96af4a4 (commit)
via 9f768708156197decac1e6dc5dedf3a730a06fad (commit)
via e135f4e0f24442e009a60f2018d95c5e8cb63e5e (commit)
via 089b68192e6fd089c03331c389da1dace039c899 (commit)
via 2c68e941a15384cecb0e3b3df6def6c3e84d0a71 (commit)
via 115a5e8861ef0a46224b2cd64568b30c884908fb (commit)
via 42a0609a6e287a82ed565413c7392d40141388ae (commit)
via 71db701269dc5d2b5eb9239828a74e9c26cd7e66 (commit)
via 6714a5e7fcf4d5fde2ecd5a7f9f6504cb5ca374b (commit)
via 985ba2d04074d830fc2e8688f2d0f59a75a1c14c (commit)
via 28225aeb0336a4872bbaa3aae5d331172f1e4068 (commit)
via b70f9ce54f1f672b423999e6c07b2f0127b76666 (commit)
via 296ac256c898c145f7e4566d86d5f09f4e966b43 (commit)
via 58fa1d8438cb613c6bf7bece8702146f3eed5205 (commit)
via bd1aa20c5878436505b31aa987473ac3fbb6395c (commit)
via 8fa57330875af11b7b61dfac6fc45080760afdf2 (commit)
via 17f521d7ffb4f3a58ca98a27395eb60d9fa34519 (commit)
via 5f42faf0818c229196b26535a0669eeb2bf27386 (commit)
via 06eb98871e82074b935d9da201118119e5719f96 (commit)
via 99887d1312315a5efb4d888251f90880846c073c (commit)
via 4ddcd079b0378d0a8556f528106ac7f60fe5cb1b (commit)
via 1e03ea4765bee6db9331a1265c8abf2c3e45caff (commit)
via a4991cffc83afea213125a24f01399d6f5707948 (commit)
via 8e31910034627dedd7259dd1e45a60768108c1e1 (commit)
via d2efca1759c9e104f67d1caf85c8e62ff06807bd (commit)
via 035b113f60302f6d9c265e6e3a63dbb3c5873153 (commit)
via 85c13201abf4a9f19dd112992a04fa65da63369c (commit)
via cefddd8efdc88c3d33e5aad4c02e96c64cbe8242 (commit)
via 0c458347e92967b5a1e6cce8267939d3ef204663 (commit)
via 54ee2b06add844efae0b0fcb97606102232974e6 (commit)
via f9347e2f053b645003d102f96a0aee49167baeef (commit)
via 5f75e2db42cea65e6f4e6b8c2b1b1dfb8e748c05 (commit)
via 29fe8719ae4f4463cf5c63ad20cf2eb1ae39cd19 (commit)
via 333a30289f47c22956bcb6a045d1cd4c797e22c3 (commit)
via 31fd95388f035bba3e70d3e8861a14511f6281e6 (commit)
via 7d7eb76d319f1ab6faeb8634f13142e7147a45ea (commit)
via 8f4b1051209f78bff358069a2a3b2f646f7a38f8 (commit)
via a518eb2b3b743259fe51b1e61d2648677e583cac (commit)
via e3483661acc3d26d4532a536bd40ed761c21d7d1 (commit)
via 1dc6c511620a852f81d5179fd2f9fbecee472e71 (commit)
via 57fcaf45fe40806dca5f001aabed9b413243183e (commit)
via afbc818e5bbc667b69e9ae9b8f080cb50ca17c53 (commit)
via ad5f111b67e2bf1c43ce9bc37e6b11b8c753f62d (commit)
via 26d20f13895fdf5c9224a7aca2b33625c511e73e (commit)
via da40bd0960806df8e2799e4fb716d41ad08b169f (commit)
via 1714431955ac7b2a1d2fd3c2cda9b806bad624cb (commit)
via da2cfffb3a3ec92c3b15841255dc704a99748fea (commit)
via df8a51cd6b8a95475dfaef08c8a56b1e1643dbfc (commit)
via 1b183730bd6ec56d51b17020e63fb2cffa2ad764 (commit)
via c9143544609d90da33eb3c2d566fc5d6a25188b2 (commit)
via d5dd5dc2838fe420c8f835975bad7052a0df55e5 (commit)
via b01b480adb45c3bbfcfab13e343e08c16854dedc (commit)
via 1965e1793598c76ceef4decd4a5a6c7755c385e3 (commit)
via 4f0c3d501d19bed5915d5d188598d3a7f1dec7f8 (commit)
via 088268fc00cb466c4b4576a2dbbaff720cb3e3af (commit)
via 73872ccc5bb6b80a6049b44b0113085a9c2b6934 (commit)
via 95c01d22571043d7b6405f928538204cf930d453 (commit)
via 76f6b358cf612e67b586452236394feabc464373 (commit)
via 5836e576fe0b78c50383cf56e1c4fb4521daeca1 (commit)
via 00cca6a192eb1ab38559bf5ed9044711ed56fc4a (commit)
via 9fcb58db8d47a42e52eead600ddabf57ab7c29ed (commit)
via 18b5ad828a4149c1b768d05e477982364002a647 (commit)
via b3e79afc9f6527927c6bf6e4ee35b11e6f20185e (commit)
via bb17ad738ed9c890881361232e0dfd84e50a529d (commit)
via cdd57a73f2f90d6b14b8652ab94adc14e4b99c8b (commit)
via 3c2266151711f532491a2369de37086bd7234dce (commit)
via 05f0008349efe56755c580e1e060115587aab352 (commit)
via 8afc85aabb9563da4de17b0b5f7d4fe574e9ad8d (commit)
via 3950ffc9481c25262f2db2b08a0f74664c433734 (commit)
via 1275e305f6d13785687c72b8609592094a067cef (commit)
via 7cc2336a974945d03649d924cf805fedf92bad41 (commit)
via f7029eca4ddc301167078827a6bed2219dd282a6 (commit)
via 92ac046a8a7a0f5a3a88b8656bb171701af840a0 (commit)
via 5d340bee3506b37f0d5f0b695dfa37661ad0fbb6 (commit)
via 9df819a55f00e893f2783c1e048a6d3c1cd391b2 (commit)
from 10176dd693120301ccc9695310c6ecbd7714cf79 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 95e5ccacf6c1193b313fa90a6d39baafa2ba67d8
Merge: 10176dd69 e73008e45
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 24 15:28:16 2018 -0400
Merge branch 'master' into 14075-uploadfiles
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --cc sdk/cwl/setup.py
index bdbe77829,e452ce264..2b7b31b9f
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@@ -36,12 -36,11 +36,13 @@@ setup(name='arvados-cwl-runner'
'cwltool==1.0.20180806194258',
'schema-salad==2.7.20180719125426',
'typing >= 3.6.4',
- 'ruamel.yaml >=0.13.11, <0.16',
+ # Need to limit ruamel.yaml version to 0.15.26 because of bug
+ # https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping
+ 'ruamel.yaml >=0.13.11, <= 0.15.26',
'arvados-python-client>=1.1.4.20180607143841',
'setuptools',
- 'ciso8601 >=1.0.6, <2.0.0'
+ 'ciso8601 >=1.0.6, <2.0.0',
+ 'subprocess32>=3.5.1',
],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
diff --cc sdk/cwl/tests/test_container.py
index 0209d2eba,3f8a32816..69f3ae046
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@@ -3,8 -3,8 +3,9 @@@
# SPDX-License-Identifier: Apache-2.0
import arvados_cwl
+ import arvados_cwl.context
from arvados_cwl.arvdocker import arv_docker_clear_cache
+import arvados.config
import logging
import mock
import unittest
@@@ -21,35 -21,30 +22,56 @@@ if not os.getenv('ARVADOS_DEBUG')
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-
+class CollectionMock(object):
+ def __init__(self, vwdmock, *args, **kwargs):
+ self.vwdmock = vwdmock
+ self.count = 0
+
+ def open(self, *args, **kwargs):
+ self.count += 1
+ return self.vwdmock.open(*args, **kwargs)
+
+ def copy(self, *args, **kwargs):
+ self.count += 1
+ self.vwdmock.copy(*args, **kwargs)
+
+ def save_new(self, *args, **kwargs):
+ pass
+
+ def __len__(self):
+ return self.count
+
+ def portable_data_hash(self):
+ if self.count == 0:
+ return arvados.config.EMPTY_BLOCK_LOCATOR
+ else:
+ return "99999999999999999999999999999996+99"
+
+
class TestContainer(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": Loader({}),
+ "metadata": {"cwlVersion": "v1.0"}})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "containers",
+ "basedir": "",
+ "name": "test_run_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "tmpdir": "/tmp",
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
diff --cc sdk/cwl/tests/test_job.py
index 1a3f6a9c8,4473b88ca..20efe1513
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@@ -331,28 -382,21 +383,22 @@@ class TestWorkflow(unittest.TestCase)
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
- document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
- document_loader.fetch_text = document_loader.fetcher.fetch_text
- document_loader.check_exists = document_loader.fetcher.check_exists
+ loadingContext, runtimeContext = self.helper(runner)
- tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
+ tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
- mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+ mockc = mock.MagicMock()
+ mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
+ mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
- it.next().run()
- it.next().run()
+ it = arvtool.job({}, mock.MagicMock(), runtimeContext)
+
+ it.next().run(runtimeContext)
+ it.next().run(runtimeContext)
with open("tests/wf/scatter2_subwf.cwl") as f:
subwf = StripYAMLComments(f.read())
@@@ -418,27 -463,19 +465,19 @@@
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
- document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
- document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
- document_loader.fetch_text = document_loader.fetcher.fetch_text
- document_loader.check_exists = document_loader.fetcher.check_exists
+ loadingContext, runtimeContext = self.helper(runner)
- tool, metadata = document_loader.resolve_ref("tests/wf/echo-wf.cwl")
+ tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
- mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+ mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
- it.next().run()
- it.next().run()
+ it = arvtool.job({}, mock.MagicMock(), runtimeContext)
+ it.next().run(runtimeContext)
+ it.next().run(runtimeContext)
with open("tests/wf/echo-subwf.cwl") as f:
subwf = StripYAMLComments(f.read())
diff --cc sdk/python/arvados/commands/run.py
index 5063d75f2,c4748fa99..96f5bdd44
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@@ -218,50 -205,24 +219,50 @@@ def uploadfiles(files, api, dry_run=Fal
for root, dirs, iterfiles in os.walk(localpath):
root = root[len(pathprefix):]
for src in iterfiles:
- write_file(collection, pathprefix, os.path.join(root, src))
+ write_file(collection, pathprefix, os.path.join(root, src), not packed)
- filters=[["portable_data_hash", "=", collection.portable_data_hash()]]
- if name:
- filters.append(["name", "like", name+"%"])
- if project:
- filters.append(["owner_uuid", "=", project])
+ pdh = None
+ if len(collection) > 0:
+ # non-empty collection
+ filters = [["portable_data_hash", "=", collection.portable_data_hash()]]
+ name_pdh = "%s (%s)" % (name, collection.portable_data_hash())
+ if name:
+ filters.append(["name", "=", name_pdh])
+ if project:
+ filters.append(["owner_uuid", "=", project])
- exists = api.collections().list(filters=filters, limit=1).execute(num_retries=num_retries)
+ # do the list / create in a loop with up to 2 tries as we are using `ensure_unique_name=False`
+ # and there is a potential race with other workflows that may have created the collection
+ # between when we list it and find it does not exist and when we attempt to create it.
+ tries = 2
+ while pdh is None and tries > 0:
+ exists = api.collections().list(filters=filters, limit=1).execute(num_retries=num_retries)
- if exists["items"]:
- item = exists["items"][0]
- pdh = item["portable_data_hash"]
- logger.info("Using collection %s (%s)", pdh, item["uuid"])
- elif len(collection) > 0:
- collection.save_new(name=name, owner_uuid=project, ensure_unique_name=True)
+ if exists["items"]:
+ item = exists["items"][0]
+ pdh = item["portable_data_hash"]
+ logger.info("Using collection %s (%s)", pdh, item["uuid"])
+ else:
+ try:
+ collection.save_new(name=name_pdh, owner_uuid=project, ensure_unique_name=False)
+ pdh = collection.portable_data_hash()
+ logger.info("Uploaded to %s (%s)", pdh, collection.manifest_locator())
+ except arvados.errors.ApiError as ae:
+ tries -= 1
+ if pdh is None:
+ # Something weird going on here, probably a collection
+ # with a conflicting name but wrong PDH. We won't
+ # able to reuse it but we still need to save our
+ # collection, so so save it with unique name.
+ logger.info("Name conflict on '%s', existing collection has an unexpected portable data hash", name_pdh)
+ collection.save_new(name=name_pdh, owner_uuid=project, ensure_unique_name=True)
+ pdh = collection.portable_data_hash()
+ logger.info("Uploaded to %s (%s)", pdh, collection.manifest_locator())
+ else:
+ # empty collection
pdh = collection.portable_data_hash()
- logger.info("Uploaded to %s (%s)", pdh, collection.manifest_locator())
+ assert (pdh == config.EMPTY_BLOCK_LOCATOR), "Empty collection portable_data_hash did not have expected locator, was %s" % pdh
+ logger.info("Using empty collection %s", pdh)
for c in files:
c.keepref = "%s/%s" % (pdh, c.fn)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list