[ARVADOS] updated: 1.3.0-1540-gd4ed3e646
Git user
git at public.curoverse.com
Wed Aug 14 15:10:25 UTC 2019
Summary of changes:
apps/workbench/fpm-info.sh | 4 +-
.../pipeline_instances_controller_test.rb | 103 -
.../test/controllers/projects_controller_test.rb | 26 -
.../test/helpers/repository_stub_helper.rb | 1 -
.../test/integration/application_layout_test.rb | 2 +
apps/workbench/test/integration/jobs_test.rb | 95 +-
.../test/integration/pipeline_instances_test.rb | 369 ---
.../test/integration/pipeline_templates_test.rb | 29 -
apps/workbench/test/integration/projects_test.rb | 1 -
.../test/integration/repositories_browse_test.rb | 23 -
apps/workbench/test/integration/websockets_test.rb | 61 -
apps/workbench/test/integration/work_units_test.rb | 30 -
apps/workbench/test/unit/disabled_api_test.rb | 8 +-
build/package-build-dockerfiles/centos7/Dockerfile | 4 -
build/package-build-dockerfiles/debian9/Dockerfile | 4 -
.../ubuntu1604/Dockerfile | 4 -
.../ubuntu1804/Dockerfile | 4 -
build/run-build-packages-one-target.sh | 1 -
build/run-build-packages.sh | 29 +-
build/run-build-test-packages-one-target.sh | 12 +-
build/run-library.sh | 183 +-
cmd/arvados-client/cmd.go | 16 +-
crunch_scripts/GATK2-VariantFiltration | 64 -
crunch_scripts/GATK2-bqsr | 103 -
crunch_scripts/GATK2-merge-call | 242 --
crunch_scripts/GATK2-realign | 163 --
crunch_scripts/arvados-bcbio-nextgen.py | 145 --
crunch_scripts/arvados_bwa.py | 115 -
crunch_scripts/arvados_gatk2.py | 52 -
crunch_scripts/arvados_ipc.py | 51 -
crunch_scripts/arvados_picard.py | 42 -
crunch_scripts/arvados_samtools.py | 110 -
crunch_scripts/bwa-aln | 127 -
crunch_scripts/bwa-index | 41 -
crunch_scripts/collection-merge | 49 -
crunch_scripts/crunchrunner | 10 -
crunch_scripts/crunchutil/__init__.py | 0
crunch_scripts/crunchutil/robust_put.py | 56 -
crunch_scripts/crunchutil/subst.py | 102 -
crunch_scripts/crunchutil/vwd.py | 107 -
crunch_scripts/cwl-runner | 117 -
crunch_scripts/decompress-all.py | 64 -
crunch_scripts/file-select | 18 -
crunch_scripts/grep | 24 -
crunch_scripts/hash | 37 -
crunch_scripts/pgp-survey-import | 119 -
crunch_scripts/pgp-survey-parse | 22 -
crunch_scripts/picard-gatk2-prep | 211 --
crunch_scripts/pyrtg.py | 75 -
crunch_scripts/rtg-fasta2sdf | 27 -
crunch_scripts/rtg-fastq2sdf | 45 -
crunch_scripts/rtg-map | 41 -
crunch_scripts/rtg-snp | 34 -
crunch_scripts/run-command | 458 ----
crunch_scripts/split-fastq.py | 70 -
crunch_scripts/test/task_output_dir | 19 -
doc/_config.yml | 22 +-
doc/_includes/_arv_run_redirection.liquid | 27 -
doc/admin/upgrading.html.textile.liquid | 14 +-
doc/api/crunch-scripts.html.textile.liquid | 2 +
doc/api/execution.html.textile.liquid | 7 -
doc/api/methods/humans.html.textile.liquid | 2 +
doc/api/methods/job_tasks.html.textile.liquid | 2 +
doc/api/methods/jobs.html.textile.liquid | 4 +-
.../methods/pipeline_instances.html.textile.liquid | 2 +
.../methods/pipeline_templates.html.textile.liquid | 2 +
doc/api/methods/specimens.html.textile.liquid | 2 +
doc/api/methods/traits.html.textile.liquid | 2 +
.../install-compute-node.html.textile.liquid | 112 -
.../install-crunch-dispatch.html.textile.liquid | 207 --
doc/install/install-keep-web.html.textile.liquid | 129 +-
doc/sdk/cli/subcommands.html.textile.liquid | 95 -
.../crunch-utility-libraries.html.textile.liquid | 228 --
doc/user/cwl/cwl-run-options.html.textile.liquid | 26 +-
doc/user/cwl/cwl-versions.html.textile.liquid | 14 -
.../examples/crunch-examples.html.textile.liquid | 102 -
.../reference/job-pipeline-ref.html.textile.liquid | 14 -
doc/user/topics/arv-run.html.textile.liquid | 163 --
.../crunch-tools-overview.html.textile.liquid | 70 -
doc/user/topics/run-command.html.textile.liquid | 319 ---
...nning-pipeline-command-line.html.textile.liquid | 58 -
.../topics/tutorial-parallel.html.textile.liquid | 85 -
.../tutorial-trait-search.html.textile.liquid | 278 ---
.../running-external-program.html.textile.liquid | 85 -
.../tutorial-firstscript.html.textile.liquid | 112 -
.../tutorial-submit-job.html.textile.liquid | 95 -
lib/cli/external.go | 6 +-
lib/config/config.default.yml | 69 +-
lib/config/deprecated.go | 70 +
lib/config/deprecated_test.go | 76 +
lib/config/export.go | 6 +-
lib/config/generated_config.go | 69 +-
lib/config/load.go | 6 +
lib/controller/fed_collections.go | 10 +-
lib/controller/federation/conn.go | 8 +-
lib/controller/federation_test.go | 17 +-
sdk/cli/arvados-cli.gemspec | 6 +-
sdk/cli/bin/arv | 10 -
sdk/cli/bin/arv-crunch-job | 6 -
sdk/cli/bin/arv-run-pipeline-instance | 781 ------
sdk/cli/bin/crunch-job | 2577 --------------------
sdk/cli/test/test_arv-run-pipeline-instance.rb | 37 -
sdk/cli/test/test_crunch-job.rb | 141 --
sdk/cwl/arvados_cwl/__init__.py | 35 +-
sdk/cwl/arvados_cwl/arvjob.py | 495 ----
sdk/cwl/arvados_cwl/arvtool.py | 12 -
sdk/cwl/arvados_cwl/crunch_script.py | 159 --
sdk/cwl/arvados_cwl/executor.py | 76 +-
sdk/cwl/tests/test_container.py | 2 -
sdk/cwl/tests/test_job.py | 554 -----
sdk/cwl/tests/test_submit.py | 431 ----
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf2.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 4 +-
sdk/go/arvados/config.go | 20 +-
sdk/go/arvadosclient/arvadosclient_test.go | 50 -
sdk/go/crunchrunner/crunchrunner.go | 439 ----
sdk/go/crunchrunner/crunchrunner_test.go | 478 ----
sdk/go/crunchrunner/upload.go | 241 --
sdk/go/crunchrunner/upload_test.go | 152 --
sdk/python/arvados/commands/run.py | 185 +-
sdk/python/bin/arv-run | 7 -
sdk/python/setup.py | 1 -
sdk/python/tests/nginx.conf | 20 +-
sdk/python/tests/run_test_server.py | 15 +-
sdk/python/tests/test_arv_run.py | 28 -
sdk/python/tests/test_pipeline_template.py | 62 -
sdk/python/tests/test_retry_job_helpers.py | 31 -
.../controllers/arvados/v1/job_tasks_controller.rb | 5 +
.../app/controllers/arvados/v1/jobs_controller.rb | 104 +-
.../arvados/v1/pipeline_instances_controller.rb | 10 +-
.../arvados/v1/pipeline_templates_controller.rb | 5 +
.../api/app/helpers/commit_ancestors_helper.rb | 6 -
services/api/app/helpers/commits_helper.rb | 263 ++
services/api/app/models/commit.rb | 272 ---
services/api/app/models/commit_ancestor.rb | 44 -
services/api/app/models/job.rb | 214 +-
services/api/app/models/job_task.rb | 10 +
services/api/app/models/pipeline_instance.rb | 35 +-
services/api/app/models/pipeline_template.rb | 11 +
services/api/config/arvados_config.rb | 5 -
....rb => 20190808145904_drop_commit_ancestors.rb} | 4 +-
...s.rb => 20190809135453_remove_commits_table.rb} | 4 +-
services/api/db/structure.sql | 115 +-
services/api/fpm-info.sh | 4 +-
services/api/lib/can_be_an_owner.rb | 2 +
services/api/lib/crunch_dispatch.rb | 981 --------
services/api/lib/enable_jobs_api.rb | 12 +
services/api/script/crunch-dispatch.rb | 16 -
services/api/script/crunch_failure_report.py | 222 --
services/api/script/fail-jobs.rb | 21 -
.../arvados/v1/job_reuse_controller_test.rb | 702 ------
.../functional/arvados/v1/jobs_controller_test.rb | 307 +--
.../v1/pipeline_instances_controller_test.rb | 44 -
services/api/test/helpers/git_test_helper.rb | 8 +-
.../api/test/integration/crunch_dispatch_test.rb | 47 -
services/api/test/integration/jobs_api_test.rb | 83 -
services/api/test/integration/pipeline_test.rb | 36 -
.../test/integration/serialized_encoding_test.rb | 21 -
services/api/test/unit/commit_test.rb | 70 +-
services/api/test/unit/crunch_dispatch_test.rb | 218 --
services/api/test/unit/fail_jobs_test.rb | 83 -
services/api/test/unit/job_task_test.rb | 11 -
services/api/test/unit/job_test.rb | 406 +--
services/api/test/unit/pipeline_instance_test.rb | 106 -
services/fuse/arvados_fuse/__init__.py | 14 +-
services/keep-web/cache.go | 28 +-
services/keep-web/cache_test.go | 6 +-
services/keep-web/cadaver_test.go | 4 +-
services/keep-web/doc.go | 68 +-
services/keep-web/handler.go | 9 +-
services/keep-web/handler_test.go | 86 +-
services/keep-web/keep-web.service | 1 -
services/keep-web/main.go | 117 +-
services/keep-web/server.go | 12 +-
services/keep-web/server_test.go | 22 +-
services/keep-web/status_test.go | 2 +-
services/keep-web/usage.go | 99 -
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/cluster-config.sh | 30 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 40 -
.../service/crunch-dispatch0/log/main/.gitstub | 0
.../arvbox/docker/service/crunch-dispatch0/log/run | 1 -
.../lib/arvbox/docker/service/crunch-dispatch0/run | 1 -
.../docker/service/crunch-dispatch0/run-service | 6 -
.../service/crunch-dispatch1/log/main/.gitstub | 0
.../arvbox/docker/service/crunch-dispatch1/log/run | 1 -
.../lib/arvbox/docker/service/crunch-dispatch1/run | 1 -
.../docker/service/crunch-dispatch1/run-service | 7 -
.../lib/arvbox/docker/service/keep-web/run-service | 6 +-
.../arvbox/docker/service/keepproxy/run-service | 4 +-
tools/arvbox/lib/arvbox/docker/service/nginx/run | 3 +
.../crunchstat_summary/command.py | 14 +-
.../crunchstat_summary/summarizer.py | 32 +-
...er_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz | Bin 235 -> 0 bytes
...st_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz | Bin 0 -> 274 bytes
...-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz.report} | 0
..._9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz} | Bin 663 -> 724 bytes
...xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz.report} | 0
...uest_9tee4-xvhdp-kk0ja1cl8b2kr1y.txt.gz.report} | 0
tools/crunchstat-summary/tests/test_examples.py | 47 +-
202 files changed, 1325 insertions(+), 17498 deletions(-)
delete mode 100755 crunch_scripts/GATK2-VariantFiltration
delete mode 100755 crunch_scripts/GATK2-bqsr
delete mode 100755 crunch_scripts/GATK2-merge-call
delete mode 100755 crunch_scripts/GATK2-realign
delete mode 100755 crunch_scripts/arvados-bcbio-nextgen.py
delete mode 100644 crunch_scripts/arvados_bwa.py
delete mode 100644 crunch_scripts/arvados_gatk2.py
delete mode 100644 crunch_scripts/arvados_ipc.py
delete mode 100644 crunch_scripts/arvados_picard.py
delete mode 100644 crunch_scripts/arvados_samtools.py
delete mode 100755 crunch_scripts/bwa-aln
delete mode 100755 crunch_scripts/bwa-index
delete mode 100755 crunch_scripts/collection-merge
delete mode 100755 crunch_scripts/crunchrunner
delete mode 100644 crunch_scripts/crunchutil/__init__.py
delete mode 100644 crunch_scripts/crunchutil/robust_put.py
delete mode 100644 crunch_scripts/crunchutil/subst.py
delete mode 100644 crunch_scripts/crunchutil/vwd.py
delete mode 100755 crunch_scripts/cwl-runner
delete mode 100755 crunch_scripts/decompress-all.py
delete mode 100755 crunch_scripts/file-select
delete mode 100755 crunch_scripts/grep
delete mode 100755 crunch_scripts/hash
delete mode 100755 crunch_scripts/pgp-survey-import
delete mode 100755 crunch_scripts/pgp-survey-parse
delete mode 100755 crunch_scripts/picard-gatk2-prep
delete mode 100644 crunch_scripts/pyrtg.py
delete mode 100755 crunch_scripts/rtg-fasta2sdf
delete mode 100755 crunch_scripts/rtg-fastq2sdf
delete mode 100755 crunch_scripts/rtg-map
delete mode 100755 crunch_scripts/rtg-snp
delete mode 100755 crunch_scripts/run-command
delete mode 100755 crunch_scripts/split-fastq.py
delete mode 100755 crunch_scripts/test/task_output_dir
delete mode 100644 doc/_includes/_arv_run_redirection.liquid
delete mode 100644 doc/install/install-compute-node.html.textile.liquid
delete mode 100644 doc/install/install-crunch-dispatch.html.textile.liquid
delete mode 100644 doc/sdk/python/crunch-utility-libraries.html.textile.liquid
delete mode 100644 doc/user/examples/crunch-examples.html.textile.liquid
delete mode 100644 doc/user/reference/job-pipeline-ref.html.textile.liquid
delete mode 100644 doc/user/topics/arv-run.html.textile.liquid
delete mode 100644 doc/user/topics/crunch-tools-overview.html.textile.liquid
delete mode 100644 doc/user/topics/run-command.html.textile.liquid
delete mode 100644 doc/user/topics/running-pipeline-command-line.html.textile.liquid
delete mode 100644 doc/user/topics/tutorial-parallel.html.textile.liquid
delete mode 100644 doc/user/topics/tutorial-trait-search.html.textile.liquid
delete mode 100644 doc/user/tutorials/running-external-program.html.textile.liquid
delete mode 100644 doc/user/tutorials/tutorial-firstscript.html.textile.liquid
delete mode 100644 doc/user/tutorials/tutorial-submit-job.html.textile.liquid
delete mode 100755 sdk/cli/bin/arv-crunch-job
delete mode 100755 sdk/cli/bin/arv-run-pipeline-instance
delete mode 100755 sdk/cli/bin/crunch-job
delete mode 100644 sdk/cli/test/test_arv-run-pipeline-instance.rb
delete mode 100644 sdk/cli/test/test_crunch-job.rb
delete mode 100644 sdk/cwl/arvados_cwl/arvjob.py
delete mode 100644 sdk/cwl/arvados_cwl/crunch_script.py
delete mode 100644 sdk/cwl/tests/test_job.py
delete mode 100644 sdk/go/crunchrunner/crunchrunner.go
delete mode 100644 sdk/go/crunchrunner/crunchrunner_test.go
delete mode 100644 sdk/go/crunchrunner/upload.go
delete mode 100644 sdk/go/crunchrunner/upload_test.go
delete mode 100755 sdk/python/bin/arv-run
delete mode 100644 sdk/python/tests/test_arv_run.py
delete mode 100644 sdk/python/tests/test_pipeline_template.py
delete mode 100644 services/api/app/helpers/commit_ancestors_helper.rb
delete mode 100644 services/api/app/models/commit.rb
delete mode 100644 services/api/app/models/commit_ancestor.rb
copy services/api/db/migrate/{20130315183626_add_log_to_jobs.rb => 20190808145904_drop_commit_ancestors.rb} (57%)
copy services/api/db/migrate/{20130315183626_add_log_to_jobs.rb => 20190809135453_remove_commits_table.rb} (58%)
delete mode 100644 services/api/lib/crunch_dispatch.rb
delete mode 100755 services/api/script/crunch-dispatch.rb
delete mode 100755 services/api/script/crunch_failure_report.py
delete mode 100755 services/api/script/fail-jobs.rb
delete mode 100644 services/api/test/integration/crunch_dispatch_test.rb
delete mode 100644 services/api/test/unit/crunch_dispatch_test.rb
delete mode 100644 services/api/test/unit/fail_jobs_test.rb
delete mode 100644 services/keep-web/usage.go
delete mode 100755 tools/arvbox/lib/arvbox/docker/crunch-setup.sh
delete mode 100644 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/log/main/.gitstub
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/log/run
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/run
delete mode 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/run-service
delete mode 100644 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/log/main/.gitstub
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/log/run
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/run
delete mode 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/run-service
delete mode 100644 tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz
create mode 100644 tools/crunchstat-summary/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz.report} (100%)
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz} (66%)
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz.report} (100%)
copy tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y.txt.gz.report} (100%)
via d4ed3e6460469f2766e1f1676c538d6c86e000b6 (commit)
via 86b7903116608ac3a8314437639cb61d774fb510 (commit)
via 8dae253559240fb16564fb934fece2cbcf001866 (commit)
via 309e3a3b10cd6a4e609f84fbdc87cfed3baad796 (commit)
via 1d5727fcdd6e524898d7507417456b9fedcb79dd (commit)
via bad30523725258c8460768be3239a8b02d4e35eb (commit)
via 18de568b068818cebbd708e2a3f39d2c5a65c6ee (commit)
via b64996755741a71c4f02cf3ef9ea2b7ba1d0e2e1 (commit)
via 3679c9415c3f9f925a40919c549b7cac65606800 (commit)
via ba2e6d49ca390a8a62aa77e531e5a255c93ebfcd (commit)
via a5cf4e0ea356a7ee06f67fe159484fe20cd8a184 (commit)
via a136c1ca723537feab8efb40de4fe68c099a175b (commit)
via 4031e38a48822b543e95f166003ecb01906e49dc (commit)
via 322599005b9425223195f8b903f0aa2cfc4b7db9 (commit)
via 26c7b3c97876a05ec7427bc2e5270108dcf00b17 (commit)
via d83e0937bc1d93560896ae1331486c473436b4a2 (commit)
via e9a366b15f7e9fec534cb2bdc319f0f3ed13236d (commit)
via 2774e45aba64557259fc35155c62988449a8dc72 (commit)
via 0fb04af57862494beb38f57b75a1427ee9443e25 (commit)
via 03590b0b68fe727405e46cf28d47ba50117c7f1d (commit)
via 8b873a9b3b8865a4d451263e48b49122b9c32759 (commit)
via c8f57c52224362d7621f1271774b0f2d60c55cac (commit)
via 366a2efdd0ac4630f4381f3b47d70ef155ed2df4 (commit)
via 803c1beb16c35f225d22e0dfd7f45165f13bc5db (commit)
via 5eb512e7b6ff3a0d9f14591fe6bf611dde5cc27c (commit)
via 7abf74e3ad723ed8aed4d13e794a8f6ad6f44059 (commit)
via 415910149bed2eef6ae818b7c059d413934df06e (commit)
via e0fd2e426856a6593f2c86c60831f0f867f4f83d (commit)
via 35658af99f09f2f6768583d65246429f789fc5a2 (commit)
via d7a5e6e37e9e9251177f9fad98d867f477e69537 (commit)
via 22de1507bc49617b02aa219302cecb830694adb5 (commit)
via c72420e52c085c76dead3e700e05e6fd058dccb2 (commit)
via 1ba00901283d88fc7a5c82cabdad6e5183d4bb78 (commit)
via 0efb729a71577d400d93b8db56975a4232ec9c6b (commit)
via 6893f07253b37deac06aefdaedbadfb730254026 (commit)
via 52cfc5c943a591d4f8095794820b72b697ae6f12 (commit)
via 38821eb2984f73e62bf21fd214890864a8ad9d47 (commit)
via 043185f6d16dd41135c255f0f36fb8a75203537b (commit)
via 021f8eb819919d4606d7b4c72a2497e842041526 (commit)
via 751745e032831e16c5b67cd4bd0d328cd1f11fd7 (commit)
via 23138fc621581634579294751beeff7f5e66358e (commit)
via 6a13a5f663102b77c157a157c8c4089269d3c3b1 (commit)
via 14626b1a574924a23f18073b8bb6eff829d66e5a (commit)
via fec7796668a4f4a73bb52e16e13e4504f76649f5 (commit)
via 2384aa9a6179333daa03002c36e7b25b83aab903 (commit)
via e10aa8e0c8b9c45d69832e71480cfb3d6929834e (commit)
via a8aa509cf3913c8dc91640ddcad6b25e6609517a (commit)
via 050ea7fdc6317a0fa0eeed20b0e6cb0b7fd6693b (commit)
via b96728249db70975be33bb5c5721ec5b8a0eb3ff (commit)
via 448f667a574b50da096051a0d062b9059ab3609f (commit)
via 9f6bfd78b88486d4b3384b4be05c1f5e4f546a9c (commit)
via a33badcdc6af29a87d1f960dbff8ca947329a46b (commit)
via 30e065c34db0ab9a0e824a77b1ac0a46412598e0 (commit)
via 9f912f75b3a0b75e8f3de94f4527d60f075f17fe (commit)
via f0bc73f28a14af9499dd32e906429d1bc276c969 (commit)
via 4923d561f16ed4c665e89142f01b7b1d3786ac2c (commit)
via bc1947e4aef52fe5f3aebc10dc2ea74cad86672d (commit)
via 4d56f9b913fcf41fbf89bf5016463b5353fa3a9f (commit)
via c49024ec99cc5717f7856d61f325c01c90f750a9 (commit)
via 07d92519438a592d531f2c7558cd51788da262ca (commit)
via 9a02d4117bfc1372ce5b6baedc9303f2500b3197 (commit)
via bdb3e297e65ed39f66b33a6fb4c3b3b0e17e6fd5 (commit)
via acb1d98cdfa6e33bfd3e1bb7510b78a4fd9384e2 (commit)
via 40d4eb5c5a942d82d158ab9d4d3b3fb1b44fde83 (commit)
via b93cb0b828989f9e2651ba436fb8a267224e4ca8 (commit)
via 2c7b49e8791154db907616243c930c552aeb3df3 (commit)
via 8f78a0332d530f9987f18e6c982e30b0ea8e943e (commit)
via 63756e0e8a1188e01d0c238394db71af278863cd (commit)
via 752c2ff97b3670bcc87b08816560f5a0be0b8d06 (commit)
via 5d82dc390a8e158b71c3c96cb1d0f301f3f12dbb (commit)
via 01fd373a99ab6e3ee13bf2c8ec38e48ee6f42ba3 (commit)
via 756aad23bd19f1533f38f0b9e9ee1206dc33b114 (commit)
via 2d9aa62499005abd47b5f2fa604f01d055480722 (commit)
via 5d68da382b77745e69640dfdd4e4bd9110e72f4a (commit)
via 1fa33cf0c06e07c7f3668f994f8d0def93d5ecbd (commit)
via 67daee82d7c7e3385ec252434109a3348a7c70d7 (commit)
via c2b55f26cbf4e0c963e6d30aa63c11898bfc3a09 (commit)
via d71c797d3bbfd26d571fb46a14262b5c01b95ff4 (commit)
from 1166aeb6033725709ded753a0c00f69320a9a873 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d4ed3e6460469f2766e1f1676c538d6c86e000b6
Merge: 1166aeb60 86b790311
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Wed Aug 14 10:49:48 2019 -0400
Merge branch 'master' into 14715-keepprox-config
refs #14715
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --cc doc/admin/upgrading.html.textile.liquid
index 7ea1b10fa,8c2ca7657..28f08db4d
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@@ -39,15 -39,21 +39,25 @@@ table(table table-bordered table-conden
|"v1.1.4":#v1_1_4|"v1.1.3":#v1_1_3|"v1.1.2":#v1_1_2|"v1.1.1":#v1_1_1|"v1.1.0":#v1_1_0|
|\5. "older":#older|
- h3(#master). development master (as of 2019-06-07)
+ h3(#master). development master (as of 2019-08-12)
+
+ h4. Keep-web dropped support on command line flags configuration
+
+ As we're migrating to a central cluster configuration file, the already deprecated way of getting configurations via environment variables and command line flags isn't valid anymore. Current keep-web supports both the now legacy @keep-web.yml@ config format (used by Arvados 1.4) and the new cluster config file format. Please check "keep-web's install guide":{{site.baseurl}}/install/install-keep-web.html for more details.
+
+ h4. Jobs API is read-only
+
+ (task "#15133":https://dev.arvados.org/issues/15133 ) The legacy 'jobs' API is now read-only. It has long been superceded by containers / container_requests (aka crunch v2). Arvados installations since the end of 2017 (v1.1.0) have probably only used containers, and are unaffected by this change.
+
+ So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run at .
+h4. Keepproxy configuration migration
+
+Keepproxy can now be configured using the centralized config at @/etc/arvados/config.yml at . Some configuration options are no longer supported. Please see "keepproxy's config migration guide":{{site.baseurl}}/admin/config-migration.html#keepproxy for more details.
+
h4. No longer stripping ':' from strings in serialized database columns
- (bug #15311) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
+ (bug "#15311":https://dev.arvados.org/issues/15311 ) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
You can test if any records in your database are affected by going to the API server directory and running @bundle exec rake symbols:check at . This will report which records contain fields with a leading ':' that would previously have been stripped. If there are records to be updated, you can update the database using @bundle exec rake symbols:stringify at .
diff --cc lib/config/deprecated.go
index 4e7b85ec5,019979d39..df872111d
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@@ -327,74 -327,72 +327,144 @@@ func (ldr *Loader) loadOldWebsocketConf
return nil
}
+type oldKeepProxyConfig struct {
+ Client *arvados.Client
+ Listen *string
+ DisableGet *bool
+ DisablePut *bool
+ DefaultReplicas *int
+ Timeout *arvados.Duration
+ PIDFile *string
+ Debug *bool
+ ManagementToken *string
+}
+
+const defaultKeepproxyConfigPath = "/etc/arvados/keepproxy/keepproxy.yml"
+
+func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error {
+ if ldr.KeepproxyPath == "" {
+ return nil
+ }
+ var oc oldKeepProxyConfig
+ err := ldr.loadOldConfigHelper("keepproxy", ldr.KeepproxyPath, &oc)
+ if os.IsNotExist(err) && ldr.KeepproxyPath == defaultKeepproxyConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ if oc.Listen != nil {
+ cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.DefaultReplicas != nil {
+ cluster.Collections.DefaultReplication = *oc.DefaultReplicas
+ }
+ if oc.Timeout != nil {
+ cluster.API.KeepServiceRequestTimeout = *oc.Timeout
+ }
+ if oc.Debug != nil {
+ if *oc.Debug && cluster.SystemLogs.LogLevel != "debug" {
+ cluster.SystemLogs.LogLevel = "debug"
+ } else if !*oc.Debug && cluster.SystemLogs.LogLevel != "info" {
+ cluster.SystemLogs.LogLevel = "info"
+ }
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+
+ // The following legacy options are no longer supported. If they are set to
+ // true or PIDFile has a value, error out and notify the user
+ unsupportedEntry := func(cfgEntry string) error {
+ return fmt.Errorf("the keepproxy %s configuration option is no longer supported, please remove it from your configuration file", cfgEntry)
+ }
+ if oc.DisableGet != nil && *oc.DisableGet {
+ return unsupportedEntry("DisableGet")
+ }
+ if oc.DisablePut != nil && *oc.DisablePut {
+ return unsupportedEntry("DisablePut")
+ }
+ if oc.PIDFile != nil && *oc.PIDFile != "" {
+ return unsupportedEntry("PIDFile")
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
++
+ const defaultKeepWebConfigPath = "/etc/arvados/keep-web/keep-web.yml"
+
+ type oldKeepWebConfig struct {
+ Client *arvados.Client
+
+ Listen string
+
+ AnonymousTokens []string
+ AttachmentOnlyHost string
+ TrustAllContent bool
+
+ Cache struct {
+ TTL arvados.Duration
+ UUIDTTL arvados.Duration
+ MaxCollectionEntries int
+ MaxCollectionBytes int64
+ MaxPermissionEntries int
+ MaxUUIDEntries int
+ }
+
+ // Hack to support old command line flag, which is a bool
+ // meaning "get actual token from environment".
+ deprecatedAllowAnonymous bool
+
+ // Authorization token to be included in all health check requests.
+ ManagementToken string
+ }
+
+ func (ldr *Loader) loadOldKeepWebConfig(cfg *arvados.Config) error {
+ if ldr.KeepWebPath == "" {
+ return nil
+ }
+ var oc oldKeepWebConfig
+ err := ldr.loadOldConfigHelper("keep-web", ldr.KeepWebPath, &oc)
+ if os.IsNotExist(err) && ldr.KeepWebPath == defaultKeepWebConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: oc.AttachmentOnlyHost}
+ cluster.TLS.Insecure = oc.Client.Insecure
+ cluster.ManagementToken = oc.ManagementToken
+ cluster.Collections.TrustAllContent = oc.TrustAllContent
+ cluster.Collections.WebDAVCache.TTL = oc.Cache.TTL
+ cluster.Collections.WebDAVCache.UUIDTTL = oc.Cache.UUIDTTL
+ cluster.Collections.WebDAVCache.MaxCollectionEntries = oc.Cache.MaxCollectionEntries
+ cluster.Collections.WebDAVCache.MaxCollectionBytes = oc.Cache.MaxCollectionBytes
+ cluster.Collections.WebDAVCache.MaxPermissionEntries = oc.Cache.MaxPermissionEntries
+ cluster.Collections.WebDAVCache.MaxUUIDEntries = oc.Cache.MaxUUIDEntries
+ if len(oc.AnonymousTokens) > 0 {
+ cluster.Users.AnonymousUserToken = oc.AnonymousTokens[0]
+ if len(oc.AnonymousTokens) > 1 {
+ ldr.Logger.Warn("More than 1 anonymous tokens configured, using only the first and discarding the rest.")
+ }
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+ }
diff --cc lib/config/load.go
index 309c0a615,3413e3bec..c0b44c17e
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@@ -31,9 -31,9 +31,10 @@@ type Loader struct
Path string
KeepstorePath string
+ KeepWebPath string
CrunchDispatchSlurmPath string
WebsocketPath string
+ KeepproxyPath string
configdata []byte
}
@@@ -61,9 -61,9 +62,10 @@@ func NewLoader(stdin io.Reader, logger
func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
+ flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+ flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
@@@ -135,9 -135,9 +137,12 @@@ func (ldr *Loader) MungeLegacyConfigArg
if legacyConfigArg != "-legacy-ws-config" {
ldr.WebsocketPath = ""
}
+ if legacyConfigArg != "-legacy-keepweb-config" {
+ ldr.KeepWebPath = ""
+ }
+ if legacyConfigArg != "-legacy-keepproxy-config" {
+ ldr.WebsocketPath = ""
+ }
return munged
}
@@@ -235,9 -235,9 +240,10 @@@ func (ldr *Loader) Load() (*arvados.Con
// legacy config file for the current component
for _, err := range []error{
ldr.loadOldKeepstoreConfig(&cfg),
+ ldr.loadOldKeepWebConfig(&cfg),
ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
ldr.loadOldWebsocketConfig(&cfg),
+ ldr.loadOldKeepproxyConfig(&cfg),
} {
if err != nil {
return nil, err
diff --cc tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 3444e61e1,58bedd284..34a0c2d75
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@@ -76,12 -79,10 +79,14 @@@ Clusters
ExternalURL: "https://$localip:${services[workbench2-ssl]}"
SSO:
ExternalURL: "https://$localip:${services[sso]}"
+ Keepproxy:
+ InternalURLs:
+ "http://localhost:${services[keepproxy]}/": {}
+ ExternalURL: "http://$localip:${services[keepproxy-ssl]}/"
Websocket:
ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
+ InternalURLs:
+ "http://localhost:${services[websockets]}": {}
GitSSH:
ExternalURL: "ssh://git@$localip:"
GitHTTP:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list