[ARVADOS] updated: 1.3.0-46-g5d05b79a2
Git user
git at public.curoverse.com
Fri Dec 7 17:03:56 EST 2018
Summary of changes:
.licenseignore | 3 +
apps/workbench/Gemfile.lock | 2 +-
apps/workbench/app/controllers/users_controller.rb | 2 +-
.../app/controllers/work_units_controller.rb | 18 +-
apps/workbench/app/helpers/application_helper.rb | 3 +-
.../test/controllers/projects_controller_test.rb | 18 +
.../test/controllers/users_controller_test.rb | 8 +
build/build.list | 6 +-
...t-package-python27-python-arvados-cwl-runner.sh | 4 +
build/run-build-docker-jobs-image.sh | 31 +-
build/run-build-packages.sh | 31 +-
build/run-library.sh | 2 +-
doc/_config.yml | 7 +-
doc/_includes/_federated_cwl.liquid | 1 +
doc/_includes/_shards_yml.liquid | 1 +
doc/admin/upgrading.html.textile.liquid | 10 +
doc/api/methods.html.textile.liquid | 6 +-
.../methods/container_requests.html.textile.liquid | 2 +-
doc/index.html.liquid | 2 +-
doc/install/install-controller.html.textile.liquid | 2 +-
doc/install/install-keep-web.html.textile.liquid | 4 +
doc/install/install-keepproxy.html.textile.liquid | 22 +-
doc/sdk/python/arvados-fuse.html.textile.liquid | 2 +-
doc/user/cwl/cwl-extensions.html.textile.liquid | 25 +
doc/user/cwl/cwl-run-options.html.textile.liquid | 57 +-
doc/user/cwl/federated-workflow.odg | Bin 0 -> 12089 bytes
doc/user/cwl/federated-workflow.svg | 239 +++++++
.../cwl/federated-workflows.html.textile.liquid | 57 ++
doc/user/cwl/federated/cat.cwl | 14 +
doc/user/cwl/federated/federated.cwl | 87 +++
doc/user/cwl/federated/file-on-clsr1.dat | 1 +
doc/user/cwl/federated/file-on-clsr2.dat | 1 +
doc/user/cwl/federated/file-on-clsr3.dat | 1 +
doc/user/cwl/federated/md5sum.cwl | 21 +
doc/user/cwl/federated/shards.yml | 18 +
doc/user/index.html.textile.liquid | 7 +-
.../tutorials/intro-crunch.html.textile.liquid | 1 -
.../tutorial-keep-get.html.textile.liquid | 13 +
...orial-keep-mount-gnu-linux.html.textile.liquid} | 4 +-
.../tutorial-keep-mount-os-x.html.textile.liquid | 24 +
...tutorial-keep-mount-windows.html.textile.liquid | 24 +
docker/jobs/apt.arvados.org.list | 1 -
lib/dispatchcloud/dispatcher.go | 24 +-
lib/dispatchcloud/dispatcher_test.go | 36 +
lib/dispatchcloud/scheduler/fix_stale_locks.go | 2 +-
lib/dispatchcloud/scheduler/interfaces.go | 2 +-
lib/dispatchcloud/scheduler/run_queue_test.go | 2 +-
lib/dispatchcloud/worker/pool.go | 6 +-
lib/dispatchcloud/worker/worker.go | 1 +
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 767 ++-------------------
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 29 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 49 +-
sdk/cwl/arvados_cwl/arvdocker.py | 11 +-
sdk/cwl/arvados_cwl/arvjob.py | 7 +-
sdk/cwl/arvados_cwl/arvtool.py | 81 ++-
sdk/cwl/arvados_cwl/arvworkflow.py | 334 +++++----
sdk/cwl/arvados_cwl/context.py | 8 +
sdk/cwl/arvados_cwl/crunch_script.py | 2 +-
sdk/cwl/arvados_cwl/{__init__.py => executor.py} | 411 +++--------
sdk/cwl/arvados_cwl/fsaccess.py | 35 +-
sdk/cwl/arvados_cwl/pathmapper.py | 10 +-
sdk/cwl/arvados_cwl/runner.py | 19 +-
sdk/cwl/arvados_cwl/task_queue.py | 52 +-
sdk/cwl/arvados_version.py | 6 +-
sdk/cwl/gittaggers.py | 6 +-
sdk/cwl/setup.py | 10 +-
sdk/cwl/tests/arvados-tests.sh | 2 +-
sdk/cwl/tests/federation/README | 44 ++
.../tests/federation/arvbox-make-federation.cwl | 72 ++
sdk/cwl/tests/federation/arvbox/fed-config.cwl | 66 ++
sdk/cwl/tests/federation/arvbox/mkdir.cwl | 47 ++
sdk/cwl/tests/federation/arvbox/setup-user.cwl | 34 +
sdk/cwl/tests/federation/arvbox/setup_user.py | 40 ++
sdk/cwl/tests/federation/arvbox/start.cwl | 72 ++
sdk/cwl/tests/federation/arvbox/stop.cwl | 17 +
sdk/cwl/tests/federation/cases/base-case.cwl | 31 +
sdk/cwl/tests/federation/cases/cat.cwl | 14 +
sdk/cwl/tests/federation/cases/hint-on-tool.cwl | 28 +
sdk/cwl/tests/federation/cases/hint-on-wf.cwl | 30 +
.../tests/federation/cases/md5sum-tool-hint.cwl | 24 +
sdk/cwl/tests/federation/cases/md5sum.cwl | 21 +
sdk/cwl/tests/federation/cases/remote-case.cwl | 31 +
.../tests/federation/cases/rev-input-to-output.cwl | 27 +
sdk/cwl/tests/federation/cases/rev.cwl | 20 +
.../federation/cases/runner-home-step-remote.cwl | 29 +
.../federation/cases/runner-remote-step-home.cwl | 29 +
sdk/cwl/tests/federation/cases/scatter-gather.cwl | 37 +
.../tests/federation/cases/threestep-remote.cwl | 50 ++
.../tests/federation/cases/twostep-both-remote.cwl | 41 ++
.../federation/cases/twostep-home-to-remote.cwl | 41 ++
.../cases/twostep-remote-copy-to-home.cwl | 41 ++
.../federation/cases/twostep-remote-to-home.cwl | 41 ++
sdk/cwl/tests/federation/data/base-case-input.txt | 16 +
sdk/cwl/tests/federation/data/hint-on-tool.txt | 16 +
sdk/cwl/tests/federation/data/hint-on-wf.txt | 16 +
.../tests/federation/data/remote-case-input.txt | 16 +
.../data/runner-home-step-remote-input.txt | 16 +
.../data/runner-remote-step-home-input.txt | 16 +
.../tests/federation/data/scatter-gather-s1.txt | 16 +
.../tests/federation/data/scatter-gather-s2.txt | 16 +
.../tests/federation/data/scatter-gather-s3.txt | 16 +
sdk/cwl/tests/federation/data/threestep-remote.txt | 16 +
.../tests/federation/data/twostep-both-remote.txt | 16 +
.../federation/data/twostep-home-to-remote.txt | 16 +
.../data/twostep-remote-copy-to-home.txt | 16 +
.../federation/data/twostep-remote-to-home.txt | 16 +
sdk/cwl/tests/federation/framework/check-exist.cwl | 42 ++
sdk/cwl/tests/federation/framework/check_exist.py | 25 +
sdk/cwl/tests/federation/framework/dockerbuild.cwl | 21 +
sdk/cwl/tests/federation/framework/prepare.cwl | 48 ++
sdk/cwl/tests/federation/framework/prepare.py | 41 ++
sdk/cwl/tests/federation/framework/run-acr.cwl | 56 ++
sdk/cwl/tests/federation/framework/testcase.cwl | 77 +++
sdk/cwl/tests/federation/main.cwl | 545 +++++++++++++++
sdk/cwl/tests/test_container.py | 40 +-
sdk/cwl/tests/test_fsaccess.py | 78 +--
sdk/cwl/tests/test_job.py | 7 +-
sdk/cwl/tests/test_make_output.py | 3 +-
sdk/cwl/tests/test_pathmapper.py | 9 +-
sdk/cwl/tests/test_submit.py | 238 +++++--
sdk/cwl/tests/test_tq.py | 40 +-
sdk/cwl/tests/wf/expect_packed.cwl | 3 +-
sdk/cwl/tests/wf/submit_wf_packed.cwl | 1 +
sdk/cwl/tests/wf/submit_wf_runner_resources.cwl | 1 +
sdk/dev-jobs.dockerfile | 2 +-
sdk/pam/arvados_version.py | 6 +-
sdk/python/arvados_version.py | 6 +-
sdk/python/gittaggers.py | 6 +-
sdk/python/setup.py | 2 +-
sdk/python/tests/nginx.conf | 11 +
sdk/ruby/arvados.gemspec | 2 +-
services/api/Gemfile.lock | 2 +-
.../controllers/arvados/v1/schema_controller.rb | 1 +
services/api/app/models/arvados_model.rb | 15 +-
services/api/app/models/container.rb | 8 -
services/api/app/models/user.rb | 2 +-
.../20180917200000_replace_full_text_indexes.rb | 14 +
services/api/db/structure.sql | 16 +-
services/api/test/unit/container_test.rb | 9 +-
services/crunch-run/background.go | 7 +-
services/crunch-run/crunchrun.go | 16 +-
services/dockercleaner/arvados_version.py | 6 +-
services/fuse/arvados_version.py | 6 +-
services/keep-web/handler.go | 40 +-
services/keep-web/handler_test.go | 2 +-
services/login-sync/arvados-login-sync.gemspec | 2 +-
services/nodemanager/arvados_version.py | 6 +-
.../58118E89F3A912897C070ADBF76221572C52609D.asc | 106 +++
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 5 +-
tools/crunchstat-summary/arvados_version.py | 6 +-
151 files changed, 3833 insertions(+), 1593 deletions(-)
create mode 120000 doc/_includes/_federated_cwl.liquid
create mode 120000 doc/_includes/_shards_yml.liquid
create mode 100644 doc/user/cwl/federated-workflow.odg
create mode 100644 doc/user/cwl/federated-workflow.svg
create mode 100644 doc/user/cwl/federated-workflows.html.textile.liquid
create mode 100644 doc/user/cwl/federated/cat.cwl
create mode 100644 doc/user/cwl/federated/federated.cwl
create mode 100644 doc/user/cwl/federated/file-on-clsr1.dat
create mode 100644 doc/user/cwl/federated/file-on-clsr2.dat
create mode 100644 doc/user/cwl/federated/file-on-clsr3.dat
create mode 100644 doc/user/cwl/federated/md5sum.cwl
create mode 100644 doc/user/cwl/federated/shards.yml
rename doc/user/tutorials/{tutorial-keep-mount.html.textile.liquid => tutorial-keep-mount-gnu-linux.html.textile.liquid} (94%)
create mode 100644 doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
create mode 100644 doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
copy sdk/cwl/arvados_cwl/{__init__.py => executor.py} (66%)
create mode 100644 sdk/cwl/tests/federation/README
create mode 100644 sdk/cwl/tests/federation/arvbox-make-federation.cwl
create mode 100644 sdk/cwl/tests/federation/arvbox/fed-config.cwl
create mode 100644 sdk/cwl/tests/federation/arvbox/mkdir.cwl
create mode 100644 sdk/cwl/tests/federation/arvbox/setup-user.cwl
create mode 100644 sdk/cwl/tests/federation/arvbox/setup_user.py
create mode 100644 sdk/cwl/tests/federation/arvbox/start.cwl
create mode 100644 sdk/cwl/tests/federation/arvbox/stop.cwl
create mode 100644 sdk/cwl/tests/federation/cases/base-case.cwl
create mode 100644 sdk/cwl/tests/federation/cases/cat.cwl
create mode 100644 sdk/cwl/tests/federation/cases/hint-on-tool.cwl
create mode 100644 sdk/cwl/tests/federation/cases/hint-on-wf.cwl
create mode 100644 sdk/cwl/tests/federation/cases/md5sum-tool-hint.cwl
create mode 100644 sdk/cwl/tests/federation/cases/md5sum.cwl
create mode 100644 sdk/cwl/tests/federation/cases/remote-case.cwl
create mode 100644 sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
create mode 100644 sdk/cwl/tests/federation/cases/rev.cwl
create mode 100644 sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
create mode 100644 sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
create mode 100644 sdk/cwl/tests/federation/cases/scatter-gather.cwl
create mode 100644 sdk/cwl/tests/federation/cases/threestep-remote.cwl
create mode 100644 sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
create mode 100644 sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
create mode 100644 sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
create mode 100644 sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
create mode 100644 sdk/cwl/tests/federation/data/base-case-input.txt
create mode 100644 sdk/cwl/tests/federation/data/hint-on-tool.txt
create mode 100644 sdk/cwl/tests/federation/data/hint-on-wf.txt
create mode 100644 sdk/cwl/tests/federation/data/remote-case-input.txt
create mode 100644 sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
create mode 100644 sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
create mode 100644 sdk/cwl/tests/federation/data/scatter-gather-s1.txt
create mode 100644 sdk/cwl/tests/federation/data/scatter-gather-s2.txt
create mode 100644 sdk/cwl/tests/federation/data/scatter-gather-s3.txt
create mode 100644 sdk/cwl/tests/federation/data/threestep-remote.txt
create mode 100644 sdk/cwl/tests/federation/data/twostep-both-remote.txt
create mode 100644 sdk/cwl/tests/federation/data/twostep-home-to-remote.txt
create mode 100644 sdk/cwl/tests/federation/data/twostep-remote-copy-to-home.txt
create mode 100644 sdk/cwl/tests/federation/data/twostep-remote-to-home.txt
create mode 100644 sdk/cwl/tests/federation/framework/check-exist.cwl
create mode 100644 sdk/cwl/tests/federation/framework/check_exist.py
create mode 100644 sdk/cwl/tests/federation/framework/dockerbuild.cwl
create mode 100644 sdk/cwl/tests/federation/framework/prepare.cwl
create mode 100644 sdk/cwl/tests/federation/framework/prepare.py
create mode 100644 sdk/cwl/tests/federation/framework/run-acr.cwl
create mode 100644 sdk/cwl/tests/federation/framework/testcase.cwl
create mode 100755 sdk/cwl/tests/federation/main.cwl
create mode 100644 services/api/db/migrate/20180917200000_replace_full_text_indexes.rb
create mode 100644 tools/arvbox/lib/arvbox/docker/58118E89F3A912897C070ADBF76221572C52609D.asc
via 5d05b79a21ee85fa6ec53f9bc88e6ddb54029f04 (commit)
via 262871175bbe08f6ab1b7051bf9c5dfdd74283ac (commit)
via a3dd7b47276e3d509eecfb930b06309d0315ac0d (commit)
via c746a03b6b08865796b5ef722f33d4b744a9c694 (commit)
via cc952178056bf6d29471f6986306fb673dcf394a (commit)
via 10bf8762c5d882ab5e9ff4c6e316878caebc6bf8 (commit)
via 0b2772025dd5132beaebeac33d3be2e56890766e (commit)
via f6e063ed8afcdd868fe1eec2fa86ac8f0f61e18b (commit)
via c2a086701c35ae71c1fa876221ff9cc3220ed22e (commit)
via 5dbc1ae3d451f904654a2a61e5df620808ac175d (commit)
via fdf6033451b12862a8305f4e3a4b06fbbdd498e1 (commit)
via 32ea7036d2cb438f5c313cff73cfe17610a3b8b9 (commit)
via 5a4cb7afb56eeeceba5ba7e710e4a7ccc5f9ae36 (commit)
via 533dbbd1d3780fc17c50b62f5b9adb1c5c005a92 (commit)
via 8f924c10a36dc073c33f390eee6a2d2cd069859f (commit)
via 45b8d59207773c0e4b3a5902f4171e6c841d755a (commit)
via 0b2ea830158b66817289b0f7a1f0ff4daa5dc565 (commit)
via 5090fa0891aa67f8373831a2c87ba69b65078b0d (commit)
via 79bc7fbc9581c5ee89be4dd23bc01e8713a1ddee (commit)
via f8ea35a26789594253cff761dce87dd51ff9e89c (commit)
via 37d8512b93cff033af35935b918deaaf6c353bce (commit)
via 4ad50255921b571d8e7748b4c6c098b53d803183 (commit)
via 5fed5bf2c830b76db0aa7a2d7c787ae6941da53f (commit)
via 2bb34027cb209d11557dbbf9f8140f9549bfc88f (commit)
via 3131146af966744a8f19e56be2d4a870bc4ab615 (commit)
via cc059aea76194417f5aaf37073b3ae530219b11e (commit)
via 2ab2ec271e09c54f6e7e1d4c3adbd1d76e56f1cd (commit)
via 88fdc1784fb2e1233cbabcbcb612da3f787ebddc (commit)
via 13e4f194c414887746db9255da3fb406b94d857f (commit)
via f61eb76622c6fc0b01fd41648392b652ac8ca767 (commit)
via eff370384f2301c5673820690ed0ae039182f4d6 (commit)
via be64ccd6498eb08960310e19a6ef2d840a33dffc (commit)
via 90f4141a1087c33d78ee9031d3309f03ae69c285 (commit)
via 0d22de74ea069003b49a9d52878d5f1ac04d71bb (commit)
via 2909f39ea214bc5b06ceeeb8d367db37a97e2197 (commit)
via 7896fc39410a07bdf48aa1f430e787538a79fb3a (commit)
via ff48e70fd7cc41997a27c0b04bb38149a6abfa0d (commit)
via f9038f303e54dd99e87e99f9631a9097cc93e591 (commit)
via afaca89ea95ab66f770ea5e878875eca11dc1c9b (commit)
via a7b09c6956e5a98df99b7fa4322c69e502f430c6 (commit)
via 1b183250a9d496cebe189360f7e0c867be0ca62d (commit)
via 76e9e480a8379189067cf1fd1d492a90099504a1 (commit)
via 9c0ea426167e01ea69ff022a811803aa95a302d4 (commit)
via 436b256c1439df98cd4a4d27cb6a15fb3557256b (commit)
via de7e1227dc7b935b73bc857ecaf0099403248c97 (commit)
via 7c775b158554b67e75d74ff1235957c0150d6a1c (commit)
via 27791e57e407a9472b1690f951022e13fe8b2fb6 (commit)
via 8d8847e070f588b5d85ac2d7123fd929b4d417cd (commit)
via 42cb6a6d7679c5dc90adc14da57bb5691930e0f0 (commit)
via eb58fd945645f5a670c761f7046b10885941167e (commit)
via 00bb1461d14cfc02e6ec2c74d622b7b6b716e775 (commit)
via 1566e86653994ceec5a6238ad0568fa272cb5d31 (commit)
via 3fd34f1f780ed70d98afe5eaf8a655b60e0c0abd (commit)
via f24e1c6ca8d5636c1c747200dc87a0077b445707 (commit)
via efaa4cc24be924ade579dd16ec08f53a45e24d7b (commit)
via 04727095afb18f4b9d99b431764d0af3997e5836 (commit)
via fdd933cb69ea44d6c705b47be62f0f985eff742c (commit)
via aa7307f0bfc73066d4f2d84724139a6b1fc21bb1 (commit)
via d8333e9e64e0dd48d3c697a7d0df73c3b306cf6c (commit)
via 7127d6c5f8ceb03ec5008a17d320f8054f3e2edb (commit)
via 8299fa11d1e813ea52e3ea482214d12044119bc3 (commit)
via 1d6540e48c83e0a740fd2a8bd63f99671d40488a (commit)
via 17311124babf9222a0205520b97adacbd7fef16b (commit)
via 601bf8bbefecb146651bac1306bf2464f0105b15 (commit)
via 8ec9c69eab7b914c0ea6d71dc8648f9a1f1c8255 (commit)
via c290df61bac294cd76d35b506a20bb9e76919f0e (commit)
via ca677782adf5d9334eb98b696359505d7ee8ef9f (commit)
via 90d69e2c02044c9d6effa9b2d52ca37b479c0c0d (commit)
via 267477f569cd2632ea14d6fdd467760c669fc087 (commit)
via eb59eb576b565c653fa2d09c8513c887e38d8aa4 (commit)
via 580c55d60829f17118f0c8e57d1d72c84a71b62b (commit)
via d738b8c32940af71462b8aac9bbfdbbc0f2f2642 (commit)
via 22cad1a48a8bea5a6f46c4241e139b03ee32e877 (commit)
via 01c15b155aa4d16a1d86a20bd4d30353ecba6f15 (commit)
via 4b402178829f3b09c34b3870869f738a9a407fa4 (commit)
via f11d620f06a284e47b12d1413bf9805ae5b488e1 (commit)
via 8352a6863cdcc2794108a71fd5b64e7b75f9dd20 (commit)
via d353e9be958d39a1aaf6ca83f269d077c74c8848 (commit)
via 55b409d9dd91ff540dcc8dbdf524a2bc031e0093 (commit)
via 3081e02da59e46d5e913a0ec3ea228e57c5c5e00 (commit)
via 200cc5eb5d3e6fbc93977a7dd11a263ad1fcffb6 (commit)
via 3f2642a715911b9e6b01026403ee7eefa82d366c (commit)
via ebed37b04288c946739d4266e8867b08908edc36 (commit)
via a98388fb52e598e3f72cfd7e3cf58b9a8f965b9e (commit)
via 06d48f573885309144895fb8c686c48fecc06115 (commit)
via fb96caaa6b4e00f61e5e5f7b4773c03bb30cf1bf (commit)
via e2a7e3b3a65bdcf5b761bbe9e34eceac6a51ad89 (commit)
via 0b20c7819705986dda40764bd3ac1efe9c93508d (commit)
via f390fd8626051c518520d7b405e2bc76fce94cba (commit)
via e8dc660deca318dc3b3bd3019bacde6253769e05 (commit)
via 03da0df474f95328c67ebc1ab4382cccdda464fb (commit)
via 840fb7f71017a4230988787864a523d8164b35bc (commit)
via 9c723434a4ae5e4c1e91f3f5ab91144715caed3d (commit)
via 74b4ea5c51773890c4b558b97091af2a0f0c9a7a (commit)
via fc1b915dc9d5c592b812c19e0f8f1d84c60b472a (commit)
via 10e2149ba9b004ae5c3d983bded39d4afee056d1 (commit)
via d11baa0f960525adcbaadf6d6cde3a4a1e219ae0 (commit)
via 276026f8c9bc619081be3def41ca2eba3c090cb0 (commit)
via 8acbf6d96287dbdf3ac69a4b0ef98452e4c203ed (commit)
via 2cf1740bfe7e5790752654ce1b7e0220bc194004 (commit)
via 9a1ae5161d87d47f61985145da61ea7b219fd8a6 (commit)
via 2dc16a9b84f092a2a5a06c3c29fbfadb6e725772 (commit)
via 674ba49f7bce861326094231d4f4eb590abbb7ce (commit)
via 7f23b24598ddb69bd85598d26de37322801547b8 (commit)
via 4d81515a381239b4b31ad7a41ed72c93dcadf08b (commit)
via e32de15f4de447dae7b3b6b91efff07ba53b91ed (commit)
via 4161dfe7f8f4ea790d286cd1745893e0a250a02f (commit)
via 2b8d56f5ecb5b6b620b439c3d2aae5c8066b4cac (commit)
via e28fb5b584952c5ef61efc6d02f86286a1e50dc5 (commit)
via 803fbc7e350dfa6b70ac63c7bf5f9046a0625155 (commit)
via 8f1185e3709f99475611c7f60122b4d2ebc67e5c (commit)
via 5b31294369715be7d5676b4009ed881c232f35cb (commit)
via 90d1b5a78f311bf9b85f992e59e783871c64ffeb (commit)
via c2989afb1ed2397676212804272cb128236758cf (commit)
via b48ecbe3e278981ee2ed0df98a5918b4c8e2151c (commit)
via 9359b63514047e816e69fb049fbd792c06bede24 (commit)
via 1396e111dcdb9a265d80e02291c29d7bc2966c2a (commit)
via 860b7ed9f1562da5b602d21e7996c25b1361a64a (commit)
via 0939c5719af9601ae6b95f6d230cbc4cace122cc (commit)
via 8e36cb0c7568f687e0287c06d987553d2921ad8b (commit)
via f5fed1ca0cd3a881a78ad6c607e17b5e1732815a (commit)
via a945f4969016be5b74496f320e954b9f32baad8c (commit)
via de619aa99c6f022da45840ffd62085683a33a0d0 (commit)
via 128b6c914e9ac66744e36b4428c5235f3bde7089 (commit)
via 1b7e38873458e9b94830658f384bd39daf6f95cd (commit)
via eb0390f26c6079c6f77fdd02fc3835fe8f3fddb7 (commit)
via 721fb4f8cfcff3b9f51b6a81299ef20e33397fe4 (commit)
via 261ae14f92fb3299f2469630757e52e4a7fc8fb2 (commit)
from df5c912a9eb5af7222e5446bc437ee97262542c8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5d05b79a21ee85fa6ec53f9bc88e6ddb54029f04
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 17:03:15 2018 -0500
14360: Add missing argument to -kill flag.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 2d56e0956..a43b96ed8 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -385,7 +385,7 @@ func (wp *Pool) kill(wkr *worker, uuid string) {
"Instance": wkr.instance,
})
logger.Debug("killing process")
- stdout, stderr, err := wkr.executor.Execute("crunch-run --kill "+uuid, nil)
+ stdout, stderr, err := wkr.executor.Execute("crunch-run --kill 15 "+uuid, nil)
if err != nil {
logger.WithFields(logrus.Fields{
"stderr": string(stderr),
commit 262871175bbe08f6ab1b7051bf9c5dfdd74283ac
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 16:44:24 2018 -0500
14360: Rename -detached -> -no-detach.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/background.go b/services/crunch-run/background.go
index 08eb6b0a3..be5b2e685 100644
--- a/services/crunch-run/background.go
+++ b/services/crunch-run/background.go
@@ -32,7 +32,7 @@ type procinfo struct {
}
// Detach acquires a lock for the given uuid, and starts the current
-// program as a child process (with -detached prepended to the given
+// program as a child process (with -no-detach prepended to the given
// arguments so the child knows not to detach again). The lock is
// passed along to the child process.
func Detach(uuid string, args []string, stdout, stderr io.Writer) int {
@@ -62,7 +62,7 @@ func detach(uuid string, args []string, stdout, stderr io.Writer) error {
}
defer errfile.Close()
- cmd := exec.Command(args[0], append([]string{"-detached"}, args[1:]...)...)
+ cmd := exec.Command(args[0], append([]string{"-no-detach"}, args[1:]...)...)
cmd.Stdout = outfile
cmd.Stderr = errfile
// Child inherits lockfile.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 1c6c58009..46d2d2068 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1748,22 +1748,22 @@ func main() {
getVersion := flag.Bool("version", false, "Print version information and exit.")
flag.Duration("check-containerd", 0, "Ignored. Exists for compatibility with older versions.")
- detached := false
- if len(os.Args) > 1 && os.Args[1] == "-detached" {
+ ignoreDetachFlag := false
+ if len(os.Args) > 1 && os.Args[1] == "-no-detach" {
// This process was invoked by a parent process, which
// has passed along its own arguments, including
- // -detach, after the leading -detached flag. Strip
- // the leading -detached flag (it's not recognized by
- // flag.Parse()) ... and remember not to detach all
- // over again in this process.
+ // -detach, after the leading -no-detach flag. Strip
+ // the leading -no-detach flag (it's not recognized by
+ // flag.Parse()) and ignore the -detach flag that
+ // comes later.
os.Args = append([]string{os.Args[0]}, os.Args[2:]...)
- detached = true
+ ignoreDetachFlag = true
}
flag.Parse()
switch {
- case *detach && !detached:
+ case *detach && !ignoreDetachFlag:
os.Exit(Detach(flag.Arg(0), os.Args, os.Stdout, os.Stderr))
case *kill >= 0:
os.Exit(KillProcess(flag.Arg(0), syscall.Signal(*kill), os.Stdout, os.Stderr))
commit a3dd7b47276e3d509eecfb930b06309d0315ac0d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 16:20:28 2018 -0500
14360: Disable management APIs if config has no ManagementToken.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 97aacf604..1a47e8490 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -126,15 +126,21 @@ func (disp *dispatcher) initialize() {
disp.pool = worker.NewPool(disp.logger, disp.reg, disp.instanceSet, disp.newExecutor, disp.Cluster)
disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, arvClient)
- mux := http.NewServeMux()
- mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
- mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
- metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
- ErrorLog: disp.logger,
- })
- mux.Handle("/metrics", metricsH)
- mux.Handle("/metrics.json", metricsH)
- disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
+ if disp.Cluster.ManagementToken == "" {
+ disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "Management API authentication is not configured", http.StatusForbidden)
+ })
+ } else {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/arvados/v1/dispatch/containers", disp.apiContainers)
+ mux.HandleFunc("/arvados/v1/dispatch/instances", disp.apiInstances)
+ metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+ ErrorLog: disp.logger,
+ })
+ mux.Handle("/metrics", metricsH)
+ mux.Handle("/metrics.json", metricsH)
+ disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
+ }
}
func (disp *dispatcher) run() {
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 83797e984..ed987154e 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -10,6 +10,7 @@ import (
"io"
"io/ioutil"
"math/rand"
+ "net/http"
"net/http/httptest"
"os"
"regexp"
@@ -324,7 +325,40 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
}
}
+func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
+ drivers["test"] = s.stubDriver
+ s.cluster.ManagementToken = "abcdefgh"
+ for _, token := range []string{"abc", ""} {
+ req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ resp := httptest.NewRecorder()
+ s.disp.ServeHTTP(resp, req)
+ if token == "" {
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusForbidden)
+ }
+ }
+}
+
+func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
+ drivers["test"] = s.stubDriver
+ s.cluster.ManagementToken = ""
+ for _, token := range []string{"abc", ""} {
+ req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ resp := httptest.NewRecorder()
+ s.disp.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusForbidden)
+ }
+}
+
func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
+ s.cluster.ManagementToken = "abcdefgh"
s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
drivers["test"] = s.stubDriver
@@ -341,9 +375,11 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
}
getInstances := func() instancesResponse {
req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
+ req.Header.Set("Authorization", "Bearer abcdefgh")
resp := httptest.NewRecorder()
s.disp.ServeHTTP(resp, req)
var sr instancesResponse
+ c.Check(resp.Code, check.Equals, http.StatusOK)
err := json.Unmarshal(resp.Body.Bytes(), &sr)
c.Check(err, check.IsNil)
return sr
commit c746a03b6b08865796b5ef722f33d4b744a9c694
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 16:01:08 2018 -0500
14360: Rename Workers -> CountWorkers.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/scheduler/fix_stale_locks.go b/lib/dispatchcloud/scheduler/fix_stale_locks.go
index 985941090..264f9e4ec 100644
--- a/lib/dispatchcloud/scheduler/fix_stale_locks.go
+++ b/lib/dispatchcloud/scheduler/fix_stale_locks.go
@@ -28,7 +28,7 @@ waiting:
// If all workers have been contacted, unlock
// containers that aren't claimed by any
// worker.
- unlock = sch.pool.Workers()[worker.StateUnknown] == 0
+ unlock = sch.pool.CountWorkers()[worker.StateUnknown] == 0
case <-timeout.C:
// Give up and unlock the containers, even
// though they might be working.
diff --git a/lib/dispatchcloud/scheduler/interfaces.go b/lib/dispatchcloud/scheduler/interfaces.go
index 467247ad9..59700c393 100644
--- a/lib/dispatchcloud/scheduler/interfaces.go
+++ b/lib/dispatchcloud/scheduler/interfaces.go
@@ -32,7 +32,7 @@ type ContainerQueue interface {
type WorkerPool interface {
Running() map[string]time.Time
Unallocated() map[arvados.InstanceType]int
- Workers() map[worker.State]int
+ CountWorkers() map[worker.State]int
AtQuota() bool
Create(arvados.InstanceType) error
Shutdown(arvados.InstanceType) bool
diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go
index 35db70522..aff0e227a 100644
--- a/lib/dispatchcloud/scheduler/run_queue_test.go
+++ b/lib/dispatchcloud/scheduler/run_queue_test.go
@@ -72,7 +72,7 @@ func (p *stubPool) Shutdown(arvados.InstanceType) bool {
p.shutdowns++
return false
}
-func (p *stubPool) Workers() map[worker.State]int {
+func (p *stubPool) CountWorkers() map[worker.State]int {
return map[worker.State]int{
worker.StateBooting: len(p.unalloc) - len(p.idle),
worker.StateIdle: len(p.idle),
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 37add6d3d..2d56e0956 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -304,8 +304,8 @@ func (wp *Pool) Shutdown(it arvados.InstanceType) bool {
return false
}
-// Workers returns the current number of workers in each state.
-func (wp *Pool) Workers() map[State]int {
+// CountWorkers returns the current number of workers in each state.
+func (wp *Pool) CountWorkers() map[State]int {
wp.setupOnce.Do(wp.setup)
wp.mtx.Lock()
defer wp.mtx.Unlock()
commit cc952178056bf6d29471f6986306fb673dcf394a
Merge: 10bf8762c c2a086701
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 15:25:39 2018 -0500
14360: Merge branch 'master' into 14360-dispatch-cloud
refs #14360
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc .licenseignore
index a0127cfa3,83c81b2fc..06519a98e
--- a/.licenseignore
+++ b/.licenseignore
@@@ -69,4 -71,4 +71,5 @@@ sdk/R/NAMESPAC
sdk/R/.Rbuildignore
sdk/R/ArvadosR.Rproj
*.Rd
+lib/dispatchcloud/test/sshkey_*
+ *.asc
commit 10bf8762c5d882ab5e9ff4c6e316878caebc6bf8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 15:24:52 2018 -0500
14360: Notify pool on worker shutdown.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/worker/worker.go b/lib/dispatchcloud/worker/worker.go
index c8e6e86f8..85104a13a 100644
--- a/lib/dispatchcloud/worker/worker.go
+++ b/lib/dispatchcloud/worker/worker.go
@@ -305,6 +305,7 @@ func (wkr *worker) shutdown() {
wkr.updated = now
wkr.destroyed = now
wkr.state = StateShutdown
+ go wkr.wp.notify()
go func() {
err := wkr.instance.Destroy()
if err != nil {
commit 0b2772025dd5132beaebeac33d3be2e56890766e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 14:55:36 2018 -0500
14360: Fix container UUID missing in proc info file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/background.go b/services/crunch-run/background.go
index deaf09cec..08eb6b0a3 100644
--- a/services/crunch-run/background.go
+++ b/services/crunch-run/background.go
@@ -80,6 +80,7 @@ func detach(uuid string, args []string, stdout, stderr io.Writer) error {
w := io.MultiWriter(stdout, lockfile)
err = json.NewEncoder(w).Encode(procinfo{
+ UUID: uuid,
PID: cmd.Process.Pid,
Stdout: outfile.Name(),
Stderr: errfile.Name(),
commit f6e063ed8afcdd868fe1eec2fa86ac8f0f61e18b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 7 14:54:08 2018 -0500
14360: Use nonblocking mode when checking for stale lock.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/background.go b/services/crunch-run/background.go
index ee054f056..deaf09cec 100644
--- a/services/crunch-run/background.go
+++ b/services/crunch-run/background.go
@@ -160,7 +160,7 @@ func ListProcesses(stdout, stderr io.Writer) int {
// acquisition during races, e.g., by connecting to a
// unix socket or checking /proc/$pid/fd/$n ->
// lockfile.
- err = syscall.Flock(int(f.Fd()), syscall.LOCK_SH)
+ err = syscall.Flock(int(f.Fd()), syscall.LOCK_SH|syscall.LOCK_NB)
if err == nil {
// lockfile is stale
err := os.Remove(path)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list