[ARVADOS] updated: 1.3.0-1563-ga64396990
Git user
git at public.curoverse.com
Thu Sep 5 15:16:54 UTC 2019
Summary of changes:
.../app/controllers/actions_controller.rb | 4 +-
.../app/controllers/application_controller.rb | 38 +-
.../app/controllers/collections_controller.rb | 14 +-
.../controllers/container_requests_controller.rb | 8 +-
.../workbench/app/controllers/groups_controller.rb | 4 +-
apps/workbench/app/controllers/jobs_controller.rb | 4 +-
.../controllers/pipeline_instances_controller.rb | 6 +-
.../app/controllers/projects_controller.rb | 2 +-
.../app/controllers/repositories_controller.rb | 4 +-
.../app/controllers/trash_items_controller.rb | 10 +-
apps/workbench/app/controllers/users_controller.rb | 27 +-
.../app/controllers/virtual_machines_controller.rb | 2 +-
.../controllers/work_unit_templates_controller.rb | 4 +-
.../app/controllers/work_units_controller.rb | 6 +-
.../app/helpers/pipeline_instances_helper.rb | 2 +-
apps/workbench/app/models/container_work_unit.rb | 4 +-
apps/workbench/app/models/pipeline_instance.rb | 4 +-
.../app/models/pipeline_instance_work_unit.rb | 2 +-
apps/workbench/fpm-info.sh | 4 +-
.../pipeline_instances_controller_test.rb | 103 -
.../test/controllers/projects_controller_test.rb | 26 -
.../test/helpers/repository_stub_helper.rb | 1 -
.../test/integration/application_layout_test.rb | 2 +
apps/workbench/test/integration/jobs_test.rb | 95 +-
.../test/integration/pipeline_instances_test.rb | 369 ---
.../test/integration/pipeline_templates_test.rb | 29 -
apps/workbench/test/integration/projects_test.rb | 1 -
.../test/integration/repositories_browse_test.rb | 23 -
apps/workbench/test/integration/websockets_test.rb | 61 -
apps/workbench/test/integration/work_units_test.rb | 30 -
apps/workbench/test/unit/disabled_api_test.rb | 8 +-
build/package-build-dockerfiles/centos7/Dockerfile | 4 -
build/package-build-dockerfiles/debian9/Dockerfile | 4 -
.../ubuntu1604/Dockerfile | 4 -
.../ubuntu1804/Dockerfile | 4 -
build/run-build-packages-one-target.sh | 1 -
build/run-build-packages.sh | 29 +-
build/run-build-test-packages-one-target.sh | 12 +-
build/run-library.sh | 183 +-
build/run-tests.sh | 1 +
cmd/arvados-client/cmd.go | 16 +-
crunch_scripts/GATK2-VariantFiltration | 64 -
crunch_scripts/GATK2-bqsr | 103 -
crunch_scripts/GATK2-merge-call | 242 --
crunch_scripts/GATK2-realign | 163 --
crunch_scripts/arvados-bcbio-nextgen.py | 145 --
crunch_scripts/arvados_bwa.py | 115 -
crunch_scripts/arvados_gatk2.py | 52 -
crunch_scripts/arvados_ipc.py | 51 -
crunch_scripts/arvados_picard.py | 42 -
crunch_scripts/arvados_samtools.py | 110 -
crunch_scripts/bwa-aln | 127 -
crunch_scripts/bwa-index | 41 -
crunch_scripts/collection-merge | 49 -
crunch_scripts/crunchrunner | 10 -
crunch_scripts/crunchutil/__init__.py | 0
crunch_scripts/crunchutil/robust_put.py | 56 -
crunch_scripts/crunchutil/subst.py | 102 -
crunch_scripts/crunchutil/vwd.py | 107 -
crunch_scripts/cwl-runner | 117 -
crunch_scripts/decompress-all.py | 64 -
crunch_scripts/file-select | 18 -
crunch_scripts/grep | 24 -
crunch_scripts/hash | 37 -
crunch_scripts/pgp-survey-import | 119 -
crunch_scripts/pgp-survey-parse | 22 -
crunch_scripts/picard-gatk2-prep | 211 --
crunch_scripts/pyrtg.py | 75 -
crunch_scripts/rtg-fasta2sdf | 27 -
crunch_scripts/rtg-fastq2sdf | 45 -
crunch_scripts/rtg-map | 41 -
crunch_scripts/rtg-snp | 34 -
crunch_scripts/run-command | 458 ----
crunch_scripts/split-fastq.py | 70 -
crunch_scripts/test/task_output_dir | 19 -
doc/_config.yml | 22 +-
doc/_includes/_arv_run_redirection.liquid | 27 -
doc/admin/config-migration.html.textile.liquid | 9 +
doc/admin/upgrading.html.textile.liquid | 22 +-
doc/api/crunch-scripts.html.textile.liquid | 2 +
doc/api/execution.html.textile.liquid | 7 -
doc/api/methods/humans.html.textile.liquid | 2 +
doc/api/methods/job_tasks.html.textile.liquid | 2 +
doc/api/methods/jobs.html.textile.liquid | 4 +-
.../methods/pipeline_instances.html.textile.liquid | 2 +
.../methods/pipeline_templates.html.textile.liquid | 2 +
doc/api/methods/specimens.html.textile.liquid | 2 +
doc/api/methods/traits.html.textile.liquid | 2 +
.../install-arv-git-httpd.html.textile.liquid | 42 +-
.../install-compute-node.html.textile.liquid | 112 -
.../install-crunch-dispatch.html.textile.liquid | 207 --
doc/install/install-keep-web.html.textile.liquid | 129 +-
doc/install/install-keepproxy.html.textile.liquid | 59 +-
doc/sdk/cli/subcommands.html.textile.liquid | 95 -
.../crunch-utility-libraries.html.textile.liquid | 228 --
doc/user/cwl/cwl-run-options.html.textile.liquid | 26 +-
doc/user/cwl/cwl-versions.html.textile.liquid | 14 -
.../examples/crunch-examples.html.textile.liquid | 102 -
.../reference/job-pipeline-ref.html.textile.liquid | 14 -
doc/user/topics/arv-run.html.textile.liquid | 163 --
.../crunch-tools-overview.html.textile.liquid | 70 -
doc/user/topics/run-command.html.textile.liquid | 319 ---
...nning-pipeline-command-line.html.textile.liquid | 58 -
.../topics/tutorial-parallel.html.textile.liquid | 85 -
.../tutorial-trait-search.html.textile.liquid | 278 ---
.../running-external-program.html.textile.liquid | 85 -
.../tutorial-firstscript.html.textile.liquid | 112 -
.../tutorial-submit-job.html.textile.liquid | 95 -
lib/cli/external.go | 6 +-
lib/config/config.default.yml | 81 +-
lib/config/deprecated.go | 183 ++
lib/config/deprecated_keepstore.go | 3 +
lib/config/deprecated_test.go | 165 ++
lib/config/export.go | 7 +-
lib/config/generated_config.go | 81 +-
lib/config/load.go | 18 +
lib/controller/fed_collections.go | 10 +-
lib/controller/federation/conn.go | 8 +-
lib/controller/federation_test.go | 17 +-
lib/service/cmd.go | 2 +-
sdk/cli/arvados-cli.gemspec | 6 +-
sdk/cli/bin/arv | 10 -
sdk/cli/bin/arv-crunch-job | 6 -
sdk/cli/bin/arv-run-pipeline-instance | 781 ------
sdk/cli/bin/crunch-job | 2577 --------------------
sdk/cli/test/test_arv-run-pipeline-instance.rb | 37 -
sdk/cli/test/test_crunch-job.rb | 141 --
sdk/cwl/arvados_cwl/__init__.py | 35 +-
sdk/cwl/arvados_cwl/arvjob.py | 495 ----
sdk/cwl/arvados_cwl/arvtool.py | 12 -
sdk/cwl/arvados_cwl/crunch_script.py | 159 --
sdk/cwl/arvados_cwl/executor.py | 76 +-
sdk/cwl/tests/test_container.py | 2 -
sdk/cwl/tests/test_job.py | 554 -----
sdk/cwl/tests/test_submit.py | 431 ----
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf2.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 4 +-
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 4 +-
sdk/go/arvados/config.go | 23 +-
sdk/go/arvadosclient/arvadosclient_test.go | 50 -
sdk/go/arvadostest/run_servers.go | 10 +-
sdk/go/crunchrunner/crunchrunner.go | 439 ----
sdk/go/crunchrunner/crunchrunner_test.go | 478 ----
sdk/go/crunchrunner/upload.go | 241 --
sdk/go/crunchrunner/upload_test.go | 152 --
sdk/go/httpserver/request_limiter.go | 28 +-
sdk/go/httpserver/request_limiter_test.go | 4 +-
sdk/python/arvados/commands/run.py | 185 +-
sdk/python/arvados/util.py | 3 +
sdk/python/bin/arv-run | 7 -
sdk/python/setup.py | 1 -
sdk/python/tests/nginx.conf | 20 +-
sdk/python/tests/run_test_server.py | 171 +-
sdk/python/tests/test_arv_run.py | 28 -
sdk/python/tests/test_keep_client.py | 67 +-
sdk/python/tests/test_pipeline_template.py | 62 -
sdk/python/tests/test_retry_job_helpers.py | 31 -
.../controllers/arvados/v1/job_tasks_controller.rb | 5 +
.../app/controllers/arvados/v1/jobs_controller.rb | 104 +-
.../arvados/v1/pipeline_instances_controller.rb | 10 +-
.../arvados/v1/pipeline_templates_controller.rb | 5 +
.../api/app/helpers/commit_ancestors_helper.rb | 6 -
services/api/app/helpers/commits_helper.rb | 263 ++
services/api/app/models/commit.rb | 272 ---
services/api/app/models/commit_ancestor.rb | 44 -
services/api/app/models/job.rb | 214 +-
services/api/app/models/job_task.rb | 10 +
services/api/app/models/pipeline_instance.rb | 35 +-
services/api/app/models/pipeline_template.rb | 11 +
services/api/config/arvados_config.rb | 5 -
....rb => 20190808145904_drop_commit_ancestors.rb} | 4 +-
...s.rb => 20190809135453_remove_commits_table.rb} | 4 +-
services/api/db/structure.sql | 115 +-
services/api/fpm-info.sh | 4 +-
services/api/lib/can_be_an_owner.rb | 2 +
services/api/lib/crunch_dispatch.rb | 981 --------
services/api/lib/enable_jobs_api.rb | 12 +
services/api/script/crunch-dispatch.rb | 16 -
services/api/script/crunch_failure_report.py | 222 --
services/api/script/fail-jobs.rb | 21 -
.../arvados/v1/job_reuse_controller_test.rb | 702 ------
.../functional/arvados/v1/jobs_controller_test.rb | 307 +--
.../v1/pipeline_instances_controller_test.rb | 44 -
services/api/test/helpers/git_test_helper.rb | 8 +-
.../api/test/integration/crunch_dispatch_test.rb | 47 -
services/api/test/integration/jobs_api_test.rb | 83 -
services/api/test/integration/pipeline_test.rb | 36 -
.../test/integration/serialized_encoding_test.rb | 21 -
services/api/test/unit/commit_test.rb | 70 +-
services/api/test/unit/crunch_dispatch_test.rb | 218 --
services/api/test/unit/fail_jobs_test.rb | 83 -
services/api/test/unit/job_task_test.rb | 11 -
services/api/test/unit/job_test.rb | 406 +--
services/api/test/unit/pipeline_instance_test.rb | 106 -
services/arv-git-httpd/arvados-git-httpd.service | 1 -
services/arv-git-httpd/auth_handler.go | 14 +-
services/arv-git-httpd/auth_handler_test.go | 29 +-
services/arv-git-httpd/git_handler.go | 23 +-
services/arv-git-httpd/git_handler_test.go | 27 +-
services/arv-git-httpd/gitolite_test.go | 24 +-
services/arv-git-httpd/integration_test.go | 43 +-
services/arv-git-httpd/main.go | 91 +-
services/arv-git-httpd/server.go | 14 +-
services/arv-git-httpd/usage.go | 81 -
services/fuse/arvados_fuse/__init__.py | 14 +-
services/keep-web/cache.go | 28 +-
services/keep-web/cache_test.go | 6 +-
services/keep-web/cadaver_test.go | 4 +-
services/keep-web/doc.go | 68 +-
services/keep-web/handler.go | 9 +-
services/keep-web/handler_test.go | 86 +-
services/keep-web/keep-web.service | 1 -
services/keep-web/main.go | 117 +-
services/keep-web/server.go | 12 +-
services/keep-web/server_test.go | 22 +-
services/keep-web/status_test.go | 2 +-
services/keep-web/usage.go | 99 -
services/keepproxy/keepproxy.go | 194 +-
services/keepproxy/keepproxy.service | 1 -
services/keepproxy/keepproxy_test.go | 110 +-
services/keepproxy/usage.go | 90 -
services/keepstore/azure_blob_volume.go | 26 +-
services/keepstore/azure_blob_volume_test.go | 41 +-
services/keepstore/command.go | 26 +-
services/keepstore/deprecated.go | 5 -
services/keepstore/handler_test.go | 11 +-
services/keepstore/handlers.go | 8 +-
services/keepstore/metrics.go | 22 -
services/keepstore/mounts_test.go | 27 +-
services/keepstore/proxy_remote_test.go | 3 +-
services/keepstore/pull_worker.go | 2 -
services/keepstore/pull_worker_integration_test.go | 1 +
services/keepstore/pull_worker_test.go | 1 +
services/keepstore/s3_volume.go | 116 +-
services/keepstore/s3_volume_test.go | 69 +-
services/keepstore/unix_volume.go | 42 +-
services/keepstore/unix_volume_test.go | 57 +-
services/keepstore/volume.go | 24 +-
services/keepstore/volume_generic_test.go | 22 +-
services/keepstore/volume_test.go | 10 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/cluster-config.sh | 42 +-
tools/arvbox/lib/arvbox/docker/common.sh | 2 +
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 40 -
.../docker/service/arv-git-httpd/run-service | 6 +-
.../service/crunch-dispatch0/log/main/.gitstub | 0
.../arvbox/docker/service/crunch-dispatch0/log/run | 1 -
.../lib/arvbox/docker/service/crunch-dispatch0/run | 1 -
.../docker/service/crunch-dispatch0/run-service | 6 -
.../service/crunch-dispatch1/log/main/.gitstub | 0
.../arvbox/docker/service/crunch-dispatch1/log/run | 1 -
.../lib/arvbox/docker/service/crunch-dispatch1/run | 1 -
.../docker/service/crunch-dispatch1/run-service | 7 -
.../lib/arvbox/docker/service/keep-web/run-service | 6 +-
.../arvbox/docker/service/keepproxy/run-service | 6 +-
tools/arvbox/lib/arvbox/docker/service/nginx/run | 43 +
.../crunchstat_summary/command.py | 14 +-
.../crunchstat_summary/summarizer.py | 32 +-
...er_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz | Bin 235 -> 0 bytes
...st_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz | Bin 0 -> 274 bytes
...-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz.report} | 0
..._9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz} | Bin 663 -> 724 bytes
...xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz.report} | 0
...uest_9tee4-xvhdp-kk0ja1cl8b2kr1y.txt.gz.report} | 0
tools/crunchstat-summary/tests/test_examples.py | 47 +-
266 files changed, 2401 insertions(+), 18518 deletions(-)
delete mode 100755 crunch_scripts/GATK2-VariantFiltration
delete mode 100755 crunch_scripts/GATK2-bqsr
delete mode 100755 crunch_scripts/GATK2-merge-call
delete mode 100755 crunch_scripts/GATK2-realign
delete mode 100755 crunch_scripts/arvados-bcbio-nextgen.py
delete mode 100644 crunch_scripts/arvados_bwa.py
delete mode 100644 crunch_scripts/arvados_gatk2.py
delete mode 100644 crunch_scripts/arvados_ipc.py
delete mode 100644 crunch_scripts/arvados_picard.py
delete mode 100644 crunch_scripts/arvados_samtools.py
delete mode 100755 crunch_scripts/bwa-aln
delete mode 100755 crunch_scripts/bwa-index
delete mode 100755 crunch_scripts/collection-merge
delete mode 100755 crunch_scripts/crunchrunner
delete mode 100644 crunch_scripts/crunchutil/__init__.py
delete mode 100644 crunch_scripts/crunchutil/robust_put.py
delete mode 100644 crunch_scripts/crunchutil/subst.py
delete mode 100644 crunch_scripts/crunchutil/vwd.py
delete mode 100755 crunch_scripts/cwl-runner
delete mode 100755 crunch_scripts/decompress-all.py
delete mode 100755 crunch_scripts/file-select
delete mode 100755 crunch_scripts/grep
delete mode 100755 crunch_scripts/hash
delete mode 100755 crunch_scripts/pgp-survey-import
delete mode 100755 crunch_scripts/pgp-survey-parse
delete mode 100755 crunch_scripts/picard-gatk2-prep
delete mode 100644 crunch_scripts/pyrtg.py
delete mode 100755 crunch_scripts/rtg-fasta2sdf
delete mode 100755 crunch_scripts/rtg-fastq2sdf
delete mode 100755 crunch_scripts/rtg-map
delete mode 100755 crunch_scripts/rtg-snp
delete mode 100755 crunch_scripts/run-command
delete mode 100755 crunch_scripts/split-fastq.py
delete mode 100755 crunch_scripts/test/task_output_dir
delete mode 100644 doc/_includes/_arv_run_redirection.liquid
delete mode 100644 doc/install/install-compute-node.html.textile.liquid
delete mode 100644 doc/install/install-crunch-dispatch.html.textile.liquid
delete mode 100644 doc/sdk/python/crunch-utility-libraries.html.textile.liquid
delete mode 100644 doc/user/examples/crunch-examples.html.textile.liquid
delete mode 100644 doc/user/reference/job-pipeline-ref.html.textile.liquid
delete mode 100644 doc/user/topics/arv-run.html.textile.liquid
delete mode 100644 doc/user/topics/crunch-tools-overview.html.textile.liquid
delete mode 100644 doc/user/topics/run-command.html.textile.liquid
delete mode 100644 doc/user/topics/running-pipeline-command-line.html.textile.liquid
delete mode 100644 doc/user/topics/tutorial-parallel.html.textile.liquid
delete mode 100644 doc/user/topics/tutorial-trait-search.html.textile.liquid
delete mode 100644 doc/user/tutorials/running-external-program.html.textile.liquid
delete mode 100644 doc/user/tutorials/tutorial-firstscript.html.textile.liquid
delete mode 100644 doc/user/tutorials/tutorial-submit-job.html.textile.liquid
delete mode 100755 sdk/cli/bin/arv-crunch-job
delete mode 100755 sdk/cli/bin/arv-run-pipeline-instance
delete mode 100755 sdk/cli/bin/crunch-job
delete mode 100644 sdk/cli/test/test_arv-run-pipeline-instance.rb
delete mode 100644 sdk/cli/test/test_crunch-job.rb
delete mode 100644 sdk/cwl/arvados_cwl/arvjob.py
delete mode 100644 sdk/cwl/arvados_cwl/crunch_script.py
delete mode 100644 sdk/cwl/tests/test_job.py
delete mode 100644 sdk/go/crunchrunner/crunchrunner.go
delete mode 100644 sdk/go/crunchrunner/crunchrunner_test.go
delete mode 100644 sdk/go/crunchrunner/upload.go
delete mode 100644 sdk/go/crunchrunner/upload_test.go
delete mode 100755 sdk/python/bin/arv-run
delete mode 100644 sdk/python/tests/test_arv_run.py
delete mode 100644 sdk/python/tests/test_pipeline_template.py
delete mode 100644 services/api/app/helpers/commit_ancestors_helper.rb
delete mode 100644 services/api/app/models/commit.rb
delete mode 100644 services/api/app/models/commit_ancestor.rb
copy services/api/db/migrate/{20130315183626_add_log_to_jobs.rb => 20190808145904_drop_commit_ancestors.rb} (57%)
copy services/api/db/migrate/{20130315183626_add_log_to_jobs.rb => 20190809135453_remove_commits_table.rb} (58%)
delete mode 100644 services/api/lib/crunch_dispatch.rb
delete mode 100755 services/api/script/crunch-dispatch.rb
delete mode 100755 services/api/script/crunch_failure_report.py
delete mode 100755 services/api/script/fail-jobs.rb
delete mode 100644 services/api/test/integration/crunch_dispatch_test.rb
delete mode 100644 services/api/test/unit/crunch_dispatch_test.rb
delete mode 100644 services/api/test/unit/fail_jobs_test.rb
delete mode 100644 services/arv-git-httpd/usage.go
delete mode 100644 services/keep-web/usage.go
delete mode 100644 services/keepproxy/usage.go
delete mode 100644 services/keepstore/deprecated.go
delete mode 100755 tools/arvbox/lib/arvbox/docker/crunch-setup.sh
delete mode 100644 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/log/main/.gitstub
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/log/run
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/run
delete mode 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch0/run-service
delete mode 100644 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/log/main/.gitstub
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/log/run
delete mode 120000 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/run
delete mode 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch1/run-service
delete mode 100644 tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz
create mode 100644 tools/crunchstat-summary/tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt.gz.report} (100%)
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz} (66%)
rename tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt.gz.report} (100%)
copy tools/crunchstat-summary/tests/{container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report => container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y.txt.gz.report} (100%)
discards 7528d55d6856050fcdc68aa25914facc4868df8e (commit)
via a64396990341078a85f61b6dd202f7eadaa487ef (commit)
via 79ba997c9bc1f97514dcc6f1d6016ba7524a0ae1 (commit)
via 6de58d02306425c5899506c571b90b5c6c93b629 (commit)
via 3c248f44a39ab0b792a8a91f875c4e09875bf54f (commit)
via 16a7c3c0b0bff303a6185282ab170fd495f7413d (commit)
via 5bbec701f90b8179d66e5c308c1c01e3c4dcade7 (commit)
via 3c0e4dc35b5f2fc34e050fef304cdec0cebe51ae (commit)
via 44a871434f648052a410f158fc8e09ec17c11339 (commit)
via 6a90dcb6b421380d4654d47db9b8037643039236 (commit)
via 72eeca4fb7abb97ae41e30c032e3d48074b915f7 (commit)
via a5db96feced74279d61fe8254ed38a321342da1d (commit)
via 73a68fb27a3531feed18b339ad4c66ff3d08c501 (commit)
via 59c608c6052d2bf47a49d6a649be5f407986f6a3 (commit)
via 33e1bbfb47da58c94bb705324362c4c709ac43bb (commit)
via 0de3f63a136cab2227204eb16da9ea0eb9b68349 (commit)
via 5b604a0884f7cac8330fb0e5cfae90f80799f6c3 (commit)
via 7410ffda247fc84a9e650dc441dd415f483cfa5e (commit)
via ef23c3d124b24a461f6947868a28e67e7a0a1010 (commit)
via cdc569ed1c4777d3293327ee10b8f1c8bec06c6a (commit)
via de47033cb700ba5655dc6cfde77b888e8a94e87f (commit)
via 82ef8e6c9b95804159e199d7dd9128da82366a50 (commit)
via 5ae0a422e60788a4039d17d2d8dfb60f250329c7 (commit)
via 729c7c95d0d9c8d0357143717d5f2bb8cdd743de (commit)
via d4ed3e6460469f2766e1f1676c538d6c86e000b6 (commit)
via 86b7903116608ac3a8314437639cb61d774fb510 (commit)
via 8dae253559240fb16564fb934fece2cbcf001866 (commit)
via 309e3a3b10cd6a4e609f84fbdc87cfed3baad796 (commit)
via 1d5727fcdd6e524898d7507417456b9fedcb79dd (commit)
via bad30523725258c8460768be3239a8b02d4e35eb (commit)
via 18de568b068818cebbd708e2a3f39d2c5a65c6ee (commit)
via b64996755741a71c4f02cf3ef9ea2b7ba1d0e2e1 (commit)
via 3679c9415c3f9f925a40919c549b7cac65606800 (commit)
via ba2e6d49ca390a8a62aa77e531e5a255c93ebfcd (commit)
via a5cf4e0ea356a7ee06f67fe159484fe20cd8a184 (commit)
via a136c1ca723537feab8efb40de4fe68c099a175b (commit)
via 4031e38a48822b543e95f166003ecb01906e49dc (commit)
via 322599005b9425223195f8b903f0aa2cfc4b7db9 (commit)
via 26c7b3c97876a05ec7427bc2e5270108dcf00b17 (commit)
via d83e0937bc1d93560896ae1331486c473436b4a2 (commit)
via e9a366b15f7e9fec534cb2bdc319f0f3ed13236d (commit)
via 2774e45aba64557259fc35155c62988449a8dc72 (commit)
via 0fb04af57862494beb38f57b75a1427ee9443e25 (commit)
via 03590b0b68fe727405e46cf28d47ba50117c7f1d (commit)
via 8b873a9b3b8865a4d451263e48b49122b9c32759 (commit)
via c8f57c52224362d7621f1271774b0f2d60c55cac (commit)
via 366a2efdd0ac4630f4381f3b47d70ef155ed2df4 (commit)
via 803c1beb16c35f225d22e0dfd7f45165f13bc5db (commit)
via 1166aeb6033725709ded753a0c00f69320a9a873 (commit)
via 8f9f169bf7d53ecaaa076bf4fbf60fa0f0016af4 (commit)
via 5eb512e7b6ff3a0d9f14591fe6bf611dde5cc27c (commit)
via 7abf74e3ad723ed8aed4d13e794a8f6ad6f44059 (commit)
via 415910149bed2eef6ae818b7c059d413934df06e (commit)
via e0fd2e426856a6593f2c86c60831f0f867f4f83d (commit)
via 35658af99f09f2f6768583d65246429f789fc5a2 (commit)
via d7a5e6e37e9e9251177f9fad98d867f477e69537 (commit)
via 22de1507bc49617b02aa219302cecb830694adb5 (commit)
via c72420e52c085c76dead3e700e05e6fd058dccb2 (commit)
via e7374a8dbe6467add8506d52a8d25b9f0eee16dd (commit)
via 1ba00901283d88fc7a5c82cabdad6e5183d4bb78 (commit)
via 0efb729a71577d400d93b8db56975a4232ec9c6b (commit)
via 6893f07253b37deac06aefdaedbadfb730254026 (commit)
via 52cfc5c943a591d4f8095794820b72b697ae6f12 (commit)
via 38821eb2984f73e62bf21fd214890864a8ad9d47 (commit)
via f248e19664fec1268e2a736d698acfbd6147016e (commit)
via 47d4b25da3ee62b641aa3026adf38adc22b3b65c (commit)
via 043185f6d16dd41135c255f0f36fb8a75203537b (commit)
via 021f8eb819919d4606d7b4c72a2497e842041526 (commit)
via 751745e032831e16c5b67cd4bd0d328cd1f11fd7 (commit)
via 23138fc621581634579294751beeff7f5e66358e (commit)
via 6a13a5f663102b77c157a157c8c4089269d3c3b1 (commit)
via 14626b1a574924a23f18073b8bb6eff829d66e5a (commit)
via fec7796668a4f4a73bb52e16e13e4504f76649f5 (commit)
via 2384aa9a6179333daa03002c36e7b25b83aab903 (commit)
via e10aa8e0c8b9c45d69832e71480cfb3d6929834e (commit)
via a8aa509cf3913c8dc91640ddcad6b25e6609517a (commit)
via 050ea7fdc6317a0fa0eeed20b0e6cb0b7fd6693b (commit)
via b96728249db70975be33bb5c5721ec5b8a0eb3ff (commit)
via 448f667a574b50da096051a0d062b9059ab3609f (commit)
via 9f6bfd78b88486d4b3384b4be05c1f5e4f546a9c (commit)
via a33badcdc6af29a87d1f960dbff8ca947329a46b (commit)
via 30e065c34db0ab9a0e824a77b1ac0a46412598e0 (commit)
via 9f912f75b3a0b75e8f3de94f4527d60f075f17fe (commit)
via f0bc73f28a14af9499dd32e906429d1bc276c969 (commit)
via 4923d561f16ed4c665e89142f01b7b1d3786ac2c (commit)
via bc1947e4aef52fe5f3aebc10dc2ea74cad86672d (commit)
via 4d56f9b913fcf41fbf89bf5016463b5353fa3a9f (commit)
via 100ae537a8329048452c656229750b97c78a3296 (commit)
via aabdf0fec790f9dd341af07013cc1c47ae04b876 (commit)
via 1cce0422dfc66a02e59f0c3a783562c90d0931d9 (commit)
via c49024ec99cc5717f7856d61f325c01c90f750a9 (commit)
via 07d92519438a592d531f2c7558cd51788da262ca (commit)
via 9a02d4117bfc1372ce5b6baedc9303f2500b3197 (commit)
via bdb3e297e65ed39f66b33a6fb4c3b3b0e17e6fd5 (commit)
via acb1d98cdfa6e33bfd3e1bb7510b78a4fd9384e2 (commit)
via 40d4eb5c5a942d82d158ab9d4d3b3fb1b44fde83 (commit)
via b93cb0b828989f9e2651ba436fb8a267224e4ca8 (commit)
via 2c7b49e8791154db907616243c930c552aeb3df3 (commit)
via 8f78a0332d530f9987f18e6c982e30b0ea8e943e (commit)
via 63756e0e8a1188e01d0c238394db71af278863cd (commit)
via 752c2ff97b3670bcc87b08816560f5a0be0b8d06 (commit)
via 5d82dc390a8e158b71c3c96cb1d0f301f3f12dbb (commit)
via 01fd373a99ab6e3ee13bf2c8ec38e48ee6f42ba3 (commit)
via 756aad23bd19f1533f38f0b9e9ee1206dc33b114 (commit)
via 2d9aa62499005abd47b5f2fa604f01d055480722 (commit)
via 5d68da382b77745e69640dfdd4e4bd9110e72f4a (commit)
via 1fa33cf0c06e07c7f3668f994f8d0def93d5ecbd (commit)
via 67daee82d7c7e3385ec252434109a3348a7c70d7 (commit)
via c2b55f26cbf4e0c963e6d30aa63c11898bfc3a09 (commit)
via d71c797d3bbfd26d571fb46a14262b5c01b95ff4 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (7528d55d6856050fcdc68aa25914facc4868df8e)
\
N -- N -- N (a64396990341078a85f61b6dd202f7eadaa487ef)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a64396990341078a85f61b6dd202f7eadaa487ef
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Sep 5 11:15:53 2019 -0400
13647: Use cluster config instead of custom keepstore config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d70722272..9ff8acfaa 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -470,6 +470,7 @@ stop_services() {
&& python sdk/python/tests/run_test_server.py stop \
&& all_services_stopped=1
deactivate
+ unset ARVADOS_CONFIG
}
interrupt() {
@@ -633,15 +634,12 @@ install_env() {
for d in \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
"$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -h "$d" ]] && rm "$d"
[[ -d "$d" ]] && rmdir "$d"
done
fi
- for d in \
- "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
- "$GOPATH/src/git.curoverse.com/arvados.git"; do
- [[ -h "$d" ]] && rm "$d"
- done
ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v github.com/kardianos/govendor
cd "$GOPATH/src/git.curoverse.com/arvados.git"
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index af7c57120..019d5edd0 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -85,7 +85,7 @@ func (s *CommandSuite) TestCheckOldKeepstoreConfigFile(c *check.C) {
c.Assert(err, check.IsNil)
defer os.Remove(f.Name())
- io.WriteString(f, "Debug: true\n")
+ io.WriteString(f, "Listen: :12345\nDebug: true\n")
var stdout, stderr bytes.Buffer
in := `
@@ -97,7 +97,7 @@ Clusters:
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-", "-legacy-keepstore-config", f.Name()}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 1)
c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*LogLevel: info\n\+ +LogLevel: debug\n.*`)
- c.Check(stderr.String(), check.Matches, `.*you should remove the legacy keepstore config file.*\n`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*you should remove the legacy keepstore config file.*\n`)
}
func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 24b2e450e..b02c9c9b3 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -176,6 +176,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -316,15 +325,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
# Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -741,6 +779,47 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 9eb8c40c1..0a030fb04 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -102,12 +102,6 @@ func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc
svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
}
-const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
-
-type oldKeepstoreConfig struct {
- Debug *bool
-}
-
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
if path == "" {
return nil
@@ -126,35 +120,6 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
return nil
}
-// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
- if ldr.KeepstorePath == "" {
- return nil
- }
- var oc oldKeepstoreConfig
- err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
- if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
- return nil
- } else if err != nil {
- return err
- }
-
- cluster, err := cfg.GetCluster("")
- if err != nil {
- return err
- }
-
- if v := oc.Debug; v == nil {
- } else if *v && cluster.SystemLogs.LogLevel != "debug" {
- cluster.SystemLogs.LogLevel = "debug"
- } else if !*v && cluster.SystemLogs.LogLevel != "info" {
- cluster.SystemLogs.LogLevel = "info"
- }
-
- cfg.Clusters[cluster.ClusterID] = *cluster
- return nil
-}
-
type oldCrunchDispatchSlurmConfig struct {
Client *arvados.Client
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
new file mode 100644
index 000000000..f196de0ea
--- /dev/null
+++ b/lib/config/deprecated_keepstore.go
@@ -0,0 +1,547 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "os"
+ "strconv"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
+
+type oldKeepstoreConfig struct {
+ Debug *bool
+ Listen *string
+
+ LogFormat *string
+
+ PIDFile *string
+
+ MaxBuffers *int
+ MaxRequests *int
+
+ BlobSignatureTTL *arvados.Duration
+ BlobSigningKeyFile *string
+ RequireSignatures *bool
+ SystemAuthTokenFile *string
+ EnableDelete *bool
+ TrashLifetime *arvados.Duration
+ TrashCheckInterval *arvados.Duration
+ PullWorkers *int
+ TrashWorkers *int
+ EmptyTrashWorkers *int
+ TLSCertificateFile *string
+ TLSKeyFile *string
+
+ Volumes *oldKeepstoreVolumeList
+
+ ManagementToken *string
+
+ DiscoverVolumesFromMountsFile string // not a real legacy config -- just useful for tests
+}
+
+type oldKeepstoreVolumeList []oldKeepstoreVolume
+
+type oldKeepstoreVolume struct {
+ arvados.Volume
+ Type string `json:",omitempty"`
+
+ // Azure driver configs
+ StorageAccountName string `json:",omitempty"`
+ StorageAccountKeyFile string `json:",omitempty"`
+ StorageBaseURL string `json:",omitempty"`
+ ContainerName string `json:",omitempty"`
+ AzureReplication int `json:",omitempty"`
+ RequestTimeout arvados.Duration `json:",omitempty"`
+ ListBlobsRetryDelay arvados.Duration `json:",omitempty"`
+ ListBlobsMaxAttempts int `json:",omitempty"`
+
+ // S3 driver configs
+ AccessKeyFile string `json:",omitempty"`
+ SecretKeyFile string `json:",omitempty"`
+ Endpoint string `json:",omitempty"`
+ Region string `json:",omitempty"`
+ Bucket string `json:",omitempty"`
+ LocationConstraint bool `json:",omitempty"`
+ IndexPageSize int `json:",omitempty"`
+ S3Replication int `json:",omitempty"`
+ ConnectTimeout arvados.Duration `json:",omitempty"`
+ ReadTimeout arvados.Duration `json:",omitempty"`
+ RaceWindow arvados.Duration `json:",omitempty"`
+ UnsafeDelete bool `json:",omitempty"`
+
+ // Directory driver configs
+ Root string
+ DirectoryReplication int
+ Serialize bool
+
+ // Common configs
+ ReadOnly bool `json:",omitempty"`
+ StorageClasses []string `json:",omitempty"`
+}
+
+// update config using values from an old-style keepstore config file.
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+ if ldr.KeepstorePath == "" {
+ return nil
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ return fmt.Errorf("getting hostname: %s", err)
+ }
+
+ var oc oldKeepstoreConfig
+ err = ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
+ if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ myURL := arvados.URL{Scheme: "http"}
+ if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
+ myURL.Scheme = "https"
+ }
+
+ if v := oc.Debug; v == nil {
+ } else if *v && cluster.SystemLogs.LogLevel != "debug" {
+ cluster.SystemLogs.LogLevel = "debug"
+ } else if !*v && cluster.SystemLogs.LogLevel != "info" {
+ cluster.SystemLogs.LogLevel = "info"
+ }
+
+ if v := oc.TLSCertificateFile; v != nil && "file://"+*v != cluster.TLS.Certificate {
+ cluster.TLS.Certificate = "file://" + *v
+ }
+ if v := oc.TLSKeyFile; v != nil && "file://"+*v != cluster.TLS.Key {
+ cluster.TLS.Key = "file://" + *v
+ }
+ if v := oc.Listen; v != nil {
+ if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+ // already listed
+ myURL.Host = *v
+ } else if len(*v) > 1 && (*v)[0] == ':' {
+ myURL.Host = net.JoinHostPort(hostname, (*v)[1:])
+ cluster.Services.Keepstore.InternalURLs[myURL] = arvados.ServiceInstance{}
+ } else {
+ return fmt.Errorf("unable to migrate Listen value %q from legacy keepstore config file -- remove after configuring Services.Keepstore.InternalURLs.", *v)
+ }
+ } else {
+ for url := range cluster.Services.Keepstore.InternalURLs {
+ if host, _, _ := net.SplitHostPort(url.Host); host == hostname {
+ myURL = url
+ break
+ }
+ }
+ if myURL.Host == "" {
+ return fmt.Errorf("unable to migrate legacy keepstore config: no 'Listen' key, and hostname %q does not match an entry in Services.Keepstore.InternalURLs", hostname)
+ }
+ }
+
+ if v := oc.LogFormat; v != nil && *v != cluster.SystemLogs.Format {
+ cluster.SystemLogs.Format = *v
+ }
+ if v := oc.MaxBuffers; v != nil && *v != cluster.API.MaxKeepBlockBuffers {
+ cluster.API.MaxKeepBlockBuffers = *v
+ }
+ if v := oc.MaxRequests; v != nil && *v != cluster.API.MaxConcurrentRequests {
+ cluster.API.MaxConcurrentRequests = *v
+ }
+ if v := oc.BlobSignatureTTL; v != nil && *v != cluster.Collections.BlobSigningTTL {
+ cluster.Collections.BlobSigningTTL = *v
+ }
+ if v := oc.BlobSigningKeyFile; v != nil {
+ buf, err := ioutil.ReadFile(*v)
+ if err != nil {
+ return fmt.Errorf("error reading BlobSigningKeyFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.Collections.BlobSigningKey {
+ cluster.Collections.BlobSigningKey = key
+ }
+ }
+ if v := oc.RequireSignatures; v != nil && *v != cluster.Collections.BlobSigning {
+ cluster.Collections.BlobSigning = *v
+ }
+ if v := oc.SystemAuthTokenFile; v != nil {
+ f, err := os.Open(*v)
+ if err != nil {
+ return fmt.Errorf("error opening SystemAuthTokenFile: %s", err)
+ }
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ if err != nil {
+ return fmt.Errorf("error reading SystemAuthTokenFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.SystemRootToken {
+ cluster.SystemRootToken = key
+ }
+ }
+ if v := oc.EnableDelete; v != nil && *v != cluster.Collections.BlobTrash {
+ cluster.Collections.BlobTrash = *v
+ }
+ if v := oc.TrashLifetime; v != nil && *v != cluster.Collections.BlobTrashLifetime {
+ cluster.Collections.BlobTrashLifetime = *v
+ }
+ if v := oc.TrashCheckInterval; v != nil && *v != cluster.Collections.BlobTrashCheckInterval {
+ cluster.Collections.BlobTrashCheckInterval = *v
+ }
+ if v := oc.TrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobTrashConcurrency = *v
+ }
+ if v := oc.EmptyTrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobDeleteConcurrency = *v
+ }
+ if v := oc.PullWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobReplicateConcurrency = *v
+ }
+ if v := oc.Volumes; v == nil {
+ ldr.Logger.Warn("no volumes in legacy config; discovering local directory volumes")
+ err := ldr.discoverLocalVolumes(cluster, oc.DiscoverVolumesFromMountsFile, myURL)
+ if err != nil {
+ return fmt.Errorf("error discovering local directory volumes: %s", err)
+ }
+ } else {
+ for i, oldvol := range *v {
+ var accessViaHosts map[arvados.URL]arvados.VolumeAccess
+ oldUUID, found := ldr.alreadyMigrated(oldvol, cluster.Volumes, myURL)
+ if found {
+ accessViaHosts = cluster.Volumes[oldUUID].AccessViaHosts
+ writers := false
+ for _, va := range accessViaHosts {
+ if !va.ReadOnly {
+ writers = true
+ }
+ }
+ if writers || len(accessViaHosts) == 0 {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ if len(accessViaHosts) > 0 {
+ cluster.Volumes[oldUUID].AccessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ }
+ continue
+ }
+ }
+ var newvol arvados.Volume
+ if found {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ newvol = cluster.Volumes[oldUUID]
+ // Remove the old entry. It will be
+ // added back below, possibly with a
+ // new UUID.
+ delete(cluster.Volumes, oldUUID)
+ } else {
+ var params interface{}
+ switch oldvol.Type {
+ case "S3":
+ accesskeydata, err := ioutil.ReadFile(oldvol.AccessKeyFile)
+ if err != nil && oldvol.AccessKeyFile != "" {
+ return fmt.Errorf("error reading AccessKeyFile: %s", err)
+ }
+ secretkeydata, err := ioutil.ReadFile(oldvol.SecretKeyFile)
+ if err != nil && oldvol.SecretKeyFile != "" {
+ return fmt.Errorf("error reading SecretKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "S3",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.S3Replication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.S3VolumeDriverParameters{
+ AccessKey: string(bytes.TrimSpace(accesskeydata)),
+ SecretKey: string(bytes.TrimSpace(secretkeydata)),
+ Endpoint: oldvol.Endpoint,
+ Region: oldvol.Region,
+ Bucket: oldvol.Bucket,
+ LocationConstraint: oldvol.LocationConstraint,
+ IndexPageSize: oldvol.IndexPageSize,
+ ConnectTimeout: oldvol.ConnectTimeout,
+ ReadTimeout: oldvol.ReadTimeout,
+ RaceWindow: oldvol.RaceWindow,
+ UnsafeDelete: oldvol.UnsafeDelete,
+ }
+ case "Azure":
+ keydata, err := ioutil.ReadFile(oldvol.StorageAccountKeyFile)
+ if err != nil && oldvol.StorageAccountKeyFile != "" {
+ return fmt.Errorf("error reading StorageAccountKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.AzureReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.AzureVolumeDriverParameters{
+ StorageAccountName: oldvol.StorageAccountName,
+ StorageAccountKey: string(bytes.TrimSpace(keydata)),
+ StorageBaseURL: oldvol.StorageBaseURL,
+ ContainerName: oldvol.ContainerName,
+ RequestTimeout: oldvol.RequestTimeout,
+ ListBlobsRetryDelay: oldvol.ListBlobsRetryDelay,
+ ListBlobsMaxAttempts: oldvol.ListBlobsMaxAttempts,
+ }
+ case "Directory":
+ newvol = arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.DirectoryReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.DirectoryVolumeDriverParameters{
+ Root: oldvol.Root,
+ Serialize: oldvol.Serialize,
+ }
+ default:
+ return fmt.Errorf("unsupported volume type %q", oldvol.Type)
+ }
+ dp, err := json.Marshal(params)
+ if err != nil {
+ return err
+ }
+ newvol.DriverParameters = json.RawMessage(dp)
+ if newvol.Replication < 1 {
+ newvol.Replication = 1
+ }
+ }
+ if accessViaHosts == nil {
+ accessViaHosts = make(map[arvados.URL]arvados.VolumeAccess, 1)
+ }
+ accessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ newvol.AccessViaHosts = accessViaHosts
+
+ volUUID := oldUUID
+ if oldvol.ReadOnly {
+ } else if oc.Listen == nil {
+ ldr.Logger.Warn("cannot find optimal volume UUID because Listen address is not given in legacy keepstore config")
+ } else if uuid, _, err := findKeepServicesItem(cluster, *oc.Listen); err != nil {
+ ldr.Logger.WithError(err).Warn("cannot find optimal volume UUID: failed to find a matching keep_service listing for this legacy keepstore config")
+ } else if len(uuid) != 27 {
+ ldr.Logger.WithField("UUID", uuid).Warn("cannot find optimal volume UUID: keep_service UUID does not have expected format")
+ } else {
+ rendezvousUUID := cluster.ClusterID + "-nyw5e-" + uuid[12:]
+ if _, ok := cluster.Volumes[rendezvousUUID]; ok {
+ ldr.Logger.Warn("suggesting a random volume UUID because the volume ID matching our keep_service UUID is already in use")
+ } else {
+ volUUID = rendezvousUUID
+ }
+ }
+ if volUUID == "" {
+ volUUID = newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithField("UUID", volUUID).Infof("suggesting a random volume UUID for volume #%d in legacy config", i)
+ }
+ cluster.Volumes[volUUID] = newvol
+ }
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+func (ldr *Loader) alreadyMigrated(oldvol oldKeepstoreVolume, newvols map[string]arvados.Volume, myURL arvados.URL) (string, bool) {
+ for uuid, newvol := range newvols {
+ if oldvol.Type != newvol.Driver {
+ continue
+ }
+ switch oldvol.Type {
+ case "S3":
+ var params arvados.S3VolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Endpoint == params.Endpoint &&
+ oldvol.Region == params.Region &&
+ oldvol.Bucket == params.Bucket &&
+ oldvol.LocationConstraint == params.LocationConstraint {
+ return uuid, true
+ }
+ case "Azure":
+ var params arvados.AzureVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.StorageAccountName == params.StorageAccountName &&
+ oldvol.StorageBaseURL == params.StorageBaseURL &&
+ oldvol.ContainerName == params.ContainerName {
+ return uuid, true
+ }
+ case "Directory":
+ var params arvados.DirectoryVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Root == params.Root {
+ if _, ok := newvol.AccessViaHosts[myURL]; ok {
+ return uuid, true
+ }
+ }
+ }
+ }
+ return "", false
+}
+
+func (ldr *Loader) discoverLocalVolumes(cluster *arvados.Cluster, mountsFile string, myURL arvados.URL) error {
+ if mountsFile == "" {
+ mountsFile = "/proc/mounts"
+ }
+ f, err := os.Open(mountsFile)
+ if err != nil {
+ return fmt.Errorf("error opening %s: %s", mountsFile, err)
+ }
+ defer f.Close()
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ args := strings.Fields(scanner.Text())
+ dev, mount := args[0], args[1]
+ if mount == "/" {
+ continue
+ }
+ if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
+ continue
+ }
+ keepdir := mount + "/keep"
+ if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
+ continue
+ }
+
+ ro := false
+ for _, fsopt := range strings.Split(args[3], ",") {
+ if fsopt == "ro" {
+ ro = true
+ }
+ }
+
+ uuid := newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithFields(logrus.Fields{
+ "UUID": uuid,
+ "Driver": "Directory",
+ "DriverParameters.Root": keepdir,
+ "DriverParameters.Serialize": false,
+ "ReadOnly": ro,
+ "Replication": 1,
+ }).Warn("adding local directory volume")
+
+ p, err := json.Marshal(arvados.DirectoryVolumeDriverParameters{
+ Root: keepdir,
+ Serialize: false,
+ })
+ if err != nil {
+ panic(err)
+ }
+ cluster.Volumes[uuid] = arvados.Volume{
+ Driver: "Directory",
+ DriverParameters: p,
+ ReadOnly: ro,
+ Replication: 1,
+ AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
+ myURL: {ReadOnly: ro},
+ },
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("reading %s: %s", mountsFile, err)
+ }
+ return nil
+}
+
+func array2boolmap(keys []string) map[string]bool {
+ m := map[string]bool{}
+ for _, k := range keys {
+ m[k] = true
+ }
+ return m
+}
+
+func newUUID(clusterID, infix string) string {
+ randint, err := rand.Int(rand.Reader, big.NewInt(0).Exp(big.NewInt(36), big.NewInt(15), big.NewInt(0)))
+ if err != nil {
+ panic(err)
+ }
+ randstr := randint.Text(36)
+ for len(randstr) < 15 {
+ randstr = "0" + randstr
+ }
+ return fmt.Sprintf("%s-%s-%s", clusterID, infix, randstr)
+}
+
+// Return the UUID and URL for the controller's keep_services listing
+// corresponding to this host/process.
+func findKeepServicesItem(cluster *arvados.Cluster, listen string) (uuid string, url arvados.URL, err error) {
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return
+ }
+ client.AuthToken = cluster.SystemRootToken
+ var svcList arvados.KeepServiceList
+ err = client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ if err != nil {
+ return
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("error getting hostname: %s", err)
+ return
+ }
+ for _, ks := range svcList.Items {
+ if ks.ServiceType != "proxy" && keepServiceIsMe(ks, hostname, listen) {
+ url := arvados.URL{
+ Scheme: "http",
+ Host: net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+ }
+ if ks.ServiceSSLFlag {
+ url.Scheme = "https"
+ }
+ return ks.UUID, url, nil
+ }
+ }
+ err = errors.New("failed to find a keep_services entry that matches the current host/port")
+ return
+}
+
+var localhostOrAllInterfaces = map[string]bool{
+ "localhost": true,
+ "127.0.0.1": true,
+ "::1": true,
+ "::": true,
+ "": true,
+}
+
+// Return true if the given KeepService entry matches the given
+// hostname and (keepstore config file) listen address.
+//
+// If the KeepService host is some variant of "localhost", we assume
+// this is a testing or single-node environment, ignore the given
+// hostname, and return true if the port numbers match.
+//
+// The hostname isn't assumed to be a FQDN: a hostname "foo.bar" will
+// match a KeepService host "foo.bar", but also "foo.bar.example",
+// "foo.bar.example.org", etc.
+func keepServiceIsMe(ks arvados.KeepService, hostname string, listen string) bool {
+ // Extract the port name/number from listen, and resolve it to
+ // a port number to compare with ks.ServicePort.
+ _, listenport, err := net.SplitHostPort(listen)
+ if err != nil && strings.HasPrefix(listen, ":") {
+ listenport = listen[1:]
+ }
+ if lp, err := net.LookupPort("tcp", listenport); err != nil {
+ return false
+ } else if !(lp == ks.ServicePort ||
+ (lp == 0 && ks.ServicePort == 80)) {
+ return false
+ }
+
+ kshost := strings.ToLower(ks.ServiceHost)
+ return localhostOrAllInterfaces[kshost] || strings.HasPrefix(kshost+".", strings.ToLower(hostname)+".")
+}
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
new file mode 100644
index 000000000..be4c643f4
--- /dev/null
+++ b/lib/config/deprecated_keepstore_test.go
@@ -0,0 +1,685 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "text/template"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+type KeepstoreMigrationSuite struct {
+ hostname string // blank = use test system's hostname
+}
+
+var _ = check.Suite(&KeepstoreMigrationSuite{})
+
+func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
+ keepstorefile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstorefile.Name())
+ _, err = io.WriteString(keepstorefile, keepstoreconfig)
+ c.Assert(err, check.IsNil)
+ err = keepstorefile.Close()
+ c.Assert(err, check.IsNil)
+
+ gotldr := testLoader(c, clusterconfig, nil)
+ gotldr.KeepstorePath = keepstorefile.Name()
+ expectedldr := testLoader(c, expectedconfig, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
+ keyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keyfile.Name())
+ io.WriteString(keyfile, "blobsigningkey\n")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ s.checkEquivalentWithKeepstoreConfig(c, `
+Listen: ":12345"
+Debug: true
+LogFormat: text
+MaxBuffers: 1234
+MaxRequests: 2345
+BlobSignatureTTL: 123m
+BlobSigningKeyFile: `+keyfile.Name()+`
+`, `
+Clusters:
+ z1111:
+ {}
+`, `
+Clusters:
+ z1111:
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://`+hostname+`:12345": {}
+ SystemLogs:
+ Format: text
+ LogLevel: debug
+ API:
+ MaxKeepBlockBuffers: 1234
+ MaxConcurrentRequests: 2345
+ Collections:
+ BlobSigningTTL: 123m
+ BlobSigningKey: blobsigningkey
+`)
+}
+
+func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
+ tmpd, err := ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tmpd)
+ err = os.Mkdir(tmpd+"/keep", 0777)
+ c.Assert(err, check.IsNil)
+
+ tmpf, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmpf.Name())
+
+ // read/write
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: false,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // read-only
+ tmpf.Seek(0, os.SEEK_SET)
+ tmpf.Truncate(0)
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
+ accesskeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(accesskeyfile.Name())
+ io.WriteString(accesskeyfile, "accesskeydata\n")
+
+ secretkeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(secretkeyfile.Name())
+ io.WriteString(secretkeyfile, "secretkeydata\n")
+
+ // s3, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 1,
+ }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
+
+ // s3, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+ AccessKeyFile: `+accesskeyfile.Name()+`
+ SecretKeyFile: `+secretkeyfile.Name()+`
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: testbucket
+ LocationConstraint: true
+ IndexPageSize: 1234
+ S3Replication: 4
+ ConnectTimeout: 3m
+ ReadTimeout: 4m
+ RaceWindow: 5m
+ UnsafeDelete: true
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 4,
+ }, &arvados.S3VolumeDriverParameters{
+ AccessKey: "accesskeydata",
+ SecretKey: "secretkeydata",
+ Endpoint: "https://storage.googleapis.com",
+ Region: "us-east-1z",
+ Bucket: "testbucket",
+ LocationConstraint: true,
+ IndexPageSize: 1234,
+ ConnectTimeout: arvados.Duration(time.Minute * 3),
+ ReadTimeout: arvados.Duration(time.Minute * 4),
+ RaceWindow: arvados.Duration(time.Minute * 5),
+ UnsafeDelete: true,
+ }, &arvados.S3VolumeDriverParameters{})
+
+ // azure, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+`, arvados.Volume{
+ Driver: "Azure",
+ Replication: 1,
+ }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
+
+ // azure, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+ ReadOnly: true
+ StorageAccountName: storageacctname
+ StorageAccountKeyFile: `+secretkeyfile.Name()+`
+ StorageBaseURL: https://example.example
+ ContainerName: testctr
+ LocationConstraint: true
+ AzureReplication: 4
+ RequestTimeout: 3m
+ ListBlobsRetryDelay: 4m
+ ListBlobsMaxAttempts: 5
+`, arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.AzureVolumeDriverParameters{
+ StorageAccountName: "storageacctname",
+ StorageAccountKey: "secretkeydata",
+ StorageBaseURL: "https://example.example",
+ ContainerName: "testctr",
+ RequestTimeout: arvados.Duration(time.Minute * 3),
+ ListBlobsRetryDelay: arvados.Duration(time.Minute * 4),
+ ListBlobsMaxAttempts: 5,
+ }, &arvados.AzureVolumeDriverParameters{})
+
+ // directory, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ Root: /tmp/xyzzy
+`, arvados.Volume{
+ Driver: "Directory",
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // directory, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ ReadOnly: true
+ Root: /tmp/xyzzy
+ DirectoryReplication: 4
+ Serialize: true
+`, arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ Serialize: true,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
+ hostname := s.hostname
+ if hostname == "" {
+ h, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ hostname = h
+ }
+
+ oldconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(oldconfig.Name())
+ io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
+ if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
+ // Prevent tests from looking at the real /proc/mounts on the test host.
+ io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
+ }
+
+ ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
+ ldr.KeepstorePath = oldconfig.Name()
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc := cfg.Clusters["z1111"]
+ c.Check(cc.Volumes, check.HasLen, 1)
+ for uuid, v := range cc.Volumes {
+ c.Check(uuid, check.HasLen, 27)
+ c.Check(v.Driver, check.Equals, expectvol.Driver)
+ c.Check(v.Replication, check.Equals, expectvol.Replication)
+
+ avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+ c.Check(ok, check.Equals, true)
+ c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
+
+ err := json.Unmarshal(v.DriverParameters, paramsdst)
+ c.Check(err, check.IsNil)
+ c.Check(paramsdst, check.DeepEquals, expectparams)
+ }
+}
+
+// How we handle a volume from a legacy keepstore config file depends
+// on whether it's writable, whether a volume using the same cloud
+// backend already exists in the cluster config, and (if so) whether
+// it already has an AccessViaHosts entry for this host.
+//
+// In all cases, we should end up with an AccessViaHosts entry for
+// this host, to indicate that the current host's volumes have been
+// migrated.
+
+// Same backend already referenced in cluster config, this host
+// already listed in AccessViaHosts --> no change, except possibly
+// updating the ReadOnly flag on the AccessViaHosts entry.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ S3Replication: 3
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Writable volume, same cloud backend already referenced in cluster
+// config --> change UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+`)
+ c.Check(after, check.HasLen, len(before))
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "readonlyonother")
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same cloud backend not yet referenced --> add a
+// new volume, with UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
+ c.Check(newvol.Replication, check.Equals, 3)
+
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, but this host isn't in AccessViaHosts --> add a new
+// volume, with UUID to match this keepstore's UUID (filesystem-backed
+// volumes are assumed to be different on different hosts, even if
+// paths are the same).
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sdd
+ DirectoryReplication: 2
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.DirectoryVolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Root, check.Equals, "/data/sdd")
+ c.Check(newvol.Replication, check.Equals, 2)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, and this host is already listed in AccessViaHosts
+// --> already migrated, don't change anything.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sde
+ DirectoryReplication: 2
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Multiple writable cloud-backed volumes --> one of them will get a
+// UUID matching this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: first-bucket-to-migrate
+ S3Replication: 3
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: second-bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 2)
+ // Sort by bucket name (so "first" comes before "second")
+ params := map[string]arvados.S3VolumeDriverParameters{}
+ for _, uuid := range newuuids {
+ var p arvados.S3VolumeDriverParameters
+ json.Unmarshal(after[uuid].DriverParameters, &p)
+ params[uuid] = p
+ }
+ sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
+ newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
+ params0, params1 := params[newuuids[0]], params[newuuids[1]]
+
+ c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
+ c.Check(newvol0.Replication, check.Equals, 3)
+
+ c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
+ c.Check(newvol1.Replication, check.Equals, 3)
+
+ // Don't care which one gets the special UUID
+ if newuuids[0] != expectUUID {
+ c.Check(newuuids[1], check.Equals, expectUUID)
+ }
+}
+
+// Non-writable volume, same cloud backend already referenced in
+// cluster config --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+ ReadOnly: true
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Writable volume, same cloud backend already writable by another
+// keepstore server --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ S3Replication: 3
+ ReadOnly: false
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Non-writable volume, same cloud backend not already referenced in
+// cluster config --> assign a new random volume UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: differentbucket
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "differentbucket")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+ c.Check(ok, check.Equals, true)
+}
+
+const clusterConfigForKeepstoreMigrationTest = `
+Clusters:
+ zzzzz:
+ SystemRootToken: ` + arvadostest.AdminToken + `
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://{{.hostname}}:12345": {}
+ Controller:
+ ExternalURL: "https://{{.controller}}"
+ TLS:
+ Insecure: true
+ Volumes:
+
+ zzzzz-nyw5e-alreadymigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ Replication: 3
+
+ zzzzz-nyw5e-readonlyonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {ReadOnly: true}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ Replication: 3
+
+ zzzzz-nyw5e-writableonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ Replication: 3
+
+ zzzzz-nyw5e-localfilesystem:
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sdd
+ Replication: 1
+
+ zzzzz-nyw5e-localismigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sde
+ Replication: 1
+`
+
+// Determine the effect of combining the given legacy keepstore config
+// YAML (just the "Volumes" entries of an old keepstore config file)
+// with the example clusterConfigForKeepstoreMigrationTest config.
+//
+// Return two Volumes configs -- one without loading
+// keepstoreconfigdata ("before") and one with ("after") -- for the
+// caller to compare.
+func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreVolumesYAML string) (before, after map[string]arvados.Volume) {
+ ldr := testLoader(c, s.clusterConfigYAML(c), nil)
+ cBefore, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ keepstoreconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstoreconfig.Name())
+ io.WriteString(keepstoreconfig, keepstoreVolumesYAML)
+
+ ldr = testLoader(c, s.clusterConfigYAML(c), nil)
+ ldr.KeepstorePath = keepstoreconfig.Name()
+ cAfter, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
+}
+
+func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
+
+ var clusterconfigdata bytes.Buffer
+ err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
+ "hostname": hostname,
+ "controller": os.Getenv("ARVADOS_API_HOST"),
+ })
+ c.Assert(err, check.IsNil)
+
+ return clusterconfigdata.String()
+}
+
+// Return the uuids of volumes that appear in "after" but not
+// "before".
+//
+// Assert the returned slice has at least minAdded entries.
+func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
+ for uuid := range after {
+ if _, ok := before[uuid]; !ok {
+ uuids = append(uuids, uuid)
+ }
+ }
+ if len(uuids) < minAdded {
+ c.Assert(uuids, check.HasLen, minAdded)
+ }
+ return
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
+ for uuid, port := range s.getTestKeepstorePorts(c) {
+ c.Assert(uuid, check.HasLen, 27)
+ return port, "zzzzz-nyw5e-" + uuid[12:]
+ }
+ c.Fatal("getTestKeepstorePorts() returned nothing")
+ return 0, ""
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePorts(c *check.C) map[string]int {
+ client := arvados.NewClientFromEnv()
+ var svcList arvados.KeepServiceList
+ err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ c.Assert(err, check.IsNil)
+ ports := map[string]int{}
+ for _, ks := range svcList.Items {
+ if ks.ServiceType == "disk" {
+ ports[ks.UUID] = ks.ServicePort
+ }
+ }
+ return ports
+}
+
+func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
+ for i, trial := range []struct {
+ match bool
+ hostname string
+ listen string
+ serviceHost string
+ servicePort int
+ }{
+ {true, "keep0", "keep0", "keep0", 80},
+ {true, "keep0", "[::1]:http", "keep0", 80},
+ {true, "keep0", "[::]:http", "keep0", 80},
+ {true, "keep0", "keep0:25107", "keep0", 25107},
+ {true, "keep0", ":25107", "keep0", 25107},
+ {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "Keep0.domain.example", 25107},
+ {true, "keep0", ":http", "keep0.domain.example", 80},
+ {true, "keep0", ":25107", "localhost", 25107},
+ {true, "keep0", ":25107", "::1", 25107},
+ {false, "keep0", ":25107", "keep0", 1111}, // different port
+ {false, "keep0", ":25107", "localhost", 1111}, // different port
+ {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
+ {false, "keep0", ":bogussss", "keep0", 25107}, // unresolvable port
+ {false, "keep0", ":25107", "keep1", 25107}, // different hostname
+ {false, "keep1", ":25107", "keep10", 25107}, // different hostname (prefix, but not on a "." boundary)
+ } {
+ c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
+ }
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 5dda0ba94..ea9b50d03 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -47,7 +47,7 @@ func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvado
func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
- s.checkEquivalent(c, `
+ checkEquivalent(c, `
Clusters:
z1111:
NodeProfiles:
diff --git a/lib/config/export.go b/lib/config/export.go
index 6eb4fbe5f..57f62fa83 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -63,8 +63,10 @@ var whitelist = map[string]bool{
"API": true,
"API.AsyncPermissionsUpdateInterval": false,
"API.DisabledAPIs": false,
+ "API.MaxConcurrentRequests": false,
"API.MaxIndexDatabaseRead": false,
"API.MaxItemsPerResponse": true,
+ "API.MaxKeepBlockBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
"API.RailsSessionSecretToken": false,
@@ -81,6 +83,12 @@ var whitelist = map[string]bool{
"Collections.BlobSigning": true,
"Collections.BlobSigningKey": false,
"Collections.BlobSigningTTL": true,
+ "Collections.BlobTrash": false,
+ "Collections.BlobTrashLifetime": false,
+ "Collections.BlobTrashConcurrency": false,
+ "Collections.BlobTrashCheckInterval": false,
+ "Collections.BlobDeleteConcurrency": false,
+ "Collections.BlobReplicateConcurrency": false,
"Collections.CollectionVersioning": false,
"Collections.DefaultReplication": true,
"Collections.DefaultTrashLifetime": true,
@@ -150,6 +158,16 @@ var whitelist = map[string]bool{
"Users.NewUsersAreActive": false,
"Users.UserNotifierEmailFrom": false,
"Users.UserProfileNotificationAddress": false,
+ "Volumes": true,
+ "Volumes.*": true,
+ "Volumes.*.*": false,
+ "Volumes.*.AccessViaHosts": true,
+ "Volumes.*.AccessViaHosts.*": true,
+ "Volumes.*.AccessViaHosts.*.ReadOnly": true,
+ "Volumes.*.ReadOnly": true,
+ "Volumes.*.Replication": true,
+ "Volumes.*.StorageClasses": true,
+ "Volumes.*.StorageClasses.*": false,
"Workbench": true,
"Workbench.ActivationContactLink": false,
"Workbench.APIClientConnectTimeout": true,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8a5b4610c..f33f5aa8a 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -182,6 +182,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -322,15 +331,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
# Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -747,6 +785,47 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index c7289350e..17e0af7ba 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -321,7 +321,7 @@ Clusters:
}
func (s *LoadSuite) TestMovedKeys(c *check.C) {
- s.checkEquivalent(c, `# config has old keys only
+ checkEquivalent(c, `# config has old keys only
Clusters:
zzzzz:
RequestLimits:
@@ -334,7 +334,7 @@ Clusters:
MaxRequestAmplification: 3
MaxItemsPerResponse: 999
`)
- s.checkEquivalent(c, `# config has both old and new keys; old values win
+ checkEquivalent(c, `# config has both old and new keys; old values win
Clusters:
zzzzz:
RequestLimits:
@@ -352,30 +352,45 @@ Clusters:
`)
}
-func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
- got, err := testLoader(c, goty, nil).Load()
+func checkEquivalent(c *check.C, goty, expectedy string) {
+ gotldr := testLoader(c, goty, nil)
+ expectedldr := testLoader(c, expectedy, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func checkEqualYAML(c *check.C, got, expected interface{}) {
+ expectedyaml, err := yaml.Marshal(expected)
c.Assert(err, check.IsNil)
- expected, err := testLoader(c, expectedy, nil).Load()
+ gotyaml, err := yaml.Marshal(got)
c.Assert(err, check.IsNil)
- if !c.Check(got, check.DeepEquals, expected) {
+ if !bytes.Equal(gotyaml, expectedyaml) {
cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
- for _, obj := range []interface{}{expected, got} {
- y, _ := yaml.Marshal(obj)
+ for _, y := range [][]byte{expectedyaml, gotyaml} {
pr, pw, err := os.Pipe()
c.Assert(err, check.IsNil)
defer pr.Close()
- go func() {
- io.Copy(pw, bytes.NewBuffer(y))
+ go func(data []byte) {
+ pw.Write(data)
pw.Close()
- }()
+ }(y)
cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
}
diff, err := cmd.CombinedOutput()
+ // diff should report differences and exit non-zero.
+ c.Check(err, check.NotNil)
c.Log(string(diff))
- c.Check(err, check.IsNil)
+ c.Error("got != expected; see diff (-expected +got) above")
}
}
+func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
+ got, err := gotldr.Load()
+ c.Assert(err, check.IsNil)
+ expected, err := expectedldr.Load()
+ c.Assert(err, check.IsNil)
+ checkEqualYAML(c, got, expected)
+}
+
func checkListKeys(path string, x interface{}) (err error) {
v := reflect.Indirect(reflect.ValueOf(x))
switch v.Kind() {
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index ae6ac70e9..7ab38c6ca 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -11,11 +11,12 @@ import (
"git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/prometheus/client_golang/prometheus"
)
var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
ac, err := arvados.NewClientFromConfig(cluster)
if err != nil {
return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
@@ -25,6 +26,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) ser
Context: ctx,
ArvClient: ac,
AuthToken: token,
+ Registry: reg,
}
go d.Start()
return d
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 731c6d25d..f0aa83c2e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -48,10 +48,10 @@ type dispatcher struct {
Context context.Context
ArvClient *arvados.Client
AuthToken string
+ Registry *prometheus.Registry
InstanceSetID cloud.InstanceSetID
logger logrus.FieldLogger
- reg *prometheus.Registry
instanceSet cloud.InstanceSet
pool pool
queue scheduler.ContainerQueue
@@ -132,14 +132,13 @@ func (disp *dispatcher) initialize() {
disp.sshKey = key
}
- disp.reg = prometheus.NewRegistry()
- instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.reg)
+ instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.Registry)
if err != nil {
disp.logger.Fatalf("error initializing driver: %s", err)
}
disp.instanceSet = instanceSet
- disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.reg, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
- disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, disp.ArvClient)
+ disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.Registry, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
+ disp.queue = container.NewQueue(disp.logger, disp.Registry, disp.typeChooser, disp.ArvClient)
if disp.Cluster.ManagementToken == "" {
disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -154,7 +153,7 @@ func (disp *dispatcher) initialize() {
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/drain", disp.apiInstanceDrain)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/run", disp.apiInstanceRun)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/kill", disp.apiInstanceKill)
- metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+ metricsH := promhttp.HandlerFor(disp.Registry, promhttp.HandlerOpts{
ErrorLog: disp.logger,
})
mux.Handler("GET", "/metrics", metricsH)
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index b6737bc55..c410e5368 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -22,6 +22,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-systemd/daemon"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -30,7 +31,7 @@ type Handler interface {
CheckHealth() error
}
-type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler
+type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string, registry *prometheus.Registry) Handler
type command struct {
newHandler NewHandlerFunc
@@ -68,7 +69,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -87,22 +87,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
if err != nil {
return 1
}
- log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+
+ // Now that we've read the config, replace the bootstrap
+ // logger with a new one according to the logging config.
+ log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
+ logger := log.WithFields(logrus.Fields{
"PID": os.Getpid(),
})
- ctx := ctxlog.Context(c.ctx, log)
+ ctx := ctxlog.Context(c.ctx, logger)
- listen, err := getListenAddr(cluster.Services, c.svcName)
+ listenURL, err := getListenAddr(cluster.Services, c.svcName)
if err != nil {
return 1
}
+ ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
if cluster.SystemRootToken == "" {
- log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
cluster.SystemRootToken = os.Getenv("ARVADOS_API_TOKEN")
}
if cluster.Services.Controller.ExternalURL.Host == "" {
- log.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
u, err := url.Parse("https://" + os.Getenv("ARVADOS_API_HOST"))
if err != nil {
err = fmt.Errorf("ARVADOS_API_HOST: %s", err)
@@ -114,27 +119,42 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
}
}
- handler := c.newHandler(ctx, cluster, cluster.SystemRootToken)
+ reg := prometheus.NewRegistry()
+ handler := c.newHandler(ctx, cluster, cluster.SystemRootToken, reg)
if err = handler.CheckHealth(); err != nil {
return 1
}
+
+ instrumented := httpserver.Instrument(reg, log,
+ httpserver.HandlerWithContext(ctx,
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.HandlerWithContext(ctx,
- httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
+ Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
},
- Addr: listen,
+ Addr: listenURL.Host,
+ }
+ if listenURL.Scheme == "https" {
+ tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger)
+ if err != nil {
+ logger.WithError(err).Errorf("cannot start %s service on %s", c.svcName, listenURL.String())
+ return 1
+ }
+ srv.TLSConfig = tlsconfig
}
err = srv.Start()
if err != nil {
return 1
}
- log.WithFields(logrus.Fields{
+ logger.WithFields(logrus.Fields{
+ "URL": listenURL,
"Listen": srv.Addr,
"Service": c.svcName,
}).Info("listening")
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.WithError(err).Errorf("error notifying init daemon")
+ logger.WithError(err).Errorf("error notifying init daemon")
}
go func() {
<-ctx.Done()
@@ -149,20 +169,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) {
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
- return "", fmt.Errorf("unknown service name %q", prog)
+ return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
}
for url := range svc.InternalURLs {
if strings.HasPrefix(url.Host, "localhost:") {
- return url.Host, nil
+ return url, nil
}
listener, err := net.Listen("tcp", url.Host)
if err == nil {
listener.Close()
- return url.Host, nil
+ return url, nil
}
}
- return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
+ return arvados.URL{}, fmt.Errorf("configuration does not enable the %s service on this host", prog)
+}
+
+type contextKeyURL struct{}
+
+func URLFromContext(ctx context.Context) (arvados.URL, bool) {
+ u, ok := ctx.Value(contextKeyURL{}).(arvados.URL)
+ return u, ok
}
diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index bb7c5c51d..ef047bc9d 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -8,14 +8,17 @@ package service
import (
"bytes"
"context"
+ "crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"os"
"testing"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
@@ -38,7 +41,7 @@ func (*Suite) TestCommand(c *check.C) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string) Handler {
+ cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
c.Check(ctx.Value("foo"), check.Equals, "bar")
c.Check(token, check.Equals, "abcde")
return &testHandler{ctx: ctx, healthCheck: healthCheck}
@@ -62,12 +65,77 @@ func (*Suite) TestCommand(c *check.C) {
c.Check(stderr.String(), check.Matches, `(?ms).*"msg":"CheckHealth called".*`)
}
+func (*Suite) TestTLS(c *check.C) {
+ cwd, err := os.Getwd()
+ c.Assert(err, check.IsNil)
+
+ stdin := bytes.NewBufferString(`
+Clusters:
+ zzzzz:
+ SystemRootToken: abcde
+ Services:
+ Controller:
+ ExternalURL: "https://localhost:12345"
+ InternalURLs: {"https://localhost:12345": {}}
+ TLS:
+ Key: file://` + cwd + `/../../services/api/tmp/self-signed.key
+ Certificate: file://` + cwd + `/../../services/api/tmp/self-signed.pem
+`)
+
+ called := make(chan bool)
+ cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
+ return &testHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("ok"))
+ close(called)
+ })}
+ })
+
+ exited := make(chan bool)
+ var stdout, stderr bytes.Buffer
+ go func() {
+ cmd.RunCommand("arvados-controller", []string{"-config", "-"}, stdin, &stdout, &stderr)
+ close(exited)
+ }()
+ got := make(chan bool)
+ go func() {
+ defer close(got)
+ client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+ for range time.NewTicker(time.Millisecond).C {
+ resp, err := client.Get("https://localhost:12345")
+ if err != nil {
+ c.Log(err)
+ continue
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Logf("status %d, body %s", resp.StatusCode, string(body))
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ break
+ }
+ }()
+ select {
+ case <-called:
+ case <-exited:
+ c.Error("command exited without calling handler")
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
+ select {
+ case <-got:
+ case <-exited:
+ c.Error("command exited before client received response")
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
+ c.Log(stderr.String())
+}
+
type testHandler struct {
ctx context.Context
+ handler http.Handler
healthCheck chan bool
}
-func (th *testHandler) ServeHTTP(http.ResponseWriter, *http.Request) {}
+func (th *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { th.handler.ServeHTTP(w, r) }
func (th *testHandler) CheckHealth() error {
ctxlog.FromContext(th.ctx).Info("CheckHealth called")
select {
diff --git a/lib/service/tls.go b/lib/service/tls.go
new file mode 100644
index 000000000..5f14bc5e8
--- /dev/null
+++ b/lib/service/tls.go
@@ -0,0 +1,81 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package service
+
+import (
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+ currentCert := make(chan *tls.Certificate, 1)
+ loaded := false
+
+ key, cert := cluster.TLS.Key, cluster.TLS.Certificate
+ if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
+ return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+ }
+ key, cert = key[7:], cert[7:]
+
+ update := func() error {
+ cert, err := tls.LoadX509KeyPair(cert, key)
+ if err != nil {
+ return fmt.Errorf("error loading X509 key pair: %s", err)
+ }
+ if loaded {
+ // Throw away old cert
+ <-currentCert
+ }
+ currentCert <- &cert
+ loaded = true
+ return nil
+ }
+ err := update()
+ if err != nil {
+ return nil, err
+ }
+
+ go func() {
+ reload := make(chan os.Signal, 1)
+ signal.Notify(reload, syscall.SIGHUP)
+ for range reload {
+ err := update()
+ if err != nil {
+ logger.WithError(err).Warn("error updating TLS certificate")
+ }
+ }
+ }()
+
+ // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
+ return &tls.Config{
+ PreferServerCipherSuites: true,
+ CurvePreferences: []tls.CurveID{
+ tls.CurveP256,
+ tls.X25519,
+ },
+ MinVersion: tls.VersionTLS12,
+ CipherSuites: []uint16{
+ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ },
+ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ cert := <-currentCert
+ currentCert <- cert
+ return cert, nil
+ },
+ }, nil
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 5a18972f5..c51f609f8 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -81,6 +81,8 @@ type Cluster struct {
DisabledAPIs StringSet
MaxIndexDatabaseRead int
MaxItemsPerResponse int
+ MaxConcurrentRequests int
+ MaxKeepBlockBuffers int
MaxRequestAmplification int
MaxRequestSize int
RailsSessionSecretToken string
@@ -96,13 +98,19 @@ type Cluster struct {
UnloggedAttributes StringSet
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]struct {
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ BlobTrash bool
+ BlobTrashLifetime Duration
+ BlobTrashCheckInterval Duration
+ BlobTrashConcurrency int
+ BlobDeleteConcurrency int
+ BlobReplicateConcurrency int
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
Value interface{}
Function string
Protected bool
@@ -157,6 +165,7 @@ type Cluster struct {
UserNotifierEmailFrom string
UserProfileNotificationAddress string
}
+ Volumes map[string]Volume
Workbench struct {
ActivationContactLink string
APIClientConnectTimeout Duration
@@ -196,6 +205,48 @@ type Cluster struct {
EnableBetaController14287 bool
}
+type Volume struct {
+ AccessViaHosts map[URL]VolumeAccess
+ ReadOnly bool
+ Replication int
+ StorageClasses map[string]bool
+ Driver string
+ DriverParameters json.RawMessage
+}
+
+type S3VolumeDriverParameters struct {
+ AccessKey string
+ SecretKey string
+ Endpoint string
+ Region string
+ Bucket string
+ LocationConstraint bool
+ IndexPageSize int
+ ConnectTimeout Duration
+ ReadTimeout Duration
+ RaceWindow Duration
+ UnsafeDelete bool
+}
+
+type AzureVolumeDriverParameters struct {
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string
+ ContainerName string
+ RequestTimeout Duration
+ ListBlobsRetryDelay Duration
+ ListBlobsMaxAttempts int
+}
+
+type DirectoryVolumeDriverParameters struct {
+ Root string
+ Serialize bool
+}
+
+type VolumeAccess struct {
+ ReadOnly bool
+}
+
type Services struct {
Composer Service
Controller Service
@@ -239,6 +290,10 @@ func (su URL) MarshalText() ([]byte, error) {
return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
}
+func (su URL) String() string {
+ return (*url.URL)(&su).String()
+}
+
type ServiceInstance struct{}
type PostgreSQL struct {
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 0c866354a..e0ae1758d 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -23,11 +23,11 @@ type KeepService struct {
}
type KeepMount struct {
- UUID string `json:"uuid"`
- DeviceID string `json:"device_id"`
- ReadOnly bool `json:"read_only"`
- Replication int `json:"replication"`
- StorageClasses []string `json:"storage_classes"`
+ UUID string `json:"uuid"`
+ DeviceID string `json:"device_id"`
+ ReadOnly bool `json:"read_only"`
+ Replication int `json:"replication"`
+ StorageClasses map[string]bool `json:"storage_classes"`
}
// KeepServiceList is an arvados#keepServiceList record
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index 490a7f3e0..5b01db5c4 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -111,16 +111,16 @@ func StopAPI() {
}
// StartKeep starts the given number of keep servers,
-// optionally with -enforce-permissions enabled.
-// Use numKeepServers = 2 and enforcePermissions = false under all normal circumstances.
-func StartKeep(numKeepServers int, enforcePermissions bool) {
+// optionally with --keep-blob-signing enabled.
+// Use numKeepServers = 2 and blobSigning = false under all normal circumstances.
+func StartKeep(numKeepServers int, blobSigning bool) {
cwd, _ := os.Getwd()
defer os.Chdir(cwd)
chdirToPythonTests()
cmdArgs := []string{"run_test_server.py", "start_keep", "--num-keep-servers", strconv.Itoa(numKeepServers)}
- if enforcePermissions {
- cmdArgs = append(cmdArgs, "--keep-enforce-permissions")
+ if blobSigning {
+ cmdArgs = append(cmdArgs, "--keep-blob-signing")
}
bgRun(exec.Command("python", cmdArgs...))
diff --git a/sdk/go/ctxlog/log.go b/sdk/go/ctxlog/log.go
index e66eeadee..a17ad8d83 100644
--- a/sdk/go/ctxlog/log.go
+++ b/sdk/go/ctxlog/log.go
@@ -11,7 +11,6 @@ import (
"os"
"github.com/sirupsen/logrus"
- check "gopkg.in/check.v1"
)
var (
@@ -41,7 +40,7 @@ func FromContext(ctx context.Context) logrus.FieldLogger {
// New returns a new logger with the indicated format and
// level.
-func New(out io.Writer, format, level string) logrus.FieldLogger {
+func New(out io.Writer, format, level string) *logrus.Logger {
logger := logrus.New()
logger.Out = out
setFormat(logger, format)
@@ -49,7 +48,7 @@ func New(out io.Writer, format, level string) logrus.FieldLogger {
return logger
}
-func TestLogger(c *check.C) logrus.FieldLogger {
+func TestLogger(c interface{ Log(...interface{}) }) *logrus.Logger {
logger := logrus.New()
logger.Out = &logWriter{c.Log}
setFormat(logger, "text")
diff --git a/sdk/go/httpserver/httpserver.go b/sdk/go/httpserver/httpserver.go
index a94146f85..627e04f0b 100644
--- a/sdk/go/httpserver/httpserver.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -43,7 +43,12 @@ func (srv *Server) Start() error {
srv.cond = sync.NewCond(mutex.RLocker())
srv.running = true
go func() {
- err = srv.Serve(tcpKeepAliveListener{srv.listener})
+ lnr := tcpKeepAliveListener{srv.listener}
+ if srv.TLSConfig != nil {
+ err = srv.ServeTLS(lnr, "", "")
+ } else {
+ err = srv.Serve(lnr)
+ }
if !srv.wantDown {
srv.err = err
}
diff --git a/sdk/go/httpserver/request_limiter.go b/sdk/go/httpserver/request_limiter.go
index e7192d5b4..23e6e016d 100644
--- a/sdk/go/httpserver/request_limiter.go
+++ b/sdk/go/httpserver/request_limiter.go
@@ -6,6 +6,9 @@ package httpserver
import (
"net/http"
+ "sync/atomic"
+
+ "github.com/prometheus/client_golang/prometheus"
)
// RequestCounter is an http.Handler that tracks the number of
@@ -24,19 +27,45 @@ type RequestCounter interface {
type limiterHandler struct {
requests chan struct{}
handler http.Handler
+ count int64 // only used if cap(requests)==0
}
// NewRequestLimiter returns a RequestCounter that delegates up to
// maxRequests at a time to the given handler, and responds 503 to all
// incoming requests beyond that limit.
-func NewRequestLimiter(maxRequests int, handler http.Handler) RequestCounter {
- return &limiterHandler{
+//
+// "concurrent_requests" and "max_concurrent_requests" metrics are
+// registered with the given reg, if reg is not nil.
+func NewRequestLimiter(maxRequests int, handler http.Handler, reg *prometheus.Registry) RequestCounter {
+ h := &limiterHandler{
requests: make(chan struct{}, maxRequests),
handler: handler,
}
+ if reg != nil {
+ reg.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "concurrent_requests",
+ Help: "Number of requests in progress",
+ },
+ func() float64 { return float64(h.Current()) },
+ ))
+ reg.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "max_concurrent_requests",
+ Help: "Maximum number of concurrent requests",
+ },
+ func() float64 { return float64(h.Max()) },
+ ))
+ }
+ return h
}
func (h *limiterHandler) Current() int {
+ if cap(h.requests) == 0 {
+ return int(atomic.LoadInt64(&h.count))
+ }
return len(h.requests)
}
@@ -45,6 +74,11 @@ func (h *limiterHandler) Max() int {
}
func (h *limiterHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if cap(h.requests) == 0 {
+ atomic.AddInt64(&h.count, 1)
+ h.handler.ServeHTTP(resp, req)
+ atomic.AddInt64(&h.count, -1)
+ }
select {
case h.requests <- struct{}{}:
default:
diff --git a/sdk/go/httpserver/request_limiter_test.go b/sdk/go/httpserver/request_limiter_test.go
index afa4e3faa..64d1f3d4c 100644
--- a/sdk/go/httpserver/request_limiter_test.go
+++ b/sdk/go/httpserver/request_limiter_test.go
@@ -31,7 +31,7 @@ func newTestHandler(maxReqs int) *testHandler {
func TestRequestLimiter1(t *testing.T) {
h := newTestHandler(10)
- l := NewRequestLimiter(1, h)
+ l := NewRequestLimiter(1, h, nil)
var wg sync.WaitGroup
resps := make([]*httptest.ResponseRecorder, 10)
for i := 0; i < 10; i++ {
@@ -91,7 +91,7 @@ func TestRequestLimiter1(t *testing.T) {
func TestRequestLimiter10(t *testing.T) {
h := newTestHandler(10)
- l := NewRequestLimiter(10, h)
+ l := NewRequestLimiter(10, h, nil)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 34342059f..212e1309f 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -399,9 +399,9 @@ def get_config():
with open(os.environ["ARVADOS_CONFIG"]) as f:
return yaml.safe_load(f)
-def internal_port_from_config(service):
+def internal_port_from_config(service, idx=0):
return int(urlparse(
- list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0]).
+ sorted(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys()))[idx]).
netloc.split(":")[1])
def external_port_from_config(service):
@@ -444,47 +444,42 @@ def stop_ws():
return
kill_server_pid(_pidfile('ws'))
-def _start_keep(n, keep_args):
- keep0 = tempfile.mkdtemp()
- port = find_available_port()
- keep_cmd = ["keepstore",
- "-volume={}".format(keep0),
- "-listen=:{}".format(port),
- "-pid="+_pidfile('keep{}'.format(n))]
-
- for arg, val in keep_args.items():
- keep_cmd.append("{}={}".format(arg, val))
-
+def _start_keep(n, blob_signing=False):
+ datadir = os.path.join(TEST_TMPDIR, "keep%d.data"%n)
+ if os.path.exists(datadir):
+ shutil.rmtree(datadir)
+ os.mkdir(datadir)
+ port = internal_port_from_config("Keepstore", idx=n)
+ conf = os.path.join(TEST_TMPDIR, "keep%d.yaml"%n)
+ confdata = get_config()
+ confdata['Clusters']['zzzzz']['Services']['Keepstore']['InternalURLs'] = {"http://127.0.0.1:%d"%port: {}}
+ confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing
+ with open(conf, 'w') as f:
+ yaml.safe_dump(confdata, f)
+ f.truncate()
+ legacyconf = os.path.join(TEST_TMPDIR, "keep%d-legacy.yaml"%n)
+ with open(legacyconf, 'w') as f:
+ yaml.safe_dump({"Listen": ":%d"%port}, f)
+ f.truncate()
+ keep_cmd = ["keepstore", "-config", conf, "-legacy-keepstore-config", legacyconf]
+
+ print('logfilename is %s'%_logfilename('keep{}'.format(n)), file=sys.stderr)
with open(_logfilename('keep{}'.format(n)), 'a') as logf:
with open('/dev/null') as _stdin:
- kp0 = subprocess.Popen(
+ child = subprocess.Popen(
keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+ print('child.pid is %d'%child.pid, file=sys.stderr)
with open(_pidfile('keep{}'.format(n)), 'w') as f:
- f.write(str(kp0.pid))
-
- with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
- f.write(keep0)
+ f.write(str(child.pid))
_wait_until_port_listens(port)
return port
-def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
+def run_keep(num_servers=2, **kwargs):
stop_keep(num_servers)
- keep_args = {}
- if not blob_signing_key:
- blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
- with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
- keep_args['-blob-signing-key-file'] = f.name
- f.write(blob_signing_key)
- keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
- with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
- keep_args['-data-manager-token-file'] = f.name
- f.write(auth_token('data_manager'))
- keep_args['-never-delete'] = 'false'
-
api = arvados.api(
version='v1',
host=os.environ['ARVADOS_API_HOST'],
@@ -497,7 +492,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
api.keep_disks().delete(uuid=d['uuid']).execute()
for d in range(0, num_servers):
- port = _start_keep(d, keep_args)
+ port = _start_keep(d, **kwargs)
svc = api.keep_services().create(body={'keep_service': {
'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
'service_host': 'localhost',
@@ -522,12 +517,6 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
def _stop_keep(n):
kill_server_pid(_pidfile('keep{}'.format(n)))
- if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
- with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
- shutil.rmtree(r.read(), True)
- os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
- if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
- os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
def stop_keep(num_servers=2):
for n in range(0, num_servers):
@@ -663,6 +652,8 @@ def setup_config():
git_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
keepproxy_external_port = find_available_port()
+ keepstore0_port = find_available_port()
+ keepstore1_port = find_available_port()
keep_web_port = find_available_port()
keep_web_external_port = find_available_port()
keep_web_dl_port = find_available_port()
@@ -678,45 +669,51 @@ def setup_config():
services = {
"RailsAPI": {
"InternalURLs": {
- "https://%s:%s"%(localhost, rails_api_port): {}
- }
+ "https://%s:%s"%(localhost, rails_api_port): {},
+ },
},
"Controller": {
"ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, controller_port): {}
- }
+ "http://%s:%s"%(localhost, controller_port): {},
+ },
},
"Websocket": {
"ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, websocket_port): {}
- }
+ "http://%s:%s"%(localhost, websocket_port): {},
+ },
},
"GitHTTP": {
"ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
"InternalURLs": {
"http://%s:%s"%(localhost, git_httpd_port): {}
- }
+ },
+ },
+ "Keepstore": {
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, keepstore0_port): {},
+ "http://%s:%s"%(localhost, keepstore1_port): {},
+ },
},
"Keepproxy": {
"ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keepproxy_port): {}
- }
+ "http://%s:%s"%(localhost, keepproxy_port): {},
+ },
},
"WebDAV": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_port): {}
- }
+ "http://%s:%s"%(localhost, keep_web_port): {},
+ },
},
"WebDAVDownload": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_dl_port): {}
- }
- }
+ "http://%s:%s"%(localhost, keep_web_dl_port): {},
+ },
+ },
}
config = {
@@ -724,30 +721,52 @@ def setup_config():
"zzzzz": {
"EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
"ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
+ "SystemRootToken": auth_token('data_manager'),
"API": {
- "RequestTimeout": "30s"
+ "RequestTimeout": "30s",
},
"SystemLogs": {
- "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
+ "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
},
"PostgreSQL": {
"Connection": pgconnection,
},
"TLS": {
- "Insecure": True
+ "Insecure": True,
},
"Services": services,
"Users": {
- "AnonymousUserToken": auth_token('anonymous')
+ "AnonymousUserToken": auth_token('anonymous'),
},
"Collections": {
- "TrustAllContent": True
+ "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
+ "TrustAllContent": True,
},
"Git": {
- "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
- }
- }
- }
+ "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git'),
+ },
+ "Volumes": {
+ "zzzzz-nyw5e-000000000000000": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore1_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep0.data"),
+ },
+ },
+ "zzzzz-nyw5e-111111111111111": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore0_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep1.data"),
+ },
+ },
+ },
+ },
+ },
}
conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
@@ -864,7 +883,7 @@ if __name__ == "__main__":
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
- parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
+ parser.add_argument('--keep-blob-signing', action="store_true", help="Enable blob signing for keepstore servers")
args = parser.parse_args()
@@ -895,7 +914,7 @@ if __name__ == "__main__":
elif args.action == 'stop_controller':
stop_controller()
elif args.action == 'start_keep':
- run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
+ run_keep(blob_signing=args.keep_blob_signing, num_servers=args.num_keep_servers)
elif args.action == 'stop_keep':
stop_keep(num_servers=args.num_keep_servers)
elif args.action == 'start_keep_proxy':
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index d6b3a2a12..80e6987b3 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -130,8 +130,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
- KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
- 'enforce_permissions': True}
+ KEEP_SERVER = {'blob_signing': True}
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
@@ -173,70 +172,6 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
unsigned_bar_locator)
-# KeepOptionalPermission: starts Keep with --permission-key-file
-# but not --enforce-permissions (i.e. generate signatures on PUT
-# requests, but do not require them for GET requests)
-#
-# All of these requests should succeed when permissions are optional:
-# * authenticated request, signed locator
-# * authenticated request, unsigned locator
-# * unauthenticated request, signed locator
-# * unauthenticated request, unsigned locator
-class KeepOptionalPermission(run_test_server.TestCaseWithServers):
- MAIN_SERVER = {}
- KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
- 'enforce_permissions': False}
-
- @classmethod
- def setUpClass(cls):
- super(KeepOptionalPermission, cls).setUpClass()
- run_test_server.authorize_with("admin")
- cls.api_client = arvados.api('v1')
-
- def setUp(self):
- super(KeepOptionalPermission, self).setUp()
- self.keep_client = arvados.KeepClient(api_client=self.api_client,
- proxy='', local_store='')
-
- def _put_foo_and_check(self):
- signed_locator = self.keep_client.put('foo')
- self.assertRegex(
- signed_locator,
- r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
- 'invalid locator from Keep.put("foo"): ' + signed_locator)
- return signed_locator
-
- def test_KeepAuthenticatedSignedTest(self):
- signed_locator = self._put_foo_and_check()
- self.assertEqual(self.keep_client.get(signed_locator),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepAuthenticatedUnsignedTest(self):
- signed_locator = self._put_foo_and_check()
- self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepUnauthenticatedSignedTest(self):
- # Check that signed GET requests work even when permissions
- # enforcement is off.
- signed_locator = self._put_foo_and_check()
- self.keep_client.api_token = ''
- self.assertEqual(self.keep_client.get(signed_locator),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepUnauthenticatedUnsignedTest(self):
- # Since --enforce-permissions is not in effect, GET requests
- # need not be authenticated.
- signed_locator = self._put_foo_and_check()
- self.keep_client.api_token = ''
- self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
-
class KeepProxyTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
KEEP_SERVER = {}
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 3c17b3bd0..b52b706b2 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -7,8 +7,8 @@ package main
import (
"bytes"
"context"
+ "encoding/json"
"errors"
- "flag"
"fmt"
"io"
"io/ioutil"
@@ -22,99 +22,94 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-const (
- azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
- azureDefaultListBlobsMaxAttempts = 12
- azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
-)
-
-var (
- azureMaxGetBytes int
- azureStorageAccountName string
- azureStorageAccountKeyFile string
- azureStorageReplication int
- azureWriteRaceInterval = 15 * time.Second
- azureWriteRacePollTime = time.Second
-)
+func init() {
+ driver["Azure"] = newAzureBlobVolume
+}
-func readKeyFromFile(file string) (string, error) {
- buf, err := ioutil.ReadFile(file)
+func newAzureBlobVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &AzureBlobVolume{
+ StorageBaseURL: storage.DefaultBaseURL,
+ RequestTimeout: azureDefaultRequestTimeout,
+ WriteRaceInterval: azureDefaultWriteRaceInterval,
+ WriteRacePollTime: azureDefaultWriteRacePollTime,
+ cluster: cluster,
+ volume: volume,
+ logger: logger,
+ metrics: metrics,
+ }
+ err := json.Unmarshal(volume.DriverParameters, &v)
if err != nil {
- return "", errors.New("reading key from " + file + ": " + err.Error())
+ return nil, err
}
- accountKey := strings.TrimSpace(string(buf))
- if accountKey == "" {
- return "", errors.New("empty account key in " + file)
+ if v.ListBlobsRetryDelay == 0 {
+ v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
+ }
+ if v.ListBlobsMaxAttempts == 0 {
+ v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
+ }
+ if v.ContainerName == "" || v.StorageAccountName == "" || v.StorageAccountKey == "" {
+ return nil, errors.New("DriverParameters: ContainerName, StorageAccountName, and StorageAccountKey must be provided")
+ }
+ azc, err := storage.NewClient(v.StorageAccountName, v.StorageAccountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
+ if err != nil {
+ return nil, fmt.Errorf("creating Azure storage client: %s", err)
+ }
+ v.azClient = azc
+ v.azClient.Sender = &singleSender{}
+ v.azClient.HTTPClient = &http.Client{
+ Timeout: time.Duration(v.RequestTimeout),
+ }
+ bs := v.azClient.GetBlobService()
+ v.container = &azureContainer{
+ ctr: bs.GetContainerReference(v.ContainerName),
}
- return accountKey, nil
-}
-
-type azureVolumeAdder struct {
- *Config
-}
-// String implements flag.Value
-func (s *azureVolumeAdder) String() string {
- return "-"
+ if ok, err := v.container.Exists(); err != nil {
+ return nil, err
+ } else if !ok {
+ return nil, fmt.Errorf("Azure container %q does not exist: %s", v.ContainerName, err)
+ }
+ return v, v.check()
}
-func (s *azureVolumeAdder) Set(containerName string) error {
- s.Config.Volumes = append(s.Config.Volumes, &AzureBlobVolume{
- ContainerName: containerName,
- StorageAccountName: azureStorageAccountName,
- StorageAccountKeyFile: azureStorageAccountKeyFile,
- AzureReplication: azureStorageReplication,
- ReadOnly: deprecated.flagReadonly,
- })
+func (v *AzureBlobVolume) check() error {
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
return nil
}
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &AzureBlobVolume{} })
-
- flag.Var(&azureVolumeAdder{theConfig},
- "azure-storage-container-volume",
- "Use the given container as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &azureStorageAccountName,
- "azure-storage-account-name",
- "",
- "Azure storage account name used for subsequent --azure-storage-container-volume arguments.")
- flag.StringVar(
- &azureStorageAccountKeyFile,
- "azure-storage-account-key-file",
- "",
- "`File` containing the account key used for subsequent --azure-storage-container-volume arguments.")
- flag.IntVar(
- &azureStorageReplication,
- "azure-storage-replication",
- 3,
- "Replication level to report to clients when data is stored in an Azure container.")
- flag.IntVar(
- &azureMaxGetBytes,
- "azure-max-get-bytes",
- BlockSize,
- fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
-}
+const (
+ azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+ azureDefaultListBlobsMaxAttempts = 12
+ azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
+ azureDefaultWriteRaceInterval = arvados.Duration(15 * time.Second)
+ azureDefaultWriteRacePollTime = arvados.Duration(time.Second)
+)
// An AzureBlobVolume stores and retrieves blocks in an Azure Blob
// container.
type AzureBlobVolume struct {
- StorageAccountName string
- StorageAccountKeyFile string
- StorageBaseURL string // "" means default, "core.windows.net"
- ContainerName string
- AzureReplication int
- ReadOnly bool
- RequestTimeout arvados.Duration
- StorageClasses []string
- ListBlobsRetryDelay arvados.Duration
- ListBlobsMaxAttempts int
-
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string // "" means default, "core.windows.net"
+ ContainerName string
+ RequestTimeout arvados.Duration
+ ListBlobsRetryDelay arvados.Duration
+ ListBlobsMaxAttempts int
+ MaxGetBytes int
+ WriteRaceInterval arvados.Duration
+ WriteRacePollTime arvados.Duration
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
azClient storage.Client
container *azureContainer
}
@@ -127,84 +122,13 @@ func (*singleSender) Send(c *storage.Client, req *http.Request) (resp *http.Resp
return c.HTTPClient.Do(req)
}
-// Examples implements VolumeWithExamples.
-func (*AzureBlobVolume) Examples() []Volume {
- return []Volume{
- &AzureBlobVolume{
- StorageAccountName: "example-account-name",
- StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
- ContainerName: "example-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- &AzureBlobVolume{
- StorageAccountName: "cn-account-name",
- StorageAccountKeyFile: "/etc/azure_cn_storage_account_key.txt",
- StorageBaseURL: "core.chinacloudapi.cn",
- ContainerName: "cn-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- }
-}
-
// Type implements Volume.
func (v *AzureBlobVolume) Type() string {
return "Azure"
}
-// Start implements Volume.
-func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
- if v.ListBlobsRetryDelay == 0 {
- v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
- }
- if v.ListBlobsMaxAttempts == 0 {
- v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
- }
- if v.ContainerName == "" {
- return errors.New("no container name given")
- }
- if v.StorageAccountName == "" || v.StorageAccountKeyFile == "" {
- return errors.New("StorageAccountName and StorageAccountKeyFile must be given")
- }
- accountKey, err := readKeyFromFile(v.StorageAccountKeyFile)
- if err != nil {
- return err
- }
- if v.StorageBaseURL == "" {
- v.StorageBaseURL = storage.DefaultBaseURL
- }
- v.azClient, err = storage.NewClient(v.StorageAccountName, accountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
- if err != nil {
- return fmt.Errorf("creating Azure storage client: %s", err)
- }
- v.azClient.Sender = &singleSender{}
-
- if v.RequestTimeout == 0 {
- v.RequestTimeout = azureDefaultRequestTimeout
- }
- v.azClient.HTTPClient = &http.Client{
- Timeout: time.Duration(v.RequestTimeout),
- }
- bs := v.azClient.GetBlobService()
- v.container = &azureContainer{
- ctr: bs.GetContainerReference(v.ContainerName),
- }
-
- if ok, err := v.container.Exists(); err != nil {
- return err
- } else if !ok {
- return fmt.Errorf("Azure container %q does not exist", v.ContainerName)
- }
- // Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(lbls)
-
- return nil
-}
-
-// DeviceID returns a globally unique ID for the storage container.
-func (v *AzureBlobVolume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage container.
+func (v *AzureBlobVolume) GetDeviceID() string {
return "azure://" + v.StorageBaseURL + "/" + v.StorageAccountName + "/" + v.ContainerName
}
@@ -245,14 +169,14 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
if !haveDeadline {
t, err := v.Mtime(loc)
if err != nil {
- log.Print("Got empty block (possible race) but Mtime failed: ", err)
+ ctxlog.FromContext(ctx).Print("Got empty block (possible race) but Mtime failed: ", err)
break
}
- deadline = t.Add(azureWriteRaceInterval)
+ deadline = t.Add(v.WriteRaceInterval.Duration())
if time.Now().After(deadline) {
break
}
- log.Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
+ ctxlog.FromContext(ctx).Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
haveDeadline = true
} else if time.Now().After(deadline) {
break
@@ -260,12 +184,12 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
select {
case <-ctx.Done():
return 0, ctx.Err()
- case <-time.After(azureWriteRacePollTime):
+ case <-time.After(v.WriteRacePollTime.Duration()):
}
size, err = v.get(ctx, loc, buf)
}
if haveDeadline {
- log.Printf("Race ended with size==%d", size)
+ ctxlog.FromContext(ctx).Printf("Race ended with size==%d", size)
}
return size, err
}
@@ -273,8 +197,15 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
+
+ pieceSize := BlockSize
+ if v.MaxGetBytes > 0 && v.MaxGetBytes < BlockSize {
+ pieceSize = v.MaxGetBytes
+ }
+
+ pieces := 1
expectSize := len(buf)
- if azureMaxGetBytes < BlockSize {
+ if pieceSize < BlockSize {
// Unfortunately the handler doesn't tell us how long the blob
// is expected to be, so we have to ask Azure.
props, err := v.container.GetBlobProperties(loc)
@@ -285,6 +216,7 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
return 0, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
}
expectSize = int(props.ContentLength)
+ pieces = (expectSize + pieceSize - 1) / pieceSize
}
if expectSize == 0 {
@@ -293,7 +225,6 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// We'll update this actualSize if/when we get the last piece.
actualSize := -1
- pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
errors := make(chan error, pieces)
var wg sync.WaitGroup
wg.Add(pieces)
@@ -308,8 +239,8 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// interrupted as a result.
go func(p int) {
defer wg.Done()
- startPos := p * azureMaxGetBytes
- endPos := startPos + azureMaxGetBytes
+ startPos := p * pieceSize
+ endPos := startPos + pieceSize
if endPos > expectSize {
endPos = expectSize
}
@@ -412,7 +343,7 @@ func (v *AzureBlobVolume) Compare(ctx context.Context, loc string, expect []byte
// Put stores a Keep block as a block blob in the container.
func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
// Send the block data through a pipe, so that (if we need to)
@@ -441,7 +372,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
+ ctxlog.FromContext(ctx).Debugf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// io.Copy() to read. If so, un-stick it. This means
// CreateBlockBlobFromReader will get corrupt data,
@@ -450,7 +381,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
+ ctxlog.FromContext(ctx).Debugf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
return ctx.Err()
case err := <-errChan:
return err
@@ -459,7 +390,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
// Touch updates the last-modified property of a block blob.
func (v *AzureBlobVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
trashed, metadata, err := v.checkTrashed(loc)
@@ -508,7 +439,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
continue
}
modtime := time.Time(b.Properties.LastModified)
- if b.Properties.ContentLength == 0 && modtime.Add(azureWriteRaceInterval).After(time.Now()) {
+ if b.Properties.ContentLength == 0 && modtime.Add(v.WriteRaceInterval.Duration()).After(time.Now()) {
// A new zero-length blob is probably
// just a new non-empty blob that
// hasn't committed its data yet (see
@@ -535,7 +466,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
resp, err = v.container.ListBlobs(params)
err = v.translateError(err)
if err == VolumeBusyError {
- log.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
+ v.logger.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
time.Sleep(time.Duration(v.ListBlobsRetryDelay))
continue
} else {
@@ -547,7 +478,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
// Trash a Keep block.
func (v *AzureBlobVolume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -562,12 +493,12 @@ func (v *AzureBlobVolume) Trash(loc string) error {
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
// If TrashLifetime == 0, just delete it
- if theConfig.TrashLifetime == 0 {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{
IfMatch: props.Etag,
})
@@ -575,7 +506,7 @@ func (v *AzureBlobVolume) Trash(loc string) error {
// Otherwise, mark as trash
return v.container.SetBlobMetadata(loc, storage.BlobMetadata{
- "expires_at": fmt.Sprintf("%d", time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()),
+ "expires_at": fmt.Sprintf("%d", time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()),
}, &storage.SetBlobMetadataOptions{
IfMatch: props.Etag,
})
@@ -613,23 +544,6 @@ func (v *AzureBlobVolume) String() string {
return fmt.Sprintf("azure-storage-container:%+q", v.ContainerName)
}
-// Writable returns true, unless the -readonly flag was on when the
-// volume was added.
-func (v *AzureBlobVolume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the replication level of the container, as
-// specified by the -azure-storage-replication argument.
-func (v *AzureBlobVolume) Replication() int {
- return v.AzureReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *AzureBlobVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// If possible, translate an Azure SDK error to a recognizable error
// like os.ErrNotExist.
func (v *AzureBlobVolume) translateError(err error) error {
@@ -656,6 +570,10 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
@@ -670,7 +588,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
if err != nil {
- log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
+ v.logger.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
return
}
@@ -682,7 +600,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
IfMatch: b.Properties.Etag,
})
if err != nil {
- log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+ v.logger.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
return
}
atomic.AddInt64(&blocksDeleted, 1)
@@ -690,8 +608,8 @@ func (v *AzureBlobVolume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan storage.Blob, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan storage.Blob, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -705,7 +623,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
for page := 1; ; page++ {
resp, err := v.listBlobs(page, params)
if err != nil {
- log.Printf("EmptyTrash: ListBlobs: %v", err)
+ v.logger.Printf("EmptyTrash: ListBlobs: %v", err)
break
}
for _, b := range resp.Blobs {
@@ -719,7 +637,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
close(todo)
wg.Wait()
- log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+ v.logger.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
}
// InternalStats returns bucket I/O and API call counters.
@@ -748,7 +666,6 @@ func (s *azureBlobStats) TickErr(err error) {
if err, ok := err.(storage.AzureStorageServiceError); ok {
errType = errType + fmt.Sprintf(" %d (%s)", err.StatusCode, err.Code)
}
- log.Printf("errType %T, err %s", err, err)
s.statsTicker.TickErr(err, errType)
}
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 8d02def14..3539c3067 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -24,13 +24,13 @@ import (
"strconv"
"strings"
"sync"
- "testing"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/Azure/azure-sdk-for-go/storage"
- "github.com/ghodss/yaml"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -66,14 +66,16 @@ type azBlob struct {
type azStubHandler struct {
sync.Mutex
+ logger logrus.FieldLogger
blobs map[string]*azBlob
race chan chan struct{}
didlist503 bool
}
-func newAzStubHandler() *azStubHandler {
+func newAzStubHandler(c *check.C) *azStubHandler {
return &azStubHandler{
- blobs: make(map[string]*azBlob),
+ blobs: make(map[string]*azBlob),
+ logger: ctxlog.TestLogger(c),
}
}
@@ -117,7 +119,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
h.Lock()
defer h.Unlock()
if azureTestDebug {
- defer log.Printf("azStubHandler: %+v", r)
+ defer h.logger.Printf("azStubHandler: %+v", r)
}
path := strings.Split(r.URL.Path, "/")
@@ -128,7 +130,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
}
if err := r.ParseForm(); err != nil {
- log.Printf("azStubHandler(%+v): %s", r, err)
+ h.logger.Printf("azStubHandler(%+v): %s", r, err)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -184,13 +186,13 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
case r.Method == "PUT" && r.Form.Get("comp") == "block":
// "Put Block" API
if !blobExists {
- log.Printf("Got block for nonexistent blob: %+v", r)
+ h.logger.Printf("Got block for nonexistent blob: %+v", r)
rw.WriteHeader(http.StatusBadRequest)
return
}
blockID, err := base64.StdEncoding.DecodeString(r.Form.Get("blockid"))
if err != nil || len(blockID) == 0 {
- log.Printf("Invalid blockid: %+q", r.Form.Get("blockid"))
+ h.logger.Printf("Invalid blockid: %+q", r.Form.Get("blockid"))
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -200,14 +202,14 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// "Put Block List" API
bl := &blockListRequestBody{}
if err := xml.Unmarshal(body, bl); err != nil {
- log.Printf("xml Unmarshal: %s", err)
+ h.logger.Printf("xml Unmarshal: %s", err)
rw.WriteHeader(http.StatusBadRequest)
return
}
for _, encBlockID := range bl.Uncommitted {
blockID, err := base64.StdEncoding.DecodeString(encBlockID)
if err != nil || len(blockID) == 0 || blob.Uncommitted[string(blockID)] == nil {
- log.Printf("Invalid blockid: %+q", encBlockID)
+ h.logger.Printf("Invalid blockid: %+q", encBlockID)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -223,7 +225,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// sets metadata headers only as a way to bump Etag
// and Last-Modified.
if !blobExists {
- log.Printf("Got metadata for nonexistent blob: %+v", r)
+ h.logger.Printf("Got metadata for nonexistent blob: %+v", r)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -269,7 +271,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
rw.Header().Set("Content-Length", strconv.Itoa(len(data)))
if r.Method == "GET" {
if _, err := rw.Write(data); err != nil {
- log.Printf("write %+q: %s", data, err)
+ h.logger.Printf("write %+q: %s", data, err)
}
}
h.unlockAndRace()
@@ -333,12 +335,12 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
}
buf, err := xml.Marshal(resp)
if err != nil {
- log.Print(err)
+ h.logger.Error(err)
rw.WriteHeader(http.StatusInternalServerError)
}
rw.Write(buf)
default:
- log.Printf("azStubHandler: not implemented: %+v Body:%+q", r, body)
+ h.logger.Printf("azStubHandler: not implemented: %+v Body:%+q", r, body)
rw.WriteHeader(http.StatusNotImplemented)
}
}
@@ -347,6 +349,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// tries to connect to "devstoreaccount1.blob.127.0.0.1:46067", and
// in such cases transparently dials "127.0.0.1:46067" instead.
type azStubDialer struct {
+ logger logrus.FieldLogger
net.Dialer
}
@@ -355,7 +358,7 @@ var localHostPortRe = regexp.MustCompile(`(127\.0\.0\.1|localhost|\[::1\]):\d+`)
func (d *azStubDialer) Dial(network, address string) (net.Conn, error) {
if hp := localHostPortRe.FindString(address); hp != "" {
if azureTestDebug {
- log.Println("azStubDialer: dial", hp, "instead of", address)
+ d.logger.Debug("azStubDialer: dial", hp, "instead of", address)
}
address = hp
}
@@ -369,29 +372,24 @@ type TestableAzureBlobVolume struct {
t TB
}
-func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableAzureBlobVolume {
- azHandler := newAzStubHandler()
+func (s *StubbedAzureBlobSuite) newTestableAzureBlobVolume(t TB, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs) *TestableAzureBlobVolume {
+ azHandler := newAzStubHandler(t.(*check.C))
azStub := httptest.NewServer(azHandler)
var azClient storage.Client
+ var err error
container := azureTestContainer
if container == "" {
// Connect to stub instead of real Azure storage service
stubURLBase := strings.Split(azStub.URL, "://")[1]
- var err error
if azClient, err = storage.NewClient(fakeAccountName, fakeAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil {
t.Fatal(err)
}
container = "fakecontainername"
} else {
// Connect to real Azure storage service
- accountKey, err := readKeyFromFile(azureStorageAccountKeyFile)
- if err != nil {
- t.Fatal(err)
- }
- azClient, err = storage.NewBasicClient(azureStorageAccountName, accountKey)
- if err != nil {
+ if azClient, err = storage.NewBasicClient(os.Getenv("ARVADOS_TEST_AZURE_ACCOUNT_NAME"), os.Getenv("ARVADOS_TEST_AZURE_ACCOUNT_KEY")); err != nil {
t.Fatal(err)
}
}
@@ -400,12 +398,19 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
bs := azClient.GetBlobService()
v := &AzureBlobVolume{
ContainerName: container,
- ReadOnly: readonly,
- AzureReplication: replication,
+ WriteRaceInterval: arvados.Duration(time.Millisecond),
+ WriteRacePollTime: arvados.Duration(time.Nanosecond),
ListBlobsMaxAttempts: 2,
ListBlobsRetryDelay: arvados.Duration(time.Millisecond),
azClient: azClient,
container: &azureContainer{ctr: bs.GetContainerReference(container)},
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(t),
+ metrics: metrics,
+ }
+ if err = v.check(); err != nil {
+ t.Fatal(err)
}
return &TestableAzureBlobVolume{
@@ -419,84 +424,45 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
var _ = check.Suite(&StubbedAzureBlobSuite{})
type StubbedAzureBlobSuite struct {
- volume *TestableAzureBlobVolume
origHTTPTransport http.RoundTripper
}
func (s *StubbedAzureBlobSuite) SetUpTest(c *check.C) {
s.origHTTPTransport = http.DefaultTransport
http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
+ Dial: (&azStubDialer{logger: ctxlog.TestLogger(c)}).Dial,
}
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
-
- s.volume = NewTestableAzureBlobVolume(c, false, 3)
}
func (s *StubbedAzureBlobSuite) TearDownTest(c *check.C) {
- s.volume.Teardown()
http.DefaultTransport = s.origHTTPTransport
}
-func TestAzureBlobVolumeWithGeneric(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeWithGeneric(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableAzureBlobVolume(t, cluster, volume, metrics)
})
}
-func TestAzureBlobVolumeConcurrentRanges(t *testing.T) {
- defer func(b int) {
- azureMaxGetBytes = b
- }(azureMaxGetBytes)
-
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeConcurrentRanges(c *check.C) {
// Test (BlockSize mod azureMaxGetBytes)==0 and !=0 cases
- for _, azureMaxGetBytes = range []int{2 << 22, 2<<22 - 1} {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+ for _, b := range []int{2 << 22, 2<<22 - 1} {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ v := s.newTestableAzureBlobVolume(t, cluster, volume, metrics)
+ v.MaxGetBytes = b
+ return v
})
}
}
-func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, true, azureStorageReplication)
+func (s *StubbedAzureBlobSuite) TestReadonlyAzureBlobVolumeWithGeneric(c *check.C) {
+ DoGenericVolumeTests(c, false, func(c TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableAzureBlobVolume(c, cluster, volume, metrics)
})
}
-func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeRangeFenceposts(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
for _, size := range []int{
@@ -514,47 +480,27 @@ func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
hash := fmt.Sprintf("%x", md5.Sum(data))
err := v.Put(context.Background(), hash, data)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
gotData := make([]byte, len(data))
gotLen, err := v.Get(context.Background(), hash, gotData)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
gotHash := fmt.Sprintf("%x", md5.Sum(gotData))
if gotLen != size {
- t.Errorf("length mismatch: got %d != %d", gotLen, size)
+ c.Errorf("length mismatch: got %d != %d", gotLen, size)
}
if gotHash != hash {
- t.Errorf("hash mismatch: got %s != %s", gotHash, hash)
- }
- }
-}
-
-func TestAzureBlobVolumeReplication(t *testing.T) {
- for r := 1; r <= 4; r++ {
- v := NewTestableAzureBlobVolume(t, false, r)
- defer v.Teardown()
- if n := v.Replication(); n != r {
- t.Errorf("Got replication %d, expected %d", n, r)
+ c.Errorf("hash mismatch: got %s != %s", gotHash, hash)
}
}
}
-func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeCreateBlobRace(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
- azureWriteRaceInterval = time.Second
- azureWriteRacePollTime = time.Millisecond
-
var wg sync.WaitGroup
v.azHandler.race = make(chan chan struct{})
@@ -564,7 +510,7 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
defer wg.Done()
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
}()
continuePut := make(chan struct{})
@@ -576,7 +522,7 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
buf := make([]byte, len(TestBlock))
_, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
}()
// Wait for the stub's Get to get the empty blob
@@ -588,26 +534,18 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
wg.Wait()
}
-func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeCreateBlobRaceDeadline(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
+ v.AzureBlobVolume.WriteRaceInterval.Set("2s")
+ v.AzureBlobVolume.WriteRacePollTime.Set("5ms")
defer v.Teardown()
- azureWriteRaceInterval = 2 * time.Second
- azureWriteRacePollTime = 5 * time.Millisecond
-
v.PutRaw(TestHash, nil)
buf := new(bytes.Buffer)
v.IndexTo("", buf)
if buf.Len() != 0 {
- t.Errorf("Index %+q should be empty", buf.Bytes())
+ c.Errorf("Index %+q should be empty", buf.Bytes())
}
v.TouchWithDate(TestHash, time.Now().Add(-1982*time.Millisecond))
@@ -618,56 +556,49 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
buf := make([]byte, BlockSize)
n, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Error(err)
+ c.Error(err)
return
}
if n != 0 {
- t.Errorf("Got %+q, expected empty buf", buf[:n])
+ c.Errorf("Got %+q, expected empty buf", buf[:n])
}
}()
select {
case <-allDone:
case <-time.After(time.Second):
- t.Error("Get should have stopped waiting for race when block was 2s old")
+ c.Error("Get should have stopped waiting for race when block was 2s old")
}
buf.Reset()
v.IndexTo("", buf)
if !bytes.HasPrefix(buf.Bytes(), []byte(TestHash+"+0")) {
- t.Errorf("Index %+q should have %+q", buf.Bytes(), TestHash+"+0")
+ c.Errorf("Index %+q should have %+q", buf.Bytes(), TestHash+"+0")
}
}
-func TestAzureBlobVolumeContextCancelGet(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelGet(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
v.PutRaw(TestHash, TestBlock)
_, err := v.Get(ctx, TestHash, make([]byte, BlockSize))
return err
})
}
-func TestAzureBlobVolumeContextCancelPut(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelPut(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
return v.Put(ctx, TestHash, make([]byte, BlockSize))
})
}
-func TestAzureBlobVolumeContextCancelCompare(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelCompare(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
v.PutRaw(TestHash, TestBlock)
return v.Compare(ctx, TestHash, TestBlock2)
})
}
-func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Context, *TestableAzureBlobVolume) error) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) testAzureBlobVolumeContextCancel(c *check.C, testFunc func(context.Context, *TestableAzureBlobVolume) error) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
v.azHandler.race = make(chan chan struct{})
@@ -677,15 +608,15 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
defer close(allDone)
err := testFunc(ctx, v)
if err != context.Canceled {
- t.Errorf("got %T %q, expected %q", err, err, context.Canceled)
+ c.Errorf("got %T %q, expected %q", err, err, context.Canceled)
}
}()
releaseHandler := make(chan struct{})
select {
case <-allDone:
- t.Error("testFunc finished without waiting for v.azHandler.race")
+ c.Error("testFunc finished without waiting for v.azHandler.race")
case <-time.After(10 * time.Second):
- t.Error("timed out waiting to enter handler")
+ c.Error("timed out waiting to enter handler")
case v.azHandler.race <- releaseHandler:
}
@@ -693,7 +624,7 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
select {
case <-time.After(10 * time.Second):
- t.Error("timed out waiting to cancel")
+ c.Error("timed out waiting to cancel")
case <-allDone:
}
@@ -703,8 +634,11 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
}
func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
+ volume := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
+ defer volume.Teardown()
+
stats := func() string {
- buf, err := json.Marshal(s.volume.InternalStats())
+ buf, err := json.Marshal(volume.InternalStats())
c.Check(err, check.IsNil)
return string(buf)
}
@@ -713,37 +647,25 @@ func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
c.Check(stats(), check.Matches, `.*"Errors":0,.*`)
loc := "acbd18db4cc2f85cedef654fccc4a4d8"
- _, err := s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err := volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
c.Check(stats(), check.Matches, `.*"storage\.AzureStorageServiceError 404 \(404 Not Found\)":[^0].*`)
c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
- err = s.volume.Put(context.Background(), loc, []byte("foo"))
+ err = volume.Put(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"InBytes":6,.*`)
}
-func (s *StubbedAzureBlobSuite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: Azure
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
-
func (v *TestableAzureBlobVolume) PutRaw(locator string, data []byte) {
v.azHandler.PutRaw(v.ContainerName, locator, data)
}
@@ -760,17 +682,6 @@ func (v *TestableAzureBlobVolume) ReadWriteOperationLabelValues() (r, w string)
return "get", "create"
}
-func (v *TestableAzureBlobVolume) DeviceID() string {
- // Dummy device id for testing purposes
- return "azure://azure_blob_volume_test"
-}
-
-func (v *TestableAzureBlobVolume) Start(vm *volumeMetricsVecs) error {
- // Override original Start() to be able to assign CounterVecs with a dummy DeviceID
- v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(prometheus.Labels{"device_id": v.DeviceID()})
- return nil
-}
-
func makeEtag() string {
return fmt.Sprintf("0x%x", rand.Int63())
}
diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go
index d2e7c9ebd..623693cd1 100644
--- a/services/keepstore/bufferpool.go
+++ b/services/keepstore/bufferpool.go
@@ -8,9 +8,12 @@ import (
"sync"
"sync/atomic"
"time"
+
+ "github.com/sirupsen/logrus"
)
type bufferPool struct {
+ log logrus.FieldLogger
// limiter has a "true" placeholder for each in-use buffer.
limiter chan bool
// allocated is the number of bytes currently allocated to buffers.
@@ -19,9 +22,9 @@ type bufferPool struct {
sync.Pool
}
-func newBufferPool(count int, bufSize int) *bufferPool {
- p := bufferPool{}
- p.New = func() interface{} {
+func newBufferPool(log logrus.FieldLogger, count int, bufSize int) *bufferPool {
+ p := bufferPool{log: log}
+ p.Pool.New = func() interface{} {
atomic.AddUint64(&p.allocated, uint64(bufSize))
return make([]byte, bufSize)
}
@@ -34,13 +37,13 @@ func (p *bufferPool) Get(size int) []byte {
case p.limiter <- true:
default:
t0 := time.Now()
- log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
+ p.log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
p.limiter <- true
- log.Printf("waited %v for a buffer", time.Since(t0))
+ p.log.Printf("waited %v for a buffer", time.Since(t0))
}
buf := p.Pool.Get().([]byte)
if cap(buf) < size {
- log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
+ p.log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
}
return buf[:size]
}
diff --git a/services/keepstore/bufferpool_test.go b/services/keepstore/bufferpool_test.go
index 21b03edd4..2afa0ddb8 100644
--- a/services/keepstore/bufferpool_test.go
+++ b/services/keepstore/bufferpool_test.go
@@ -5,8 +5,10 @@
package main
import (
+ "context"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
. "gopkg.in/check.v1"
)
@@ -17,16 +19,16 @@ type BufferPoolSuite struct{}
// Initialize a default-sized buffer pool for the benefit of test
// suites that don't run main().
func init() {
- bufs = newBufferPool(theConfig.MaxBuffers, BlockSize)
+ bufs = newBufferPool(ctxlog.FromContext(context.Background()), 12, BlockSize)
}
// Restore sane default after bufferpool's own tests
func (s *BufferPoolSuite) TearDownTest(c *C) {
- bufs = newBufferPool(theConfig.MaxBuffers, BlockSize)
+ bufs = newBufferPool(ctxlog.FromContext(context.Background()), 12, BlockSize)
}
func (s *BufferPoolSuite) TestBufferPoolBufSize(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
b1 := bufs.Get(1)
bufs.Get(2)
bufs.Put(b1)
@@ -35,14 +37,14 @@ func (s *BufferPoolSuite) TestBufferPoolBufSize(c *C) {
}
func (s *BufferPoolSuite) TestBufferPoolUnderLimit(c *C) {
- bufs := newBufferPool(3, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 3, 10)
b1 := bufs.Get(10)
bufs.Get(10)
testBufferPoolRace(c, bufs, b1, "Get")
}
func (s *BufferPoolSuite) TestBufferPoolAtLimit(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
b1 := bufs.Get(10)
bufs.Get(10)
testBufferPoolRace(c, bufs, b1, "Put")
@@ -66,7 +68,7 @@ func testBufferPoolRace(c *C, bufs *bufferPool, unused []byte, expectWin string)
}
func (s *BufferPoolSuite) TestBufferPoolReuse(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
bufs.Get(10)
last := bufs.Get(10)
// The buffer pool is allowed to throw away unused buffers
diff --git a/services/keepstore/command.go b/services/keepstore/command.go
new file mode 100644
index 000000000..81716f6dd
--- /dev/null
+++ b/services/keepstore/command.go
@@ -0,0 +1,218 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/lib/config"
+ "git.curoverse.com/arvados.git/lib/service"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
+)
+
+var (
+ version = "dev"
+ Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler)
+)
+
+func main() {
+ os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background()))
+ if !ok {
+ return 2
+ }
+ return Command.RunCommand(prog, args, stdin, stdout, stderr)
+}
+
+// Parse keepstore command line flags, and return equivalent
+// service.Command flags. The second return value ("ok") is true if
+// all provided flags were successfully converted.
+func convertKeepstoreFlagsToServiceFlags(args []string, lgr logrus.FieldLogger) ([]string, bool) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.String("listen", "", "Services.Keepstore.InternalURLs")
+ flags.Int("max-buffers", 0, "API.MaxKeepBlockBuffers")
+ flags.Int("max-requests", 0, "API.MaxConcurrentRequests")
+ flags.Bool("never-delete", false, "Collections.BlobTrash")
+ flags.Bool("enforce-permissions", false, "Collections.BlobSigning")
+ flags.String("permission-key-file", "", "Collections.BlobSigningKey")
+ flags.String("blob-signing-key-file", "", "Collections.BlobSigningKey")
+ flags.String("data-manager-token-file", "", "SystemRootToken")
+ flags.Int("permission-ttl", 0, "Collections.BlobSigningTTL")
+ flags.Int("blob-signature-ttl", 0, "Collections.BlobSigningTTL")
+ flags.String("trash-lifetime", "", "Collections.BlobTrashLifetime")
+ flags.Bool("serialize", false, "Volumes.*.DriverParameters.Serialize")
+ flags.Bool("readonly", false, "Volumes.*.ReadOnly")
+ flags.String("pid", "", "-")
+ flags.String("trash-check-interval", "", "Collections.BlobTrashCheckInterval")
+
+ flags.String("azure-storage-container-volume", "", "Volumes.*.Driver")
+ flags.String("azure-storage-account-name", "", "Volumes.*.DriverParameters.StorageAccountName")
+ flags.String("azure-storage-account-key-file", "", "Volumes.*.DriverParameters.StorageAccountKey")
+ flags.String("azure-storage-replication", "", "Volumes.*.Replication")
+ flags.String("azure-max-get-bytes", "", "Volumes.*.DriverParameters.MaxDataReadSize")
+
+ flags.String("s3-bucket-volume", "", "Volumes.*.DriverParameters.Bucket")
+ flags.String("s3-region", "", "Volumes.*.DriverParameters.Region")
+ flags.String("s3-endpoint", "", "Volumes.*.DriverParameters.Endpoint")
+ flags.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKey")
+ flags.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretKey")
+ flags.String("s3-race-window", "", "Volumes.*.DriverParameters.RaceWindow")
+ flags.String("s3-replication", "", "Volumes.*.Replication")
+ flags.String("s3-unsafe-delete", "", "Volumes.*.DriverParameters.UnsafeDelete")
+
+ flags.String("volume", "", "Volumes")
+
+ flags.String("config", "", "")
+ flags.String("legacy-keepstore-config", "", "")
+
+ err := flags.Parse(args)
+ if err == flag.ErrHelp {
+ return []string{"-help"}, true
+ } else if err != nil {
+ return nil, false
+ }
+
+ args = nil
+ ok := true
+ flags.Visit(func(f *flag.Flag) {
+ if f.Name == "config" || f.Name == "legacy-keepstore-config" {
+ args = append(args, "-"+f.Name, f.Value.String())
+ } else if f.Usage == "-" {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported", f.Name)
+ } else {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported -- use Clusters.*.%s in cluster config file instead", f.Name, f.Usage)
+ }
+ })
+ if !ok {
+ return nil, false
+ }
+
+ flags = flag.NewFlagSet("", flag.ExitOnError)
+ loader := config.NewLoader(nil, lgr)
+ loader.SetupFlags(flags)
+ return loader.MungeLegacyConfigArgs(lgr, args, "-legacy-keepstore-config"), true
+}
+
+type handler struct {
+ http.Handler
+ Cluster *arvados.Cluster
+ Logger logrus.FieldLogger
+
+ pullq *WorkQueue
+ trashq *WorkQueue
+ volmgr *RRVolumeManager
+ keepClient *keepclient.KeepClient
+
+ err error
+ setupOnce sync.Once
+}
+
+func (h *handler) CheckHealth() error {
+ return h.err
+}
+
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ var h handler
+ serviceURL, ok := service.URLFromContext(ctx)
+ if !ok {
+ return service.ErrorHandler(ctx, cluster, errors.New("BUG: no URL from service.URLFromContext"))
+ }
+ err := h.setup(ctx, cluster, token, reg, serviceURL)
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, err)
+ }
+ return &h
+}
+
+func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry, serviceURL arvados.URL) error {
+ h.Cluster = cluster
+ h.Logger = ctxlog.FromContext(ctx)
+ if h.Cluster.API.MaxKeepBlockBuffers <= 0 {
+ return fmt.Errorf("MaxBuffers must be greater than zero")
+ }
+ bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlockBuffers, BlockSize)
+
+ if h.Cluster.API.MaxConcurrentRequests < 1 {
+ h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlockBuffers * 2
+ h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlockBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
+ }
+
+ if h.Cluster.Collections.BlobSigningKey != "" {
+ } else if h.Cluster.Collections.BlobSigning {
+ return errors.New("cannot enable Collections.BlobSigning with no Collections.BlobSigningKey")
+ } else {
+ h.Logger.Warn("Running without a blob signing key. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions. To fix this, configure Collections.BlobSigning and Collections.BlobSigningKey.")
+ }
+
+ if len(h.Cluster.Volumes) == 0 {
+ return errors.New("no volumes configured")
+ }
+
+ h.Logger.Printf("keepstore %s starting, pid %d", version, os.Getpid())
+
+ // Start a round-robin VolumeManager with the configured volumes.
+ vm, err := makeRRVolumeManager(h.Logger, h.Cluster, serviceURL, newVolumeMetricsVecs(reg))
+ if err != nil {
+ return err
+ }
+ if len(vm.readables) == 0 {
+ return fmt.Errorf("no volumes configured for %s", serviceURL)
+ }
+ h.volmgr = vm
+
+ // Initialize the pullq and workers
+ h.pullq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobReplicateConcurrency; i++ {
+ go h.runPullWorker(h.pullq)
+ }
+
+ // Initialize the trashq and workers
+ h.trashq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
+ go RunTrashWorker(h.volmgr, h.Cluster, h.trashq)
+ }
+
+ // Set up routes and metrics
+ h.Handler = MakeRESTRouter(ctx, cluster, reg, vm, h.pullq, h.trashq)
+
+ // Initialize keepclient for pull workers
+ c, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return err
+ }
+ ac, err := arvadosclient.New(c)
+ if err != nil {
+ return err
+ }
+ h.keepClient = &keepclient.KeepClient{
+ Arvados: ac,
+ Want_replicas: 1,
+ }
+ h.keepClient.Arvados.ApiToken = fmt.Sprintf("%x", rand.Int63())
+
+ if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 {
+ go emptyTrash(h.volmgr.writables, d)
+ }
+
+ return nil
+}
diff --git a/services/keepstore/command_test.go b/services/keepstore/command_test.go
new file mode 100644
index 000000000..ad2aa0957
--- /dev/null
+++ b/services/keepstore/command_test.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (*CommandSuite) TestLegacyConfigPath(c *check.C) {
+ var stdin, stdout, stderr bytes.Buffer
+ tmp, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmp.Name())
+ tmp.Write([]byte("Listen: \"1.2.3.4.5:invalidport\"\n"))
+ tmp.Close()
+ exited := runCommand("keepstore", []string{"-config", tmp.Name()}, &stdin, &stdout, &stderr)
+ c.Check(exited, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `(?ms).*unable to migrate Listen value.*`)
+}
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
deleted file mode 100644
index 43a219111..000000000
--- a/services/keepstore/config.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "strings"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
-)
-
-type Config struct {
- Debug bool
- Listen string
-
- LogFormat string
-
- PIDFile string
-
- MaxBuffers int
- MaxRequests int
-
- BlobSignatureTTL arvados.Duration
- BlobSigningKeyFile string
- RequireSignatures bool
- SystemAuthTokenFile string
- EnableDelete bool
- TrashLifetime arvados.Duration
- TrashCheckInterval arvados.Duration
- PullWorkers int
- TrashWorkers int
- EmptyTrashWorkers int
- TLSCertificateFile string
- TLSKeyFile string
-
- Volumes VolumeList
-
- blobSigningKey []byte
- systemAuthToken string
- debugLogf func(string, ...interface{})
-
- ManagementToken string
-}
-
-var (
- theConfig = DefaultConfig()
- formatter = map[string]logrus.Formatter{
- "text": &logrus.TextFormatter{
- FullTimestamp: true,
- TimestampFormat: rfc3339NanoFixed,
- },
- "json": &logrus.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- },
- }
- log = logrus.StandardLogger()
-)
-
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
-// DefaultConfig returns the default configuration.
-func DefaultConfig() *Config {
- return &Config{
- Listen: ":25107",
- LogFormat: "json",
- MaxBuffers: 128,
- RequireSignatures: true,
- BlobSignatureTTL: arvados.Duration(14 * 24 * time.Hour),
- TrashLifetime: arvados.Duration(14 * 24 * time.Hour),
- TrashCheckInterval: arvados.Duration(24 * time.Hour),
- Volumes: []Volume{},
- }
-}
-
-// Start should be called exactly once: after setting all public
-// fields, and before using the config.
-func (cfg *Config) Start(reg *prometheus.Registry) error {
- if cfg.Debug {
- log.Level = logrus.DebugLevel
- cfg.debugLogf = log.Printf
- cfg.debugLogf("debugging enabled")
- } else {
- log.Level = logrus.InfoLevel
- cfg.debugLogf = func(string, ...interface{}) {}
- }
-
- f := formatter[strings.ToLower(cfg.LogFormat)]
- if f == nil {
- return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
- }
- log.Formatter = f
-
- if cfg.MaxBuffers < 0 {
- return fmt.Errorf("MaxBuffers must be greater than zero")
- }
- bufs = newBufferPool(cfg.MaxBuffers, BlockSize)
-
- if cfg.MaxRequests < 1 {
- cfg.MaxRequests = cfg.MaxBuffers * 2
- log.Printf("MaxRequests <1 or not specified; defaulting to MaxBuffers * 2 == %d", cfg.MaxRequests)
- }
-
- if cfg.BlobSigningKeyFile != "" {
- buf, err := ioutil.ReadFile(cfg.BlobSigningKeyFile)
- if err != nil {
- return fmt.Errorf("reading blob signing key file: %s", err)
- }
- cfg.blobSigningKey = bytes.TrimSpace(buf)
- if len(cfg.blobSigningKey) == 0 {
- return fmt.Errorf("blob signing key file %q is empty", cfg.BlobSigningKeyFile)
- }
- } else if cfg.RequireSignatures {
- return fmt.Errorf("cannot enable RequireSignatures (-enforce-permissions) without a blob signing key")
- } else {
- log.Println("Running without a blob signing key. Block locators " +
- "returned by this server will not be signed, and will be rejected " +
- "by a server that enforces permissions.")
- log.Println("To fix this, use the BlobSigningKeyFile config entry.")
- }
-
- if fn := cfg.SystemAuthTokenFile; fn != "" {
- buf, err := ioutil.ReadFile(fn)
- if err != nil {
- return fmt.Errorf("cannot read system auth token file %q: %s", fn, err)
- }
- cfg.systemAuthToken = strings.TrimSpace(string(buf))
- }
-
- if cfg.EnableDelete {
- log.Print("Trash/delete features are enabled. WARNING: this has not " +
- "been extensively tested. You should disable this unless you can afford to lose data.")
- }
-
- if len(cfg.Volumes) == 0 {
- if (&unixVolumeAdder{cfg}).Discover() == 0 {
- return fmt.Errorf("no volumes found")
- }
- }
- vm := newVolumeMetricsVecs(reg)
- for _, v := range cfg.Volumes {
- if err := v.Start(vm); err != nil {
- return fmt.Errorf("volume %s: %s", v, err)
- }
- log.Printf("Using volume %v (writable=%v)", v, v.Writable())
- }
- return nil
-}
-
-// VolumeTypes is built up by init() funcs in the source files that
-// define the volume types.
-var VolumeTypes = []func() VolumeWithExamples{}
-
-type VolumeList []Volume
-
-// UnmarshalJSON -- given an array of objects -- deserializes each
-// object as the volume type indicated by the object's Type field.
-func (vl *VolumeList) UnmarshalJSON(data []byte) error {
- typeMap := map[string]func() VolumeWithExamples{}
- for _, factory := range VolumeTypes {
- t := factory().Type()
- if _, ok := typeMap[t]; ok {
- log.Fatalf("volume type %+q is claimed by multiple VolumeTypes", t)
- }
- typeMap[t] = factory
- }
-
- var mapList []map[string]interface{}
- err := json.Unmarshal(data, &mapList)
- if err != nil {
- return err
- }
- for _, mapIn := range mapList {
- typeIn, ok := mapIn["Type"].(string)
- if !ok {
- return fmt.Errorf("invalid volume type %+v", mapIn["Type"])
- }
- factory, ok := typeMap[typeIn]
- if !ok {
- return fmt.Errorf("unsupported volume type %+q", typeIn)
- }
- data, err := json.Marshal(mapIn)
- if err != nil {
- return err
- }
- vol := factory()
- err = json.Unmarshal(data, vol)
- if err != nil {
- return err
- }
- *vl = append(*vl, vol)
- }
- return nil
-}
-
-// MarshalJSON adds a "Type" field to each volume corresponding to its
-// Type().
-func (vl *VolumeList) MarshalJSON() ([]byte, error) {
- data := []byte{'['}
- for _, vs := range *vl {
- j, err := json.Marshal(vs)
- if err != nil {
- return nil, err
- }
- if len(data) > 1 {
- data = append(data, byte(','))
- }
- t, err := json.Marshal(vs.Type())
- if err != nil {
- panic(err)
- }
- data = append(data, j[0])
- data = append(data, []byte(`"Type":`)...)
- data = append(data, t...)
- data = append(data, byte(','))
- data = append(data, j[1:]...)
- }
- return append(data, byte(']')), nil
-}
diff --git a/services/keepstore/config_test.go b/services/keepstore/config_test.go
deleted file mode 100644
index e3b0ffc22..000000000
--- a/services/keepstore/config_test.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "github.com/sirupsen/logrus"
-)
-
-func init() {
- log.Level = logrus.DebugLevel
- theConfig.debugLogf = log.Printf
-}
diff --git a/services/keepstore/deprecated.go b/services/keepstore/deprecated.go
deleted file mode 100644
index d1377978a..000000000
--- a/services/keepstore/deprecated.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
-)
-
-type deprecatedOptions struct {
- flagSerializeIO bool
- flagReadonly bool
- neverDelete bool
- signatureTTLSeconds int
-}
-
-var deprecated = deprecatedOptions{
- neverDelete: !theConfig.EnableDelete,
- signatureTTLSeconds: int(theConfig.BlobSignatureTTL.Duration() / time.Second),
-}
-
-func (depr *deprecatedOptions) beforeFlagParse(cfg *Config) {
- flag.StringVar(&cfg.Listen, "listen", cfg.Listen, "see Listen configuration")
- flag.IntVar(&cfg.MaxBuffers, "max-buffers", cfg.MaxBuffers, "see MaxBuffers configuration")
- flag.IntVar(&cfg.MaxRequests, "max-requests", cfg.MaxRequests, "see MaxRequests configuration")
- flag.BoolVar(&depr.neverDelete, "never-delete", depr.neverDelete, "see EnableDelete configuration")
- flag.BoolVar(&cfg.RequireSignatures, "enforce-permissions", cfg.RequireSignatures, "see RequireSignatures configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "permission-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "blob-signing-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.SystemAuthTokenFile, "data-manager-token-file", cfg.SystemAuthTokenFile, "see SystemAuthToken`File` configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "permission-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "blob-signature-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.Var(&cfg.TrashLifetime, "trash-lifetime", "see TrashLifetime configuration")
- flag.BoolVar(&depr.flagSerializeIO, "serialize", depr.flagSerializeIO, "serialize read and write operations on the following volumes.")
- flag.BoolVar(&depr.flagReadonly, "readonly", depr.flagReadonly, "do not write, delete, or touch anything on the following volumes.")
- flag.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "see `PIDFile` configuration")
- flag.Var(&cfg.TrashCheckInterval, "trash-check-interval", "see TrashCheckInterval configuration")
-}
-
-func (depr *deprecatedOptions) afterFlagParse(cfg *Config) {
- cfg.BlobSignatureTTL = arvados.Duration(depr.signatureTTLSeconds) * arvados.Duration(time.Second)
- cfg.EnableDelete = !depr.neverDelete
-}
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index ad907ef10..251ad0a1d 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -23,16 +23,49 @@ import (
"os"
"regexp"
"strings"
- "testing"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
-var testCluster = &arvados.Cluster{
- ClusterID: "zzzzz",
+var testServiceURL = func() arvados.URL {
+ return arvados.URL{Host: "localhost:12345", Scheme: "http"}
+}()
+
+func testCluster(t TB) *arvados.Cluster {
+ cfg, err := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(t)).Load()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cluster.SystemRootToken = arvadostest.DataManagerToken
+ cluster.ManagementToken = arvadostest.ManagementToken
+ cluster.Collections.BlobSigning = false
+ return cluster
+}
+
+var _ = check.Suite(&HandlerSuite{})
+
+type HandlerSuite struct {
+ cluster *arvados.Cluster
+ handler *handler
+}
+
+func (s *HandlerSuite) SetUpTest(c *check.C) {
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Replication: 1, Driver: "mock"},
+ "zzzzz-nyw5e-111111111111111": {Replication: 1, Driver: "mock"},
+ }
+ s.handler = &handler{}
}
// A RequestTester represents the parameters for an HTTP request to
@@ -52,46 +85,41 @@ type RequestTester struct {
// - permissions on, authenticated request, expired locator
// - permissions on, authenticated request, signed locator, transient error from backend
//
-func TestGetHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestGetHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
- if err := vols[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
- }
+ vols := s.handler.volmgr.AllWritable()
+ err := vols[0].Put(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.IsNil)
// Create locators for testing.
// Turn on permission settings so we can generate signed locators.
- theConfig.RequireSignatures = true
- theConfig.blobSigningKey = []byte(knownKey)
- theConfig.BlobSignatureTTL.Set("5m")
+ s.cluster.Collections.BlobSigning = true
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
var (
unsignedLocator = "/" + TestHash
- validTimestamp = time.Now().Add(theConfig.BlobSignatureTTL.Duration())
+ validTimestamp = time.Now().Add(s.cluster.Collections.BlobSigningTTL.Duration())
expiredTimestamp = time.Now().Add(-time.Hour)
- signedLocator = "/" + SignLocator(TestHash, knownToken, validTimestamp)
- expiredLocator = "/" + SignLocator(TestHash, knownToken, expiredTimestamp)
+ signedLocator = "/" + SignLocator(s.cluster, TestHash, knownToken, validTimestamp)
+ expiredLocator = "/" + SignLocator(s.cluster, TestHash, knownToken, expiredTimestamp)
)
// -----------------
// Test unauthenticated request with permissions off.
- theConfig.RequireSignatures = false
+ s.cluster.Collections.BlobSigning = false
// Unauthenticated request, unsigned locator
// => OK
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "GET",
uri: unsignedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, unsigned locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, unsigned locator",
string(TestBlock),
response)
@@ -99,58 +127,58 @@ func TestGetHandler(t *testing.T) {
receivedLen := response.Header().Get("Content-Length")
expectedLen := fmt.Sprintf("%d", len(TestBlock))
if receivedLen != expectedLen {
- t.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
+ c.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
}
// ----------------
// Permissions: on.
- theConfig.RequireSignatures = true
+ s.cluster.Collections.BlobSigning = true
// Authenticated request, signed locator
// => OK
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated request, signed locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Authenticated request, signed locator", string(TestBlock), response)
receivedLen = response.Header().Get("Content-Length")
expectedLen = fmt.Sprintf("%d", len(TestBlock))
if receivedLen != expectedLen {
- t.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
+ c.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
}
// Authenticated request, unsigned locator
// => PermissionError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: unsignedLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t, "unsigned locator", PermissionError.HTTPCode, response)
+ ExpectStatusCode(c, "unsigned locator", PermissionError.HTTPCode, response)
// Unauthenticated request, signed locator
// => PermissionError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, signed locator",
PermissionError.HTTPCode, response)
// Authenticated request, expired locator
// => ExpiredError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: expiredLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated request, expired locator",
ExpiredError.HTTPCode, response)
@@ -158,16 +186,16 @@ func TestGetHandler(t *testing.T) {
// => 503 Server busy (transient error)
// Set up the block owning volume to respond with errors
- vols[0].(*MockVolume).Bad = true
- vols[0].(*MockVolume).BadVolumeError = VolumeBusyError
- response = IssueRequest(&RequestTester{
+ vols[0].Volume.(*MockVolume).Bad = true
+ vols[0].Volume.(*MockVolume).BadVolumeError = VolumeBusyError
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
apiToken: knownToken,
})
// A transient error from one volume while the other doesn't find the block
// should make the service return a 503 so that clients can retry.
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Volume backend busy",
503, response)
}
@@ -177,44 +205,42 @@ func TestGetHandler(t *testing.T) {
// - with server key, authenticated request, unsigned locator
// - with server key, unauthenticated request, unsigned locator
//
-func TestPutHandler(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// --------------
// No server key.
+ s.cluster.Collections.BlobSigningKey = ""
+
// Unauthenticated request, no server key
// => OK (unsigned response)
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, no server key", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, no server key",
TestHashPutResp, response)
// ------------------
// With a server key.
- theConfig.blobSigningKey = []byte(knownKey)
- theConfig.BlobSignatureTTL.Set("5m")
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
// When a permission key is available, the locator returned
// from an authenticated PUT request will be signed.
// Authenticated PUT, signed locator
// => OK (signed response)
- response = IssueRequest(
+ response = IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
@@ -222,76 +248,72 @@ func TestPutHandler(t *testing.T) {
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated PUT, signed locator, with server key",
http.StatusOK, response)
responseLocator := strings.TrimSpace(response.Body.String())
- if VerifySignature(responseLocator, knownToken) != nil {
- t.Errorf("Authenticated PUT, signed locator, with server key:\n"+
+ if VerifySignature(s.cluster, responseLocator, knownToken) != nil {
+ c.Errorf("Authenticated PUT, signed locator, with server key:\n"+
"response '%s' does not contain a valid signature",
responseLocator)
}
// Unauthenticated PUT, unsigned locator
// => OK
- response = IssueRequest(
+ response = IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated PUT, unsigned locator, with server key",
http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated PUT, unsigned locator, with server key",
TestHashPutResp, response)
}
-func TestPutAndDeleteSkipReadonlyVolumes(t *testing.T) {
- defer teardown()
- theConfig.systemAuthToken = "fake-data-manager-token"
- vols := []*MockVolume{CreateMockVolume(), CreateMockVolume()}
- vols[0].Readonly = true
- KeepVM = MakeRRVolumeManager([]Volume{vols[0], vols[1]})
- defer KeepVM.Close()
- IssueRequest(
+func (s *HandlerSuite) TestPutAndDeleteSkipReadonlyVolumes(c *check.C) {
+ s.cluster.Volumes["zzzzz-nyw5e-000000000000000"] = arvados.Volume{Driver: "mock", ReadOnly: true}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+
+ s.cluster.SystemRootToken = "fake-data-manager-token"
+ IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
- defer func(orig bool) {
- theConfig.EnableDelete = orig
- }(theConfig.EnableDelete)
- theConfig.EnableDelete = true
- IssueRequest(
+
+ s.cluster.Collections.BlobTrash = true
+ IssueRequest(s.handler,
&RequestTester{
method: "DELETE",
uri: "/" + TestHash,
requestBody: TestBlock,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
})
type expect struct {
- volnum int
+ volid string
method string
callcount int
}
for _, e := range []expect{
- {0, "Get", 0},
- {0, "Compare", 0},
- {0, "Touch", 0},
- {0, "Put", 0},
- {0, "Delete", 0},
- {1, "Get", 0},
- {1, "Compare", 1},
- {1, "Touch", 1},
- {1, "Put", 1},
- {1, "Delete", 1},
+ {"zzzzz-nyw5e-000000000000000", "Get", 0},
+ {"zzzzz-nyw5e-000000000000000", "Compare", 0},
+ {"zzzzz-nyw5e-000000000000000", "Touch", 0},
+ {"zzzzz-nyw5e-000000000000000", "Put", 0},
+ {"zzzzz-nyw5e-000000000000000", "Delete", 0},
+ {"zzzzz-nyw5e-111111111111111", "Get", 0},
+ {"zzzzz-nyw5e-111111111111111", "Compare", 1},
+ {"zzzzz-nyw5e-111111111111111", "Touch", 1},
+ {"zzzzz-nyw5e-111111111111111", "Put", 1},
+ {"zzzzz-nyw5e-111111111111111", "Delete", 1},
} {
- if calls := vols[e.volnum].CallCount(e.method); calls != e.callcount {
- t.Errorf("Got %d %s() on vol %d, expect %d", calls, e.method, e.volnum, e.callcount)
+ if calls := s.handler.volmgr.mountMap[e.volid].Volume.(*MockVolume).CallCount(e.method); calls != e.callcount {
+ c.Errorf("Got %d %s() on vol %s, expect %d", calls, e.method, e.volid, e.callcount)
}
}
}
@@ -307,22 +329,18 @@ func TestPutAndDeleteSkipReadonlyVolumes(t *testing.T) {
// The only /index requests that should succeed are those issued by the
// superuser. They should pass regardless of the value of RequireSignatures.
//
-func TestIndexHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestIndexHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- // Set up Keep volumes and populate them.
// Include multiple blocks on different volumes, and
// some metadata files (which should be omitted from index listings)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
vols[1].Put(context.Background(), TestHash2, TestBlock2)
vols[0].Put(context.Background(), TestHash+".meta", []byte("metadata"))
vols[1].Put(context.Background(), TestHash2+".meta", []byte("metadata"))
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
unauthenticatedReq := &RequestTester{
method: "GET",
@@ -336,7 +354,7 @@ func TestIndexHandler(t *testing.T) {
superuserReq := &RequestTester{
method: "GET",
uri: "/index",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
unauthPrefixReq := &RequestTester{
method: "GET",
@@ -350,76 +368,76 @@ func TestIndexHandler(t *testing.T) {
superuserPrefixReq := &RequestTester{
method: "GET",
uri: "/index/" + TestHash[0:3],
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserNoSuchPrefixReq := &RequestTester{
method: "GET",
uri: "/index/abcd",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserInvalidPrefixReq := &RequestTester{
method: "GET",
uri: "/index/xyz",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
// -------------------------------------------------------------
// Only the superuser should be allowed to issue /index requests.
// ---------------------------
- // RequireSignatures enabled
+ // BlobSigning enabled
// This setting should not affect tests passing.
- theConfig.RequireSignatures = true
+ s.cluster.Collections.BlobSigning = true
// unauthenticated /index request
// => UnauthorizedError
- response := IssueRequest(unauthenticatedReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, unauthenticatedReq)
+ ExpectStatusCode(c,
"RequireSignatures on, unauthenticated request",
UnauthorizedError.HTTPCode,
response)
// unauthenticated /index/prefix request
// => UnauthorizedError
- response = IssueRequest(unauthPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, unauthPrefixReq)
+ ExpectStatusCode(c,
"permissions on, unauthenticated /index/prefix request",
UnauthorizedError.HTTPCode,
response)
// authenticated /index request, non-superuser
// => UnauthorizedError
- response = IssueRequest(authenticatedReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, authenticatedReq)
+ ExpectStatusCode(c,
"permissions on, authenticated request, non-superuser",
UnauthorizedError.HTTPCode,
response)
// authenticated /index/prefix request, non-superuser
// => UnauthorizedError
- response = IssueRequest(authPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, authPrefixReq)
+ ExpectStatusCode(c,
"permissions on, authenticated /index/prefix request, non-superuser",
UnauthorizedError.HTTPCode,
response)
// superuser /index request
// => OK
- response = IssueRequest(superuserReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
// ----------------------------
- // RequireSignatures disabled
+ // BlobSigning disabled
// Valid Request should still pass.
- theConfig.RequireSignatures = false
+ s.cluster.Collections.BlobSigning = false
// superuser /index request
// => OK
- response = IssueRequest(superuserReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
@@ -428,15 +446,15 @@ func TestIndexHandler(t *testing.T) {
TestHash2 + `\+\d+ \d+\n\n$`
match, _ := regexp.MatchString(expected, response.Body.String())
if !match {
- t.Errorf(
+ c.Errorf(
"permissions on, superuser request: expected %s, got:\n%s",
expected, response.Body.String())
}
// superuser /index/prefix request
// => OK
- response = IssueRequest(superuserPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
@@ -444,27 +462,27 @@ func TestIndexHandler(t *testing.T) {
expected = `^` + TestHash + `\+\d+ \d+\n\n$`
match, _ = regexp.MatchString(expected, response.Body.String())
if !match {
- t.Errorf(
+ c.Errorf(
"permissions on, superuser /index/prefix request: expected %s, got:\n%s",
expected, response.Body.String())
}
// superuser /index/{no-such-prefix} request
// => OK
- response = IssueRequest(superuserNoSuchPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserNoSuchPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
if "\n" != response.Body.String() {
- t.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
+ c.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
}
// superuser /index/{invalid-prefix} request
// => StatusBadRequest
- response = IssueRequest(superuserInvalidPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserInvalidPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusBadRequest,
response)
@@ -496,27 +514,21 @@ func TestIndexHandler(t *testing.T) {
// (test for 200 OK, response with copies_deleted=0, copies_failed=1,
// confirm block not deleted)
//
-func TestDeleteHandler(t *testing.T) {
- defer teardown()
-
- // Set up Keep volumes and populate them.
- // Include multiple blocks on different volumes, and
- // some metadata files (which should be omitted from index listings)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestDeleteHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
// Explicitly set the BlobSignatureTTL to 0 for these
// tests, to ensure the MockVolume deletes the blocks
// even though they have just been created.
- theConfig.BlobSignatureTTL = arvados.Duration(0)
+ s.cluster.Collections.BlobSigningTTL = arvados.Duration(0)
var userToken = "NOT DATA MANAGER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
- theConfig.EnableDelete = true
+ s.cluster.Collections.BlobTrash = true
unauthReq := &RequestTester{
method: "DELETE",
@@ -532,26 +544,26 @@ func TestDeleteHandler(t *testing.T) {
superuserExistingBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserNonexistentBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash2,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
// Unauthenticated request returns PermissionError.
var response *httptest.ResponseRecorder
- response = IssueRequest(unauthReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, unauthReq)
+ ExpectStatusCode(c,
"unauthenticated request",
PermissionError.HTTPCode,
response)
// Authenticated non-admin request returns PermissionError.
- response = IssueRequest(userReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, userReq)
+ ExpectStatusCode(c,
"authenticated non-admin request",
PermissionError.HTTPCode,
response)
@@ -563,24 +575,24 @@ func TestDeleteHandler(t *testing.T) {
}
var responseDc, expectedDc deletecounter
- response = IssueRequest(superuserNonexistentBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserNonexistentBlockReq)
+ ExpectStatusCode(c,
"data manager request, nonexistent block",
http.StatusNotFound,
response)
- // Authenticated admin request for existing block while EnableDelete is false.
- theConfig.EnableDelete = false
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ // Authenticated admin request for existing block while BlobTrash is false.
+ s.cluster.Collections.BlobTrash = false
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"authenticated request, existing block, method disabled",
MethodDisabledError.HTTPCode,
response)
- theConfig.EnableDelete = true
+ s.cluster.Collections.BlobTrash = true
// Authenticated admin request for existing block.
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"data manager request, existing block",
http.StatusOK,
response)
@@ -588,7 +600,7 @@ func TestDeleteHandler(t *testing.T) {
expectedDc = deletecounter{1, 0}
json.NewDecoder(response.Body).Decode(&responseDc)
if responseDc != expectedDc {
- t.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
+ c.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
expectedDc, responseDc)
}
// Confirm the block has been deleted
@@ -596,16 +608,16 @@ func TestDeleteHandler(t *testing.T) {
_, err := vols[0].Get(context.Background(), TestHash, buf)
var blockDeleted = os.IsNotExist(err)
if !blockDeleted {
- t.Error("superuserExistingBlockReq: block not deleted")
+ c.Error("superuserExistingBlockReq: block not deleted")
}
// A DELETE request on a block newer than BlobSignatureTTL
// should return success but leave the block on the volume.
vols[0].Put(context.Background(), TestHash, TestBlock)
- theConfig.BlobSignatureTTL = arvados.Duration(time.Hour)
+ s.cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour)
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"data manager request, existing block",
http.StatusOK,
response)
@@ -613,13 +625,13 @@ func TestDeleteHandler(t *testing.T) {
expectedDc = deletecounter{1, 0}
json.NewDecoder(response.Body).Decode(&responseDc)
if responseDc != expectedDc {
- t.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
+ c.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
expectedDc, responseDc)
}
// Confirm the block has NOT been deleted.
_, err = vols[0].Get(context.Background(), TestHash, buf)
if err != nil {
- t.Errorf("testing delete on new block: %s\n", err)
+ c.Errorf("testing delete on new block: %s\n", err)
}
}
@@ -650,29 +662,33 @@ func TestDeleteHandler(t *testing.T) {
// pull list simultaneously. Make sure that none of them return 400
// Bad Request and that pullq.GetList() returns a valid list.
//
-func TestPullHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestPullHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- var userToken = "USER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ // Replace the router's pullq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ pullq := NewWorkQueue()
+ s.handler.Handler.(*router).pullq = pullq
- pullq = NewWorkQueue()
+ var userToken = "USER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
goodJSON := []byte(`[
{
- "locator":"locator_with_two_servers",
+ "locator":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+12345",
"servers":[
- "server1",
- "server2"
+ "http://server1",
+ "http://server2"
]
},
{
- "locator":"locator_with_no_servers",
+ "locator":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+12345",
"servers":[]
},
{
- "locator":"",
- "servers":["empty_locator"]
+ "locator":"cccccccccccccccccccccccccccccccc+12345",
+ "servers":["http://server1"]
}
]`)
@@ -699,34 +715,39 @@ func TestPullHandler(t *testing.T) {
},
{
"Valid pull request from the data manager",
- RequestTester{"/pull", theConfig.systemAuthToken, "PUT", goodJSON},
+ RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 pull requests\n",
},
{
"Invalid pull request from the data manager",
- RequestTester{"/pull", theConfig.systemAuthToken, "PUT", badJSON},
+ RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
}
for _, tst := range testcases {
- response := IssueRequest(&tst.req)
- ExpectStatusCode(t, tst.name, tst.responseCode, response)
- ExpectBody(t, tst.name, tst.responseBody, response)
+ response := IssueRequest(s.handler, &tst.req)
+ ExpectStatusCode(c, tst.name, tst.responseCode, response)
+ ExpectBody(c, tst.name, tst.responseBody, response)
}
// The Keep pull manager should have received one good list with 3
// requests on it.
for i := 0; i < 3; i++ {
- item := <-pullq.NextItem
+ var item interface{}
+ select {
+ case item = <-pullq.NextItem:
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
if _, ok := item.(PullRequest); !ok {
- t.Errorf("item %v could not be parsed as a PullRequest", item)
+ c.Errorf("item %v could not be parsed as a PullRequest", item)
}
}
- expectChannelEmpty(t, pullq.NextItem)
+ expectChannelEmpty(c, pullq.NextItem)
}
// TestTrashHandler
@@ -756,13 +777,16 @@ func TestPullHandler(t *testing.T) {
// pull list simultaneously. Make sure that none of them return 400
// Bad Request and that replica.Dump() returns a valid list.
//
-func TestTrashHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestTrashHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ // Replace the router's trashq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ trashq := NewWorkQueue()
+ s.handler.Handler.(*router).trashq = trashq
var userToken = "USER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
-
- trashq = NewWorkQueue()
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
goodJSON := []byte(`[
{
@@ -803,22 +827,22 @@ func TestTrashHandler(t *testing.T) {
},
{
"Valid trash list from the data manager",
- RequestTester{"/trash", theConfig.systemAuthToken, "PUT", goodJSON},
+ RequestTester{"/trash", s.cluster.SystemRootToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 trash requests\n",
},
{
"Invalid trash list from the data manager",
- RequestTester{"/trash", theConfig.systemAuthToken, "PUT", badJSON},
+ RequestTester{"/trash", s.cluster.SystemRootToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
}
for _, tst := range testcases {
- response := IssueRequest(&tst.req)
- ExpectStatusCode(t, tst.name, tst.responseCode, response)
- ExpectBody(t, tst.name, tst.responseBody, response)
+ response := IssueRequest(s.handler, &tst.req)
+ ExpectStatusCode(c, tst.name, tst.responseCode, response)
+ ExpectBody(c, tst.name, tst.responseBody, response)
}
// The trash collector should have received one good list with 3
@@ -826,11 +850,11 @@ func TestTrashHandler(t *testing.T) {
for i := 0; i < 3; i++ {
item := <-trashq.NextItem
if _, ok := item.(TrashRequest); !ok {
- t.Errorf("item %v could not be parsed as a TrashRequest", item)
+ c.Errorf("item %v could not be parsed as a TrashRequest", item)
}
}
- expectChannelEmpty(t, trashq.NextItem)
+ expectChannelEmpty(c, trashq.NextItem)
}
// ====================
@@ -839,75 +863,71 @@ func TestTrashHandler(t *testing.T) {
// IssueTestRequest executes an HTTP request described by rt, to a
// REST router. It returns the HTTP response to the request.
-func IssueRequest(rt *RequestTester) *httptest.ResponseRecorder {
+func IssueRequest(handler http.Handler, rt *RequestTester) *httptest.ResponseRecorder {
response := httptest.NewRecorder()
body := bytes.NewReader(rt.requestBody)
req, _ := http.NewRequest(rt.method, rt.uri, body)
if rt.apiToken != "" {
req.Header.Set("Authorization", "OAuth2 "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter(testCluster, prometheus.NewRegistry())
- loggingRouter.ServeHTTP(response, req)
+ handler.ServeHTTP(response, req)
return response
}
-func IssueHealthCheckRequest(rt *RequestTester) *httptest.ResponseRecorder {
+func IssueHealthCheckRequest(handler http.Handler, rt *RequestTester) *httptest.ResponseRecorder {
response := httptest.NewRecorder()
body := bytes.NewReader(rt.requestBody)
req, _ := http.NewRequest(rt.method, rt.uri, body)
if rt.apiToken != "" {
req.Header.Set("Authorization", "Bearer "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter(testCluster, prometheus.NewRegistry())
- loggingRouter.ServeHTTP(response, req)
+ handler.ServeHTTP(response, req)
return response
}
// ExpectStatusCode checks whether a response has the specified status code,
// and reports a test failure if not.
func ExpectStatusCode(
- t *testing.T,
+ c *check.C,
testname string,
expectedStatus int,
response *httptest.ResponseRecorder) {
if response.Code != expectedStatus {
- t.Errorf("%s: expected status %d, got %+v",
+ c.Errorf("%s: expected status %d, got %+v",
testname, expectedStatus, response)
}
}
func ExpectBody(
- t *testing.T,
+ c *check.C,
testname string,
expectedBody string,
response *httptest.ResponseRecorder) {
if expectedBody != "" && response.Body.String() != expectedBody {
- t.Errorf("%s: expected response body '%s', got %+v",
+ c.Errorf("%s: expected response body '%s', got %+v",
testname, expectedBody, response)
}
}
// See #7121
-func TestPutNeedsOnlyOneBuffer(t *testing.T) {
- defer teardown()
- KeepVM = MakeTestVolumeManager(1)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutNeedsOnlyOneBuffer(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
defer func(orig *bufferPool) {
bufs = orig
}(bufs)
- bufs = newBufferPool(1, BlockSize)
+ bufs = newBufferPool(ctxlog.TestLogger(c), 1, BlockSize)
ok := make(chan struct{})
go func() {
for i := 0; i < 2; i++ {
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"TestPutNeedsOnlyOneBuffer", http.StatusOK, response)
}
ok <- struct{}{}
@@ -916,34 +936,30 @@ func TestPutNeedsOnlyOneBuffer(t *testing.T) {
select {
case <-ok:
case <-time.After(time.Second):
- t.Fatal("PUT deadlocks with MaxBuffers==1")
+ c.Fatal("PUT deadlocks with MaxBuffers==1")
}
}
// Invoke the PutBlockHandler a bunch of times to test for bufferpool resource
// leak.
-func TestPutHandlerNoBufferleak(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutHandlerNoBufferleak(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
ok := make(chan bool)
go func() {
- for i := 0; i < theConfig.MaxBuffers+1; i++ {
+ for i := 0; i < s.cluster.API.MaxKeepBlockBuffers+1; i++ {
// Unauthenticated request, no server key
// => OK (unsigned response)
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"TestPutHandlerBufferleak", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"TestPutHandlerBufferleak",
TestHashPutResp, response)
}
@@ -952,7 +968,7 @@ func TestPutHandlerNoBufferleak(t *testing.T) {
select {
case <-time.After(20 * time.Second):
// If the buffer pool leaks, the test goroutine hangs.
- t.Fatal("test did not finish, assuming pool leaked")
+ c.Fatal("test did not finish, assuming pool leaked")
case <-ok:
}
}
@@ -966,23 +982,18 @@ func (r *notifyingResponseRecorder) CloseNotify() <-chan bool {
return r.closer
}
-func TestGetHandlerClientDisconnect(t *testing.T) {
- defer func(was bool) {
- theConfig.RequireSignatures = was
- }(theConfig.RequireSignatures)
- theConfig.RequireSignatures = false
+func (s *HandlerSuite) TestGetHandlerClientDisconnect(c *check.C) {
+ s.cluster.Collections.BlobSigning = false
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
defer func(orig *bufferPool) {
bufs = orig
}(bufs)
- bufs = newBufferPool(1, BlockSize)
+ bufs = newBufferPool(ctxlog.TestLogger(c), 1, BlockSize)
defer bufs.Put(bufs.Get(BlockSize))
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- if err := KeepVM.AllWritable()[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
+ if err := s.handler.volmgr.AllWritable()[0].Put(context.Background(), TestHash, TestBlock); err != nil {
+ c.Error(err)
}
resp := ¬ifyingResponseRecorder{
@@ -990,7 +1001,7 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
closer: make(chan bool, 1),
}
if _, ok := http.ResponseWriter(resp).(http.CloseNotifier); !ok {
- t.Fatal("notifyingResponseRecorder is broken")
+ c.Fatal("notifyingResponseRecorder is broken")
}
// If anyone asks, the client has disconnected.
resp.closer <- true
@@ -998,52 +1009,48 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
ok := make(chan struct{})
go func() {
req, _ := http.NewRequest("GET", fmt.Sprintf("/%s+%d", TestHash, len(TestBlock)), nil)
- MakeRESTRouter(testCluster, prometheus.NewRegistry()).ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
ok <- struct{}{}
}()
select {
case <-time.After(20 * time.Second):
- t.Fatal("request took >20s, close notifier must be broken")
+ c.Fatal("request took >20s, close notifier must be broken")
case <-ok:
}
- ExpectStatusCode(t, "client disconnect", http.StatusServiceUnavailable, resp.ResponseRecorder)
- for i, v := range KeepVM.AllWritable() {
- if calls := v.(*MockVolume).called["GET"]; calls != 0 {
- t.Errorf("volume %d got %d calls, expected 0", i, calls)
+ ExpectStatusCode(c, "client disconnect", http.StatusServiceUnavailable, resp.ResponseRecorder)
+ for i, v := range s.handler.volmgr.AllWritable() {
+ if calls := v.Volume.(*MockVolume).called["GET"]; calls != 0 {
+ c.Errorf("volume %d got %d calls, expected 0", i, calls)
}
}
}
// Invoke the GetBlockHandler a bunch of times to test for bufferpool resource
// leak.
-func TestGetHandlerNoBufferLeak(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestGetHandlerNoBufferLeak(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
if err := vols[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
+ c.Error(err)
}
ok := make(chan bool)
go func() {
- for i := 0; i < theConfig.MaxBuffers+1; i++ {
+ for i := 0; i < s.cluster.API.MaxKeepBlockBuffers+1; i++ {
// Unauthenticated request, unsigned locator
// => OK
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "GET",
uri: unsignedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, unsigned locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, unsigned locator",
string(TestBlock),
response)
@@ -1053,45 +1060,41 @@ func TestGetHandlerNoBufferLeak(t *testing.T) {
select {
case <-time.After(20 * time.Second):
// If the buffer pool leaks, the test goroutine hangs.
- t.Fatal("test did not finish, assuming pool leaked")
+ c.Fatal("test did not finish, assuming pool leaked")
case <-ok:
}
}
-func TestPutReplicationHeader(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestPutReplicationHeader(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- resp := IssueRequest(&RequestTester{
+ resp := IssueRequest(s.handler, &RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
if r := resp.Header().Get("X-Keep-Replicas-Stored"); r != "1" {
- t.Errorf("Got X-Keep-Replicas-Stored: %q, expected %q", r, "1")
+ c.Logf("%#v", resp)
+ c.Errorf("Got X-Keep-Replicas-Stored: %q, expected %q", r, "1")
}
}
-func TestUntrashHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestUntrashHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// Set up Keep volumes
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
// unauthenticatedReq => UnauthorizedError
unauthenticatedReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
}
- response := IssueRequest(unauthenticatedReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, unauthenticatedReq)
+ ExpectStatusCode(c,
"Unauthenticated request",
UnauthorizedError.HTTPCode,
response)
@@ -1103,8 +1106,8 @@ func TestUntrashHandler(t *testing.T) {
apiToken: knownToken,
}
- response = IssueRequest(notDataManagerReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, notDataManagerReq)
+ ExpectStatusCode(c,
"Non-datamanager token",
UnauthorizedError.HTTPCode,
response)
@@ -1113,10 +1116,10 @@ func TestUntrashHandler(t *testing.T) {
datamanagerWithBadHashReq := &RequestTester{
method: "PUT",
uri: "/untrash/thisisnotalocator",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerWithBadHashReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerWithBadHashReq)
+ ExpectStatusCode(c,
"Bad locator in untrash request",
http.StatusBadRequest,
response)
@@ -1125,10 +1128,10 @@ func TestUntrashHandler(t *testing.T) {
datamanagerWrongMethodReq := &RequestTester{
method: "GET",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerWrongMethodReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerWrongMethodReq)
+ ExpectStatusCode(c,
"Only PUT method is supported for untrash",
http.StatusMethodNotAllowed,
response)
@@ -1137,60 +1140,57 @@ func TestUntrashHandler(t *testing.T) {
datamanagerReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerReq)
+ ExpectStatusCode(c,
"",
http.StatusOK,
response)
expected := "Successfully untrashed on: [MockVolume],[MockVolume]"
if response.Body.String() != expected {
- t.Errorf(
+ c.Errorf(
"Untrash response mismatched: expected %s, got:\n%s",
expected, response.Body.String())
}
}
-func TestUntrashHandlerWithNoWritableVolumes(t *testing.T) {
- defer teardown()
-
- // Set up readonly Keep volumes
- vols := []*MockVolume{CreateMockVolume(), CreateMockVolume()}
- vols[0].Readonly = true
- vols[1].Readonly = true
- KeepVM = MakeRRVolumeManager([]Volume{vols[0], vols[1]})
- defer KeepVM.Close()
-
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+func (s *HandlerSuite) TestUntrashHandlerWithNoWritableVolumes(c *check.C) {
+ // Change all volumes to read-only
+ for uuid, v := range s.cluster.Volumes {
+ v.ReadOnly = true
+ s.cluster.Volumes[uuid] = v
+ }
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// datamanagerReq => StatusOK
datamanagerReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response := IssueRequest(datamanagerReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, datamanagerReq)
+ ExpectStatusCode(c,
"No writable volumes",
http.StatusNotFound,
response)
}
-func TestHealthCheckPing(t *testing.T) {
- theConfig.ManagementToken = arvadostest.ManagementToken
+func (s *HandlerSuite) TestHealthCheckPing(c *check.C) {
+ s.cluster.ManagementToken = arvadostest.ManagementToken
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
pingReq := &RequestTester{
method: "GET",
uri: "/_health/ping",
apiToken: arvadostest.ManagementToken,
}
- response := IssueHealthCheckRequest(pingReq)
- ExpectStatusCode(t,
+ response := IssueHealthCheckRequest(s.handler, pingReq)
+ ExpectStatusCode(c,
"",
http.StatusOK,
response)
want := `{"health":"OK"}`
if !strings.Contains(response.Body.String(), want) {
- t.Errorf("expected response to include %s: got %s", want, response.Body.String())
+ c.Errorf("expected response to include %s: got %s", want, response.Body.String())
}
}
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 72088e2b5..86504422d 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,23 +27,31 @@ import (
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
type router struct {
*mux.Router
- limiter httpserver.RequestCounter
cluster *arvados.Cluster
+ logger logrus.FieldLogger
remoteProxy remoteProxy
metrics *nodeMetrics
+ volmgr *RRVolumeManager
+ pullq *WorkQueue
+ trashq *WorkQueue
}
// MakeRESTRouter returns a new router that forwards all Keep requests
// to the appropriate handlers.
-func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
+func MakeRESTRouter(ctx context.Context, cluster *arvados.Cluster, reg *prometheus.Registry, volmgr *RRVolumeManager, pullq, trashq *WorkQueue) http.Handler {
rtr := &router{
Router: mux.NewRouter(),
cluster: cluster,
+ logger: ctxlog.FromContext(ctx),
metrics: &nodeMetrics{reg: reg},
+ volmgr: volmgr,
+ pullq: pullq,
+ trashq: trashq,
}
rtr.HandleFunc(
@@ -52,12 +61,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
rtr.handleGET).Methods("GET", "HEAD")
rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handlePUT).Methods("PUT")
- rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleDELETE).Methods("DELETE")
// List all blocks stored here. Privileged client only.
- rtr.HandleFunc(`/index`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index`, rtr.handleIndex).Methods("GET", "HEAD")
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
- rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
// Internals/debugging info (runtime.MemStats)
rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
@@ -67,20 +76,20 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// List mounts: UUID, readonly, tier, device ID, ...
rtr.HandleFunc(`/mounts`, rtr.MountsHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.IndexHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.IndexHandler).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.handleIndex).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.handleIndex).Methods("GET")
// Replace the current pull queue.
- rtr.HandleFunc(`/pull`, PullHandler).Methods("PUT")
+ rtr.HandleFunc(`/pull`, rtr.handlePull).Methods("PUT")
// Replace the current trash queue.
- rtr.HandleFunc(`/trash`, TrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/trash`, rtr.handleTrash).Methods("PUT")
// Untrash moves blocks from trash back into store
- rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, UntrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, rtr.handleUntrash).Methods("PUT")
rtr.Handle("/_health/{check}", &health.Handler{
- Token: theConfig.ManagementToken,
+ Token: cluster.ManagementToken,
Prefix: "/_health/",
}).Methods("GET")
@@ -88,17 +97,11 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// 400 Bad Request.
rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
- rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
rtr.metrics.setupBufferPoolMetrics(bufs)
- rtr.metrics.setupWorkQueueMetrics(pullq, "pull")
- rtr.metrics.setupWorkQueueMetrics(trashq, "trash")
- rtr.metrics.setupRequestMetrics(rtr.limiter)
-
- instrumented := httpserver.Instrument(rtr.metrics.reg, log,
- httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), log),
- httpserver.AddRequestIDs(httpserver.LogRequests(rtr.limiter))))
- return instrumented.ServeAPI(theConfig.ManagementToken, instrumented)
+ rtr.metrics.setupWorkQueueMetrics(rtr.pullq, "pull")
+ rtr.metrics.setupWorkQueueMetrics(rtr.trashq, "trash")
+
+ return rtr
}
// BadRequestHandler is a HandleFunc to address bad requests.
@@ -112,13 +115,13 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
locator := req.URL.Path[1:]
if strings.Contains(locator, "+R") && !strings.Contains(locator, "+A") {
- rtr.remoteProxy.Get(ctx, resp, req, rtr.cluster)
+ rtr.remoteProxy.Get(ctx, resp, req, rtr.cluster, rtr.volmgr)
return
}
- if theConfig.RequireSignatures {
+ if rtr.cluster.Collections.BlobSigning {
locator := req.URL.Path[1:] // strip leading slash
- if err := VerifySignature(locator, GetAPIToken(req)); err != nil {
+ if err := VerifySignature(rtr.cluster, locator, GetAPIToken(req)); err != nil {
http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
return
}
@@ -138,7 +141,7 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
}
defer bufs.Put(buf)
- size, err := GetBlock(ctx, mux.Vars(req)["hash"], buf, resp)
+ size, err := GetBlock(ctx, rtr.volmgr, mux.Vars(req)["hash"], buf, resp)
if err != nil {
code := http.StatusInternalServerError
if err, ok := err.(*KeepError); ok {
@@ -160,7 +163,6 @@ func contextForResponse(parent context.Context, resp http.ResponseWriter) (conte
go func(c <-chan bool) {
select {
case <-c:
- theConfig.debugLogf("cancel context")
cancel()
case <-ctx.Done():
}
@@ -210,7 +212,7 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
return
}
- if len(KeepVM.AllWritable()) == 0 {
+ if len(rtr.volmgr.AllWritable()) == 0 {
http.Error(resp, FullError.Error(), FullError.HTTPCode)
return
}
@@ -228,7 +230,7 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
return
}
- replication, err := PutBlock(ctx, buf, hash)
+ replication, err := PutBlock(ctx, rtr.volmgr, buf, hash)
bufs.Put(buf)
if err != nil {
@@ -244,9 +246,9 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// return it to the client.
returnHash := fmt.Sprintf("%s+%d", hash, req.ContentLength)
apiToken := GetAPIToken(req)
- if theConfig.blobSigningKey != nil && apiToken != "" {
- expiry := time.Now().Add(theConfig.BlobSignatureTTL.Duration())
- returnHash = SignLocator(returnHash, apiToken, expiry)
+ if rtr.cluster.Collections.BlobSigningKey != "" && apiToken != "" {
+ expiry := time.Now().Add(rtr.cluster.Collections.BlobSigningTTL.Duration())
+ returnHash = SignLocator(rtr.cluster, returnHash, apiToken, expiry)
}
resp.Header().Set("X-Keep-Replicas-Stored", strconv.Itoa(replication))
resp.Write([]byte(returnHash + "\n"))
@@ -254,8 +256,8 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// IndexHandler responds to "/index", "/index/{prefix}", and
// "/mounts/{uuid}/blocks" requests.
-func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
- if !IsSystemAuth(GetAPIToken(req)) {
+func (rtr *router) handleIndex(resp http.ResponseWriter, req *http.Request) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -268,14 +270,14 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
uuid := mux.Vars(req)["uuid"]
- var vols []Volume
+ var vols []*VolumeMount
if uuid == "" {
- vols = KeepVM.AllReadable()
- } else if v := KeepVM.Lookup(uuid, false); v == nil {
+ vols = rtr.volmgr.AllReadable()
+ } else if mnt := rtr.volmgr.Lookup(uuid, false); mnt == nil {
http.Error(resp, "mount not found", http.StatusNotFound)
return
} else {
- vols = []Volume{v}
+ vols = []*VolumeMount{mnt}
}
for _, v := range vols {
@@ -303,9 +305,9 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
// MountsHandler responds to "GET /mounts" requests.
func (rtr *router) MountsHandler(resp http.ResponseWriter, req *http.Request) {
- err := json.NewEncoder(resp).Encode(KeepVM.Mounts())
+ err := json.NewEncoder(resp).Encode(rtr.volmgr.Mounts())
if err != nil {
- http.Error(resp, err.Error(), http.StatusInternalServerError)
+ httpserver.Error(resp, err.Error(), http.StatusInternalServerError)
}
}
@@ -368,32 +370,28 @@ func (rtr *router) StatusHandler(resp http.ResponseWriter, req *http.Request) {
// populate the given NodeStatus struct with current values.
func (rtr *router) readNodeStatus(st *NodeStatus) {
st.Version = version
- vols := KeepVM.AllReadable()
+ vols := rtr.volmgr.AllReadable()
if cap(st.Volumes) < len(vols) {
st.Volumes = make([]*volumeStatusEnt, len(vols))
}
st.Volumes = st.Volumes[:0]
for _, vol := range vols {
var internalStats interface{}
- if vol, ok := vol.(InternalStatser); ok {
+ if vol, ok := vol.Volume.(InternalStatser); ok {
internalStats = vol.InternalStats()
}
st.Volumes = append(st.Volumes, &volumeStatusEnt{
Label: vol.String(),
Status: vol.Status(),
InternalStats: internalStats,
- //VolumeStats: KeepVM.VolumeStats(vol),
+ //VolumeStats: rtr.volmgr.VolumeStats(vol),
})
}
st.BufferPool.Alloc = bufs.Alloc()
st.BufferPool.Cap = bufs.Cap()
st.BufferPool.Len = bufs.Len()
- st.PullQueue = getWorkQueueStatus(pullq)
- st.TrashQueue = getWorkQueueStatus(trashq)
- if rtr.limiter != nil {
- st.RequestsCurrent = rtr.limiter.Current()
- st.RequestsMax = rtr.limiter.Max()
- }
+ st.PullQueue = getWorkQueueStatus(rtr.pullq)
+ st.TrashQueue = getWorkQueueStatus(rtr.trashq)
}
// return a WorkQueueStatus for the given queue. If q is nil (which
@@ -407,7 +405,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
return q.Status()
}
-// DeleteHandler processes DELETE requests.
+// handleDELETE processes DELETE requests.
//
// DELETE /{hash:[0-9a-f]{32} will delete the block with the specified hash
// from all connected volumes.
@@ -418,7 +416,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// a PermissionError.
//
// Upon receiving a valid request from an authorized user,
-// DeleteHandler deletes all copies of the specified block on local
+// handleDELETE deletes all copies of the specified block on local
// writable volumes.
//
// Response format:
@@ -434,17 +432,17 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// where d and f are integers representing the number of blocks that
// were successfully and unsuccessfully deleted.
//
-func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleDELETE(resp http.ResponseWriter, req *http.Request) {
hash := mux.Vars(req)["hash"]
// Confirm that this user is an admin and has a token with unlimited scope.
var tok = GetAPIToken(req)
- if tok == "" || !CanDelete(tok) {
+ if tok == "" || !rtr.canDelete(tok) {
http.Error(resp, PermissionError.Error(), PermissionError.HTTPCode)
return
}
- if !theConfig.EnableDelete {
+ if !rtr.cluster.Collections.BlobTrash {
http.Error(resp, MethodDisabledError.Error(), MethodDisabledError.HTTPCode)
return
}
@@ -456,7 +454,7 @@ func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
Deleted int `json:"copies_deleted"`
Failed int `json:"copies_failed"`
}
- for _, vol := range KeepVM.AllWritable() {
+ for _, vol := range rtr.volmgr.AllWritable() {
if err := vol.Trash(hash); err == nil {
result.Deleted++
} else if os.IsNotExist(err) {
@@ -530,9 +528,9 @@ type PullRequest struct {
}
// PullHandler processes "PUT /pull" requests for the data manager.
-func PullHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handlePull(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -556,7 +554,7 @@ func PullHandler(resp http.ResponseWriter, req *http.Request) {
for _, p := range pr {
plist.PushBack(p)
}
- pullq.ReplaceQueue(plist)
+ rtr.pullq.ReplaceQueue(plist)
}
// TrashRequest consists of a block locator and its Mtime
@@ -569,9 +567,9 @@ type TrashRequest struct {
}
// TrashHandler processes /trash requests.
-func TrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleTrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -595,27 +593,27 @@ func TrashHandler(resp http.ResponseWriter, req *http.Request) {
for _, t := range trash {
tlist.PushBack(t)
}
- trashq.ReplaceQueue(tlist)
+ rtr.trashq.ReplaceQueue(tlist)
}
// UntrashHandler processes "PUT /untrash/{hash:[0-9a-f]{32}}" requests for the data manager.
-func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleUntrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
hash := mux.Vars(req)["hash"]
- if len(KeepVM.AllWritable()) == 0 {
+ if len(rtr.volmgr.AllWritable()) == 0 {
http.Error(resp, "No writable volumes", http.StatusNotFound)
return
}
var untrashedOn, failedOn []string
var numNotFound int
- for _, vol := range KeepVM.AllWritable() {
+ for _, vol := range rtr.volmgr.AllWritable() {
err := vol.Untrash(hash)
if os.IsNotExist(err) {
@@ -629,12 +627,12 @@ func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
}
}
- if numNotFound == len(KeepVM.AllWritable()) {
+ if numNotFound == len(rtr.volmgr.AllWritable()) {
http.Error(resp, "Block not found on any of the writable volumes", http.StatusNotFound)
return
}
- if len(failedOn) == len(KeepVM.AllWritable()) {
+ if len(failedOn) == len(rtr.volmgr.AllWritable()) {
http.Error(resp, "Failed to untrash on all writable volumes", http.StatusInternalServerError)
} else {
respBody := "Successfully untrashed on: " + strings.Join(untrashedOn, ",")
@@ -664,11 +662,11 @@ func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
// If the block found does not have the correct MD5 hash, returns
// DiskHashError.
//
-func GetBlock(ctx context.Context, hash string, buf []byte, resp http.ResponseWriter) (int, error) {
+func GetBlock(ctx context.Context, volmgr *RRVolumeManager, hash string, buf []byte, resp http.ResponseWriter) (int, error) {
// Attempt to read the requested hash from a keep volume.
errorToCaller := NotFoundError
- for _, vol := range KeepVM.AllReadable() {
+ for _, vol := range volmgr.AllReadable() {
size, err := vol.Get(ctx, hash, buf)
select {
case <-ctx.Done():
@@ -738,7 +736,7 @@ func GetBlock(ctx context.Context, hash string, buf []byte, resp http.ResponseWr
// all writes failed). The text of the error message should
// provide as much detail as possible.
//
-func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
+func PutBlock(ctx context.Context, volmgr *RRVolumeManager, block []byte, hash string) (int, error) {
// Check that BLOCK's checksum matches HASH.
blockhash := fmt.Sprintf("%x", md5.Sum(block))
if blockhash != hash {
@@ -749,7 +747,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// If we already have this data, it's intact on disk, and we
// can update its timestamp, return success. If we have
// different data with the same hash, return failure.
- if n, err := CompareAndTouch(ctx, hash, block); err == nil || err == CollisionError {
+ if n, err := CompareAndTouch(ctx, volmgr, hash, block); err == nil || err == CollisionError {
return n, err
} else if ctx.Err() != nil {
return 0, ErrClientDisconnect
@@ -757,16 +755,16 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// Choose a Keep volume to write to.
// If this volume fails, try all of the volumes in order.
- if vol := KeepVM.NextWritable(); vol != nil {
- if err := vol.Put(ctx, hash, block); err == nil {
- return vol.Replication(), nil // success!
+ if mnt := volmgr.NextWritable(); mnt != nil {
+ if err := mnt.Put(ctx, hash, block); err == nil {
+ return mnt.Replication, nil // success!
}
if ctx.Err() != nil {
return 0, ErrClientDisconnect
}
}
- writables := KeepVM.AllWritable()
+ writables := volmgr.AllWritable()
if len(writables) == 0 {
log.Print("No writable volumes.")
return 0, FullError
@@ -779,7 +777,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
return 0, ErrClientDisconnect
}
if err == nil {
- return vol.Replication(), nil // success!
+ return vol.Replication, nil // success!
}
if err != FullError {
// The volume is not full but the
@@ -803,10 +801,10 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// the relevant block's modification time in order to protect it from
// premature garbage collection. Otherwise, it returns a non-nil
// error.
-func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error) {
+func CompareAndTouch(ctx context.Context, volmgr *RRVolumeManager, hash string, buf []byte) (int, error) {
var bestErr error = NotFoundError
- for _, vol := range KeepVM.AllWritable() {
- err := vol.Compare(ctx, hash, buf)
+ for _, mnt := range volmgr.AllWritable() {
+ err := mnt.Compare(ctx, hash, buf)
if ctx.Err() != nil {
return 0, ctx.Err()
} else if err == CollisionError {
@@ -815,7 +813,7 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
// to tell which one is wanted if we have
// both, so there's no point writing it even
// on a different volume.)
- log.Printf("%s: Compare(%s): %s", vol, hash, err)
+ log.Printf("%s: Compare(%s): %s", mnt.Volume, hash, err)
return 0, err
} else if os.IsNotExist(err) {
// Block does not exist. This is the only
@@ -825,16 +823,16 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
// Couldn't open file, data is corrupt on
// disk, etc.: log this abnormal condition,
// and try the next volume.
- log.Printf("%s: Compare(%s): %s", vol, hash, err)
+ log.Printf("%s: Compare(%s): %s", mnt.Volume, hash, err)
continue
}
- if err := vol.Touch(hash); err != nil {
- log.Printf("%s: Touch %s failed: %s", vol, hash, err)
+ if err := mnt.Touch(hash); err != nil {
+ log.Printf("%s: Touch %s failed: %s", mnt.Volume, hash, err)
bestErr = err
continue
}
// Compare and Touch both worked --> done.
- return vol.Replication(), nil
+ return mnt.Replication, nil
}
return 0, bestErr
}
@@ -875,15 +873,15 @@ func IsExpired(timestampHex string) bool {
return time.Unix(ts, 0).Before(time.Now())
}
-// CanDelete returns true if the user identified by apiToken is
+// canDelete returns true if the user identified by apiToken is
// allowed to delete blocks.
-func CanDelete(apiToken string) bool {
+func (rtr *router) canDelete(apiToken string) bool {
if apiToken == "" {
return false
}
// Blocks may be deleted only when Keep has been configured with a
// data manager.
- if IsSystemAuth(apiToken) {
+ if rtr.isSystemAuth(apiToken) {
return true
}
// TODO(twp): look up apiToken with the API server
@@ -892,8 +890,8 @@ func CanDelete(apiToken string) bool {
return false
}
-// IsSystemAuth returns true if the given token is allowed to perform
+// isSystemAuth returns true if the given token is allowed to perform
// system level actions like deleting data.
-func IsSystemAuth(token string) bool {
- return token != "" && token == theConfig.systemAuthToken
+func (rtr *router) isSystemAuth(token string) bool {
+ return token != "" && token == rtr.cluster.SystemRootToken
}
diff --git a/services/keepstore/handlers_with_generic_volume_test.go b/services/keepstore/handlers_with_generic_volume_test.go
deleted file mode 100644
index 4ffb7f8f1..000000000
--- a/services/keepstore/handlers_with_generic_volume_test.go
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
-)
-
-// A TestableVolumeManagerFactory creates a volume manager with at least two TestableVolume instances.
-// The factory function, and the TestableVolume instances it returns, can use "t" to write
-// logs, fail the current test, etc.
-type TestableVolumeManagerFactory func(t TB) (*RRVolumeManager, []TestableVolume)
-
-// DoHandlersWithGenericVolumeTests runs a set of handler tests with a
-// Volume Manager comprised of TestableVolume instances.
-// It calls factory to create a volume manager with TestableVolume
-// instances for each test case, to avoid leaking state between tests.
-func DoHandlersWithGenericVolumeTests(t TB, factory TestableVolumeManagerFactory) {
- testGetBlock(t, factory, TestHash, TestBlock)
- testGetBlock(t, factory, EmptyHash, EmptyBlock)
- testPutRawBadDataGetBlock(t, factory, TestHash, TestBlock, []byte("baddata"))
- testPutRawBadDataGetBlock(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
- testPutBlock(t, factory, TestHash, TestBlock)
- testPutBlock(t, factory, EmptyHash, EmptyBlock)
- testPutBlockCorrupt(t, factory, TestHash, TestBlock, []byte("baddata"))
- testPutBlockCorrupt(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
-}
-
-// Setup RRVolumeManager with TestableVolumes
-func setupHandlersWithGenericVolumeTest(t TB, factory TestableVolumeManagerFactory) []TestableVolume {
- vm, testableVolumes := factory(t)
- KeepVM = vm
-
- for _, v := range testableVolumes {
- defer v.Teardown()
- }
- defer KeepVM.Close()
-
- return testableVolumes
-}
-
-// Put a block using PutRaw in just one volume and Get it using GetBlock
-func testGetBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put testBlock in one volume
- testableVolumes[1].PutRaw(testHash, testBlock)
-
- // Get should pass
- buf := make([]byte, len(testBlock))
- n, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error while getting block %s", err)
- }
- if bytes.Compare(buf[:n], testBlock) != 0 {
- t.Errorf("Put succeeded but Get returned %+v, expected %+v", buf[:n], testBlock)
- }
-}
-
-// Put a bad block using PutRaw and get it.
-func testPutRawBadDataGetBlock(t TB, factory TestableVolumeManagerFactory,
- testHash string, testBlock []byte, badData []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put bad data for testHash in both volumes
- testableVolumes[0].PutRaw(testHash, badData)
- testableVolumes[1].PutRaw(testHash, badData)
-
- // Get should fail
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err == nil {
- t.Fatalf("Got %+q, expected error while getting corrupt block %v", buf[:size], testHash)
- }
-}
-
-// Invoke PutBlock twice to ensure CompareAndTouch path is tested.
-func testPutBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
- setupHandlersWithGenericVolumeTest(t, factory)
-
- // PutBlock
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock: %s", err)
- }
-
- // Check that PutBlock succeeds again even after CompareAndTouch
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock: %s", err)
- }
-
- // Check that PutBlock stored the data as expected
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
- } else if bytes.Compare(buf[:size], testBlock) != 0 {
- t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf[:size])
- }
-}
-
-// Put a bad block using PutRaw, overwrite it using PutBlock and get it.
-func testPutBlockCorrupt(t TB, factory TestableVolumeManagerFactory,
- testHash string, testBlock []byte, badData []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put bad data for testHash in both volumes
- testableVolumes[0].PutRaw(testHash, badData)
- testableVolumes[1].PutRaw(testHash, badData)
-
- // Check that PutBlock with good data succeeds
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock for %q: %s", testHash, err)
- }
-
- // Put succeeded and overwrote the badData in one volume,
- // and Get should return the testBlock now, ignoring the bad data.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
- } else if bytes.Compare(buf[:size], testBlock) != 0 {
- t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf[:size])
- }
-}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index fcbdddacb..f2973b586 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -5,24 +5,9 @@
package main
import (
- "flag"
- "fmt"
- "net"
- "os"
- "os/signal"
- "syscall"
"time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/config"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "github.com/coreos/go-systemd/daemon"
- "github.com/prometheus/client_golang/prometheus"
)
-var version = "dev"
-
// A Keep "block" is 64MB.
const BlockSize = 64 * 1024 * 1024
@@ -30,9 +15,6 @@ const BlockSize = 64 * 1024 * 1024
// in order to permit writes.
const MinFreeKilobytes = BlockSize / 1024
-// ProcMounts /proc/mounts
-var ProcMounts = "/proc/mounts"
-
var bufs *bufferPool
// KeepError types.
@@ -65,184 +47,11 @@ func (e *KeepError) Error() string {
return e.ErrMsg
}
-// ========================
-// Internal data structures
-//
-// These global variables are used by multiple parts of the
-// program. They are good candidates for moving into their own
-// packages.
-
-// The Keep VolumeManager maintains a list of available volumes.
-// Initialized by the --volumes flag (or by FindKeepVolumes).
-var KeepVM VolumeManager
-
-// The pull list manager and trash queue are threadsafe queues which
-// support atomic update operations. The PullHandler and TrashHandler
-// store results from Data Manager /pull and /trash requests here.
-//
-// See the Keep and Data Manager design documents for more details:
-// https://arvados.org/projects/arvados/wiki/Keep_Design_Doc
-// https://arvados.org/projects/arvados/wiki/Data_Manager_Design_Doc
-//
-var pullq *WorkQueue
-var trashq *WorkQueue
-
-func main() {
- deprecated.beforeFlagParse(theConfig)
-
- dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flag.Bool("version", false, "Print version information and exit.")
-
- defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
- var configPath string
- flag.StringVar(
- &configPath,
- "config",
- defaultConfigPath,
- "YAML or JSON configuration file `path`")
- flag.Usage = usage
- flag.Parse()
-
- // Print version information if requested
- if *getVersion {
- fmt.Printf("keepstore %s\n", version)
- return
- }
-
- deprecated.afterFlagParse(theConfig)
-
- err := config.LoadFile(theConfig, configPath)
- if err != nil && (!os.IsNotExist(err) || configPath != defaultConfigPath) {
- log.Fatal(err)
- }
-
- if *dumpConfig {
- log.Fatal(config.DumpAndExit(theConfig))
- }
-
- log.Printf("keepstore %s started", version)
-
- metricsRegistry := prometheus.NewRegistry()
-
- err = theConfig.Start(metricsRegistry)
- if err != nil {
- log.Fatal(err)
- }
-
- if pidfile := theConfig.PIDFile; pidfile != "" {
- f, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0777)
- if err != nil {
- log.Fatalf("open pidfile (%s): %s", pidfile, err)
- }
- defer f.Close()
- err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
- if err != nil {
- log.Fatalf("flock pidfile (%s): %s", pidfile, err)
- }
- defer os.Remove(pidfile)
- err = f.Truncate(0)
- if err != nil {
- log.Fatalf("truncate pidfile (%s): %s", pidfile, err)
- }
- _, err = fmt.Fprint(f, os.Getpid())
- if err != nil {
- log.Fatalf("write pidfile (%s): %s", pidfile, err)
- }
- err = f.Sync()
- if err != nil {
- log.Fatalf("sync pidfile (%s): %s", pidfile, err)
- }
- }
-
- var cluster *arvados.Cluster
- cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
- if err != nil && os.IsNotExist(err) {
- log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
- cluster = &arvados.Cluster{
- ClusterID: "xxxxx",
- }
- } else if err != nil {
- log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
- } else {
- cluster, err = cfg.GetCluster("")
- if err != nil {
- log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
- }
- }
-
- log.Println("keepstore starting, pid", os.Getpid())
- defer log.Println("keepstore exiting, pid", os.Getpid())
-
- // Start a round-robin VolumeManager with the volumes we have found.
- KeepVM = MakeRRVolumeManager(theConfig.Volumes)
-
- // Middleware/handler stack
- router := MakeRESTRouter(cluster, metricsRegistry)
-
- // Set up a TCP listener.
- listener, err := net.Listen("tcp", theConfig.Listen)
- if err != nil {
- log.Fatal(err)
- }
-
- // Initialize keepclient for pull workers
- keepClient := &keepclient.KeepClient{
- Arvados: &arvadosclient.ArvadosClient{},
- Want_replicas: 1,
- }
-
- // Initialize the pullq and workers
- pullq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
- go RunPullWorker(pullq, keepClient)
- }
-
- // Initialize the trashq and workers
- trashq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
- go RunTrashWorker(trashq)
- }
-
- // Start emptyTrash goroutine
- doneEmptyingTrash := make(chan bool)
- go emptyTrash(doneEmptyingTrash, theConfig.TrashCheckInterval.Duration())
-
- // Shut down the server gracefully (by closing the listener)
- // if SIGTERM is received.
- term := make(chan os.Signal, 1)
- go func(sig <-chan os.Signal) {
- s := <-sig
- log.Println("caught signal:", s)
- doneEmptyingTrash <- true
- listener.Close()
- }(term)
- signal.Notify(term, syscall.SIGTERM)
- signal.Notify(term, syscall.SIGINT)
-
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
- }
- log.Println("listening at", listener.Addr())
- srv := &server{}
- srv.Handler = router
- srv.Serve(listener)
-}
-
// Periodically (once per interval) invoke EmptyTrash on all volumes.
-func emptyTrash(done <-chan bool, interval time.Duration) {
- ticker := time.NewTicker(interval)
-
- for {
- select {
- case <-ticker.C:
- for _, v := range theConfig.Volumes {
- if v.Writable() {
- v.EmptyTrash()
- }
- }
- case <-done:
- ticker.Stop()
- return
+func emptyTrash(mounts []*VolumeMount, interval time.Duration) {
+ for range time.NewTicker(interval).C {
+ for _, v := range mounts {
+ v.EmptyTrash()
}
}
}
diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 8b448e72c..728c6fded 100644
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -6,7 +6,6 @@
Description=Arvados Keep Storage Daemon
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/keepstore/keepstore.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
diff --git a/services/keepstore/keepstore_test.go b/services/keepstore/keepstore_test.go
deleted file mode 100644
index d1d380466..000000000
--- a/services/keepstore/keepstore_test.go
+++ /dev/null
@@ -1,456 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "io/ioutil"
- "os"
- "path"
- "regexp"
- "sort"
- "strings"
- "testing"
-
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-)
-
-var TestBlock = []byte("The quick brown fox jumps over the lazy dog.")
-var TestHash = "e4d909c290d0fb1ca068ffaddf22cbd0"
-var TestHashPutResp = "e4d909c290d0fb1ca068ffaddf22cbd0+44\n"
-
-var TestBlock2 = []byte("Pack my box with five dozen liquor jugs.")
-var TestHash2 = "f15ac516f788aec4f30932ffb6395c39"
-
-var TestBlock3 = []byte("Now is the time for all good men to come to the aid of their country.")
-var TestHash3 = "eed29bbffbc2dbe5e5ee0bb71888e61f"
-
-// BadBlock is used to test collisions and corruption.
-// It must not match any test hashes.
-var BadBlock = []byte("The magic words are squeamish ossifrage.")
-
-// Empty block
-var EmptyHash = "d41d8cd98f00b204e9800998ecf8427e"
-var EmptyBlock = []byte("")
-
-// TODO(twp): Tests still to be written
-//
-// * TestPutBlockFull
-// - test that PutBlock returns 503 Full if the filesystem is full.
-// (must mock FreeDiskSpace or Statfs? use a tmpfs?)
-//
-// * TestPutBlockWriteErr
-// - test the behavior when Write returns an error.
-// - Possible solutions: use a small tmpfs and a high
-// MIN_FREE_KILOBYTES to trick PutBlock into attempting
-// to write a block larger than the amount of space left
-// - use an interface to mock ioutil.TempFile with a File
-// object that always returns an error on write
-//
-// ========================================
-// GetBlock tests.
-// ========================================
-
-// TestGetBlock
-// Test that simple block reads succeed.
-//
-func TestGetBlock(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- if err := vols[1].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
- }
-
- // Check that GetBlock returns success.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != nil {
- t.Errorf("GetBlock error: %s", err)
- }
- if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Errorf("got %v, expected %v", buf[:size], TestBlock)
- }
-}
-
-// TestGetBlockMissing
-// GetBlock must return an error when the block is not found.
-//
-func TestGetBlockMissing(t *testing.T) {
- defer teardown()
-
- // Create two empty test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that GetBlock returns failure.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != NotFoundError {
- t.Errorf("Expected NotFoundError, got %v, err %v", buf[:size], err)
- }
-}
-
-// TestGetBlockCorrupt
-// GetBlock must return an error when a corrupted block is requested
-// (the contents of the file do not checksum to its hash).
-//
-func TestGetBlockCorrupt(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes and store a corrupt block in one.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- vols[0].Put(context.Background(), TestHash, BadBlock)
-
- // Check that GetBlock returns failure.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != DiskHashError {
- t.Errorf("Expected DiskHashError, got %v (buf: %v)", err, buf[:size])
- }
-}
-
-// ========================================
-// PutBlock tests
-// ========================================
-
-// TestPutBlockOK
-// PutBlock can perform a simple block write and returns success.
-//
-func TestPutBlockOK(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that PutBlock stores the data as expected.
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
-
- vols := KeepVM.AllReadable()
- buf := make([]byte, BlockSize)
- n, err := vols[1].Get(context.Background(), TestHash, buf)
- if err != nil {
- t.Fatalf("Volume #0 Get returned error: %v", err)
- }
- if string(buf[:n]) != string(TestBlock) {
- t.Fatalf("PutBlock stored '%s', Get retrieved '%s'",
- string(TestBlock), string(buf[:n]))
- }
-}
-
-// TestPutBlockOneVol
-// PutBlock still returns success even when only one of the known
-// volumes is online.
-//
-func TestPutBlockOneVol(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes, but cripple one of them.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
- vols[0].(*MockVolume).Bad = true
- vols[0].(*MockVolume).BadVolumeError = errors.New("Bad volume")
-
- // Check that PutBlock stores the data as expected.
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
-
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != nil {
- t.Fatalf("GetBlock: %v", err)
- }
- if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Fatalf("PutBlock stored %+q, GetBlock retrieved %+q",
- TestBlock, buf[:size])
- }
-}
-
-// TestPutBlockMD5Fail
-// Check that PutBlock returns an error if passed a block and hash that
-// do not match.
-//
-func TestPutBlockMD5Fail(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that PutBlock returns the expected error when the hash does
- // not match the block.
- if _, err := PutBlock(context.Background(), BadBlock, TestHash); err != RequestHashError {
- t.Errorf("Expected RequestHashError, got %v", err)
- }
-
- // Confirm that GetBlock fails to return anything.
- if result, err := GetBlock(context.Background(), TestHash, make([]byte, BlockSize), nil); err != NotFoundError {
- t.Errorf("GetBlock succeeded after a corrupt block store (result = %s, err = %v)",
- string(result), err)
- }
-}
-
-// TestPutBlockCorrupt
-// PutBlock should overwrite corrupt blocks on disk when given
-// a PUT request with a good block.
-//
-func TestPutBlockCorrupt(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Store a corrupted block under TestHash.
- vols := KeepVM.AllWritable()
- vols[0].Put(context.Background(), TestHash, BadBlock)
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Errorf("PutBlock: n %d err %v", n, err)
- }
-
- // The block on disk should now match TestBlock.
- buf := make([]byte, BlockSize)
- if size, err := GetBlock(context.Background(), TestHash, buf, nil); err != nil {
- t.Errorf("GetBlock: %v", err)
- } else if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Errorf("Got %+q, expected %+q", buf[:size], TestBlock)
- }
-}
-
-// TestPutBlockCollision
-// PutBlock returns a 400 Collision error when attempting to
-// store a block that collides with another block on disk.
-//
-func TestPutBlockCollision(t *testing.T) {
- defer teardown()
-
- // These blocks both hash to the MD5 digest cee9a457e790cf20d4bdaa6d69f01e41.
- b1 := arvadostest.MD5CollisionData[0]
- b2 := arvadostest.MD5CollisionData[1]
- locator := arvadostest.MD5CollisionMD5
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Store one block, then attempt to store the other. Confirm that
- // PutBlock reported a CollisionError.
- if _, err := PutBlock(context.Background(), b1, locator); err != nil {
- t.Error(err)
- }
- if _, err := PutBlock(context.Background(), b2, locator); err == nil {
- t.Error("PutBlock did not report a collision")
- } else if err != CollisionError {
- t.Errorf("PutBlock returned %v", err)
- }
-}
-
-// TestPutBlockTouchFails
-// When PutBlock is asked to PUT an existing block, but cannot
-// modify the timestamp, it should write a second block.
-//
-func TestPutBlockTouchFails(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
- vols := KeepVM.AllWritable()
-
- // Store a block and then make the underlying volume bad,
- // so a subsequent attempt to update the file timestamp
- // will fail.
- vols[0].Put(context.Background(), TestHash, BadBlock)
- oldMtime, err := vols[0].Mtime(TestHash)
- if err != nil {
- t.Fatalf("vols[0].Mtime(%s): %s\n", TestHash, err)
- }
-
- // vols[0].Touch will fail on the next call, so the volume
- // manager will store a copy on vols[1] instead.
- vols[0].(*MockVolume).Touchable = false
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
- vols[0].(*MockVolume).Touchable = true
-
- // Now the mtime on the block on vols[0] should be unchanged, and
- // there should be a copy of the block on vols[1].
- newMtime, err := vols[0].Mtime(TestHash)
- if err != nil {
- t.Fatalf("vols[0].Mtime(%s): %s\n", TestHash, err)
- }
- if !newMtime.Equal(oldMtime) {
- t.Errorf("mtime was changed on vols[0]:\noldMtime = %v\nnewMtime = %v\n",
- oldMtime, newMtime)
- }
- buf := make([]byte, BlockSize)
- n, err := vols[1].Get(context.Background(), TestHash, buf)
- if err != nil {
- t.Fatalf("vols[1]: %v", err)
- }
- if bytes.Compare(buf[:n], TestBlock) != 0 {
- t.Errorf("new block does not match test block\nnew block = %v\n", buf[:n])
- }
-}
-
-func TestDiscoverTmpfs(t *testing.T) {
- var tempVols [4]string
- var err error
-
- // Create some directories suitable for using as keep volumes.
- for i := range tempVols {
- if tempVols[i], err = ioutil.TempDir("", "findvol"); err != nil {
- t.Fatal(err)
- }
- defer os.RemoveAll(tempVols[i])
- tempVols[i] = tempVols[i] + "/keep"
- if err = os.Mkdir(tempVols[i], 0755); err != nil {
- t.Fatal(err)
- }
- }
-
- // Set up a bogus ProcMounts file.
- f, err := ioutil.TempFile("", "keeptest")
- if err != nil {
- t.Fatal(err)
- }
- defer os.Remove(f.Name())
- for i, vol := range tempVols {
- // Add readonly mount points at odd indexes.
- var opts string
- switch i % 2 {
- case 0:
- opts = "rw,nosuid,nodev,noexec"
- case 1:
- opts = "nosuid,nodev,noexec,ro"
- }
- fmt.Fprintf(f, "tmpfs %s tmpfs %s 0 0\n", path.Dir(vol), opts)
- }
- f.Close()
- ProcMounts = f.Name()
-
- cfg := &Config{}
- added := (&unixVolumeAdder{cfg}).Discover()
-
- if added != len(cfg.Volumes) {
- t.Errorf("Discover returned %d, but added %d volumes",
- added, len(cfg.Volumes))
- }
- if added != len(tempVols) {
- t.Errorf("Discover returned %d but we set up %d volumes",
- added, len(tempVols))
- }
- for i, tmpdir := range tempVols {
- if tmpdir != cfg.Volumes[i].(*UnixVolume).Root {
- t.Errorf("Discover returned %s, expected %s\n",
- cfg.Volumes[i].(*UnixVolume).Root, tmpdir)
- }
- if expectReadonly := i%2 == 1; expectReadonly != cfg.Volumes[i].(*UnixVolume).ReadOnly {
- t.Errorf("Discover added %s with readonly=%v, should be %v",
- tmpdir, !expectReadonly, expectReadonly)
- }
- }
-}
-
-func TestDiscoverNone(t *testing.T) {
- defer teardown()
-
- // Set up a bogus ProcMounts file with no Keep vols.
- f, err := ioutil.TempFile("", "keeptest")
- if err != nil {
- t.Fatal(err)
- }
- defer os.Remove(f.Name())
- fmt.Fprintln(f, "rootfs / rootfs opts 0 0")
- fmt.Fprintln(f, "sysfs /sys sysfs opts 0 0")
- fmt.Fprintln(f, "proc /proc proc opts 0 0")
- fmt.Fprintln(f, "udev /dev devtmpfs opts 0 0")
- fmt.Fprintln(f, "devpts /dev/pts devpts opts 0 0")
- f.Close()
- ProcMounts = f.Name()
-
- cfg := &Config{}
- added := (&unixVolumeAdder{cfg}).Discover()
- if added != 0 || len(cfg.Volumes) != 0 {
- t.Fatalf("got %d, %v; expected 0, []", added, cfg.Volumes)
- }
-}
-
-// TestIndex
-// Test an /index request.
-func TestIndex(t *testing.T) {
- defer teardown()
-
- // Set up Keep volumes and populate them.
- // Include multiple blocks on different volumes, and
- // some metadata files.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- vols[0].Put(context.Background(), TestHash, TestBlock)
- vols[1].Put(context.Background(), TestHash2, TestBlock2)
- vols[0].Put(context.Background(), TestHash3, TestBlock3)
- vols[0].Put(context.Background(), TestHash+".meta", []byte("metadata"))
- vols[1].Put(context.Background(), TestHash2+".meta", []byte("metadata"))
-
- buf := new(bytes.Buffer)
- vols[0].IndexTo("", buf)
- vols[1].IndexTo("", buf)
- indexRows := strings.Split(string(buf.Bytes()), "\n")
- sort.Strings(indexRows)
- sortedIndex := strings.Join(indexRows, "\n")
- expected := `^\n` + TestHash + `\+\d+ \d+\n` +
- TestHash3 + `\+\d+ \d+\n` +
- TestHash2 + `\+\d+ \d+$`
-
- match, err := regexp.MatchString(expected, sortedIndex)
- if err == nil {
- if !match {
- t.Errorf("IndexLocators returned:\n%s", string(buf.Bytes()))
- }
- } else {
- t.Errorf("regexp.MatchString: %s", err)
- }
-}
-
-// ========================================
-// Helper functions for unit tests.
-// ========================================
-
-// MakeTestVolumeManager returns a RRVolumeManager with the specified
-// number of MockVolumes.
-func MakeTestVolumeManager(numVolumes int) VolumeManager {
- vols := make([]Volume, numVolumes)
- for i := range vols {
- vols[i] = CreateMockVolume()
- }
- return MakeRRVolumeManager(vols)
-}
-
-// teardown cleans up after each test.
-func teardown() {
- theConfig.systemAuthToken = ""
- theConfig.RequireSignatures = false
- theConfig.blobSigningKey = nil
- KeepVM = nil
-}
diff --git a/services/keepstore/metrics.go b/services/keepstore/metrics.go
index 235c41891..b2f0aa663 100644
--- a/services/keepstore/metrics.go
+++ b/services/keepstore/metrics.go
@@ -7,7 +7,6 @@ package main
import (
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
)
@@ -66,27 +65,6 @@ func (m *nodeMetrics) setupWorkQueueMetrics(q *WorkQueue, qName string) {
))
}
-func (m *nodeMetrics) setupRequestMetrics(rc httpserver.RequestCounter) {
- m.reg.MustRegister(prometheus.NewGaugeFunc(
- prometheus.GaugeOpts{
- Namespace: "arvados",
- Subsystem: "keepstore",
- Name: "concurrent_requests",
- Help: "Number of requests in progress",
- },
- func() float64 { return float64(rc.Current()) },
- ))
- m.reg.MustRegister(prometheus.NewGaugeFunc(
- prometheus.GaugeOpts{
- Namespace: "arvados",
- Subsystem: "keepstore",
- Name: "max_concurrent_requests",
- Help: "Maximum number of concurrent requests",
- },
- func() float64 { return float64(rc.Max()) },
- ))
-}
-
type volumeMetricsVecs struct {
ioBytes *prometheus.CounterVec
errCounters *prometheus.CounterVec
diff --git a/services/keepstore/mounts_test.go b/services/keepstore/mounts_test.go
index 7c932ee02..9b5606b5c 100644
--- a/services/keepstore/mounts_test.go
+++ b/services/keepstore/mounts_test.go
@@ -12,59 +12,39 @@ import (
"net/http/httptest"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
-var _ = check.Suite(&MountsSuite{})
+func (s *HandlerSuite) TestMounts(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
-type MountsSuite struct {
- vm VolumeManager
- rtr http.Handler
-}
-
-func (s *MountsSuite) SetUpTest(c *check.C) {
- s.vm = MakeTestVolumeManager(2)
- KeepVM = s.vm
- theConfig = DefaultConfig()
- theConfig.systemAuthToken = arvadostest.DataManagerToken
- theConfig.ManagementToken = arvadostest.ManagementToken
- r := prometheus.NewRegistry()
- theConfig.Start(r)
- s.rtr = MakeRESTRouter(testCluster, r)
-}
-
-func (s *MountsSuite) TearDownTest(c *check.C) {
- s.vm.Close()
- KeepVM = nil
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
-}
-
-func (s *MountsSuite) TestMounts(c *check.C) {
- vols := s.vm.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
vols[1].Put(context.Background(), TestHash2, TestBlock2)
resp := s.call("GET", "/mounts", "", nil)
c.Check(resp.Code, check.Equals, http.StatusOK)
var mntList []struct {
- UUID string `json:"uuid"`
- DeviceID string `json:"device_id"`
- ReadOnly bool `json:"read_only"`
- Replication int `json:"replication"`
- StorageClasses []string `json:"storage_classes"`
+ UUID string `json:"uuid"`
+ DeviceID string `json:"device_id"`
+ ReadOnly bool `json:"read_only"`
+ Replication int `json:"replication"`
+ StorageClasses map[string]bool `json:"storage_classes"`
}
+ c.Log(resp.Body.String())
err := json.Unmarshal(resp.Body.Bytes(), &mntList)
c.Assert(err, check.IsNil)
c.Assert(len(mntList), check.Equals, 2)
for _, m := range mntList {
c.Check(len(m.UUID), check.Equals, 27)
- c.Check(m.UUID[:12], check.Equals, "zzzzz-ivpuk-")
+ c.Check(m.UUID[:12], check.Equals, "zzzzz-nyw5e-")
c.Check(m.DeviceID, check.Equals, "mock-device-id")
c.Check(m.ReadOnly, check.Equals, false)
c.Check(m.Replication, check.Equals, 1)
- c.Check(m.StorageClasses, check.DeepEquals, []string{"default"})
+ c.Check(m.StorageClasses, check.DeepEquals, map[string]bool{"default": true})
}
c.Check(mntList[0].UUID, check.Not(check.Equals), mntList[1].UUID)
@@ -103,7 +83,12 @@ func (s *MountsSuite) TestMounts(c *check.C) {
c.Check(resp.Body.String(), check.Equals, "\n")
}
-func (s *MountsSuite) TestMetrics(c *check.C) {
+func (s *HandlerSuite) TestMetrics(c *check.C) {
+ reg := prometheus.NewRegistry()
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", reg, testServiceURL), check.IsNil)
+ instrumented := httpserver.Instrument(reg, ctxlog.TestLogger(c), s.handler.Handler)
+ s.handler.Handler = instrumented.ServeAPI(s.cluster.ManagementToken, instrumented)
+
s.call("PUT", "/"+TestHash, "", TestBlock)
s.call("PUT", "/"+TestHash2, "", TestBlock2)
resp := s.call("GET", "/metrics.json", "", nil)
@@ -145,8 +130,6 @@ func (s *MountsSuite) TestMetrics(c *check.C) {
}
}
}
- c.Check(found["request_duration_seconds"], check.Equals, true)
- c.Check(found["time_to_status_seconds"], check.Equals, true)
metricsNames := []string{
"arvados_keepstore_bufferpool_inuse_buffers",
@@ -154,25 +137,22 @@ func (s *MountsSuite) TestMetrics(c *check.C) {
"arvados_keepstore_bufferpool_allocated_bytes",
"arvados_keepstore_pull_queue_inprogress_entries",
"arvados_keepstore_pull_queue_pending_entries",
- "arvados_keepstore_concurrent_requests",
- "arvados_keepstore_max_concurrent_requests",
"arvados_keepstore_trash_queue_inprogress_entries",
"arvados_keepstore_trash_queue_pending_entries",
"request_duration_seconds",
- "time_to_status_seconds",
}
for _, m := range metricsNames {
_, ok := names[m]
- c.Check(ok, check.Equals, true)
+ c.Check(ok, check.Equals, true, check.Commentf("checking metric %q", m))
}
}
-func (s *MountsSuite) call(method, path, tok string, body []byte) *httptest.ResponseRecorder {
+func (s *HandlerSuite) call(method, path, tok string, body []byte) *httptest.ResponseRecorder {
resp := httptest.NewRecorder()
req, _ := http.NewRequest(method, path, bytes.NewReader(body))
if tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
return resp
}
diff --git a/services/keepstore/perms.go b/services/keepstore/perms.go
index 49a231685..e2155f94f 100644
--- a/services/keepstore/perms.go
+++ b/services/keepstore/perms.go
@@ -5,14 +5,16 @@
package main
import (
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
// SignLocator takes a blobLocator, an apiToken and an expiry time, and
// returns a signed locator string.
-func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
- return keepclient.SignLocator(blobLocator, apiToken, expiry, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func SignLocator(cluster *arvados.Cluster, blobLocator, apiToken string, expiry time.Time) string {
+ return keepclient.SignLocator(blobLocator, apiToken, expiry, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
}
// VerifySignature returns nil if the signature on the signedLocator
@@ -20,8 +22,8 @@ func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
// either ExpiredError (if the timestamp has expired, which is
// something the client could have figured out independently) or
// PermissionError.
-func VerifySignature(signedLocator, apiToken string) error {
- err := keepclient.VerifySignature(signedLocator, apiToken, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func VerifySignature(cluster *arvados.Cluster, signedLocator, apiToken string) error {
+ err := keepclient.VerifySignature(signedLocator, apiToken, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
if err == keepclient.ErrSignatureExpired {
return ExpiredError
} else if err != nil {
diff --git a/services/keepstore/perms_test.go b/services/keepstore/perms_test.go
index dd57faf27..6ec4887ce 100644
--- a/services/keepstore/perms_test.go
+++ b/services/keepstore/perms_test.go
@@ -6,10 +6,10 @@ package main
import (
"strconv"
- "testing"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ check "gopkg.in/check.v1"
)
const (
@@ -30,44 +30,34 @@ const (
knownSignedLocator = knownLocator + knownSigHint
)
-func TestSignLocator(t *testing.T) {
- defer func(b []byte) {
- theConfig.blobSigningKey = b
- }(theConfig.blobSigningKey)
-
+func (s *HandlerSuite) TestSignLocator(c *check.C) {
tsInt, err := strconv.ParseInt(knownTimestamp, 16, 0)
if err != nil {
- t.Fatal(err)
+ c.Fatal(err)
}
t0 := time.Unix(tsInt, 0)
- theConfig.BlobSignatureTTL = knownSignatureTTL
-
- theConfig.blobSigningKey = []byte(knownKey)
- if x := SignLocator(knownLocator, knownToken, t0); x != knownSignedLocator {
- t.Fatalf("Got %+q, expected %+q", x, knownSignedLocator)
+ s.cluster.Collections.BlobSigningTTL = knownSignatureTTL
+ s.cluster.Collections.BlobSigningKey = knownKey
+ if x := SignLocator(s.cluster, knownLocator, knownToken, t0); x != knownSignedLocator {
+ c.Fatalf("Got %+q, expected %+q", x, knownSignedLocator)
}
- theConfig.blobSigningKey = []byte("arbitrarykey")
- if x := SignLocator(knownLocator, knownToken, t0); x == knownSignedLocator {
- t.Fatalf("Got same signature %+q, even though blobSigningKey changed", x)
+ s.cluster.Collections.BlobSigningKey = "arbitrarykey"
+ if x := SignLocator(s.cluster, knownLocator, knownToken, t0); x == knownSignedLocator {
+ c.Fatalf("Got same signature %+q, even though blobSigningKey changed", x)
}
}
-func TestVerifyLocator(t *testing.T) {
- defer func(b []byte) {
- theConfig.blobSigningKey = b
- }(theConfig.blobSigningKey)
-
- theConfig.BlobSignatureTTL = knownSignatureTTL
-
- theConfig.blobSigningKey = []byte(knownKey)
- if err := VerifySignature(knownSignedLocator, knownToken); err != nil {
- t.Fatal(err)
+func (s *HandlerSuite) TestVerifyLocator(c *check.C) {
+ s.cluster.Collections.BlobSigningTTL = knownSignatureTTL
+ s.cluster.Collections.BlobSigningKey = knownKey
+ if err := VerifySignature(s.cluster, knownSignedLocator, knownToken); err != nil {
+ c.Fatal(err)
}
- theConfig.blobSigningKey = []byte("arbitrarykey")
- if err := VerifySignature(knownSignedLocator, knownToken); err == nil {
- t.Fatal("Verified signature even with wrong blobSigningKey")
+ s.cluster.Collections.BlobSigningKey = "arbitrarykey"
+ if err := VerifySignature(s.cluster, knownSignedLocator, knownToken); err == nil {
+ c.Fatal("Verified signature even with wrong blobSigningKey")
}
}
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
index 1f82f3f4f..fac9c542f 100644
--- a/services/keepstore/proxy_remote.go
+++ b/services/keepstore/proxy_remote.go
@@ -25,7 +25,7 @@ type remoteProxy struct {
mtx sync.Mutex
}
-func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.Request, cluster *arvados.Cluster) {
+func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.Request, cluster *arvados.Cluster, volmgr *RRVolumeManager) {
// Intervening proxies must not return a cached GET response
// to a prior request if a X-Keep-Signature request header has
// been added or changed.
@@ -49,6 +49,8 @@ func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.R
Buffer: buf[:0],
ResponseWriter: w,
Context: ctx,
+ Cluster: cluster,
+ VolumeManager: volmgr,
}
defer rrc.Close()
w = rrc
@@ -145,10 +147,12 @@ var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^\+]*`)
// local volume, adds a response header with a locally-signed locator,
// and finally writes the data through.
type remoteResponseCacher struct {
- Locator string
- Token string
- Buffer []byte
- Context context.Context
+ Locator string
+ Token string
+ Buffer []byte
+ Context context.Context
+ Cluster *arvados.Cluster
+ VolumeManager *RRVolumeManager
http.ResponseWriter
statusCode int
}
@@ -173,7 +177,7 @@ func (rrc *remoteResponseCacher) Close() error {
rrc.ResponseWriter.Write(rrc.Buffer)
return nil
}
- _, err := PutBlock(rrc.Context, rrc.Buffer, rrc.Locator[:32])
+ _, err := PutBlock(rrc.Context, rrc.VolumeManager, rrc.Buffer, rrc.Locator[:32])
if rrc.Context.Err() != nil {
// If caller hung up, log that instead of subsequent/misleading errors.
http.Error(rrc.ResponseWriter, rrc.Context.Err().Error(), http.StatusGatewayTimeout)
@@ -193,7 +197,8 @@ func (rrc *remoteResponseCacher) Close() error {
}
unsigned := localOrRemoteSignature.ReplaceAllLiteralString(rrc.Locator, "")
- signed := SignLocator(unsigned, rrc.Token, time.Now().Add(theConfig.BlobSignatureTTL.Duration()))
+ expiry := time.Now().Add(rrc.Cluster.Collections.BlobSigningTTL.Duration())
+ signed := SignLocator(rrc.Cluster, unsigned, rrc.Token, expiry)
if signed == unsigned {
err = errors.New("could not sign locator")
http.Error(rrc.ResponseWriter, err.Error(), http.StatusInternalServerError)
diff --git a/services/keepstore/proxy_remote_test.go b/services/keepstore/proxy_remote_test.go
index 6c22d1d32..6483d6cf0 100644
--- a/services/keepstore/proxy_remote_test.go
+++ b/services/keepstore/proxy_remote_test.go
@@ -5,6 +5,7 @@
package main
import (
+ "context"
"crypto/md5"
"encoding/json"
"fmt"
@@ -28,8 +29,7 @@ var _ = check.Suite(&ProxyRemoteSuite{})
type ProxyRemoteSuite struct {
cluster *arvados.Cluster
- vm VolumeManager
- rtr http.Handler
+ handler *handler
remoteClusterID string
remoteBlobSigningKey []byte
@@ -87,7 +87,9 @@ func (s *ProxyRemoteSuite) SetUpTest(c *check.C) {
s.remoteKeepproxy = httptest.NewServer(http.HandlerFunc(s.remoteKeepproxyHandler))
s.remoteAPI = httptest.NewUnstartedServer(http.HandlerFunc(s.remoteAPIHandler))
s.remoteAPI.StartTLS()
- s.cluster = arvados.IntegrationTestCluster()
+ s.cluster = testCluster(c)
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.SystemRootToken = arvadostest.DataManagerToken
s.cluster.RemoteClusters = map[string]arvados.RemoteCluster{
s.remoteClusterID: arvados.RemoteCluster{
Host: strings.Split(s.remoteAPI.URL, "//")[1],
@@ -96,21 +98,12 @@ func (s *ProxyRemoteSuite) SetUpTest(c *check.C) {
Insecure: true,
},
}
- s.vm = MakeTestVolumeManager(2)
- KeepVM = s.vm
- theConfig = DefaultConfig()
- theConfig.systemAuthToken = arvadostest.DataManagerToken
- theConfig.blobSigningKey = []byte(knownKey)
- r := prometheus.NewRegistry()
- theConfig.Start(r)
- s.rtr = MakeRESTRouter(s.cluster, r)
+ s.cluster.Volumes = map[string]arvados.Volume{"zzzzz-nyw5e-000000000000000": {Driver: "mock"}}
+ s.handler = &handler{}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
}
func (s *ProxyRemoteSuite) TearDownTest(c *check.C) {
- s.vm.Close()
- KeepVM = nil
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
s.remoteAPI.Close()
s.remoteKeepproxy.Close()
}
@@ -191,7 +184,7 @@ func (s *ProxyRemoteSuite) TestProxyRemote(c *check.C) {
req.Header.Set("X-Keep-Signature", trial.xKeepSignature)
}
resp = httptest.NewRecorder()
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(s.remoteKeepRequests, check.Equals, trial.expectRemoteReqs)
c.Check(resp.Code, check.Equals, trial.expectCode)
if resp.Code == http.StatusOK {
@@ -210,13 +203,13 @@ func (s *ProxyRemoteSuite) TestProxyRemote(c *check.C) {
c.Check(locHdr, check.Not(check.Equals), "")
c.Check(locHdr, check.Not(check.Matches), `.*\+R.*`)
- c.Check(VerifySignature(locHdr, trial.token), check.IsNil)
+ c.Check(VerifySignature(s.cluster, locHdr, trial.token), check.IsNil)
// Ensure block can be requested using new signature
req = httptest.NewRequest("GET", "/"+locHdr, nil)
req.Header.Set("Authorization", "Bearer "+trial.token)
resp = httptest.NewRecorder()
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(s.remoteKeepRequests, check.Equals, trial.expectRemoteReqs)
}
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 42b5d5889..100d08838 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -6,7 +6,6 @@ package main
import (
"context"
- "crypto/rand"
"fmt"
"io"
"io/ioutil"
@@ -18,15 +17,15 @@ import (
// RunPullWorker receives PullRequests from pullq, invokes
// PullItemAndProcess on each one. After each PR, it logs a message
// indicating whether the pull was successful.
-func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
+func (h *handler) runPullWorker(pullq *WorkQueue) {
for item := range pullq.NextItem {
pr := item.(PullRequest)
- err := PullItemAndProcess(pr, keepClient)
+ err := h.pullItemAndProcess(pr)
pullq.DoneItem <- struct{}{}
if err == nil {
- log.Printf("Pull %s success", pr)
+ h.Logger.Printf("Pull %s success", pr)
} else {
- log.Printf("Pull %s error: %s", pr, err)
+ h.Logger.Printf("Pull %s error: %s", pr, err)
}
}
}
@@ -39,28 +38,28 @@ func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
// only attempt to write the data to the corresponding
// volume. Otherwise it writes to any local volume, as a PUT request
// would.
-func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClient) error {
- var vol Volume
+func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
+ var vol *VolumeMount
if uuid := pullRequest.MountUUID; uuid != "" {
- vol = KeepVM.Lookup(pullRequest.MountUUID, true)
+ vol = h.volmgr.Lookup(pullRequest.MountUUID, true)
if vol == nil {
return fmt.Errorf("pull req has nonexistent mount: %v", pullRequest)
}
}
- keepClient.Arvados.ApiToken = randomToken
-
+ // Make a private copy of keepClient so we can set
+ // ServiceRoots to the source servers specified in the pull
+ // request.
+ keepClient := *h.keepClient
serviceRoots := make(map[string]string)
for _, addr := range pullRequest.Servers {
serviceRoots[addr] = addr
}
keepClient.SetServiceRoots(serviceRoots, nil, nil)
- // Generate signature with a random token
- expiresAt := time.Now().Add(60 * time.Second)
- signedLocator := SignLocator(pullRequest.Locator, randomToken, expiresAt)
+ signedLocator := SignLocator(h.Cluster, pullRequest.Locator, keepClient.Arvados.ApiToken, time.Now().Add(time.Minute))
- reader, contentLen, _, err := GetContent(signedLocator, keepClient)
+ reader, contentLen, _, err := GetContent(signedLocator, &keepClient)
if err != nil {
return err
}
@@ -78,8 +77,7 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
return fmt.Errorf("Content not found for: %s", signedLocator)
}
- writePulledBlock(vol, readContent, pullRequest.Locator)
- return nil
+ return writePulledBlock(h.volmgr, vol, readContent, pullRequest.Locator)
}
// Fetch the content for the given locator using keepclient.
@@ -87,24 +85,11 @@ var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (
return keepClient.Get(signedLocator)
}
-var writePulledBlock = func(volume Volume, data []byte, locator string) {
- var err error
+var writePulledBlock = func(volmgr *RRVolumeManager, volume Volume, data []byte, locator string) error {
if volume != nil {
- err = volume.Put(context.Background(), locator, data)
+ return volume.Put(context.Background(), locator, data)
} else {
- _, err = PutBlock(context.Background(), data, locator)
- }
- if err != nil {
- log.Printf("error writing pulled block %q: %s", locator, err)
+ _, err := PutBlock(context.Background(), volmgr, data, locator)
+ return err
}
}
-
-var randomToken = func() string {
- const alphaNumeric = "0123456789abcdefghijklmnopqrstuvwxyz"
- var bytes = make([]byte, 36)
- rand.Read(bytes)
- for i, b := range bytes {
- bytes[i] = alphaNumeric[b%byte(len(alphaNumeric))]
- }
- return (string(bytes))
-}()
diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index 231a4c0ab..a35b744c5 100644
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -6,20 +6,18 @@ package main
import (
"bytes"
+ "context"
"errors"
"io"
"io/ioutil"
- "os"
"strings"
- "testing"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
-var keepClient *keepclient.KeepClient
-
type PullWorkIntegrationTestData struct {
Name string
Locator string
@@ -27,55 +25,31 @@ type PullWorkIntegrationTestData struct {
GetError string
}
-func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
- // start api and keep servers
- arvadostest.StartAPI()
+func (s *HandlerSuite) setupPullWorkerIntegrationTest(c *check.C, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
arvadostest.StartKeep(2, false)
-
- // make arvadosclient
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- t.Fatalf("Error creating arv: %s", err)
- }
-
- // keep client
- keepClient, err = keepclient.MakeKeepClient(arv)
- if err != nil {
- t.Fatalf("error creating KeepClient: %s", err)
- }
- keepClient.Want_replicas = 1
- keepClient.RefreshServiceDiscovery()
-
- // discover keep services
- var servers []string
- for _, host := range keepClient.LocalRoots() {
- servers = append(servers, host)
- }
-
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// Put content if the test needs it
if wantData {
- locator, _, err := keepClient.PutB([]byte(testData.Content))
+ locator, _, err := s.handler.keepClient.PutB([]byte(testData.Content))
if err != nil {
- t.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
+ c.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
}
if locator == "" {
- t.Errorf("No locator found after putting test data")
+ c.Errorf("No locator found after putting test data")
}
}
// Create pullRequest for the test
pullRequest := PullRequest{
Locator: testData.Locator,
- Servers: servers,
}
return pullRequest
}
// Do a get on a block that is not existing in any of the keep servers.
// Expect "block not found" error.
-func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) {
+func (s *HandlerSuite) TestPullWorkerIntegration_GetNonExistingLocator(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
testData := PullWorkIntegrationTestData{
Name: "TestPullWorkerIntegration_GetLocator",
Locator: "5d41402abc4b2a76b9719d911017c592",
@@ -83,16 +57,17 @@ func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) {
GetError: "Block not found",
}
- pullRequest := SetupPullWorkerIntegrationTest(t, testData, false)
+ pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false)
defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
- performPullWorkerIntegrationTest(testData, pullRequest, t)
+ s.performPullWorkerIntegrationTest(testData, pullRequest, c)
}
// Do a get on a block that exists on one of the keep servers.
// The setup method will create this block before doing the get.
-func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) {
+func (s *HandlerSuite) TestPullWorkerIntegration_GetExistingLocator(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
testData := PullWorkIntegrationTestData{
Name: "TestPullWorkerIntegration_GetLocator",
Locator: "5d41402abc4b2a76b9719d911017c592",
@@ -100,24 +75,23 @@ func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) {
GetError: "",
}
- pullRequest := SetupPullWorkerIntegrationTest(t, testData, true)
+ pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true)
defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
- performPullWorkerIntegrationTest(testData, pullRequest, t)
+ s.performPullWorkerIntegrationTest(testData, pullRequest, c)
}
// Perform the test.
// The test directly invokes the "PullItemAndProcess" rather than
// putting an item on the pullq so that the errors can be verified.
-func performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, t *testing.T) {
+func (s *HandlerSuite) performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, c *check.C) {
// Override writePulledBlock to mock PutBlock functionality
- defer func(orig func(Volume, []byte, string)) { writePulledBlock = orig }(writePulledBlock)
- writePulledBlock = func(v Volume, content []byte, locator string) {
- if string(content) != testData.Content {
- t.Errorf("writePulledBlock invoked with unexpected data. Expected: %s; Found: %s", testData.Content, content)
- }
+ defer func(orig func(*RRVolumeManager, Volume, []byte, string) error) { writePulledBlock = orig }(writePulledBlock)
+ writePulledBlock = func(_ *RRVolumeManager, _ Volume, content []byte, _ string) error {
+ c.Check(string(content), check.Equals, testData.Content)
+ return nil
}
// Override GetContent to mock keepclient Get functionality
@@ -132,15 +106,15 @@ func performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pull
return rdr, int64(len(testData.Content)), "", nil
}
- err := PullItemAndProcess(pullRequest, keepClient)
+ err := s.handler.pullItemAndProcess(pullRequest)
if len(testData.GetError) > 0 {
if (err == nil) || (!strings.Contains(err.Error(), testData.GetError)) {
- t.Errorf("Got error %v, expected %v", err, testData.GetError)
+ c.Errorf("Got error %v, expected %v", err, testData.GetError)
}
} else {
if err != nil {
- t.Errorf("Got error %v, expected nil", err)
+ c.Errorf("Got error %v, expected nil", err)
}
}
}
diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go
index 8e667e048..6a7a0b7a8 100644
--- a/services/keepstore/pull_worker_test.go
+++ b/services/keepstore/pull_worker_test.go
@@ -6,21 +6,26 @@ package main
import (
"bytes"
+ "context"
"errors"
"io"
"io/ioutil"
"net/http"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"github.com/prometheus/client_golang/prometheus"
. "gopkg.in/check.v1"
+ check "gopkg.in/check.v1"
)
var _ = Suite(&PullWorkerTestSuite{})
type PullWorkerTestSuite struct {
+ cluster *arvados.Cluster
+ handler *handler
+
testPullLists map[string]string
readContent string
readError error
@@ -29,7 +34,16 @@ type PullWorkerTestSuite struct {
}
func (s *PullWorkerTestSuite) SetUpTest(c *C) {
- theConfig.systemAuthToken = "arbitrary data manager token"
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Driver: "mock"},
+ "zzzzz-nyw5e-111111111111111": {Driver: "mock"},
+ }
+ s.cluster.Collections.BlobReplicateConcurrency = 1
+
+ s.handler = &handler{}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+
s.readContent = ""
s.readError = nil
s.putContent = []byte{}
@@ -39,27 +53,6 @@ func (s *PullWorkerTestSuite) SetUpTest(c *C) {
// This behavior is verified using these two maps in the
// "TestPullWorkerPullList_with_two_items_latest_replacing_old"
s.testPullLists = make(map[string]string)
-
- KeepVM = MakeTestVolumeManager(2)
-
- // Normally the pull queue and workers are started by main()
- // -- tests need to set up their own.
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, IsNil)
- keepClient, err := keepclient.MakeKeepClient(arv)
- c.Assert(err, IsNil)
- pullq = NewWorkQueue()
- go RunPullWorker(pullq, keepClient)
-}
-
-func (s *PullWorkerTestSuite) TearDownTest(c *C) {
- KeepVM.Close()
- KeepVM = nil
- pullq.Close()
- pullq = nil
- teardown()
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
}
var firstPullList = []byte(`[
@@ -100,9 +93,10 @@ type PullWorkerTestData struct {
// Ensure MountUUID in a pull list is correctly translated to a Volume
// argument passed to writePulledBlock().
func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
- defer func(f func(Volume, []byte, string)) {
+ defer func(f func(*RRVolumeManager, Volume, []byte, string) error) {
writePulledBlock = f
}(writePulledBlock)
+ pullq := s.handler.Handler.(*router).pullq
for _, spec := range []struct {
sendUUID string
@@ -113,17 +107,18 @@ func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
expectVolume: nil,
},
{
- sendUUID: KeepVM.Mounts()[0].UUID,
- expectVolume: KeepVM.Mounts()[0].volume,
+ sendUUID: s.handler.volmgr.Mounts()[0].UUID,
+ expectVolume: s.handler.volmgr.Mounts()[0].Volume,
},
} {
- writePulledBlock = func(v Volume, _ []byte, _ string) {
+ writePulledBlock = func(_ *RRVolumeManager, v Volume, _ []byte, _ string) error {
c.Check(v, Equals, spec.expectVolume)
+ return nil
}
- resp := IssueRequest(&RequestTester{
+ resp := IssueRequest(s.handler, &RequestTester{
uri: "/pull",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
method: "PUT",
requestBody: []byte(`[{
"locator":"acbd18db4cc2f85cedef654fccc4a4d8+3",
@@ -141,7 +136,7 @@ func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello",
@@ -155,7 +150,7 @@ func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_two_locators(c *C) {
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hola",
@@ -169,7 +164,7 @@ func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "unused",
@@ -183,7 +178,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "unused",
@@ -197,7 +192,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hello hello",
@@ -211,7 +206,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello again",
@@ -238,6 +233,8 @@ func (s *PullWorkerTestSuite) TestPullWorker_invalidToken(c *C) {
}
func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
+ pullq := s.handler.Handler.(*router).pullq
+
s.testPullLists[testData.name] = testData.responseBody
processedPullLists := make(map[string]string)
@@ -247,7 +244,7 @@ func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
GetContent = orig
}(GetContent)
GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (reader io.ReadCloser, contentLength int64, url string, err error) {
- c.Assert(getStatusItem("PullQueue", "InProgress"), Equals, float64(1))
+ c.Assert(getStatusItem(s.handler, "PullQueue", "InProgress"), Equals, float64(1))
processedPullLists[testData.name] = testData.responseBody
if testData.readError {
err = errors.New("Error getting data")
@@ -261,20 +258,21 @@ func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
}
// Override writePulledBlock to mock PutBlock functionality
- defer func(orig func(Volume, []byte, string)) { writePulledBlock = orig }(writePulledBlock)
- writePulledBlock = func(v Volume, content []byte, locator string) {
+ defer func(orig func(*RRVolumeManager, Volume, []byte, string) error) { writePulledBlock = orig }(writePulledBlock)
+ writePulledBlock = func(_ *RRVolumeManager, v Volume, content []byte, locator string) error {
if testData.putError {
s.putError = errors.New("Error putting data")
- return
+ return s.putError
}
s.putContent = content
+ return nil
}
- c.Check(getStatusItem("PullQueue", "InProgress"), Equals, float64(0))
- c.Check(getStatusItem("PullQueue", "Queued"), Equals, float64(0))
- c.Check(getStatusItem("Version"), Not(Equals), "")
+ c.Check(getStatusItem(s.handler, "PullQueue", "InProgress"), Equals, float64(0))
+ c.Check(getStatusItem(s.handler, "PullQueue", "Queued"), Equals, float64(0))
+ c.Check(getStatusItem(s.handler, "Version"), Not(Equals), "")
- response := IssueRequest(&testData.req)
+ response := IssueRequest(s.handler, &testData.req)
c.Assert(response.Code, Equals, testData.responseCode)
c.Assert(response.Body.String(), Equals, testData.responseBody)
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 4c39dcd5c..22a38e208 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -10,10 +10,12 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,188 +28,41 @@ import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-const (
- s3DefaultReadTimeout = arvados.Duration(10 * time.Minute)
- s3DefaultConnectTimeout = arvados.Duration(time.Minute)
-)
-
-var (
- // ErrS3TrashDisabled is returned by Trash if that operation
- // is impossible with the current config.
- ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
-
- s3AccessKeyFile string
- s3SecretKeyFile string
- s3RegionName string
- s3Endpoint string
- s3Replication int
- s3UnsafeDelete bool
- s3RaceWindow time.Duration
-
- s3ACL = s3.Private
-
- zeroTime time.Time
-)
-
-const (
- maxClockSkew = 600 * time.Second
- nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
-)
-
-type s3VolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (s *s3VolumeAdder) String() string {
- return "-"
-}
-
-func (s *s3VolumeAdder) Set(bucketName string) error {
- if bucketName == "" {
- return fmt.Errorf("no container name given")
- }
- if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
- return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
- }
- if deprecated.flagSerializeIO {
- log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
- }
- s.Config.Volumes = append(s.Config.Volumes, &S3Volume{
- Bucket: bucketName,
- AccessKeyFile: s3AccessKeyFile,
- SecretKeyFile: s3SecretKeyFile,
- Endpoint: s3Endpoint,
- Region: s3RegionName,
- RaceWindow: arvados.Duration(s3RaceWindow),
- S3Replication: s3Replication,
- UnsafeDelete: s3UnsafeDelete,
- ReadOnly: deprecated.flagReadonly,
- IndexPageSize: 1000,
- })
- return nil
-}
-
-func s3regions() (okList []string) {
- for r := range aws.Regions {
- okList = append(okList, r)
- }
- return
-}
-
func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &S3Volume{} })
-
- flag.Var(&s3VolumeAdder{theConfig},
- "s3-bucket-volume",
- "Use the given bucket as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &s3RegionName,
- "s3-region",
- "",
- fmt.Sprintf("AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are %+q.", s3regions()))
- flag.StringVar(
- &s3Endpoint,
- "s3-endpoint",
- "",
- "Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use \"https://storage.googleapis.com\".")
- flag.StringVar(
- &s3AccessKeyFile,
- "s3-access-key-file",
- "",
- "`File` containing the access key used for subsequent -s3-bucket-volume arguments.")
- flag.StringVar(
- &s3SecretKeyFile,
- "s3-secret-key-file",
- "",
- "`File` containing the secret key used for subsequent -s3-bucket-volume arguments.")
- flag.DurationVar(
- &s3RaceWindow,
- "s3-race-window",
- 24*time.Hour,
- "Maximum eventual consistency latency for subsequent -s3-bucket-volume arguments.")
- flag.IntVar(
- &s3Replication,
- "s3-replication",
- 2,
- "Replication level reported to clients for subsequent -s3-bucket-volume arguments.")
- flag.BoolVar(
- &s3UnsafeDelete,
- "s3-unsafe-delete",
- false,
- "EXPERIMENTAL. Enable deletion (garbage collection) even when trash lifetime is zero, even though there are known race conditions that can cause data loss.")
+ driver["S3"] = newS3Volume
}
-// S3Volume implements Volume using an S3 bucket.
-type S3Volume struct {
- AccessKeyFile string
- SecretKeyFile string
- Endpoint string
- Region string
- Bucket string
- LocationConstraint bool
- IndexPageSize int
- S3Replication int
- ConnectTimeout arvados.Duration
- ReadTimeout arvados.Duration
- RaceWindow arvados.Duration
- ReadOnly bool
- UnsafeDelete bool
- StorageClasses []string
-
- bucket *s3bucket
-
- startOnce sync.Once
-}
-
-// Examples implements VolumeWithExamples.
-func (*S3Volume) Examples() []Volume {
- return []Volume{
- &S3Volume{
- AccessKeyFile: "/etc/aws_s3_access_key.txt",
- SecretKeyFile: "/etc/aws_s3_secret_key.txt",
- Endpoint: "",
- Region: "us-east-1",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
- &S3Volume{
- AccessKeyFile: "/etc/gce_s3_access_key.txt",
- SecretKeyFile: "/etc/gce_s3_secret_key.txt",
- Endpoint: "https://storage.googleapis.com",
- Region: "",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
+func newS3Volume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &S3Volume{cluster: cluster, volume: volume, logger: logger, metrics: metrics}
+ err := json.Unmarshal(volume.DriverParameters, &v)
+ if err != nil {
+ return nil, err
}
+ return v, v.check()
}
-// Type implements Volume.
-func (*S3Volume) Type() string {
- return "S3"
-}
+func (v *S3Volume) check() error {
+ if v.Bucket == "" || v.AccessKey == "" || v.SecretKey == "" {
+ return errors.New("DriverParameters: Bucket, AccessKey, and SecretKey must be provided")
+ }
+ if v.IndexPageSize == 0 {
+ v.IndexPageSize = 1000
+ }
+ if v.RaceWindow < 0 {
+ return errors.New("DriverParameters: RaceWindow must not be negative")
+ }
-// Start populates private fields and verifies the configuration is
-// valid.
-func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
region, ok := aws.Regions[v.Region]
if v.Endpoint == "" {
if !ok {
- return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region)
+ return fmt.Errorf("unrecognized region %+q; try specifying endpoint instead", v.Region)
}
} else if ok {
return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
- "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint)
+ "specify empty endpoint or use a different region name", v.Region, v.Endpoint)
} else {
region = aws.Region{
Name: v.Region,
@@ -216,15 +71,9 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
}
}
- var err error
- var auth aws.Auth
- auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile)
- if err != nil {
- return err
- }
- auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile)
- if err != nil {
- return err
+ auth := aws.Auth{
+ AccessKey: v.AccessKey,
+ SecretKey: v.SecretKey,
}
// Zero timeouts mean "wait forever", which is a bad
@@ -250,14 +99,63 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
},
}
// Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls)
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
return nil
}
-// DeviceID returns a globally unique ID for the storage bucket.
-func (v *S3Volume) DeviceID() string {
+const (
+ s3DefaultReadTimeout = arvados.Duration(10 * time.Minute)
+ s3DefaultConnectTimeout = arvados.Duration(time.Minute)
+)
+
+var (
+ // ErrS3TrashDisabled is returned by Trash if that operation
+ // is impossible with the current config.
+ ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
+
+ s3ACL = s3.Private
+
+ zeroTime time.Time
+)
+
+const (
+ maxClockSkew = 600 * time.Second
+ nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
+)
+
+func s3regions() (okList []string) {
+ for r := range aws.Regions {
+ okList = append(okList, r)
+ }
+ return
+}
+
+// S3Volume implements Volume using an S3 bucket.
+type S3Volume struct {
+ AccessKey string
+ SecretKey string
+ Endpoint string
+ Region string
+ Bucket string
+ LocationConstraint bool
+ IndexPageSize int
+ ConnectTimeout arvados.Duration
+ ReadTimeout arvados.Duration
+ RaceWindow arvados.Duration
+ UnsafeDelete bool
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ bucket *s3bucket
+ startOnce sync.Once
+}
+
+// GetDeviceID returns a globally unique ID for the storage bucket.
+func (v *S3Volume) GetDeviceID() string {
return "s3://" + v.Endpoint + "/" + v.Bucket
}
@@ -271,7 +169,7 @@ func (v *S3Volume) getReaderWithContext(ctx context.Context, loc string) (rdr io
case <-ready:
return
case <-ctx.Done():
- theConfig.debugLogf("s3: abandoning getReader(): %s", ctx.Err())
+ v.logger.Debugf("s3: abandoning getReader(): %s", ctx.Err())
go func() {
<-ready
if err == nil {
@@ -339,11 +237,11 @@ func (v *S3Volume) Get(ctx context.Context, loc string, buf []byte) (int, error)
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
+ v.logger.Debugf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
rdr.Close()
// Must wait for ReadFull to return, to ensure it
// doesn't write to buf after we return.
- theConfig.debugLogf("s3: waiting for ReadFull() to fail")
+ v.logger.Debug("s3: waiting for ReadFull() to fail")
<-ready
return 0, ctx.Err()
case <-ready:
@@ -397,7 +295,7 @@ func (v *S3Volume) Compare(ctx context.Context, loc string, expect []byte) error
// Put writes a block.
func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
var opts s3.Options
@@ -433,7 +331,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go func() {
defer func() {
if ctx.Err() != nil {
- theConfig.debugLogf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
+ v.logger.Debugf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
}
}()
defer close(ready)
@@ -445,7 +343,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking PutReader's input away: %s", v, ctx.Err())
+ v.logger.Debugf("%s: taking PutReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// PutReader() to read. If so, un-stick it. This means
// PutReader will get corrupt data, but that's OK: the
@@ -453,7 +351,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning PutReader goroutine", v)
+ v.logger.Debugf("%s: abandoning PutReader goroutine", v)
return ctx.Err()
case <-ready:
// Unblock pipe in case PutReader did not consume it.
@@ -464,7 +362,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
// Touch sets the timestamp for the given locator to the current time.
func (v *S3Volume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
_, err := v.bucket.Head(loc, nil)
@@ -571,16 +469,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
// Trash a Keep block.
func (v *S3Volume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
- if theConfig.TrashLifetime == 0 {
- if !s3UnsafeDelete {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
+ if !v.UnsafeDelete {
return ErrS3TrashDisabled
}
return v.translateError(v.bucket.Del(loc))
@@ -615,7 +513,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
// Can't parse timestamp
return err
}
- safeWindow := t.Add(theConfig.TrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
+ safeWindow := t.Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
if safeWindow <= 0 {
// We can't count on "touch trash/X" to prolong
// trash/X's lifetime. The new timestamp might not
@@ -698,23 +596,6 @@ func (v *S3Volume) String() string {
return fmt.Sprintf("s3-bucket:%+q", v.Bucket)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *S3Volume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the storage redundancy of the underlying
-// device. Configured via command line flag.
-func (v *S3Volume) Replication() int {
- return v.S3Replication
-}
-
-// GetStorageClasses implements Volume
-func (v *S3Volume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
func (v *S3Volume) isKeepBlock(s string) bool {
@@ -751,13 +632,13 @@ func (v *S3Volume) fixRace(loc string) bool {
}
ageWhenTrashed := trashTime.Sub(recentTime)
- if ageWhenTrashed >= theConfig.BlobSignatureTTL.Duration() {
+ if ageWhenTrashed >= v.cluster.Collections.BlobSigningTTL.Duration() {
// No evidence of a race: block hasn't been written
// since it became eligible for Trash. No fix needed.
return false
}
- log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, theConfig.BlobSignatureTTL)
+ log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, v.cluster.Collections.BlobSigningTTL)
log.Printf("notice: fixRace: copying %q to %q to recover from race between Put/Touch and Trash", "recent/"+loc, loc)
err = v.safeCopy(loc, "trash/"+loc)
if err != nil {
@@ -785,6 +666,10 @@ func (v *S3Volume) translateError(err error) error {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *S3Volume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
// Define "ready to delete" as "...when EmptyTrash started".
@@ -820,8 +705,8 @@ func (v *S3Volume) EmptyTrash() {
log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
return
}
- if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
- if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
+ if trashT.Sub(recentT) < v.cluster.Collections.BlobSigningTTL.Duration() {
+ if age := startT.Sub(recentT); age >= v.cluster.Collections.BlobSigningTTL.Duration()-time.Duration(v.RaceWindow) {
// recent/loc is too old to protect
// loc from being Trashed again during
// the raceWindow that starts if we
@@ -845,7 +730,7 @@ func (v *S3Volume) EmptyTrash() {
return
}
}
- if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
+ if startT.Sub(trashT) < v.cluster.Collections.BlobTrashLifetime.Duration() {
return
}
err = v.bucket.Del(trash.Key)
@@ -872,8 +757,8 @@ func (v *S3Volume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan *s3.Key, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 6377420ff..b8c4458a5 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -10,17 +10,19 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
- "io/ioutil"
+ "log"
"net/http"
"net/http/httptest"
"os"
+ "strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/s3/s3test"
- "github.com/ghodss/yaml"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -39,34 +41,41 @@ func (c *fakeClock) Now() time.Time {
return *c.now
}
-func init() {
- // Deleting isn't safe from races, but if it's turned on
- // anyway we do expect it to pass the generic volume tests.
- s3UnsafeDelete = true
-}
-
var _ = check.Suite(&StubbedS3Suite{})
type StubbedS3Suite struct {
- volumes []*TestableS3Volume
+ s3server *httptest.Server
+ cluster *arvados.Cluster
+ handler *handler
+ volumes []*TestableS3Volume
+}
+
+func (s *StubbedS3Suite) SetUpTest(c *check.C) {
+ s.s3server = nil
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Driver: "S3"},
+ "zzzzz-nyw5e-111111111111111": {Driver: "S3"},
+ }
+ s.handler = &handler{}
}
func (s *StubbedS3Suite) TestGeneric(c *check.C) {
- DoGenericVolumeTests(c, func(t TB) TestableVolume {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
// Use a negative raceWindow so s3test's 1-second
// timestamp precision doesn't confuse fixRace.
- return s.newTestableVolume(c, -2*time.Second, false, 2)
+ return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second)
})
}
func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
- DoGenericVolumeTests(c, func(t TB) TestableVolume {
- return s.newTestableVolume(c, -2*time.Second, true, 2)
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second)
})
}
func (s *StubbedS3Suite) TestIndex(c *check.C) {
- v := s.newTestableVolume(c, 0, false, 2)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 0)
v.IndexPageSize = 3
for i := 0; i < 256; i++ {
v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111})
@@ -91,7 +100,7 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) {
}
func (s *StubbedS3Suite) TestStats(c *check.C) {
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
stats := func() string {
buf, err := json.Marshal(v.InternalStats())
c.Check(err, check.IsNil)
@@ -125,6 +134,11 @@ type blockingHandler struct {
}
func (h *blockingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "PUT" && !strings.Contains(strings.Trim(r.URL.Path, "/"), "/") {
+ // Accept PutBucket ("PUT /bucketname/"), called by
+ // newTestableVolume
+ return
+ }
if h.requested != nil {
h.requested <- r
}
@@ -164,15 +178,10 @@ func (s *StubbedS3Suite) TestPutContextCancel(c *check.C) {
func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Context, *TestableS3Volume) error) {
handler := &blockingHandler{}
- srv := httptest.NewServer(handler)
- defer srv.Close()
+ s.s3server = httptest.NewServer(handler)
+ defer s.s3server.Close()
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
- vol := *v.S3Volume
- vol.Endpoint = srv.URL
- v = &TestableS3Volume{S3Volume: &vol}
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
ctx, cancel := context.WithCancel(context.Background())
@@ -209,14 +218,10 @@ func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Con
}
func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
- defer func(tl, bs arvados.Duration) {
- theConfig.TrashLifetime = tl
- theConfig.BlobSignatureTTL = bs
- }(theConfig.TrashLifetime, theConfig.BlobSignatureTTL)
- theConfig.TrashLifetime.Set("1h")
- theConfig.BlobSignatureTTL.Set("1h")
-
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
+ s.cluster.Collections.BlobSigningTTL.Set("1h")
+
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
var none time.Time
putS3Obj := func(t time.Time, key string, data []byte) {
@@ -411,61 +416,42 @@ type TestableS3Volume struct {
serverClock *fakeClock
}
-func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume {
+func (s *StubbedS3Suite) newTestableVolume(c *check.C, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs, raceWindow time.Duration) *TestableS3Volume {
clock := &fakeClock{}
srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
c.Assert(err, check.IsNil)
+ endpoint := srv.URL()
+ if s.s3server != nil {
+ endpoint = s.s3server.URL
+ }
v := &TestableS3Volume{
S3Volume: &S3Volume{
+ AccessKey: "xxx",
+ SecretKey: "xxx",
Bucket: TestBucketName,
- Endpoint: srv.URL(),
+ Endpoint: endpoint,
Region: "test-region-1",
LocationConstraint: true,
- RaceWindow: arvados.Duration(raceWindow),
- S3Replication: replication,
- UnsafeDelete: s3UnsafeDelete,
- ReadOnly: readonly,
+ UnsafeDelete: true,
IndexPageSize: 1000,
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(c),
+ metrics: metrics,
},
c: c,
server: srv,
serverClock: clock,
}
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics)
- err = v.bucket.PutBucket(s3.ACL("private"))
- c.Assert(err, check.IsNil)
+ c.Assert(v.S3Volume.check(), check.IsNil)
+ c.Assert(v.bucket.PutBucket(s3.ACL("private")), check.IsNil)
+ // We couldn't set RaceWindow until now because check()
+ // rejects negative values.
+ v.S3Volume.RaceWindow = arvados.Duration(raceWindow)
return v
}
-func (s *StubbedS3Suite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: S3
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
-
-func (v *TestableS3Volume) Start(vm *volumeMetricsVecs) error {
- tmp, err := ioutil.TempFile("", "keepstore")
- v.c.Assert(err, check.IsNil)
- defer os.Remove(tmp.Name())
- _, err = tmp.Write([]byte("xxx\n"))
- v.c.Assert(err, check.IsNil)
- v.c.Assert(tmp.Close(), check.IsNil)
-
- v.S3Volume.AccessKeyFile = tmp.Name()
- v.S3Volume.SecretKeyFile = tmp.Name()
-
- v.c.Assert(v.S3Volume.Start(vm), check.IsNil)
- return nil
-}
-
// PutRaw skips the ContentMD5 test
func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
diff --git a/services/keepstore/server.go b/services/keepstore/server.go
deleted file mode 100644
index 3f6727712..000000000
--- a/services/keepstore/server.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "crypto/tls"
- "net"
- "net/http"
- "os"
- "os/signal"
- "syscall"
-)
-
-type server struct {
- http.Server
-
- // channel (size=1) with the current keypair
- currentCert chan *tls.Certificate
-}
-
-func (srv *server) Serve(l net.Listener) error {
- if theConfig.TLSCertificateFile == "" && theConfig.TLSKeyFile == "" {
- return srv.Server.Serve(l)
- }
- // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
- srv.TLSConfig = &tls.Config{
- GetCertificate: srv.getCertificate,
- PreferServerCipherSuites: true,
- CurvePreferences: []tls.CurveID{
- tls.CurveP256,
- tls.X25519,
- },
- MinVersion: tls.VersionTLS12,
- CipherSuites: []uint16{
- tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
- tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
- },
- }
- srv.currentCert = make(chan *tls.Certificate, 1)
- go srv.refreshCertificate(theConfig.TLSCertificateFile, theConfig.TLSKeyFile)
- return srv.Server.ServeTLS(l, "", "")
-}
-
-func (srv *server) refreshCertificate(certfile, keyfile string) {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Fatal("error loading X509 key pair")
- }
- srv.currentCert <- &cert
-
- reload := make(chan os.Signal, 1)
- signal.Notify(reload, syscall.SIGHUP)
- for range reload {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Warn("error loading X509 key pair")
- continue
- }
- // Throw away old cert and start using new one
- <-srv.currentCert
- srv.currentCert <- &cert
- }
-}
-
-func (srv *server) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
- if srv.currentCert == nil {
- panic("srv.currentCert not initialized")
- }
- cert := <-srv.currentCert
- srv.currentCert <- cert
- return cert, nil
-}
diff --git a/services/keepstore/server_test.go b/services/keepstore/server_test.go
deleted file mode 100644
index 84adf3603..000000000
--- a/services/keepstore/server_test.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
- "crypto/tls"
- "io/ioutil"
- "net"
- "net/http"
- "testing"
-)
-
-func TestTLS(t *testing.T) {
- defer func() {
- theConfig.TLSKeyFile = ""
- theConfig.TLSCertificateFile = ""
- }()
- theConfig.TLSKeyFile = "../api/tmp/self-signed.key"
- theConfig.TLSCertificateFile = "../api/tmp/self-signed.pem"
- srv := &server{}
- srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("OK"))
- })
- l, err := net.Listen("tcp", ":")
- if err != nil {
- t.Fatal(err)
- }
- defer l.Close()
- go srv.Serve(l)
- defer srv.Shutdown(context.Background())
- c := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
- resp, err := c.Get("https://" + l.Addr().String() + "/")
- if err != nil {
- t.Fatal(err)
- }
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- t.Error(err)
- }
- if !bytes.Equal(body, []byte("OK")) {
- t.Errorf("expected OK, got %q", body)
- }
-}
diff --git a/services/keepstore/status_test.go b/services/keepstore/status_test.go
index dc6efb083..7bff2584e 100644
--- a/services/keepstore/status_test.go
+++ b/services/keepstore/status_test.go
@@ -14,8 +14,8 @@ import (
// getStatusItem("foo","bar","baz") retrieves /status.json, decodes
// the response body into resp, and returns resp["foo"]["bar"]["baz"].
-func getStatusItem(keys ...string) interface{} {
- resp := IssueRequest(&RequestTester{"/status.json", "", "GET", nil})
+func getStatusItem(h *handler, keys ...string) interface{} {
+ resp := IssueRequest(h, &RequestTester{"/status.json", "", "GET", nil})
var s interface{}
json.NewDecoder(resp.Body).Decode(&s)
for _, k := range keys {
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 8a9fedfb7..ba1455ac6 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -6,6 +6,7 @@ package main
import (
"errors"
+ "log"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -17,35 +18,35 @@ import (
// Delete the block indicated by the trash request Locator
// Repeat
//
-func RunTrashWorker(trashq *WorkQueue) {
+func RunTrashWorker(volmgr *RRVolumeManager, cluster *arvados.Cluster, trashq *WorkQueue) {
for item := range trashq.NextItem {
trashRequest := item.(TrashRequest)
- TrashItem(trashRequest)
+ TrashItem(volmgr, cluster, trashRequest)
trashq.DoneItem <- struct{}{}
}
}
// TrashItem deletes the indicated block from every writable volume.
-func TrashItem(trashRequest TrashRequest) {
+func TrashItem(volmgr *RRVolumeManager, cluster *arvados.Cluster, trashRequest TrashRequest) {
reqMtime := time.Unix(0, trashRequest.BlockMtime)
- if time.Since(reqMtime) < theConfig.BlobSignatureTTL.Duration() {
+ if time.Since(reqMtime) < cluster.Collections.BlobSigningTTL.Duration() {
log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
arvados.Duration(time.Since(reqMtime)),
trashRequest.Locator,
trashRequest.BlockMtime,
reqMtime,
- theConfig.BlobSignatureTTL)
+ cluster.Collections.BlobSigningTTL)
return
}
- var volumes []Volume
+ var volumes []*VolumeMount
if uuid := trashRequest.MountUUID; uuid == "" {
- volumes = KeepVM.AllWritable()
- } else if v := KeepVM.Lookup(uuid, true); v == nil {
+ volumes = volmgr.AllWritable()
+ } else if mnt := volmgr.Lookup(uuid, true); mnt == nil {
log.Printf("warning: trash request for nonexistent mount: %v", trashRequest)
return
} else {
- volumes = []Volume{v}
+ volumes = []*VolumeMount{mnt}
}
for _, volume := range volumes {
@@ -59,8 +60,8 @@ func TrashItem(trashRequest TrashRequest) {
continue
}
- if !theConfig.EnableDelete {
- err = errors.New("skipping because EnableDelete is false")
+ if !cluster.Collections.BlobTrash {
+ err = errors.New("skipping because Collections.BlobTrash is false")
} else {
err = volume.Trash(trashRequest.Locator)
}
diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index c5a410b06..bd3743090 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -7,8 +7,10 @@ package main
import (
"container/list"
"context"
- "testing"
"time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
type TrashWorkerTestData struct {
@@ -36,8 +38,8 @@ type TrashWorkerTestData struct {
/* Delete block that does not exist in any of the keep volumes.
Expect no errors.
*/
-func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_GetNonExistingLocator(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: "5d41402abc4b2a76b9719d911017c592",
Block1: []byte("hello"),
@@ -52,14 +54,14 @@ func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block that exists on volume 1 of the keep servers.
Expect the second locator in volume 2 to be unaffected.
*/
-func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInVolume1(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -74,14 +76,14 @@ func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block that exists on volume 2 of the keep servers.
Expect the first locator in volume 1 to be unaffected.
*/
-func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInVolume2(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -96,14 +98,14 @@ func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block with matching mtime for locator in both volumes.
Expect locator to be deleted from both volumes.
*/
-func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInBothVolumes(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -118,14 +120,14 @@ func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Same locator with different Mtimes exists in both volumes.
Delete the second and expect the first to be still around.
*/
-func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -141,14 +143,14 @@ func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *test
ExpectLocator1: true,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
// Delete a block that exists on both volumes with matching mtimes,
// but specify a MountUUID in the request so it only gets deleted from
// the first volume.
-func TestTrashWorkerIntegration_SpecifyMountUUID(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_SpecifyMountUUID(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -164,15 +166,15 @@ func TestTrashWorkerIntegration_SpecifyMountUUID(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Two different locators in volume 1.
Delete one of them.
Expect the other unaffected.
*/
-func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -188,14 +190,14 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Allow default Trash Life time to be used. Thus, the newly created block
will not be deleted because its Mtime is within the trash life time.
*/
-func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -214,14 +216,14 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block with matching mtime for locator in both volumes, but EnableDelete is false,
so block won't be deleted.
*/
-func TestTrashWorkerIntegration_DisabledDelete(t *testing.T) {
- theConfig.EnableDelete = false
+func (s *HandlerSuite) TestTrashWorkerIntegration_DisabledDelete(c *check.C) {
+ s.cluster.Collections.BlobTrash = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -236,31 +238,34 @@ func TestTrashWorkerIntegration_DisabledDelete(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Perform the test */
-func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
- // Create Keep Volumes
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) performTrashWorkerTest(c *check.C, testData TrashWorkerTestData) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ // Replace the router's trashq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ trashq := NewWorkQueue()
+ s.handler.Handler.(*router).trashq = trashq
// Put test content
- vols := KeepVM.AllWritable()
+ mounts := s.handler.volmgr.AllWritable()
if testData.CreateData {
- vols[0].Put(context.Background(), testData.Locator1, testData.Block1)
- vols[0].Put(context.Background(), testData.Locator1+".meta", []byte("metadata"))
+ mounts[0].Put(context.Background(), testData.Locator1, testData.Block1)
+ mounts[0].Put(context.Background(), testData.Locator1+".meta", []byte("metadata"))
if testData.CreateInVolume1 {
- vols[0].Put(context.Background(), testData.Locator2, testData.Block2)
- vols[0].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
+ mounts[0].Put(context.Background(), testData.Locator2, testData.Block2)
+ mounts[0].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
} else {
- vols[1].Put(context.Background(), testData.Locator2, testData.Block2)
- vols[1].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
+ mounts[1].Put(context.Background(), testData.Locator2, testData.Block2)
+ mounts[1].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
}
}
- oldBlockTime := time.Now().Add(-theConfig.BlobSignatureTTL.Duration() - time.Minute)
+ oldBlockTime := time.Now().Add(-s.cluster.Collections.BlobSigningTTL.Duration() - time.Minute)
// Create TrashRequest for the test
trashRequest := TrashRequest{
@@ -268,37 +273,35 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
BlockMtime: oldBlockTime.UnixNano(),
}
if testData.SpecifyMountUUID {
- trashRequest.MountUUID = KeepVM.Mounts()[0].UUID
+ trashRequest.MountUUID = s.handler.volmgr.Mounts()[0].UUID
}
// Run trash worker and put the trashRequest on trashq
trashList := list.New()
trashList.PushBack(trashRequest)
- trashq = NewWorkQueue()
- defer trashq.Close()
if !testData.UseTrashLifeTime {
// Trash worker would not delete block if its Mtime is
// within trash life time. Back-date the block to
// allow the deletion to succeed.
- for _, v := range vols {
- v.(*MockVolume).Timestamps[testData.DeleteLocator] = oldBlockTime
+ for _, mnt := range mounts {
+ mnt.Volume.(*MockVolume).Timestamps[testData.DeleteLocator] = oldBlockTime
if testData.DifferentMtimes {
oldBlockTime = oldBlockTime.Add(time.Second)
}
}
}
- go RunTrashWorker(trashq)
+ go RunTrashWorker(s.handler.volmgr, s.cluster, trashq)
// Install gate so all local operations block until we say go
gate := make(chan struct{})
- for _, v := range vols {
- v.(*MockVolume).Gate = gate
+ for _, mnt := range mounts {
+ mnt.Volume.(*MockVolume).Gate = gate
}
assertStatusItem := func(k string, expect float64) {
- if v := getStatusItem("TrashQueue", k); v != expect {
- t.Errorf("Got %s %v, expected %v", k, v, expect)
+ if v := getStatusItem(s.handler, "TrashQueue", k); v != expect {
+ c.Errorf("Got %s %v, expected %v", k, v, expect)
}
}
@@ -309,7 +312,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
trashq.ReplaceQueue(trashList)
// Wait for worker to take request(s)
- expectEqualWithin(t, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
+ expectEqualWithin(c, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
// Ensure status.json also reports work is happening
assertStatusItem("InProgress", float64(1))
@@ -319,31 +322,31 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
close(gate)
// Wait for worker to finish
- expectEqualWithin(t, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
+ expectEqualWithin(c, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
// Verify Locator1 to be un/deleted as expected
buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testData.Locator1, buf, nil)
+ size, err := GetBlock(context.Background(), s.handler.volmgr, testData.Locator1, buf, nil)
if testData.ExpectLocator1 {
if size == 0 || err != nil {
- t.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
+ c.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
}
} else {
if size > 0 || err == nil {
- t.Errorf("Expected Locator1 to be deleted: %s", testData.Locator1)
+ c.Errorf("Expected Locator1 to be deleted: %s", testData.Locator1)
}
}
// Verify Locator2 to be un/deleted as expected
if testData.Locator1 != testData.Locator2 {
- size, err = GetBlock(context.Background(), testData.Locator2, buf, nil)
+ size, err = GetBlock(context.Background(), s.handler.volmgr, testData.Locator2, buf, nil)
if testData.ExpectLocator2 {
if size == 0 || err != nil {
- t.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)
+ c.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)
}
} else {
if size > 0 || err == nil {
- t.Errorf("Expected Locator2 to be deleted: %s", testData.Locator2)
+ c.Errorf("Expected Locator2 to be deleted: %s", testData.Locator2)
}
}
}
@@ -353,14 +356,12 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
// the trash request.
if testData.DifferentMtimes {
locatorFoundIn := 0
- for _, volume := range KeepVM.AllReadable() {
+ for _, volume := range s.handler.volmgr.AllReadable() {
buf := make([]byte, BlockSize)
if _, err := volume.Get(context.Background(), testData.Locator1, buf); err == nil {
locatorFoundIn = locatorFoundIn + 1
}
}
- if locatorFoundIn != 1 {
- t.Errorf("Found %d copies of %s, expected 1", locatorFoundIn, testData.Locator1)
- }
+ c.Check(locatorFoundIn, check.Equals, 1)
}
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 4d9e798ac..918555c2b 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -5,12 +5,13 @@
package main
import (
- "bufio"
"context"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"os"
"os/exec"
"path/filepath"
@@ -22,98 +23,52 @@ import (
"syscall"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-type unixVolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (vs *unixVolumeAdder) String() string {
- return "-"
+func init() {
+ driver["Directory"] = newDirectoryVolume
}
-func (vs *unixVolumeAdder) Set(path string) error {
- if dirs := strings.Split(path, ","); len(dirs) > 1 {
- log.Print("DEPRECATED: using comma-separated volume list.")
- for _, dir := range dirs {
- if err := vs.Set(dir); err != nil {
- return err
- }
- }
- return nil
+func newDirectoryVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &UnixVolume{cluster: cluster, volume: volume, logger: logger, metrics: metrics}
+ err := json.Unmarshal(volume.DriverParameters, &v)
+ if err != nil {
+ return nil, err
}
- vs.Config.Volumes = append(vs.Config.Volumes, &UnixVolume{
- Root: path,
- ReadOnly: deprecated.flagReadonly,
- Serialize: deprecated.flagSerializeIO,
- })
- return nil
-}
-
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &UnixVolume{} })
-
- flag.Var(&unixVolumeAdder{theConfig}, "volumes", "see Volumes configuration")
- flag.Var(&unixVolumeAdder{theConfig}, "volume", "see Volumes configuration")
+ return v, v.check()
}
-// Discover adds a UnixVolume for every directory named "keep" that is
-// located at the top level of a device- or tmpfs-backed mount point
-// other than "/". It returns the number of volumes added.
-func (vs *unixVolumeAdder) Discover() int {
- added := 0
- f, err := os.Open(ProcMounts)
- if err != nil {
- log.Fatalf("opening %s: %s", ProcMounts, err)
+func (v *UnixVolume) check() error {
+ if v.Root == "" {
+ return errors.New("DriverParameters.Root was not provided")
}
- scanner := bufio.NewScanner(f)
- for scanner.Scan() {
- args := strings.Fields(scanner.Text())
- if err := scanner.Err(); err != nil {
- log.Fatalf("reading %s: %s", ProcMounts, err)
- }
- dev, mount := args[0], args[1]
- if mount == "/" {
- continue
- }
- if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
- continue
- }
- keepdir := mount + "/keep"
- if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
- continue
- }
- // Set the -readonly flag (but only for this volume)
- // if the filesystem is mounted readonly.
- flagReadonlyWas := deprecated.flagReadonly
- for _, fsopt := range strings.Split(args[3], ",") {
- if fsopt == "ro" {
- deprecated.flagReadonly = true
- break
- }
- if fsopt == "rw" {
- break
- }
- }
- if err := vs.Set(keepdir); err != nil {
- log.Printf("adding %q: %s", keepdir, err)
- } else {
- added++
- }
- deprecated.flagReadonly = flagReadonlyWas
+ if v.Serialize {
+ v.locker = &sync.Mutex{}
+ }
+ if !strings.HasPrefix(v.Root, "/") {
+ return fmt.Errorf("DriverParameters.Root %q does not start with '/'", v.Root)
}
- return added
+
+ // Set up prometheus metrics
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
+
+ _, err := v.os.Stat(v.Root)
+ return err
}
// A UnixVolume stores and retrieves blocks in a local directory.
type UnixVolume struct {
- Root string // path to the volume's root directory
- ReadOnly bool
- Serialize bool
- DirectoryReplication int
- StorageClasses []string
+ Root string // path to the volume's root directory
+ Serialize bool
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
// something to lock during IO, typically a sync.Mutex (or nil
// to skip locking)
@@ -122,12 +77,12 @@ type UnixVolume struct {
os osWithStats
}
-// DeviceID returns a globally unique ID for the volume's root
+// GetDeviceID returns a globally unique ID for the volume's root
// directory, consisting of the filesystem's UUID and the path from
// filesystem root to storage directory, joined by "/". For example,
-// the DeviceID for a local directory "/mnt/xvda1/keep" might be
+// the device ID for a local directory "/mnt/xvda1/keep" might be
// "fa0b6166-3b55-4994-bd3f-92f4e00a1bb0/keep".
-func (v *UnixVolume) DeviceID() string {
+func (v *UnixVolume) GetDeviceID() string {
giveup := func(f string, args ...interface{}) string {
log.Printf(f+"; using blank DeviceID for volume %s", append(args, v)...)
return ""
@@ -198,50 +153,9 @@ func (v *UnixVolume) DeviceID() string {
return giveup("could not find entry in %q matching %q", udir, dev)
}
-// Examples implements VolumeWithExamples.
-func (*UnixVolume) Examples() []Volume {
- return []Volume{
- &UnixVolume{
- Root: "/mnt/local-disk",
- Serialize: true,
- DirectoryReplication: 1,
- },
- &UnixVolume{
- Root: "/mnt/network-disk",
- Serialize: false,
- DirectoryReplication: 2,
- },
- }
-}
-
-// Type implements Volume
-func (v *UnixVolume) Type() string {
- return "Directory"
-}
-
-// Start implements Volume
-func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
- if v.Serialize {
- v.locker = &sync.Mutex{}
- }
- if !strings.HasPrefix(v.Root, "/") {
- return fmt.Errorf("volume root does not start with '/': %q", v.Root)
- }
- if v.DirectoryReplication == 0 {
- v.DirectoryReplication = 1
- }
- // Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = vm.getCounterVecsFor(lbls)
-
- _, err := v.os.Stat(v.Root)
-
- return err
-}
-
// Touch sets the timestamp for the given locator to the current time
func (v *UnixVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
p := v.blockPath(loc)
@@ -349,7 +263,7 @@ func (v *UnixVolume) Put(ctx context.Context, loc string, block []byte) error {
// WriteBlock implements BlockWriter.
func (v *UnixVolume) WriteBlock(ctx context.Context, loc string, rdr io.Reader) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if v.IsFull() {
@@ -516,7 +430,7 @@ func (v *UnixVolume) Trash(loc string) error {
// Trash() will read the correct up-to-date timestamp and choose not to
// trash the file.
- if v.ReadOnly {
+ if v.volume.ReadOnly || !v.cluster.Collections.BlobTrash {
return MethodDisabledError
}
if err := v.lock(context.TODO()); err != nil {
@@ -541,21 +455,21 @@ func (v *UnixVolume) Trash(loc string) error {
// anyway (because the permission signatures have expired).
if fi, err := v.os.Stat(p); err != nil {
return err
- } else if time.Since(fi.ModTime()) < time.Duration(theConfig.BlobSignatureTTL) {
+ } else if time.Since(fi.ModTime()) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
- if theConfig.TrashLifetime == 0 {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.os.Remove(p)
}
- return v.os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()))
+ return v.os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()))
}
// Untrash moves block from trash back into store
// Look for path/{loc}.trash.{deadline} in storage,
// and rename the first such file as path/{loc}
func (v *UnixVolume) Untrash(loc string) (err error) {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -650,23 +564,6 @@ func (v *UnixVolume) String() string {
return fmt.Sprintf("[UnixVolume %s]", v.Root)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *UnixVolume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the number of replicas promised by the
-// underlying device (as specified in configuration).
-func (v *UnixVolume) Replication() int {
- return v.DirectoryReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *UnixVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// InternalStats returns I/O and filesystem ops counters.
func (v *UnixVolume) InternalStats() interface{} {
return &v.os.stats
@@ -739,6 +636,10 @@ var unixTrashLocRegexp = regexp.MustCompile(`/([0-9a-f]{32})\.trash\.(\d+)$`)
// EmptyTrash walks hierarchy looking for {hash}.trash.*
// and deletes those with deadline < now.
func (v *UnixVolume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
@@ -774,8 +675,8 @@ func (v *UnixVolume) EmptyTrash() {
info os.FileInfo
}
var wg sync.WaitGroup
- todo := make(chan dirent, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan dirent, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index 872f408cf..1ffc46513 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -16,11 +16,11 @@ import (
"strings"
"sync"
"syscall"
- "testing"
"time"
- "github.com/ghodss/yaml"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -29,32 +29,13 @@ type TestableUnixVolume struct {
t TB
}
-func NewTestableUnixVolume(t TB, serialize bool, readonly bool) *TestableUnixVolume {
- d, err := ioutil.TempDir("", "volume_test")
- if err != nil {
- t.Fatal(err)
- }
- var locker sync.Locker
- if serialize {
- locker = &sync.Mutex{}
- }
- return &TestableUnixVolume{
- UnixVolume: UnixVolume{
- Root: d,
- ReadOnly: readonly,
- locker: locker,
- },
- t: t,
- }
-}
-
// PutRaw writes a Keep block directly into a UnixVolume, even if
// the volume is readonly.
func (v *TestableUnixVolume) PutRaw(locator string, data []byte) {
defer func(orig bool) {
- v.ReadOnly = orig
- }(v.ReadOnly)
- v.ReadOnly = false
+ v.volume.ReadOnly = orig
+ }(v.volume.ReadOnly)
+ v.volume.ReadOnly = false
err := v.Put(context.Background(), locator, data)
if err != nil {
v.t.Fatal(err)
@@ -70,7 +51,7 @@ func (v *TestableUnixVolume) TouchWithDate(locator string, lastPut time.Time) {
func (v *TestableUnixVolume) Teardown() {
if err := os.RemoveAll(v.Root); err != nil {
- v.t.Fatal(err)
+ v.t.Error(err)
}
}
@@ -78,59 +59,77 @@ func (v *TestableUnixVolume) ReadWriteOperationLabelValues() (r, w string) {
return "open", "create"
}
+var _ = check.Suite(&UnixVolumeSuite{})
+
+type UnixVolumeSuite struct {
+ cluster *arvados.Cluster
+ volumes []*TestableUnixVolume
+ metrics *volumeMetricsVecs
+}
+
+func (s *UnixVolumeSuite) SetUpTest(c *check.C) {
+ s.cluster = testCluster(c)
+ s.metrics = newVolumeMetricsVecs(prometheus.NewRegistry())
+}
+
+func (s *UnixVolumeSuite) TearDownTest(c *check.C) {
+ for _, v := range s.volumes {
+ v.Teardown()
+ }
+}
+
+func (s *UnixVolumeSuite) newTestableUnixVolume(c *check.C, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs, serialize bool) *TestableUnixVolume {
+ d, err := ioutil.TempDir("", "volume_test")
+ c.Check(err, check.IsNil)
+ var locker sync.Locker
+ if serialize {
+ locker = &sync.Mutex{}
+ }
+ v := &TestableUnixVolume{
+ UnixVolume: UnixVolume{
+ Root: d,
+ locker: locker,
+ cluster: cluster,
+ volume: volume,
+ metrics: metrics,
+ },
+ t: c,
+ }
+ c.Check(v.check(), check.IsNil)
+ s.volumes = append(s.volumes, v)
+ return v
+}
+
// serialize = false; readonly = false
-func TestUnixVolumeWithGenericTests(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTests(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, false)
})
}
// serialize = false; readonly = true
-func TestUnixVolumeWithGenericTestsReadOnly(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, false, true)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTestsReadOnly(c *check.C) {
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, true)
})
}
// serialize = true; readonly = false
-func TestUnixVolumeWithGenericTestsSerialized(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, true, false)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTestsSerialized(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, false)
})
}
-// serialize = false; readonly = false
-func TestUnixVolumeHandlersWithGenericVolumeTests(t *testing.T) {
- DoHandlersWithGenericVolumeTests(t, func(t TB) (*RRVolumeManager, []TestableVolume) {
- vols := make([]Volume, 2)
- testableUnixVols := make([]TestableVolume, 2)
-
- for i := range vols {
- v := NewTestableUnixVolume(t, false, false)
- vols[i] = v
- testableUnixVols[i] = v
- }
-
- return MakeRRVolumeManager(vols), testableUnixVols
+// serialize = true; readonly = true
+func (s *UnixVolumeSuite) TestUnixVolumeHandlersWithGenericVolumeTests(c *check.C) {
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, true)
})
}
-func TestReplicationDefault1(t *testing.T) {
- v := &UnixVolume{
- Root: "/",
- ReadOnly: true,
- }
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- if err := v.Start(metrics); err != nil {
- t.Error(err)
- }
- if got := v.Replication(); got != 1 {
- t.Errorf("Replication() returned %d, expected 1 if no config given", got)
- }
-}
-
-func TestGetNotFound(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestGetNotFound(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -140,42 +139,42 @@ func TestGetNotFound(t *testing.T) {
case os.IsNotExist(err):
break
case err == nil:
- t.Errorf("Read should have failed, returned %+q", buf[:n])
+ c.Errorf("Read should have failed, returned %+q", buf[:n])
default:
- t.Errorf("Read expected ErrNotExist, got: %s", err)
+ c.Errorf("Read expected ErrNotExist, got: %s", err)
}
}
-func TestPut(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestPut(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
if buf, err := ioutil.ReadFile(p); err != nil {
- t.Error(err)
+ c.Error(err)
} else if bytes.Compare(buf, TestBlock) != 0 {
- t.Errorf("Write should have stored %s, did store %s",
+ c.Errorf("Write should have stored %s, did store %s",
string(TestBlock), string(buf))
}
}
-func TestPutBadVolume(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestPutBadVolume(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
os.Chmod(v.Root, 000)
err := v.Put(context.Background(), TestHash, TestBlock)
if err == nil {
- t.Error("Write should have failed")
+ c.Error("Write should have failed")
}
}
-func TestUnixVolumeReadonly(t *testing.T) {
- v := NewTestableUnixVolume(t, false, true)
+func (s *UnixVolumeSuite) TestUnixVolumeReadonly(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{ReadOnly: true, Replication: 1}, s.metrics, false)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -183,34 +182,34 @@ func TestUnixVolumeReadonly(t *testing.T) {
buf := make([]byte, BlockSize)
_, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Errorf("got err %v, expected nil", err)
+ c.Errorf("got err %v, expected nil", err)
}
err = v.Put(context.Background(), TestHash, TestBlock)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
err = v.Touch(TestHash)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
err = v.Trash(TestHash)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
}
-func TestIsFull(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestIsFull(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
fullPath := v.Root + "/full"
now := fmt.Sprintf("%d", time.Now().Unix())
os.Symlink(now, fullPath)
if !v.IsFull() {
- t.Errorf("%s: claims not to be full", v)
+ c.Errorf("%s: claims not to be full", v)
}
os.Remove(fullPath)
@@ -218,32 +217,32 @@ func TestIsFull(t *testing.T) {
expired := fmt.Sprintf("%d", time.Now().Unix()-3605)
os.Symlink(expired, fullPath)
if v.IsFull() {
- t.Errorf("%s: should no longer be full", v)
+ c.Errorf("%s: should no longer be full", v)
}
}
-func TestNodeStatus(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestNodeStatus(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
// Get node status and make a basic sanity check.
volinfo := v.Status()
if volinfo.MountPoint != v.Root {
- t.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.Root)
+ c.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.Root)
}
if volinfo.DeviceNum == 0 {
- t.Errorf("uninitialized device_num in %v", volinfo)
+ c.Errorf("uninitialized device_num in %v", volinfo)
}
if volinfo.BytesFree == 0 {
- t.Errorf("uninitialized bytes_free in %v", volinfo)
+ c.Errorf("uninitialized bytes_free in %v", volinfo)
}
if volinfo.BytesUsed == 0 {
- t.Errorf("uninitialized bytes_used in %v", volinfo)
+ c.Errorf("uninitialized bytes_used in %v", volinfo)
}
}
-func TestUnixVolumeGetFuncWorkerError(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncWorkerError(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -252,12 +251,12 @@ func TestUnixVolumeGetFuncWorkerError(t *testing.T) {
return mockErr
})
if err != mockErr {
- t.Errorf("Got %v, expected %v", err, mockErr)
+ c.Errorf("Got %v, expected %v", err, mockErr)
}
}
-func TestUnixVolumeGetFuncFileError(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncFileError(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
funcCalled := false
@@ -266,15 +265,15 @@ func TestUnixVolumeGetFuncFileError(t *testing.T) {
return nil
})
if err == nil {
- t.Errorf("Expected error opening non-existent file")
+ c.Errorf("Expected error opening non-existent file")
}
if funcCalled {
- t.Errorf("Worker func should not have been called")
+ c.Errorf("Worker func should not have been called")
}
}
-func TestUnixVolumeGetFuncWorkerWaitsOnMutex(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncWorkerWaitsOnMutex(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -290,55 +289,55 @@ func TestUnixVolumeGetFuncWorkerWaitsOnMutex(t *testing.T) {
select {
case mtx.AllowLock <- struct{}{}:
case <-funcCalled:
- t.Fatal("Function was called before mutex was acquired")
+ c.Fatal("Function was called before mutex was acquired")
case <-time.After(5 * time.Second):
- t.Fatal("Timed out before mutex was acquired")
+ c.Fatal("Timed out before mutex was acquired")
}
select {
case <-funcCalled:
case mtx.AllowUnlock <- struct{}{}:
- t.Fatal("Mutex was released before function was called")
+ c.Fatal("Mutex was released before function was called")
case <-time.After(5 * time.Second):
- t.Fatal("Timed out waiting for funcCalled")
+ c.Fatal("Timed out waiting for funcCalled")
}
select {
case mtx.AllowUnlock <- struct{}{}:
case <-time.After(5 * time.Second):
- t.Fatal("Timed out waiting for getFunc() to release mutex")
+ c.Fatal("Timed out waiting for getFunc() to release mutex")
}
}
-func TestUnixVolumeCompare(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeCompare(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
err := v.Compare(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Errorf("Got err %q, expected nil", err)
+ c.Errorf("Got err %q, expected nil", err)
}
err = v.Compare(context.Background(), TestHash, []byte("baddata"))
if err != CollisionError {
- t.Errorf("Got err %q, expected %q", err, CollisionError)
+ c.Errorf("Got err %q, expected %q", err, CollisionError)
}
v.Put(context.Background(), TestHash, []byte("baddata"))
err = v.Compare(context.Background(), TestHash, TestBlock)
if err != DiskHashError {
- t.Errorf("Got err %q, expected %q", err, DiskHashError)
+ c.Errorf("Got err %q, expected %q", err, DiskHashError)
}
p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
os.Chmod(p, 000)
err = v.Compare(context.Background(), TestHash, TestBlock)
if err == nil || strings.Index(err.Error(), "permission denied") < 0 {
- t.Errorf("Got err %q, expected %q", err, "permission denied")
+ c.Errorf("Got err %q, expected %q", err, "permission denied")
}
}
-func TestUnixVolumeContextCancelPut(t *testing.T) {
- v := NewTestableUnixVolume(t, true, false)
+func (s *UnixVolumeSuite) TestUnixVolumeContextCancelPut(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, true)
defer v.Teardown()
v.locker.Lock()
ctx, cancel := context.WithCancel(context.Background())
@@ -350,19 +349,19 @@ func TestUnixVolumeContextCancelPut(t *testing.T) {
}()
err := v.Put(ctx, TestHash, TestBlock)
if err != context.Canceled {
- t.Errorf("Put() returned %s -- expected short read / canceled", err)
+ c.Errorf("Put() returned %s -- expected short read / canceled", err)
}
}
-func TestUnixVolumeContextCancelGet(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeContextCancelGet(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
bpath := v.blockPath(TestHash)
v.PutRaw(TestHash, TestBlock)
os.Remove(bpath)
err := syscall.Mkfifo(bpath, 0600)
if err != nil {
- t.Fatalf("Mkfifo %s: %s", bpath, err)
+ c.Fatalf("Mkfifo %s: %s", bpath, err)
}
defer os.Remove(bpath)
ctx, cancel := context.WithCancel(context.Background())
@@ -373,35 +372,23 @@ func TestUnixVolumeContextCancelGet(t *testing.T) {
buf := make([]byte, len(TestBlock))
n, err := v.Get(ctx, TestHash, buf)
if n == len(TestBlock) || err != context.Canceled {
- t.Errorf("Get() returned %d, %s -- expected short read / canceled", n, err)
- }
-}
-
-var _ = check.Suite(&UnixVolumeSuite{})
-
-type UnixVolumeSuite struct {
- volume *TestableUnixVolume
-}
-
-func (s *UnixVolumeSuite) TearDownTest(c *check.C) {
- if s.volume != nil {
- s.volume.Teardown()
+ c.Errorf("Get() returned %d, %s -- expected short read / canceled", n, err)
}
}
func (s *UnixVolumeSuite) TestStats(c *check.C) {
- s.volume = NewTestableUnixVolume(c, false, false)
+ vol := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
stats := func() string {
- buf, err := json.Marshal(s.volume.InternalStats())
+ buf, err := json.Marshal(vol.InternalStats())
c.Check(err, check.IsNil)
return string(buf)
}
- c.Check(stats(), check.Matches, `.*"StatOps":0,.*`)
+ c.Check(stats(), check.Matches, `.*"StatOps":1,.*`) // (*UnixVolume)check() calls Stat() once
c.Check(stats(), check.Matches, `.*"Errors":0,.*`)
loc := "acbd18db4cc2f85cedef654fccc4a4d8"
- _, err := s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err := vol.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"StatOps":[^0],.*`)
c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
@@ -410,39 +397,27 @@ func (s *UnixVolumeSuite) TestStats(c *check.C) {
c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":0,.*`)
- err = s.volume.Put(context.Background(), loc, []byte("foo"))
+ err = vol.Put(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
c.Check(stats(), check.Matches, `.*"UtimesOps":0,.*`)
- err = s.volume.Touch(loc)
+ err = vol.Touch(loc)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":1,.*`)
c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = vol.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
- err = s.volume.Compare(context.Background(), loc, []byte("foo"))
+ err = vol.Compare(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"InBytes":6,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":3,.*`)
- err = s.volume.Trash(loc)
+ err = vol.Trash(loc)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":2,.*`)
}
-
-func (s *UnixVolumeSuite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: Directory
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
deleted file mode 100644
index 8e83f6ce5..000000000
--- a/services/keepstore/usage.go
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "sort"
- "strings"
-
- "github.com/ghodss/yaml"
-)
-
-func usage() {
- c := DefaultConfig()
- knownTypes := []string{}
- for _, vt := range VolumeTypes {
- c.Volumes = append(c.Volumes, vt().Examples()...)
- knownTypes = append(knownTypes, vt().Type())
- }
- exampleConfigFile, err := yaml.Marshal(c)
- if err != nil {
- panic(err)
- }
- sort.Strings(knownTypes)
- knownTypeList := strings.Join(knownTypes, ", ")
- fmt.Fprintf(os.Stderr, `
-
-keepstore provides a content-addressed data store backed by a local filesystem or networked storage.
-
-Usage: keepstore -config path/to/keepstore.yml
- keepstore [OPTIONS] -dump-config
-
-NOTE: All options (other than -config) are deprecated in favor of YAML
- configuration. Use -dump-config to translate existing
- configurations to YAML format.
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-LogFormat:
-
- Format of request/response and error logs: "json" or "text".
-
-PIDFile:
-
- Path to write PID file during startup. This file is kept open and
- locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
- one way to shut down. Exit immediately if there is an error
- opening, locking, or writing the PID file.
-
-MaxBuffers:
-
- Maximum RAM to use for data buffers, given in multiples of block
- size (64 MiB). When this limit is reached, HTTP requests requiring
- buffers (like GET and PUT) will wait for buffer space to be
- released.
-
-MaxRequests:
-
- Maximum concurrent requests. When this limit is reached, new
- requests will receive 503 responses. Note: this limit does not
- include idle connections from clients using HTTP keepalive, so it
- does not strictly limit the number of concurrent connections. If
- omitted or zero, the default is 2 * MaxBuffers.
-
-BlobSigningKeyFile:
-
- Local file containing the secret blob signing key (used to
- generate and verify blob signatures). This key should be
- identical to the API server's blob_signing_key configuration
- entry.
-
-RequireSignatures:
-
- Honor read requests only if a valid signature is provided. This
- should be true, except for development use and when migrating from
- a very old version.
-
-BlobSignatureTTL:
-
- Duration for which new permission signatures (returned in PUT
- responses) will be valid. This should be equal to the API
- server's blob_signature_ttl configuration entry.
-
-SystemAuthTokenFile:
-
- Local file containing the Arvados API token used by keep-balance
- or data manager. Delete, trash, and index requests are honored
- only for this token.
-
-EnableDelete:
-
- Enable trash and delete features. If false, trash lists will be
- accepted but blocks will not be trashed or deleted.
-
-TrashLifetime:
-
- Time duration after a block is trashed during which it can be
- recovered using an /untrash request.
-
-TrashCheckInterval:
-
- How often to check for (and delete) trashed blocks whose
- TrashLifetime has expired.
-
-TrashWorkers:
-
- Maximum number of concurrent trash operations. Default is 1, i.e.,
- trash lists are processed serially.
-
-EmptyTrashWorkers:
-
- Maximum number of concurrent block deletion operations (per
- volume) when emptying trash. Default is 1.
-
-PullWorkers:
-
- Maximum number of concurrent pull operations. Default is 1, i.e.,
- pull lists are processed serially.
-
-TLSCertificateFile:
-
- Path to server certificate file in X509 format. Enables TLS mode.
-
- Example: /var/lib/acme/live/keep0.example.com/fullchain
-
-TLSKeyFile:
-
- Path to server key file in X509 format. Enables TLS mode.
-
- The key pair is read from disk during startup, and whenever SIGHUP
- is received.
-
- Example: /var/lib/acme/live/keep0.example.com/privkey
-
-Volumes:
-
- List of storage volumes. If omitted or empty, the default is to
- use all directories named "keep" that exist in the top level
- directory of a mount point at startup time.
-
- Volume types: %s
-
- (See volume configuration examples above.)
-
-`, exampleConfigFile, knownTypeList)
-}
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 52b9b1b24..861435502 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -14,6 +14,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
)
type BlockWriter interface {
@@ -28,19 +29,12 @@ type BlockReader interface {
ReadBlock(ctx context.Context, loc string, w io.Writer) error
}
+var driver = map[string]func(*arvados.Cluster, arvados.Volume, logrus.FieldLogger, *volumeMetricsVecs) (Volume, error){}
+
// A Volume is an interface representing a Keep back-end storage unit:
// for example, a single mounted disk, a RAID array, an Amazon S3 volume,
// etc.
type Volume interface {
- // Volume type as specified in config file. Examples: "S3",
- // "Directory".
- Type() string
-
- // Do whatever private setup tasks and configuration checks
- // are needed. Return non-nil if the volume is unusable (e.g.,
- // invalid config).
- Start(vm *volumeMetricsVecs) error
-
// Get a block: copy the block data into buf, and return the
// number of bytes copied.
//
@@ -222,29 +216,13 @@ type Volume interface {
// secrets.
String() string
- // Writable returns false if all future Put, Mtime, and Delete
- // calls are expected to fail.
- //
- // If the volume is only temporarily unwritable -- or if Put
- // will fail because it is full, but Mtime or Delete can
- // succeed -- then Writable should return false.
- Writable() bool
-
- // Replication returns the storage redundancy of the
- // underlying device. It will be passed on to clients in
- // responses to PUT requests.
- Replication() int
-
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
EmptyTrash()
// Return a globally unique ID of the underlying storage
// device if possible, otherwise "".
- DeviceID() string
-
- // Get the storage classes associated with this volume
- GetStorageClasses() []string
+ GetDeviceID() string
}
// A VolumeWithExamples provides example configs to display in the
@@ -260,24 +238,24 @@ type VolumeManager interface {
// Mounts returns all mounts (volume attachments).
Mounts() []*VolumeMount
- // Lookup returns the volume under the given mount
- // UUID. Returns nil if the mount does not exist. If
- // write==true, returns nil if the volume is not writable.
- Lookup(uuid string, write bool) Volume
+ // Lookup returns the mount with the given UUID. Returns nil
+ // if the mount does not exist. If write==true, returns nil if
+ // the mount is not writable.
+ Lookup(uuid string, write bool) *VolumeMount
- // AllReadable returns all volumes.
- AllReadable() []Volume
+ // AllReadable returns all mounts.
+ AllReadable() []*VolumeMount
- // AllWritable returns all volumes that aren't known to be in
+ // AllWritable returns all mounts that aren't known to be in
// a read-only state. (There is no guarantee that a write to
// one will succeed, though.)
- AllWritable() []Volume
+ AllWritable() []*VolumeMount
// NextWritable returns the volume where the next new block
// should be written. A VolumeManager can select a volume in
// order to distribute activity across spindles, fill up disks
// with more free space, etc.
- NextWritable() Volume
+ NextWritable() *VolumeMount
// VolumeStats returns the ioStats used for tracking stats for
// the given Volume.
@@ -290,7 +268,7 @@ type VolumeManager interface {
// A VolumeMount is an attachment of a Volume to a VolumeManager.
type VolumeMount struct {
arvados.KeepMount
- volume Volume
+ Volume
}
// Generate a UUID the way API server would for a "KeepVolumeMount"
@@ -314,68 +292,85 @@ func (*VolumeMount) generateUUID() string {
type RRVolumeManager struct {
mounts []*VolumeMount
mountMap map[string]*VolumeMount
- readables []Volume
- writables []Volume
+ readables []*VolumeMount
+ writables []*VolumeMount
counter uint32
iostats map[Volume]*ioStats
}
-// MakeRRVolumeManager initializes RRVolumeManager
-func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
+func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, myURL arvados.URL, metrics *volumeMetricsVecs) (*RRVolumeManager, error) {
vm := &RRVolumeManager{
iostats: make(map[Volume]*ioStats),
}
vm.mountMap = make(map[string]*VolumeMount)
- for _, v := range volumes {
- sc := v.GetStorageClasses()
+ for uuid, cfgvol := range cluster.Volumes {
+ va, ok := cfgvol.AccessViaHosts[myURL]
+ if !ok && len(cfgvol.AccessViaHosts) > 0 {
+ continue
+ }
+ dri, ok := driver[cfgvol.Driver]
+ if !ok {
+ return nil, fmt.Errorf("volume %s: invalid driver %q", uuid, cfgvol.Driver)
+ }
+ vol, err := dri(cluster, cfgvol, logger, metrics)
+ if err != nil {
+ return nil, fmt.Errorf("error initializing volume %s: %s", uuid, err)
+ }
+ logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+
+ sc := cfgvol.StorageClasses
if len(sc) == 0 {
- sc = []string{"default"}
+ sc = map[string]bool{"default": true}
+ }
+ repl := cfgvol.Replication
+ if repl < 1 {
+ repl = 1
}
mnt := &VolumeMount{
KeepMount: arvados.KeepMount{
- UUID: (*VolumeMount)(nil).generateUUID(),
- DeviceID: v.DeviceID(),
- ReadOnly: !v.Writable(),
- Replication: v.Replication(),
+ UUID: uuid,
+ DeviceID: vol.GetDeviceID(),
+ ReadOnly: cfgvol.ReadOnly || va.ReadOnly,
+ Replication: repl,
StorageClasses: sc,
},
- volume: v,
+ Volume: vol,
}
- vm.iostats[v] = &ioStats{}
+ vm.iostats[vol] = &ioStats{}
vm.mounts = append(vm.mounts, mnt)
- vm.mountMap[mnt.UUID] = mnt
- vm.readables = append(vm.readables, v)
- if v.Writable() {
- vm.writables = append(vm.writables, v)
+ vm.mountMap[uuid] = mnt
+ vm.readables = append(vm.readables, mnt)
+ if !mnt.KeepMount.ReadOnly {
+ vm.writables = append(vm.writables, mnt)
}
}
- return vm
+ return vm, nil
}
func (vm *RRVolumeManager) Mounts() []*VolumeMount {
return vm.mounts
}
-func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) Volume {
+func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) *VolumeMount {
if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) {
- return mnt.volume
+ return mnt
} else {
return nil
}
}
// AllReadable returns an array of all readable volumes
-func (vm *RRVolumeManager) AllReadable() []Volume {
+func (vm *RRVolumeManager) AllReadable() []*VolumeMount {
return vm.readables
}
// AllWritable returns an array of all writable volumes
-func (vm *RRVolumeManager) AllWritable() []Volume {
+func (vm *RRVolumeManager) AllWritable() []*VolumeMount {
return vm.writables
}
// NextWritable returns the next writable
-func (vm *RRVolumeManager) NextWritable() Volume {
+func (vm *RRVolumeManager) NextWritable() *VolumeMount {
if len(vm.writables) == 0 {
return nil
}
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index d5a413693..683521c01 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -18,8 +18,10 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
+ "github.com/sirupsen/logrus"
)
type TB interface {
@@ -37,65 +39,96 @@ type TB interface {
// A TestableVolumeFactory returns a new TestableVolume. The factory
// function, and the TestableVolume it returns, can use "t" to write
// logs, fail the current test, etc.
-type TestableVolumeFactory func(t TB) TestableVolume
+type TestableVolumeFactory func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume
// DoGenericVolumeTests runs a set of tests that every TestableVolume
// is expected to pass. It calls factory to create a new TestableVolume
// for each test case, to avoid leaking state between tests.
-func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
- testGet(t, factory)
- testGetNoSuchBlock(t, factory)
+func DoGenericVolumeTests(t TB, readonly bool, factory TestableVolumeFactory) {
+ var s genericVolumeSuite
+ s.volume.ReadOnly = readonly
- testCompareNonexistent(t, factory)
- testCompareSameContent(t, factory, TestHash, TestBlock)
- testCompareSameContent(t, factory, EmptyHash, EmptyBlock)
- testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata"))
- testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock)
- testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock)
- testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata"))
- testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock)
- testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
+ s.testGet(t, factory)
+ s.testGetNoSuchBlock(t, factory)
- testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
- testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
- testPutMultipleBlocks(t, factory)
+ s.testCompareNonexistent(t, factory)
+ s.testCompareSameContent(t, factory, TestHash, TestBlock)
+ s.testCompareSameContent(t, factory, EmptyHash, EmptyBlock)
+ s.testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata"))
+ s.testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock)
+ s.testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock)
+ s.testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata"))
+ s.testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock)
+ s.testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
- testPutAndTouch(t, factory)
- testTouchNoSuchBlock(t, factory)
+ if !readonly {
+ s.testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
+ s.testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
+ s.testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
+ s.testPutMultipleBlocks(t, factory)
- testMtimeNoSuchBlock(t, factory)
+ s.testPutAndTouch(t, factory)
+ }
+ s.testTouchNoSuchBlock(t, factory)
+
+ s.testMtimeNoSuchBlock(t, factory)
+
+ s.testIndexTo(t, factory)
- testIndexTo(t, factory)
+ if !readonly {
+ s.testDeleteNewBlock(t, factory)
+ s.testDeleteOldBlock(t, factory)
+ }
+ s.testDeleteNoSuchBlock(t, factory)
- testDeleteNewBlock(t, factory)
- testDeleteOldBlock(t, factory)
- testDeleteNoSuchBlock(t, factory)
+ s.testStatus(t, factory)
- testStatus(t, factory)
+ s.testMetrics(t, readonly, factory)
- testMetrics(t, factory)
+ s.testString(t, factory)
- testString(t, factory)
+ if readonly {
+ s.testUpdateReadOnly(t, factory)
+ }
- testUpdateReadOnly(t, factory)
+ s.testGetConcurrent(t, factory)
+ if !readonly {
+ s.testPutConcurrent(t, factory)
- testGetConcurrent(t, factory)
- testPutConcurrent(t, factory)
+ s.testPutFullBlock(t, factory)
+ }
- testPutFullBlock(t, factory)
+ s.testTrashUntrash(t, readonly, factory)
+ s.testTrashEmptyTrashUntrash(t, factory)
+}
+
+type genericVolumeSuite struct {
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ registry *prometheus.Registry
+}
+
+func (s *genericVolumeSuite) setup(t TB) {
+ s.cluster = testCluster(t)
+ s.logger = ctxlog.TestLogger(t)
+ s.registry = prometheus.NewRegistry()
+ s.metrics = newVolumeMetricsVecs(s.registry)
+}
- testTrashUntrash(t, factory)
- testTrashEmptyTrashUntrash(t, factory)
+func (s *genericVolumeSuite) newVolume(t TB, factory TestableVolumeFactory) TestableVolume {
+ return factory(t, s.cluster, s.volume, s.logger, s.metrics)
}
// Put a test block, get it and verify content
// Test should pass for both writable and read-only volumes
-func testGet(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGet(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -113,8 +146,9 @@ func testGet(t TB, factory TestableVolumeFactory) {
// Invoke get on a block that does not exist in volume; should result in error
// Test should pass for both writable and read-only volumes
-func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
buf := make([]byte, BlockSize)
@@ -126,8 +160,9 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Compare() should return os.ErrNotExist if the block does not exist.
// Otherwise, writing new data causes CompareAndTouch() to generate
// error logs even though everything is working fine.
-func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareNonexistent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
err := v.Compare(context.Background(), TestHash, TestBlock)
@@ -138,8 +173,9 @@ func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
// Put a test block and compare the locator with same content
// Test should pass for both writable and read-only volumes
-func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(testHash, testData)
@@ -156,8 +192,9 @@ func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string
// testHash = md5(testDataA).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(testHash, testDataA)
@@ -173,8 +210,9 @@ func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash stri
// corrupted. Requires testHash = md5(testDataA) != md5(testDataB).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, testDataB)
@@ -187,14 +225,11 @@ func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testH
// Put a block and put again with same content
// Test is intended for only writable volumes
-func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
err := v.Put(context.Background(), testHash, testData)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -208,14 +243,11 @@ func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash s
// Put a block and put again with different content
// Test is intended for only writable volumes
-func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
v.PutRaw(testHash, testDataA)
putErr := v.Put(context.Background(), testHash, testDataB)
@@ -239,14 +271,11 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH
// Put and get multiple blocks
// Test is intended for only writable volumes
-func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -295,14 +324,11 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
// Test that when applying PUT to a block that already exists,
// the block's modification time is updated.
// Test is intended for only writable volumes
-func testPutAndTouch(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutAndTouch(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
if err := v.Put(context.Background(), TestHash, TestBlock); err != nil {
t.Error(err)
}
@@ -337,8 +363,9 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) {
// Touching a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if err := v.Touch(TestHash); err == nil {
@@ -348,8 +375,9 @@ func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Invoking Mtime on a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if _, err := v.Mtime("12345678901234567890123456789012"); err == nil {
@@ -362,8 +390,9 @@ func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
// * with a prefix
// * with no such prefix
// Test should pass for both writable and read-only volumes
-func testIndexTo(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testIndexTo(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
// minMtime and maxMtime are the minimum and maximum
@@ -437,14 +466,11 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block immediately after writing it (not old enough)
// should neither delete the data nor return an error.
// Test is intended for only writable volumes
-func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- theConfig.BlobSignatureTTL.Set("5m")
-
- if v.Writable() == false {
- return
- }
v.Put(context.Background(), TestHash, TestBlock)
@@ -463,17 +489,14 @@ func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block with a timestamp older than
// BlobSignatureTTL seconds in the past should delete the data.
// Test is intended for only writable volumes
-func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- theConfig.BlobSignatureTTL.Set("5m")
-
- if v.Writable() == false {
- return
- }
v.Put(context.Background(), TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
if err := v.Trash(TestHash); err != nil {
t.Error(err)
@@ -507,8 +530,9 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block that does not exist should result in error.
// Test should pass for both writable and read-only volumes
-func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if err := v.Trash(TestHash2); err == nil {
@@ -518,8 +542,9 @@ func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Invoke Status and verify that VolumeStatus is returned
// Test should pass for both writable and read-only volumes
-func testStatus(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testStatus(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
// Get node status and make a basic sanity check.
@@ -544,19 +569,14 @@ func getValueFrom(cv *prometheus.CounterVec, lbls prometheus.Labels) float64 {
return pb.GetCounter().GetValue()
}
-func testMetrics(t TB, factory TestableVolumeFactory) {
+func (s *genericVolumeSuite) testMetrics(t TB, readonly bool, factory TestableVolumeFactory) {
var err error
- v := factory(t)
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- reg := prometheus.NewRegistry()
- vm := newVolumeMetricsVecs(reg)
- err = v.Start(vm)
- if err != nil {
- t.Error("Failed Start(): ", err)
- }
- opsC, _, ioC := vm.getCounterVecsFor(prometheus.Labels{"device_id": v.DeviceID()})
+ opsC, _, ioC := s.metrics.getCounterVecsFor(prometheus.Labels{"device_id": v.GetDeviceID()})
if ioC == nil {
t.Error("ioBytes CounterVec is nil")
@@ -580,7 +600,7 @@ func testMetrics(t TB, factory TestableVolumeFactory) {
readOpCounter = getValueFrom(opsC, prometheus.Labels{"operation": readOpType})
// Test Put if volume is writable
- if v.Writable() {
+ if !readonly {
err = v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -617,8 +637,9 @@ func testMetrics(t TB, factory TestableVolumeFactory) {
// Invoke String for the volume; expect non-empty result
// Test should pass for both writable and read-only volumes
-func testString(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testString(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if id := v.String(); len(id) == 0 {
@@ -628,14 +649,11 @@ func testString(t TB, factory TestableVolumeFactory) {
// Putting, updating, touching, and deleting blocks from a read-only volume result in error.
// Test is intended for only read-only volumes
-func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == true {
- return
- }
-
v.PutRaw(TestHash, TestBlock)
buf := make([]byte, BlockSize)
@@ -676,8 +694,9 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
// Launch concurrent Gets
// Test should pass for both writable and read-only volumes
-func testGetConcurrent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGetConcurrent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -729,14 +748,11 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) {
// Launch concurrent Puts
// Test is intended for only writable volumes
-func testPutConcurrent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutConcurrent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
sem := make(chan int)
go func(sem chan int) {
err := v.Put(context.Background(), TestHash, TestBlock)
@@ -795,14 +811,11 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
}
// Write and read back a full size block
-func testPutFullBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutFullBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if !v.Writable() {
- return
- }
-
wdata := make([]byte, BlockSize)
wdata[0] = 'a'
wdata[BlockSize-1] = 'z'
@@ -825,18 +838,15 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) {
// Trash an old block - which either raises ErrNotImplemented or succeeds
// Untrash - which either raises ErrNotImplemented or succeeds
// Get - which must succeed
-func testTrashUntrash(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTrashUntrash(t TB, readonly bool, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- defer func() {
- theConfig.TrashLifetime = 0
- }()
-
- theConfig.TrashLifetime.Set("1h")
// put block and backdate it
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
buf := make([]byte, BlockSize)
n, err := v.Get(context.Background(), TestHash, buf)
@@ -849,7 +859,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
// Trash
err = v.Trash(TestHash)
- if v.Writable() == false {
+ if readonly {
if err != MethodDisabledError {
t.Fatal(err)
}
@@ -880,12 +890,10 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
}
}
-func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- defer func(orig arvados.Duration) {
- theConfig.TrashLifetime = orig
- }(theConfig.TrashLifetime)
checkGet := func() error {
buf := make([]byte, BlockSize)
@@ -918,10 +926,10 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// First set: EmptyTrash before reaching the trash deadline.
- theConfig.TrashLifetime.Set("1h")
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
err := checkGet()
if err != nil {
@@ -966,7 +974,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
// Because we Touch'ed, need to backdate again for next set of tests
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// If the only block in the trash has already been untrashed,
// most volumes will fail a subsequent Untrash with a 404, but
@@ -984,11 +992,11 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
// Untrash might have updated the timestamp, so backdate again
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// Second set: EmptyTrash after the trash deadline has passed.
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
@@ -1013,7 +1021,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// Trash it again, and this time call EmptyTrash so it really
// goes away.
// (In Azure volumes, un/trash changes Mtime, so first backdate again)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
_ = v.Trash(TestHash)
err = checkGet()
if err == nil || !os.IsNotExist(err) {
@@ -1038,9 +1046,9 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// un-trashed copy doesn't get deleted along with it.
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
@@ -1051,7 +1059,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// EmptyTrash should not delete the untrashed copy.
v.EmptyTrash()
@@ -1066,18 +1074,18 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// untrash the block whose deadline is "C".
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
}
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1h")
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 0b8af330f..62582d309 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -15,6 +15,28 @@ import (
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+var (
+ TestBlock = []byte("The quick brown fox jumps over the lazy dog.")
+ TestHash = "e4d909c290d0fb1ca068ffaddf22cbd0"
+ TestHashPutResp = "e4d909c290d0fb1ca068ffaddf22cbd0+44\n"
+
+ TestBlock2 = []byte("Pack my box with five dozen liquor jugs.")
+ TestHash2 = "f15ac516f788aec4f30932ffb6395c39"
+
+ TestBlock3 = []byte("Now is the time for all good men to come to the aid of their country.")
+ TestHash3 = "eed29bbffbc2dbe5e5ee0bb71888e61f"
+
+ // BadBlock is used to test collisions and corruption.
+ // It must not match any test hashes.
+ BadBlock = []byte("The magic words are squeamish ossifrage.")
+
+ EmptyHash = "d41d8cd98f00b204e9800998ecf8427e"
+ EmptyBlock = []byte("")
)
// A TestableVolume allows test suites to manipulate the state of an
@@ -38,6 +60,10 @@ type TestableVolume interface {
Teardown()
}
+func init() {
+ driver["mock"] = newMockVolume
+}
+
// MockVolumes are test doubles for Volumes, used to test handlers.
type MockVolume struct {
Store map[string][]byte
@@ -51,10 +77,6 @@ type MockVolume struct {
// that has been Put().
Touchable bool
- // Readonly volumes return an error for Put, Delete, and
- // Touch.
- Readonly bool
-
// Gate is a "starting gate", allowing test cases to pause
// volume operations long enough to inspect state. Every
// operation (except Status) starts by receiving from
@@ -62,15 +84,19 @@ type MockVolume struct {
// channel unblocks all operations. By default, Gate is a
// closed channel, so all operations proceed without
// blocking. See trash_worker_test.go for an example.
- Gate chan struct{}
-
- called map[string]int
- mutex sync.Mutex
+ Gate chan struct{} `json:"-"`
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ called map[string]int
+ mutex sync.Mutex
}
-// CreateMockVolume returns a non-Bad, non-Readonly, Touchable mock
+// newMockVolume returns a non-Bad, non-Readonly, Touchable mock
// volume.
-func CreateMockVolume() *MockVolume {
+func newMockVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
gate := make(chan struct{})
close(gate)
return &MockVolume{
@@ -78,10 +104,13 @@ func CreateMockVolume() *MockVolume {
Timestamps: make(map[string]time.Time),
Bad: false,
Touchable: true,
- Readonly: false,
called: map[string]int{},
Gate: gate,
- }
+ cluster: cluster,
+ volume: volume,
+ logger: logger,
+ metrics: metrics,
+ }, nil
}
// CallCount returns how many times the named method has been called.
@@ -141,7 +170,7 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
if v.Bad {
return v.BadVolumeError
}
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
v.Store[loc] = block
@@ -151,7 +180,7 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
func (v *MockVolume) Touch(loc string) error {
v.gotCall("Touch")
<-v.Gate
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if v.Touchable {
@@ -195,11 +224,11 @@ func (v *MockVolume) IndexTo(prefix string, w io.Writer) error {
func (v *MockVolume) Trash(loc string) error {
v.gotCall("Delete")
<-v.Gate
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if _, ok := v.Store[loc]; ok {
- if time.Since(v.Timestamps[loc]) < time.Duration(theConfig.BlobSignatureTTL) {
+ if time.Since(v.Timestamps[loc]) < time.Duration(v.cluster.Collections.BlobSigningTTL) {
return nil
}
delete(v.Store, loc)
@@ -208,18 +237,10 @@ func (v *MockVolume) Trash(loc string) error {
return os.ErrNotExist
}
-func (v *MockVolume) DeviceID() string {
+func (v *MockVolume) GetDeviceID() string {
return "mock-device-id"
}
-func (v *MockVolume) Type() string {
- return "Mock"
-}
-
-func (v *MockVolume) Start(vm *volumeMetricsVecs) error {
- return nil
-}
-
func (v *MockVolume) Untrash(loc string) error {
return nil
}
@@ -236,14 +257,6 @@ func (v *MockVolume) String() string {
return "[MockVolume]"
}
-func (v *MockVolume) Writable() bool {
- return !v.Readonly
-}
-
-func (v *MockVolume) Replication() int {
- return 1
-}
-
func (v *MockVolume) EmptyTrash() {
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list