[ARVADOS] updated: 64325b327855daea30553ac43bb95851bb7f860a
Git user
git at public.curoverse.com
Thu Dec 15 12:03:30 EST 2016
Summary of changes:
README.md | 1 +
apps/workbench/Gemfile | 6 +-
apps/workbench/Gemfile.lock | 16 +-
.../app/controllers/application_controller.rb | 10 +-
.../app/controllers/work_units_controller.rb | 14 +-
apps/workbench/app/models/container_work_unit.rb | 13 +-
apps/workbench/app/models/proxy_work_unit.rb | 6 +-
.../views/container_requests/_show_inputs.html.erb | 36 +--
.../pipeline_instances/_running_component.html.erb | 2 +-
.../_show_components_running.html.erb | 2 +-
.../test/controllers/projects_controller_test.rb | 21 +-
.../test/integration/application_layout_test.rb | 4 +-
.../test/integration/container_requests_test.rb | 13 +
apps/workbench/test/integration/work_units_test.rb | 8 +-
build/README | 6 +-
build/build-dev-docker-jobs-image.sh | 63 +++++
build/run-build-packages-one-target.sh | 11 +-
build/run-build-packages.sh | 41 +++-
build/run-library.sh | 17 ++
build/run-tests.sh | 6 +
doc/_config.yml | 8 +-
doc/_includes/_arvados_cwl_runner.liquid | 62 +++++
doc/_includes/_register_cwl_workflow.liquid | 21 ++
doc/_includes/_what_is_cwl.liquid | 1 +
.../methods/container_requests.html.textile.liquid | 2 +
doc/images/upload-using-workbench.png | Bin 49690 -> 23979 bytes
doc/images/workbench-dashboard.png | Bin 94257 -> 57930 bytes
doc/images/workbench-move-selected.png | Bin 47833 -> 15450 bytes
doc/user/cwl/cwl-runner.html.textile.liquid | 101 ++------
.../getting_started/workbench.html.textile.liquid | 2 +-
doc/user/topics/arv-copy.html.textile.liquid | 3 +
...nning-workflow-command-line.html.textile.liquid | 17 ++
...tutorial-pipeline-workbench.html.textile.liquid | 35 ---
...tutorial-workflow-workbench.html.textile.liquid | 27 +++
.../writing-cwl-workflow.html.textile.liquid | 28 +++
sdk/cli/bin/crunch-job | 6 +-
sdk/cli/test/test_arv-keep-get.rb | 8 +
sdk/cwl/arvados_cwl/__init__.py | 48 +++-
sdk/cwl/arvados_cwl/arvcontainer.py | 65 ++---
sdk/cwl/arvados_cwl/arvjob.py | 12 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 11 +-
sdk/cwl/arvados_cwl/done.py | 3 +
sdk/cwl/arvados_cwl/runner.py | 25 +-
sdk/cwl/setup.py | 5 +-
sdk/cwl/tests/test_container.py | 71 +-----
sdk/cwl/tests/test_submit.py | 267 +++++++++++++++++++--
sdk/dev-jobs.dockerfile | 38 +++
sdk/python/arvados/_version.py | 3 +
sdk/python/arvados/commands/arv_copy.py | 4 +
sdk/python/arvados/commands/keepdocker.py | 5 +
sdk/python/arvados/commands/ls.py | 6 +
sdk/python/arvados/commands/put.py | 4 +
sdk/python/arvados/commands/run.py | 36 ++-
sdk/python/arvados/commands/ws.py | 4 +
sdk/python/arvados/keep.py | 134 ++++++-----
sdk/python/bin/arv-get | 5 +
sdk/python/bin/arv-normalize | 14 +-
sdk/python/setup.py | 1 +
sdk/python/tests/arvados_testutil.py | 13 +
sdk/python/tests/test_arv_copy.py | 29 +++
sdk/python/tests/test_arv_keepdocker.py | 30 +++
sdk/python/tests/test_arv_ls.py | 15 +-
sdk/python/tests/test_arv_normalize.py | 27 +++
sdk/python/tests/test_arv_put.py | 10 +
sdk/python/tests/test_arv_run.py | 29 +++
sdk/python/tests/test_arv_ws.py | 15 ++
sdk/python/tests/test_keep_client.py | 76 +++---
services/api/Gemfile | 19 +-
services/api/Gemfile.lock | 239 +++++++++---------
.../api/app/controllers/application_controller.rb | 18 +-
.../arvados/v1/collections_controller.rb | 4 +-
.../controllers/arvados/v1/groups_controller.rb | 9 +-
.../controllers/arvados/v1/schema_controller.rb | 22 +-
.../arvados/v1/user_agreements_controller.rb | 3 +-
.../app/controllers/arvados/v1/users_controller.rb | 2 +-
.../arvados/v1/virtual_machines_controller.rb | 2 +-
.../api/app/controllers/database_controller.rb | 2 +-
services/api/app/middlewares/arvados_api_token.rb | 2 +-
services/api/app/models/arvados_model.rb | 9 +-
services/api/app/models/blob.rb | 4 +-
services/api/app/models/collection.rb | 15 +-
services/api/app/models/commit_ancestor.rb | 6 +-
services/api/app/models/job.rb | 42 ++--
services/api/app/models/link.rb | 1 -
services/api/app/models/log.rb | 1 -
services/api/app/models/node.rb | 6 +-
services/api/app/models/repository.rb | 2 +-
services/api/app/models/user.rb | 4 +-
services/api/config/boot.rb | 2 +-
services/api/config/initializers/inflections.rb | 8 +-
services/api/config/initializers/load_config.rb | 4 +-
.../api/config/initializers/preload_all_models.rb | 2 +-
services/api/lib/crunch_dispatch.rb | 10 +-
services/api/lib/current_api_client.rb | 10 +-
services/api/lib/eventbus.rb | 35 ++-
services/api/lib/load_param.rb | 4 +-
services/api/lib/salvage_collection.rb | 2 +-
services/api/script/arvados-git-sync.rb | 4 +-
.../api/script/migrate-gitolite-to-uuid-storage.rb | 4 +-
services/api/test/factories/user.rb | 2 +-
services/api/test/fixtures/container_requests.yml | 2 +
services/api/test/fixtures/pipeline_templates.yml | 22 ++
services/api/test/fixtures/workflows.yml | 19 ++
.../api_client_authorizations_controller_test.rb | 2 +-
.../arvados/v1/collections_controller_test.rb | 8 +-
.../api/test/functional/arvados/v1/filters_test.rb | 6 +-
.../arvados/v1/groups_controller_test.rb | 19 +-
.../functional/arvados/v1/jobs_controller_test.rb | 2 +-
.../functional/arvados/v1/links_controller_test.rb | 4 +-
.../arvados/v1/repositories_controller_test.rb | 13 +-
.../arvados/v1/schema_controller_test.rb | 2 +-
.../functional/arvados/v1/users_controller_test.rb | 4 +-
.../arvados/v1/virtual_machines_controller_test.rb | 1 -
services/api/test/helpers/users_test_helper.rb | 2 +-
.../api/test/integration/collections_api_test.rb | 14 +-
.../integration/collections_performance_test.rb | 6 +-
services/api/test/integration/cross_origin_test.rb | 2 +-
.../api/test/integration/database_reset_test.rb | 9 +-
.../api/test/integration/user_sessions_test.rb | 2 +-
services/api/test/integration/websocket_test.rb | 136 +++++++----
services/api/test/test_helper.rb | 32 ++-
services/api/test/unit/app_version_test.rb | 7 +-
services/api/test/unit/authorized_key_test.rb | 2 +-
.../api/test/unit/collection_performance_test.rb | 5 +-
services/api/test/unit/collection_test.rb | 16 +-
services/api/test/unit/commit_test.rb | 18 +-
services/api/test/unit/container_test.rb | 55 +++--
services/api/test/unit/fail_jobs_test.rb | 4 +-
services/api/test/unit/job_test.rb | 6 +-
services/api/test/unit/log_test.rb | 2 +-
services/api/test/unit/node_test.rb | 2 +-
services/api/test/unit/owner_test.rb | 3 +-
services/api/test/unit/permission_test.rb | 14 +-
services/api/test/unit/pipeline_instance_test.rb | 1 -
services/api/test/unit/salvage_collection_test.rb | 12 +-
services/api/test/unit/user_test.rb | 1 -
services/api/test/websocket_runner.rb | 53 ----
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 14 +-
.../crunch-dispatch-slurm_test.go | 5 +-
services/crunch-dispatch-slurm/squeue.go | 42 +++-
services/crunch-run/crunchrun.go | 3 +
services/fuse/arvados_fuse/_version.py | 3 +
services/fuse/arvados_fuse/command.py | 4 +
services/fuse/setup.py | 3 +-
services/fuse/tests/test_command_args.py | 16 ++
services/keep-balance/balance.go | 2 +-
services/keepstore/azure_blob_volume.go | 86 ++++++-
services/nodemanager/arvnodeman/_version.py | 3 +
.../arvnodeman/computenode/driver/gce.py | 3 +-
services/nodemanager/arvnodeman/launcher.py | 5 +
services/nodemanager/setup.py | 1 +
services/nodemanager/tests/test_arguments.py | 27 +++
services/nodemanager/tests/testutil.py | 17 +-
tools/arvbash/arvbash.sh | 124 ++++++++++
.../lib/arvbox/docker/service/api/run-service | 4 +-
155 files changed, 2081 insertions(+), 910 deletions(-)
create mode 100755 build/build-dev-docker-jobs-image.sh
create mode 100644 doc/_includes/_arvados_cwl_runner.liquid
create mode 100644 doc/_includes/_register_cwl_workflow.liquid
create mode 100644 doc/_includes/_what_is_cwl.liquid
create mode 100644 doc/user/topics/running-workflow-command-line.html.textile.liquid
delete mode 100644 doc/user/tutorials/tutorial-pipeline-workbench.html.textile.liquid
create mode 100644 doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
create mode 100644 doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
create mode 100644 sdk/dev-jobs.dockerfile
create mode 100644 sdk/python/arvados/_version.py
create mode 100644 sdk/python/tests/test_arv_copy.py
create mode 100644 sdk/python/tests/test_arv_keepdocker.py
create mode 100644 sdk/python/tests/test_arv_normalize.py
create mode 100644 sdk/python/tests/test_arv_run.py
delete mode 100644 services/api/test/websocket_runner.rb
create mode 100644 services/fuse/arvados_fuse/_version.py
create mode 100644 services/nodemanager/arvnodeman/_version.py
create mode 100644 services/nodemanager/tests/test_arguments.py
create mode 100755 tools/arvbash/arvbash.sh
via 64325b327855daea30553ac43bb95851bb7f860a (commit)
via 0514b290f5ff9a2700b599bf6fb19a468a73c3fb (commit)
via e336e28f4ce9c13aad98d059d2befb505bff365b (commit)
via 6e4f0a8e8cfe7154fe88653c48e1d4980efbdeeb (commit)
via feb463839301b0b596089e48a981660365d2c4a7 (commit)
via 08271aa69aafdc92e2e8f8f25ed7fab078deb993 (commit)
via b8e3dea8979a7dbbdb62b8ecd47f7094a7f0863c (commit)
via 177873e861d0f6975832a17cfbb9d1e21acc5cce (commit)
via 9b0654adfffaac018395de29f6e441b843d46e85 (commit)
via ba94fb47b5553b85dad8544a3d8d02dcadc90c91 (commit)
via 088314afe2987920628b9e921e7de18a2bdd5745 (commit)
via 9c1a28719df89a68b83cee07e3e0ab87c1712f69 (commit)
via 3e6ad647c3da4cb77299249d1fa25767dc66b281 (commit)
via b0f9d8dcc4cee11e4faed5a6bba593819209d7e2 (commit)
via 3d30e1a5c2b4edac70628a551ed0a34f7cf3be68 (commit)
via ceae525c9b247a5c66094c0a733b62b3038c8fea (commit)
via f88d5e28703e3de0e02a05df6d472effb03fc6a8 (commit)
via 4876169ec54cdfe24edd3b17475b6984274b086f (commit)
via de78c0abbfdd23deebd1bc6b57a63c745c1cee12 (commit)
via bdbcc4024e4320e17634fdfbdeb48a6051b0fccd (commit)
via cf6a9fc619a6c65d2bccb563c61b8a448334213c (commit)
via fee8873d0c5eeec1bd838161357679de1a3fe0cb (commit)
via 9e06ce4eff850ee428f235501cede643f094cf6d (commit)
via b67039fa8eb94e80482767eb1f8e8f54ac31823e (commit)
via 0fb93082a34a92699c9be9ed4b1c991dd4003ea1 (commit)
via d6d38552e0b39683e76060dba924b1c77c1c5b4d (commit)
via 2d4b14cf28c71d3aa7e0417a2951806c15e29fb1 (commit)
via c494f4c398457c5edbd9b2c3f1a439bf041c7ed2 (commit)
via 0f361d9a0748e2b470ed5bc73351cb2eaf3e02e3 (commit)
via 0936c2e7630c0e4316684c5b79965ad7dfaf9fa4 (commit)
via 0398eff55902b779e8d78434873a4ed846c67b5f (commit)
via e49ad3dfe5092d2a5f2371aba4d6005a54af36a0 (commit)
via 9d1f08198723f64d23473bb0ef1d6e30df1f4a1f (commit)
via df747077737cbcd62d6cfaedf17b1eaa18c1cbf8 (commit)
via 131be7089c317f4259d4645f7c1516972b3c5015 (commit)
via b9381df7b730762ee7e32a9ab58c6cbf3dcaed89 (commit)
via 21c1d7b918eb4019a8341f433979333da876218d (commit)
via b64a5695c0ec262d02a6e67099d5d076033ca584 (commit)
via beeab71df7bda32524c6afe5bf19f435ee3e0ade (commit)
via 252047244ba1c285a689e13ea0d59effa9d837ee (commit)
via 85f965a266837c9a2637c1180d8789c7d6dd0944 (commit)
via cb6baf8dc12be24ae664335dce29529252ac35df (commit)
via 49d18afe093301bbb48892e05b2c9c732e038abe (commit)
via 9f1ce358ac564abf62965cccf2bf3afa3bcfef95 (commit)
via d25dedaec8ea386c18b7f61c08a3097ba3c4f26c (commit)
via bd6b334c80cea328a51a8612d40ef16bdd6ab2e2 (commit)
via 6b2481ad7e565cf173c4eeb2df4f13cd095d23ff (commit)
via 84861876fd077f1892c6419fecfd348d7588747c (commit)
via d645da6a5744ad5dc76079d5d630c426a9ce1b8c (commit)
via 03f812007d8a28d7023dddb6f1fafb72e65ec525 (commit)
via 9e9190af52e4fc448f623aa2f4dd1fee803d99cd (commit)
via 2a4fbc88b7a16a059b5eb62bf0a3f1b13ebfc72d (commit)
via 163fc733dbe7bd13212d7df6a1563edd6d8798f4 (commit)
via 6448d08b2ebc1d54b05d50c6f27810c6722e81a4 (commit)
via 2993bc9aea20c2295d1b2b3cbecf3b1f7e06bb17 (commit)
via 87d84e41215333729a436726ee28068eb430ff38 (commit)
via 9be8a468534acaf324e9c18b831677f0ae067e60 (commit)
via b0cbc39917df391c0f0e7f31d9d46c5cc9070520 (commit)
via 0250713cc4e6d7fdf41fd7c0a99c6307e2eac72e (commit)
via 0d2bed2452f0840612be0b1bc792ffeff576d065 (commit)
via 4d164d025cd1e8c5c719b1e0f670e6cb10a6fe5e (commit)
via 66622dea0bb8725d0cbec0976175d11162b17815 (commit)
via 87143451d4c6535576fe7232729ae9ce93a26d30 (commit)
via e1b133b13f9ac50a87051d07c36a3904d6f01028 (commit)
via 5f38e62616773fcc97d795d1dd707c7fce801f2a (commit)
via 128766dfc5826eafb5768d49461019dee980baa4 (commit)
via 1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7 (commit)
via e64e03d71d1ad9c4ed9354fce55be7af17b4a56e (commit)
via 2b3af1807efa6bc4b3aaa298c1e24998d8ddd1c5 (commit)
via 79e63a733c48b04d7c9b6bcc6120af72b3f14641 (commit)
via 8cd17455f6cd86df7d9f9e66a24437aab25f0380 (commit)
via b29cc280e5220c7de3d24c3acc687be88bbcb29b (commit)
via f198e6126e38a30288763e8720f3dee1b4805677 (commit)
via 277ad30ef4824ab6363ba24f10f62d9fb6544ad7 (commit)
via 515e3f5c3dac3076217bd59545d01604008c01e6 (commit)
via 5646a899f667ee14efc32e9db84c72c0938ac6dc (commit)
via b9e5c8b32858338850da3e12ce27570b828898b3 (commit)
via bde488e7eff0e9a94cbf9709d28de13603857a3d (commit)
via dfd9492ee371ce86f7b4543106eb5060bc98928c (commit)
via dcc80dc5b02e46170400d42ca72672cceff03ba2 (commit)
via b5d3273b5aa503b22fdbeab7f8979e720e30c119 (commit)
via 7a76d3b2fc40c41e2d028daa0ee150b47421f0c5 (commit)
via 8735f2aa77b9e3c51449738e429399e4501ec94e (commit)
via 03406083ed63e5a0c118216e3e33fc8823c808a7 (commit)
via 7bf6588ce6589a194df2c7f45b9a443025c2ff67 (commit)
via d71a3ed8a91b7491df1f675389e1038d1a781e8e (commit)
via a12d68dd2e2646beb4ae68e4dd0825272e279508 (commit)
via b738c7e7357a143dade94dc5a1bad2b69bac2b27 (commit)
via 8f39c027e4895a8d872093c3dd16aa51b26a3731 (commit)
via e4664336c420836bf26f423faf2af9316302da93 (commit)
via 16942303133cb4e6be76d4d33a211d1e0ce2ea38 (commit)
via 52f9f7b3cf35efb7e8d1189b80f9b0f7afe0e111 (commit)
via 1bb30a981efa038bf07f139f996eb9d77749fced (commit)
via 1d45a45d1a1ebb0c02b4a9dd7ece73a55a6b24db (commit)
via f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a (commit)
via 86f04235021d84afa0d28d105111422e0dd15738 (commit)
via 7c2b38632ae2af8a2f0f5c8fa2e55523220a3335 (commit)
via 15635ff9fa2c964fb45467c9846ed92f7f6388b9 (commit)
via e2598332f9ecd40ab0dfd025bd1e5eba02ad8673 (commit)
via e4a51416586f73593ac68bf0d2a74c53a4875f7e (commit)
via 81ccd9264d0742ba1bf0b9dfbc31ff4f15929f3e (commit)
via 09a2e88c51e5432e607f2a38466e55b4ba15e887 (commit)
via 9f53e085c98249ba79d85ba59e6f1ca624fede10 (commit)
via c40265a873d73f03a9ca077f18fe305d883fb4a5 (commit)
via d53262e8e07785a9d2566966ee41ab8949ef1962 (commit)
via a4762b52738e12c39d93c14501a2f62463b05d07 (commit)
via 5fe0d5e7496fad7bd1c4bab0e5ca5f348c0eec63 (commit)
via a9a677e1655c461e742e46cc3c239f8605f4fc6b (commit)
via 8668135855b400e7f6047ecacd9cfa27fed723f7 (commit)
via 72912c6b25fcb10c0acc540daf4f8a25d802784b (commit)
via 8886788e82b7e45c9211f6bf3a23c601ed0b88c1 (commit)
via 279db7d614a25c5e81c6efda5c8950988682974a (commit)
via 0b228d13b944b4b94165bf22a8f796070bd711bf (commit)
via 014f72fcdac2b26f9f2bbf257707eb59c676eebb (commit)
via 1671eb078bc4d254ebdfe5ac8dd40c83b371e8ac (commit)
via 554350245276ee082417cdcfcb8fd3f4f593e00f (commit)
via c2f6922a1997ca73d46866f03474a23c76c7afa5 (commit)
via 450cf2524ff41b2a0fcea09e2e6ccbf3bdbf78ed (commit)
via c632b87902785c4bbc5242b095e53bcf657f6aae (commit)
via 61925eaac13ef601879178d6343de9f9714f3da1 (commit)
via c6b3743bae761b8802cf7e016e54833d833ba9f7 (commit)
via 722670d1113e1af551d8470841a38d8d9218b4e8 (commit)
via 2bb3fb59369abd48211a872ca1a8e49cb70be5f3 (commit)
via e34688e4a78595aaabe7307f0184df84ff703c6f (commit)
via b8a539e3a0803083b1fcbb23755fc12cffff6f31 (commit)
via 2e9514e54e31bb37c438cf5d611d49a7c15235b4 (commit)
via fd308c814745d9c4eed5c3f79b44b9f9142df28a (commit)
from e675118bd2b28ec40833d06ea384b6f1c78f3039 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 64325b327855daea30553ac43bb95851bb7f860a
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Dec 15 12:02:54 2016 -0500
10467: (azure) Abandon request if client hangs up.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 43cf83a..570e75f 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -187,7 +187,7 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
}
var deadline time.Time
haveDeadline := false
- size, err := v.get(loc, buf)
+ size, err := v.get(ctx, loc, buf)
for err == nil && size == 0 && loc != "d41d8cd98f00b204e9800998ecf8427e" {
// Seeing a brand new empty block probably means we're
// in a race with CreateBlob, which under the hood
@@ -208,8 +208,12 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
} else if time.Now().After(deadline) {
break
}
- time.Sleep(azureWriteRacePollTime)
- size, err = v.get(loc, buf)
+ select {
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ case <-time.After(azureWriteRacePollTime):
+ }
+ size, err = v.get(ctx, loc, buf)
}
if haveDeadline {
log.Printf("Race ended with size==%d", size)
@@ -217,7 +221,9 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
return size, err
}
-func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
+func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int, error) {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
expectSize := len(buf)
if azureMaxGetBytes < BlockSize {
// Unfortunately the handler doesn't tell us how long the blob
@@ -239,10 +245,18 @@ func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
// We'll update this actualSize if/when we get the last piece.
actualSize := -1
pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
- errors := make([]error, pieces)
+ errors := make(chan error, pieces)
var wg sync.WaitGroup
wg.Add(pieces)
for p := 0; p < pieces; p++ {
+ // Each goroutine retrieves one piece. If we hit an
+ // error, it is sent to the errors chan so get() can
+ // return it -- but only if the error happens before
+ // ctx is done. This way, if ctx is done before we hit
+ // any other error (e.g., requesting client has hung
+ // up), we return the original ctx.Err() instead of
+ // the secondary errors from the transfers that got
+ // interrupted as a result.
go func(p int) {
defer wg.Done()
startPos := p * azureMaxGetBytes
@@ -258,17 +272,32 @@ func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
rdr, err = v.bsClient.GetBlobRange(v.ContainerName, loc, fmt.Sprintf("%d-%d", startPos, endPos-1), nil)
}
if err != nil {
- errors[p] = err
+ if ctx.Err() == nil {
+ errors <- err
+ }
+ cancel()
return
}
- defer rdr.Close()
+ go func() {
+ // Close the reader when the client
+ // hangs up or another piece fails
+ // (possibly interrupting ReadFull())
+ // or when all pieces succeed and
+ // get() returns.
+ <-ctx.Done()
+ rdr.Close()
+ }()
n, err := io.ReadFull(rdr, buf[startPos:endPos])
if pieces == 1 && (err == io.ErrUnexpectedEOF || err == io.EOF) {
// If we don't know the actual size,
// and just tried reading 64 MiB, it's
// normal to encounter EOF.
} else if err != nil {
- errors[p] = err
+ if ctx.Err() == nil {
+ errors <- err
+ }
+ cancel()
+ return
}
if p == pieces-1 {
actualSize = startPos + n
@@ -276,10 +305,12 @@ func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
}(p)
}
wg.Wait()
- for _, err := range errors {
- if err != nil {
- return 0, v.translateError(err)
- }
+ close(errors)
+ if len(errors) > 0 {
+ return 0, v.translateError(<-errors)
+ }
+ if ctx.Err() != nil {
+ return 0, ctx.Err()
}
return actualSize, nil
}
@@ -306,7 +337,36 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
if v.ReadOnly {
return MethodDisabledError
}
- return v.bsClient.CreateBlockBlobFromReader(v.ContainerName, loc, uint64(len(block)), bytes.NewReader(block), nil)
+ // Send the block data through a pipe, so that (if we need to)
+ // we can close the pipe early and abandon our
+ // CreateBlockBlobFromReader() goroutine, without worrying
+ // about CreateBlockBlobFromReader() accessing our block
+ // buffer after we release it.
+ bufr, bufw := io.Pipe()
+ go func() {
+ io.Copy(bufw, bytes.NewReader(block))
+ bufw.Close()
+ }()
+ errChan := make(chan error)
+ go func() {
+ errChan <- v.bsClient.CreateBlockBlobFromReader(v.ContainerName, loc, uint64(len(block)), bufr, nil)
+ }()
+ select {
+ case <-ctx.Done():
+ theConfig.debugLogf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
+ // Our pipe might be stuck in Write(), waiting for
+ // io.Copy() to read. If so, un-stick it. This means
+ // CreateBlockBlobFromReader will get corrupt data,
+ // but that's OK: the size won't match, so the write
+ // will fail.
+ go io.Copy(ioutil.Discard, bufr)
+ // CloseWithError() will return once pending I/O is done.
+ bufw.CloseWithError(ctx.Err())
+ theConfig.debugLogf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
+ return ctx.Err()
+ case err := <-errChan:
+ return err
+ }
}
// Touch updates the last-modified property of a block blob.
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list