[ARVADOS] updated: 62a252a65b8983722c8545edbbd8c3e40805f81b
Git user
git at public.curoverse.com
Thu Apr 28 16:54:11 EDT 2016
Summary of changes:
.../pipeline_instances/_running_component.html.erb | 10 -
apps/workbench/test/test_helper.rb | 6 +-
build/package-build-dockerfiles/centos6/Dockerfile | 6 +-
build/package-build-dockerfiles/debian7/Dockerfile | 2 +-
build/package-build-dockerfiles/debian8/Dockerfile | 2 +-
.../ubuntu1204/Dockerfile | 2 +-
.../ubuntu1404/Dockerfile | 2 +-
build/run-build-docker-jobs-image.sh | 38 ++-
build/run-build-packages-one-target.sh | 5 +-
build/run-build-packages-python-and-ruby.sh | 203 ++++++++++++
build/run-build-packages.sh | 61 ++--
build/run-build-test-packages-one-target.sh | 111 +++++++
build/run-library.sh | 41 +++
build/run-tests.sh | 65 ++--
crunch_scripts/crunchrunner | 7 +-
crunch_scripts/cwl-runner | 92 ++++++
doc/api/schema/Job.html.textile.liquid | 1 +
doc/install/arvbox.html.textile.liquid | 11 +-
.../install-compute-node.html.textile.liquid | 4 +-
doc/install/install-keepstore.html.textile.liquid | 2 +-
.../install-shell-server.html.textile.liquid | 4 +-
docker/build_tools/Makefile | 2 +-
docker/compute/Dockerfile | 2 +-
docker/jobs/Dockerfile | 12 +-
docker/jobs/apt.arvados.org.list | 2 +-
docker/shell/Dockerfile | 2 +-
sdk/cli/bin/arv-run-pipeline-instance | 4 +-
sdk/cli/bin/crunch-job | 51 +--
sdk/cwl/arvados_cwl/__init__.py | 345 ++++++++++++++++----
sdk/cwl/setup.py | 4 +-
sdk/cwl/test_with_arvbox.sh | 7 +
sdk/cwl/tests/input/blorp.txt | 1 +
sdk/cwl/tests/submit_test_job.json | 6 +
sdk/cwl/tests/test_job.py | 167 +++++++---
sdk/cwl/tests/test_submit.py | 103 ++++++
sdk/cwl/tests/tool/blub.txt | 1 +
sdk/cwl/tests/tool/submit_tool.cwl | 19 ++
sdk/cwl/tests/wf/submit_wf.cwl | 16 +
sdk/go/arvadostest/fixtures.go | 2 +
sdk/go/crunchrunner/crunchrunner.go | 26 +-
sdk/go/keepclient/collectionreader.go | 4 +
sdk/go/keepclient/collectionreader_test.go | 1 +
sdk/go/keepclient/perms.go | 14 +-
sdk/go/keepclient/perms_test.go | 29 +-
sdk/go/manifest/manifest.go | 2 +-
sdk/go/streamer/transfer.go | 4 +-
sdk/python/arvados/arvfile.py | 2 +
sdk/python/arvados/commands/keepdocker.py | 12 +-
sdk/python/arvados/commands/run.py | 26 +-
sdk/python/arvados/events.py | 126 +++++++-
sdk/python/arvados/retry.py | 7 +-
sdk/python/setup.py | 6 +
sdk/python/tests/slow_test.py | 7 +
sdk/python/tests/test_retry.py | 2 +-
sdk/python/tests/test_stream.py | 13 +
sdk/python/tests/test_websockets.py | 129 +++++++-
sdk/ruby/arvados.gemspec | 10 +-
sdk/ruby/lib/arvados.rb | 4 +-
sdk/ruby/lib/arvados/collection.rb | 12 +-
sdk/ruby/lib/arvados/keep.rb | 8 +-
sdk/ruby/test/test_keep_manifest.rb | 15 +
.../v1/api_client_authorizations_controller.rb | 33 +-
.../app/controllers/arvados/v1/jobs_controller.rb | 5 +-
services/api/app/models/blob.rb | 11 +-
services/api/app/models/job.rb | 21 +-
services/api/config/application.default.yml | 17 +-
.../20160324144017_add_components_to_job.rb | 11 +
services/api/db/structure.sql | 7 +-
services/api/test/fixtures/jobs.yml | 29 ++
.../api_client_authorizations_controller_test.rb | 40 ++-
.../arvados/v1/groups_controller_test.rb | 6 +-
.../functional/arvados/v1/jobs_controller_test.rb | 75 +++++
services/api/test/test_helper.rb | 6 +-
services/api/test/unit/blob_test.rb | 21 +-
services/api/test/unit/job_test.rb | 1 -
services/crunch-run/crunchrun_test.go | 4 +-
services/datamanager/loggerutil/loggerutil.go | 2 +-
services/datamanager/summary/pull_list.go | 2 +-
services/dockercleaner/setup.py | 2 +-
services/fuse/arvados_fuse/fusedir.py | 5 +
services/fuse/arvados_fuse/fusefile.py | 6 +
services/fuse/setup.py | 5 +
services/fuse/tests/mount_test_base.py | 14 +-
.../performance/test_collection_performance.py | 7 +
services/fuse/tests/slow_test.py | 1 +
services/fuse/tests/test_cache.py | 45 +++
services/fuse/tests/test_mount.py | 1 -
services/keep-web/handler.go | 6 +
services/keep-web/server_test.go | 10 +-
services/keepproxy/keepproxy.go | 2 +-
services/keepstore/azure_blob_volume.go | 29 +-
services/keepstore/azure_blob_volume_test.go | 16 +-
services/keepstore/keepstore.go | 2 +-
services/keepstore/perms.go | 4 +-
services/keepstore/perms_test.go | 7 +-
services/keepstore/trash_worker_test.go | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 126 ++++----
.../arvnodeman/computenode/dispatch/slurm.py | 67 ++--
.../arvnodeman/computenode/dispatch/transitions.py | 52 +++
.../arvnodeman/computenode/driver/__init__.py | 61 ++--
.../arvnodeman/computenode/driver/azure.py | 7 +-
.../arvnodeman/computenode/driver/ec2.py | 4 +-
.../arvnodeman/computenode/driver/gce.py | 7 +-
services/nodemanager/arvnodeman/daemon.py | 22 +-
.../nodemanager/tests/test_computenode_dispatch.py | 47 +--
.../tests/test_computenode_dispatch_slurm.py | 38 ++-
.../tests/test_computenode_driver_azure.py | 24 ++
.../tests/test_computenode_driver_gce.py | 5 +
services/nodemanager/tests/test_daemon.py | 14 +-
services/nodemanager/tests/testutil.py | 46 +++
tools/arvbox/bin/arvbox | 50 +--
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 4 +
.../{service/api/run-service => api-setup.sh} | 18 +-
tools/arvbox/lib/arvbox/docker/common.sh | 9 +-
tools/arvbox/lib/arvbox/docker/createusers.sh | 6 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 9 +-
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 11 +-
.../lib/arvbox/docker/service/api/run-service | 78 +----
.../docker/service/arv-git-httpd/run-service | 12 +-
.../service/crunch-dispatch-local/run-service | 8 +-
.../lib/arvbox/docker/service/keep-web/run-service | 12 +-
.../arvbox/docker/service/keepproxy/run-service | 12 +-
.../lib/arvbox/docker/service/sdk/run-service | 4 +-
.../{workbench => websockets}/log/main/.gitstub | 0
.../service/{workbench => websockets}/log/run | 0
.../arvbox/docker/service/{sso => websockets}/run | 0
.../arvbox/docker/service/websockets/run-service | 30 ++
.../crunchstat_summary/reader.py | 1 +
.../crunchstat_summary/summarizer.py | 1 +
tools/keep-block-check/.gitignore | 1 +
tools/keep-block-check/keep-block-check.go | 239 ++++++++++++++
tools/keep-block-check/keep-block-check_test.go | 352 +++++++++++++++++++++
tools/keep-rsync/keep-rsync.go | 40 ++-
tools/keep-rsync/keep-rsync_test.go | 31 +-
135 files changed, 3042 insertions(+), 743 deletions(-)
create mode 100755 build/run-build-packages-python-and-ruby.sh
create mode 100755 build/run-build-test-packages-one-target.sh
create mode 100755 crunch_scripts/cwl-runner
create mode 100644 sdk/cwl/tests/input/blorp.txt
create mode 100644 sdk/cwl/tests/submit_test_job.json
create mode 100644 sdk/cwl/tests/test_submit.py
create mode 100644 sdk/cwl/tests/tool/blub.txt
create mode 100644 sdk/cwl/tests/tool/submit_tool.cwl
create mode 100644 sdk/cwl/tests/wf/submit_wf.cwl
create mode 100644 sdk/python/tests/slow_test.py
create mode 100644 services/api/db/migrate/20160324144017_add_components_to_job.rb
create mode 120000 services/fuse/tests/slow_test.py
create mode 100644 services/fuse/tests/test_cache.py
create mode 100644 services/nodemanager/arvnodeman/computenode/dispatch/transitions.py
copy tools/arvbox/lib/arvbox/docker/{service/api/run-service => api-setup.sh} (83%)
copy tools/arvbox/lib/arvbox/docker/service/{workbench => websockets}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{workbench => websockets}/log/run (100%)
copy tools/arvbox/lib/arvbox/docker/service/{sso => websockets}/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/websockets/run-service
create mode 100644 tools/keep-block-check/.gitignore
create mode 100644 tools/keep-block-check/keep-block-check.go
create mode 100644 tools/keep-block-check/keep-block-check_test.go
via 62a252a65b8983722c8545edbbd8c3e40805f81b (commit)
via f6838b6c52134b4e5d037b566ec9478cab332d38 (commit)
via e1de889290360f6dd5b5fdeab10cea997bcc6962 (commit)
via ac48d85ae411b1bcef0c194caae1a861eae9e929 (commit)
via 47a79960c81ea689445f2040b24cb76729afab06 (commit)
via 75c049d9561b08c09f207edb4ebd3bc6fb6460a6 (commit)
via 20beb96440f1bae29adb0132617aae4959e76c75 (commit)
via c1276bd9f83a7826f10e1752ac793d8a1cd3c47f (commit)
via 8cc6fc047506e3bead524f18416b78fe068c70fd (commit)
via a6401a8c59a582633de2ae197e71aad493de291a (commit)
via eebcb5e6a6cb98562212f30841a0411872ac3e89 (commit)
via 6279c22bd0e85db1bdb21ad9d95ae8816ac87a5a (commit)
via da6f122ed9c8ff0e79c7b3969c4f0960039b9bc7 (commit)
via 655e654e99a00a248639a910923e0ce66c16581c (commit)
via d021d4a0549210dd513d4d28735ad81db85cafbf (commit)
via f5617be935a121dc339effcaafebb569109ebe5b (commit)
via 628f2f2e1bfabdb7221badab3d5189011aee5a54 (commit)
via 08d8c1af187132692baf70a531f87c8082f72cdc (commit)
via 89e091b3ee1fe85a68b0a9a0a619ca7baf606d2d (commit)
via 3c88abd3cb33cbe80bb81a7cca779fe668036c9e (commit)
via 469102b30008b4ee73c9f44b03a6bd82a2a91fb2 (commit)
via 83332c0e546ad5e6332ce507224d9c6f64552ce6 (commit)
via 2b152757ce494145c10971c36754c473d21dfcc7 (commit)
via 0cacef6ec2b553052e3897fee3067951f65117cd (commit)
via 899af276f73189b0581b65c6d77b19f5c041312a (commit)
via 5784c9a32528a7795181a837660f7f21a7393cce (commit)
via 4d9ece42c8482948e4eca9f587cdb36f62a00e2b (commit)
via aec99288fab5ceed8dc746e62efaeee33ff38a82 (commit)
via f2e0e32c7ab0f1de29e6e90a6af87c1969dfd640 (commit)
via b2cfb1a8873ee8c4128bd3bdd9fb32b9d9e8ee6e (commit)
via d7b6ec65cfd774f0d17c96d2baf9b6620c55ad8d (commit)
via 2617eb303578ff5fdf9f24fd4a5de4bfd2144425 (commit)
via caa404bdf14b29dc592d5b8b87fc264c1cbce8ae (commit)
via 9b90fe97f8e92794856e3d730811953e1c13ea25 (commit)
via 611244398282b805188d4c334f3323807ca394c5 (commit)
via 29379beaa615d5a36032a05e71d7a2730e255c48 (commit)
via 6f1543559341624f0b9bdb880f7feb3fb8495811 (commit)
via aaab24cd029c049963acf74ff8ec3694c599ba20 (commit)
via b51781b96a666ac5e671dfed8c255ac02f7da163 (commit)
via c213eb1a85c5a0cb26000dc1d013cb11c0f8c601 (commit)
via 827f01731858dc21578f0bd35f01d9d58b72d2ef (commit)
via af754a64c48e30aec75bbcb1b741a768faacb94c (commit)
via 6a202a2765bdbac5062fd34231804527edfb6a14 (commit)
via ecf4a87ac687b713d3eb4f191f4c5ead293fa046 (commit)
via fe8143ccd8193e4d7d2098d0a61dce7127aef713 (commit)
via dfdb6060ea1a6544fe2497d7c1d18078879fe090 (commit)
via 17c23d338518f0498fb1396f24954f884a06b05b (commit)
via 8704a73054ecf639ac12cd4de4b1349ed69ae093 (commit)
via da39f7ee464b4be805591ff5901bdacaa005b0a8 (commit)
via da6e931b990c7edde55213196fc8a47f4dc4998d (commit)
via e7091ec45f8f634a52db2fba1b385e790d69e6fe (commit)
via 3dc4c3e304a1d2ccb975a43679276eed55bd49fd (commit)
via 51f64605f9b033efcc02b45376f2b7f22e360483 (commit)
via abfdd429351ad6b457667b624c8ba550ffcac999 (commit)
via 2a13fcc84b240e368787c8e94ced95d75eee0cc4 (commit)
via 26751323e77005dc158b64e86c47bbb9459e6697 (commit)
via 96f9b11c3aafad5905f57848b0a5a1ad728f0ee0 (commit)
via fd696fac8234ff799aa6ac63819e5e6ac0a8a9d4 (commit)
via 3a31612fdc5e1ccdfa91d04ff5a089d6e06c0f60 (commit)
via 065cefa6e87c8b668af5a4abc065514b84db83b5 (commit)
via 40ad5fb3f699c348222c90692b9ae27a8f716af9 (commit)
via a7e8991d92cdd798406e163171127d8285ddd4d0 (commit)
via c0b76aafff3e6883635b3bd0f186591eddac2a7d (commit)
via d4ac44b6224001cb9dc1d86e180c2ac48ef1bce6 (commit)
via e935e107a4ac6250ae64878262c3145d7a62b8e8 (commit)
via a8bbf6efd755e3a6915ecfa5d980ef76ca1ea4a5 (commit)
via f044dbbcab5f4fcefaf2952fa1b71eb68aba7999 (commit)
via f8e1022669f0ab5d5db10d0e043b17be83d879b0 (commit)
via 1ef981e08fb9905fbe4fdd294197368a3a9e5bbb (commit)
via 5026c691c3b2b402243acd4c2dd936aa7976ba2b (commit)
via 2581a27602bd98fa532926efc02d9401ee095108 (commit)
via b5dc4ae38eb6da852e167952675aff59987a995c (commit)
via 28fb859d989c295e1f44aed96a7478e8454db4d5 (commit)
via 207136fe4f9787ac7f99cba448ed5a2a05a6f8dd (commit)
via 788b8d7247da8c4592b1f9d482fff4e1509f57f3 (commit)
via 32eb510594a45b395d1a0c8dafda1a94c86c0a9f (commit)
via f735a9c42905fbc900cd1f3612f35b9438c730f2 (commit)
via 9c3de7371b820b1cecf1f3b0117773a11c5f5883 (commit)
via 55b017847fe87e00e7fb9c6bfd0444b83f8ca12c (commit)
via 65efe2535e49058ccf8250f93e958ea653f55cd8 (commit)
via 8ad62b017ea0dc84613eeb43f21f32890d9ed4ed (commit)
via d5a21b4c6198d861a09ebeefec02bf78395f2e4e (commit)
via a8a65836a931811e869a3abba2244cc529ef2dd6 (commit)
via 4ab126d1574b7db2fdb5b0cea253b2df28d7b130 (commit)
via 2d112118532f0e059bc7e72d85ee63083a98156f (commit)
via 791bfa0fabd9b37ff8b52fb934f16b47461731da (commit)
via 92d7a504629d848927d5334b0a801cd3993a585b (commit)
via ba534230bcc23aaea69e5f256693b9e143200b63 (commit)
via 95253d987794df9c27db93b1b9f8be2d32229309 (commit)
via 7ddbf6b763f15bb84d7d897d373e37e85a8c3d6b (commit)
via 7c607284a048225b86b210c8c5397ff8a983d820 (commit)
via 0c848aa00c9af71ea2c7a575d8ee04b420a5eae7 (commit)
via 10f81953efe1884f170a7a2be14e6f1c32012c86 (commit)
via 47c2e05d4d86206e15fdf7d14e6f00637dd81f94 (commit)
via a4f8073c3c8520bf0e12f335e2c829a7f7f43dd1 (commit)
via 68d551b65a837664be72bf08aad55ab76d778d07 (commit)
via c244fbb0d880b45a44300e9ed650a32a954fd7d9 (commit)
via a2dc68267b5ff974460cef796820e7f8b90e04bc (commit)
via bbb0760e909b3ceca850b3aa319011fba2c98ed0 (commit)
via f5348ab8671390e85956a9b78c5810be7c01e81e (commit)
via 86996de865c2b4d395c2a3977b05f7278924f627 (commit)
via f1e315aa701757ee8bf497805033b039b21515ba (commit)
via e13c868ed9216b4ad414adc435a9f9ed5afe2b89 (commit)
via ae6ced575905d226963516bb7780c2bae391621c (commit)
via 448ec6cc59fe837eb71c416a82b8a72a959eddc8 (commit)
via fc6f80e4d1157b4b69bad852be1a7bb921fcb91b (commit)
via 241ef75ec8b6cf5dd14ce19fa068462adaeb0386 (commit)
via 2d18e31468b269e526cb6e4918912b60a48540b2 (commit)
via f37349e9436294472bdbdec820c5ec44ebe64430 (commit)
via 9c3a1a10b702ec7ee79bb0aa1beb3c227bec95dd (commit)
via 91b379c4a76b8278354903158a65e5d47babd363 (commit)
via b8ae9ce66c24055f3bf66b0956e15c44c9dd51f0 (commit)
via 8b93280fb2fae252e29263660e9591fcaa2d8722 (commit)
via a2c147e469bc4ccc0e788f2e72b2d94aa81ba368 (commit)
via 7b2cd50f499a1f9c5b21ab6aa641f14701402885 (commit)
via 55778ad90a9b869f514e729b3f85c12383f4e2a7 (commit)
via eea5f75077ca5d3adced3ad3d90f755d0fa5b3d8 (commit)
via 34d92b238ebc107bf28dac5c7e3ce138ac84b2c1 (commit)
via 86f774031fd38bd8d34341afd007fbea9e6da740 (commit)
via cafe405682669b7ebdec8db4ee083c3ca2761827 (commit)
via dce9b03c23e89e77e3547a3a92088f332c0f747e (commit)
via 8680c8740d128a6fe71b27aad528dea890edaefa (commit)
via d48af5cfd486b9ea93b57c7f88b80e0305664b0b (commit)
via 82225f2eeb39f6798ff83e979c28698ff617d414 (commit)
via 75ed52bfd557502aa04d413cdfb4f798af7cec66 (commit)
via 798b08af218972bf9fb60101b7f85fd71951c6b7 (commit)
via de985c815dd4a81dcb5b03475452e2c93d3dadb4 (commit)
via a74883043be4bfda29010adcd3627b4ad8923a29 (commit)
via 3386dd9826cf143d078aa8985726516932fafa5b (commit)
via 9bf2ba202652163937d79341e855910f8c330d05 (commit)
via 49743c080265b270693154d7a327d0433b0a7dbe (commit)
via 2fa70753122b53e70005c8f7d2d9b0f12588aa2e (commit)
via 98ada97aef7838001ba87af08761b3abd5b011bb (commit)
via d646e4d515fff35c04544f531f5a7bc0a53a79ab (commit)
via c020dd4470a5f8cbcfe15e764553622828f69ab5 (commit)
via cdcddca613c896fd8395a4045c858945451c3fa0 (commit)
via c19a7bbf7af84f0b8d60ebe29ee9365eca36fbbb (commit)
via 9d035125ea1f8b6bd32fa6f862b32ba9308ebc66 (commit)
via c980683a243903babe9cc09cabc71e1c6229fef1 (commit)
via 057a218324fd68bf5b9d034240d225b539b393a6 (commit)
via 92c876d90253a0db23d40aa125cba335f7bf7e27 (commit)
via 505b7680f8ed8f518f1c3368eb4107e493115b23 (commit)
via f7e07c3a28d4f4e1adbd957749bd825e44a2e523 (commit)
via 3fe0e22c8b6edecd2651c5ee4db105a1e19f42e8 (commit)
via 5eabedb88ce741076167efd2305216043dd9abd5 (commit)
via e68d56d2d033c7be04294fd30cd0edb424f291fb (commit)
via cf5af834e8fb31720b459127394f6bf4e2c0158e (commit)
via db1fd09c38132bbaf04e6921bdc0de3bf28abe09 (commit)
via a3f2cb186e437bfce0031b024b2157b73ed2717d (commit)
via 436105d3e249ad8dc55fb618d0d2c0d8ed29b395 (commit)
via c22b3b27841508c534c6b287ef1508841fd892d4 (commit)
via dbafab756158fa8bbd374a9938f0f4cb21e02818 (commit)
via a7d819f660840767df3e393a30a775f445db266e (commit)
via 529a648458ceefc02d77abf470a2005a099d9907 (commit)
via 204bba7b53a2f7eba36782facf6dc8903aee120a (commit)
via 4743b2e8f9587146f24359f98b648e92d5b04e38 (commit)
via 94f5be8c86ad975ee7aa9f3df87be23fbc154dec (commit)
via e9667fff4716e4990fec560f92c1e0f4a6cb686f (commit)
via 98036435a54261258faadedeef0675eecfe4ff39 (commit)
via f2f5c6aa8117b51e9fed1a59a41b97b18ac713a8 (commit)
via dfc93aac9c256d6ebb868aeb6c2107821e9fd041 (commit)
via 4727a3b6417bd254b3e8bca1bd19a77773493572 (commit)
via 1b1e985db040147feb0145b0a6a6e3da6a7b4dbe (commit)
via 414ab8bf556c08065d8235a80822344ab5265aae (commit)
via 5c1410e60eb06769207dbafd77501bf2fa45a29b (commit)
via 0e32c4ee6281dd4a3ae186b75dd7c94c8be508c8 (commit)
via adad241671bd37f76b5dcefad1e01914f9797214 (commit)
via 087ad4d01176703f22fde4cf053427e562890c16 (commit)
via 6c01579495dd6a26ee9b05aa0accd136094d453e (commit)
via a810fa495eab5bc3356ac53a9a3aa40d69b1c77e (commit)
via 32fba9fa61361bf068e8125507e6242da4295ee5 (commit)
via 525e930dcb35b440424d53b09a9788fd8726b781 (commit)
via 48a323ca58614b7e2e100fd69fdc8a441d5d0c80 (commit)
via abd7a5952a3e852c854e3e08353b42b778a7381d (commit)
via 7a6612d56b719165d3aff84754521c9a38d9912a (commit)
via 91b6e4f4ae93850fb3cc38cf8566115a66f44b27 (commit)
via 48fce933fec998722e669d7d33a62f3c0a10b05e (commit)
via f49a4d7ff243bb3e8b15f4c5adf77d6355fb6bcd (commit)
via c7539c212173e4ce103ec8cc893f0c4f0a942ba3 (commit)
via a8af246b8c99b070fc445cda3d3a13bdf0885f24 (commit)
via d27d9df7bfb7be820b4674675a240901348cfadb (commit)
via a3d43ada117ee7e0535579fc3f1b66503f95b4c7 (commit)
via 4447bfa5edac893079894484244318a872a0d3ba (commit)
via beaea7361fdfaf3dd220e3f5999fb3c0dfc3e61c (commit)
via 8b133b596ef25d548e01820c25595803bb49d18e (commit)
via 1b7ac33e0651087b85f8583c72325d5583b98776 (commit)
via ed646f12b58e930fe018dc4caf2121294c26e375 (commit)
via d170d92ce35f402b68fc07c7cbeed2d215685837 (commit)
via b8260eb5cdc2c403083d7dbea49c45791a763e9c (commit)
via 7bf10f8b9d2246a9144ef1322e102fe227c7362e (commit)
via 9bccc21498bab7e1300af17c5844f868f119bb9c (commit)
via 3e50cdc0ff3c5c8d40d082bf99a43eae3682aa72 (commit)
via 354ab6f297bb2100b6ebd0c94fa6463a77d18844 (commit)
via 9df4d7da24ef8be639af1ab806cf833ee544fb45 (commit)
via ad775d785386d751e9b8b0d127885b56f3e016ff (commit)
via 67ef6afce1dd158938eebcf6bed9d3edeb9d73a0 (commit)
via 9507b78b62a042013a8c80ed82182a2ca6d45912 (commit)
via 39df0227516440638035168f5f89df4634f37e1a (commit)
via e1655f12ed1cfec6b5763e1db81e7ec8dd0f7a14 (commit)
via 5450d1417a31aaf2f71e16a440fcfee0a06be6c7 (commit)
via f9f7ccc13963e52b60a10d99806b00a1de66833f (commit)
via 133d751f665ccf24fde56d1f8acf23f9a3475927 (commit)
via 8539a6c20fecc7bed02f1c7f81d3d2a1c0974e1f (commit)
via cb6d05e27e37f29ea99cf6e474ee09a1479252e1 (commit)
via 46f5721a767f8e012b3e3183412bb8f9c5ff0ac6 (commit)
via 8f46e1564dd7c4f42fdccc6e5c8222ce8955405e (commit)
via 76c7406f60319da7011ac9797ae32eb22f574d4c (commit)
via 785a877a46f0c6580d46dac179ea2d4c4a6e7de3 (commit)
via 4fecea55bd6dd5befd82bf7ec5dd4fe93bd93cea (commit)
via f04cfcce18c6e4bd1faf3140ccbc835328580024 (commit)
via 9c6598596ae99cfea9ab8ced51c58d32ca306764 (commit)
via 4b10b7265345211d0db4818c2ec1c804227511f6 (commit)
via 3fbb8f2d87742f5075c1829c64cf223725cb4fd2 (commit)
via 1c2af19398b425fb249e6fa8cc909500ce1fa80f (commit)
via cf82d85482660860f143024db6c0be444e07b19a (commit)
via e66f7687796f4ed3513fbc5a469d578ce76334e0 (commit)
via 9b15259759d5e73ca51f30956527dc774c5a80cf (commit)
via 05a4b601e0f7810a0e7be2e557aa266cc8f8b4e7 (commit)
via 2d2d6ad747ccb94d77fcb11c275317af7c08f7af (commit)
via 322b1729108da208e7376b05e2396b0c9da1c654 (commit)
via 8e6e328e8f8985b763e5a26185de769ea3ad904d (commit)
via defc3f65bfeed18d4ca0beb0e2a3aabd0db6cc75 (commit)
via a87af13d2d3461470b89b0811629a077a70c1938 (commit)
via 846a2728f3cfc80efc220a60536dea16b5f0fcbd (commit)
via bdfd1e9f8bc2a3933f6b0bc7afd5cfebd815fd90 (commit)
via 86865a741f902ac008e11d1dec90b87b89822025 (commit)
via 4d5e04d4998d2fe6c33a98bb93e7ec05e559bcbd (commit)
via 30443c31bf020b6f558c4b7d954599b24c1a98ce (commit)
via 83645e26981cf9a01c72096298fd9f8dd0f5309e (commit)
via ca4958ad27277b1d4ae0226382e214fb2b1f2fec (commit)
from 316f588fdd86e4dfb0193a78f0a71859b77d1714 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 62a252a65b8983722c8545edbbd8c3e40805f81b
Author: radhika <radhika at curoverse.com>
Date: Thu Apr 28 16:50:42 2016 -0400
8556: When an azure blob is trashed, setting the expires_at metadata results in changing the blob updated time.
This is problematic because the next time around we invoke Mtime on it, we get this new time, which is wrong.
Introduced a last_write_at metadata attribute to be used by Mtime to determine the actual updated time instead.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 13cdafc..2592524 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -276,7 +276,9 @@ func (v *AzureBlobVolume) Put(loc string, block []byte) error {
if v.readonly {
return MethodDisabledError
}
- return v.bsClient.CreateBlockBlobFromReader(v.containerName, loc, uint64(len(block)), bytes.NewReader(block), nil)
+ extraHeaders := make(map[string]string)
+ extraHeaders["x-ms-meta-last_write_at"] = fmt.Sprintf("%d", time.Now().Add(trashLifetime).Unix())
+ return v.bsClient.CreateBlockBlobFromReader(v.containerName, loc, uint64(len(block)), bytes.NewReader(block), extraHeaders)
}
func (v *AzureBlobVolume) addToMetadata(loc, name, value string) error {
@@ -293,7 +295,7 @@ func (v *AzureBlobVolume) removeFromMetadata(loc, name string) error {
if err != nil {
return err
}
- delete(metadata, name)
+ metadata[name] = ""
return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
}
@@ -305,7 +307,7 @@ func (v *AzureBlobVolume) Touch(loc string) error {
if err := v.checkTrashed(loc); err != nil {
return err
}
- return v.addToMetadata(loc, "touch", fmt.Sprintf("%d", time.Now()))
+ return v.addToMetadata(loc, "last_write_at", fmt.Sprintf("%d", time.Now()))
}
// Mtime returns the last-modified property of a block blob.
@@ -313,11 +315,16 @@ func (v *AzureBlobVolume) Mtime(loc string) (time.Time, error) {
if err := v.checkTrashed(loc); err != nil {
return time.Time{}, err
}
- props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
+ metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
+ if err != nil {
+ return time.Time{}, err
+ }
+
+ last_write_at, err := strconv.ParseInt(metadata["last_write_at"], 10, 64)
if err != nil {
return time.Time{}, err
}
- return time.Parse(time.RFC1123, props.LastModified)
+ return time.Unix(last_write_at, 0), nil
}
// IndexTo writes a list of Keep blocks that are stored in the
@@ -367,27 +374,27 @@ func (v *AzureBlobVolume) Trash(loc string) error {
// we get the Etag before checking Mtime, and use If-Match to
// ensure we don't delete data if Put() or Touch() happens
// between our calls to Mtime() and DeleteBlob().
- props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
- if err != nil {
- return err
- }
if t, err := v.Mtime(loc); err != nil {
return err
} else if time.Since(t) < blobSignatureTTL {
return nil
}
if trashLifetime == 0 {
+ props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
+ if err != nil {
+ return err
+ }
return v.bsClient.DeleteBlob(v.containerName, loc, map[string]string{
"If-Match": props.Etag,
})
}
// Mark as trash
- err = v.addToMetadata(loc, "expires_at", fmt.Sprintf("%d", time.Now().Add(trashLifetime).Unix()))
+ err := v.addToMetadata(loc, "expires_at", fmt.Sprintf("%d", time.Now().Add(trashLifetime).Unix()))
if err != nil {
return err
}
return v.bsClient.CreateBlockBlobFromReader(v.containerName,
- fmt.Sprintf("trash.%d.%v", time.Now().Add(trashLifetime).Unix(), loc), 0, nil)
+ fmt.Sprintf("trash.%d.%v", time.Now().Add(trashLifetime).Unix(), loc), 0, nil, nil)
}
// Untrash a Keep block.
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 08166a8..e4a0d4c 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -66,6 +66,7 @@ func (h *azStubHandler) TouchWithDate(container, hash string, t time.Time) {
return
}
blob.Mtime = t
+ blob.Metadata["last_write_at"] = fmt.Sprintf("%d", t.Unix())
}
func (h *azStubHandler) PutRaw(container, hash string, data []byte) {
@@ -144,11 +145,18 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
}
h.unlockAndRace()
}
+ metadata := make(map[string]string)
+ for k, v := range r.Header {
+ if strings.HasPrefix(strings.ToLower(k), "x-ms-meta-") {
+ name := k[len("x-ms-meta-"):]
+ metadata[strings.ToLower(name)] = v[0]
+ }
+ }
h.blobs[container+"|"+hash] = &azBlob{
Data: body,
Mtime: time.Now(),
Uncommitted: make(map[string][]byte),
- Metadata: make(map[string]string),
+ Metadata: metadata,
Etag: makeEtag(),
}
rw.WriteHeader(http.StatusCreated)
@@ -198,10 +206,10 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusBadRequest)
return
}
- blob.Metadata = make(map[string]string)
for k, v := range r.Header {
if strings.HasPrefix(strings.ToLower(k), "x-ms-meta-") {
- blob.Metadata[k] = v[0]
+ name := k[len("x-ms-meta-"):]
+ blob.Metadata[strings.ToLower(name)] = v[0]
}
}
blob.Mtime = time.Now()
@@ -213,7 +221,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
return
}
for k, v := range blob.Metadata {
- rw.Header().Set(k, v)
+ rw.Header().Set(fmt.Sprintf("x-ms-meta-%s", k), v)
}
return
case (r.Method == "GET" || r.Method == "HEAD") && hash != "":
commit f6838b6c52134b4e5d037b566ec9478cab332d38
Merge: 1b1e985 e1de889
Author: radhika <radhika at curoverse.com>
Date: Wed Apr 27 17:38:02 2016 -0400
Merge branch 'master' into 8556-trash-untrash-azure-volume
commit 1b1e985db040147feb0145b0a6a6e3da6a7b4dbe
Merge: 316f588 0e32c4e
Author: radhika <radhika at curoverse.com>
Date: Mon Mar 28 11:21:38 2016 -0400
Merge branch 'master' into 8556-trash-untrash-azure-volume
diff --cc services/keepstore/azure_blob_volume.go
index 576f22e,f08cebf..13cdafc
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@@ -276,27 -257,9 +276,27 @@@ func (v *AzureBlobVolume) Put(loc strin
if v.readonly {
return MethodDisabledError
}
- return v.bsClient.CreateBlockBlobFromReader(v.containerName, loc, uint64(len(block)), bytes.NewReader(block))
+ return v.bsClient.CreateBlockBlobFromReader(v.containerName, loc, uint64(len(block)), bytes.NewReader(block), nil)
}
+func (v *AzureBlobVolume) addToMetadata(loc, name, value string) error {
+ metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
+ if err != nil {
+ return err
+ }
+ metadata[name] = value
+ return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
+}
+
+func (v *AzureBlobVolume) removeFromMetadata(loc, name string) error {
+ metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
+ if err != nil {
+ return err
+ }
+ delete(metadata, name)
+ return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
+}
+
// Touch updates the last-modified property of a block blob.
func (v *AzureBlobVolume) Touch(loc string) error {
if v.readonly {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list