[ARVADOS] updated: 8ae4b9342772ce4693b5961da5e83f54a36ad0e7
git at public.curoverse.com
git at public.curoverse.com
Thu Feb 12 13:53:00 EST 2015
Summary of changes:
.gitignore | 7 +-
apps/workbench/.gitignore | 1 +
apps/workbench/Gemfile | 6 +-
apps/workbench/Gemfile.lock | 4 +-
.../workbench/app/assets/javascripts/ajax_error.js | 15 +
.../workbench/app/assets/javascripts/filterable.js | 6 +-
.../app/assets/javascripts/request_shell_access.js | 10 +
.../app/controllers/projects_controller.rb | 4 +-
.../workbench/app/controllers/search_controller.rb | 3 +-
apps/workbench/app/controllers/users_controller.rb | 6 +
apps/workbench/app/helpers/application_helper.rb | 4 +-
.../app/mailers/request_shell_access_reporter.rb | 11 +
apps/workbench/app/models/collection.rb | 6 +-
.../views/application/_browser_unsupported.html | 24 ++
.../app/views/application/_show_sharing.html.erb | 9 +
.../jobs/_rerun_job_with_options_popup.html.erb | 6 +-
apps/workbench/app/views/layouts/body.html.erb | 3 +
.../app/views/pipeline_instances/show.html.erb | 4 +-
.../send_request.text.erb | 7 +
.../app/views/users/_add_ssh_key_popup.html.erb | 2 +-
.../views/users/_manage_virtual_machines.html.erb | 42 ++-
.../app/views/users/_setup_popup.html.erb | 2 +-
.../app/views/users/request_shell_access.js | 10 +
apps/workbench/app/views/users/welcome.html.erb | 4 +-
apps/workbench/config/application.default.yml | 7 +-
apps/workbench/config/routes.rb | 1 +
apps/workbench/public/browser_unsupported.js | 15 +
.../controllers/collections_controller_test.rb | 19 +
.../test/controllers/users_controller_test.rb | 23 ++
apps/workbench/test/diagnostics/pipeline_test.rb | 1 +
.../test/integration/browser_unsupported_test.rb | 17 +
.../integration/filterable_infinite_scroll_test.rb | 14 +-
apps/workbench/test/integration/jobs_test.rb | 25 +-
.../test/integration/pipeline_instances_test.rb | 70 ++--
apps/workbench/test/integration/projects_test.rb | 30 +-
.../test/integration/user_manage_account_test.rb | 38 ++
apps/workbench/test/integration_helper.rb | 17 +-
apps/workbench/test/performance_test_helper.rb | 2 +-
apps/workbench/test/support/remove_file_api.js | 1 +
apps/workbench/test/test_helper.rb | 126 +++----
doc/api/methods/collections.html.textile.liquid | 2 +-
doc/api/methods/groups.html.textile.liquid | 9 +-
doc/images/keyfeatures/chooseinputs.png | Bin 76641 -> 67586 bytes
doc/images/keyfeatures/collectionpage.png | Bin 84533 -> 68735 bytes
doc/images/keyfeatures/dashboard2.png | Bin 44396 -> 39651 bytes
doc/images/keyfeatures/graph.png | Bin 49127 -> 37727 bytes
doc/images/keyfeatures/log.png | Bin 135214 -> 94845 bytes
doc/images/keyfeatures/provenance.png | Bin 67060 -> 53567 bytes
doc/images/keyfeatures/rerun.png | Bin 73314 -> 56872 bytes
doc/images/keyfeatures/running2.png | Bin 56959 -> 40453 bytes
doc/images/keyfeatures/shared.png | Bin 62395 -> 46090 bytes
doc/images/keyfeatures/webupload.png | Bin 88061 -> 70592 bytes
doc/images/quickstart/1.png | Bin 47239 -> 36164 bytes
doc/images/quickstart/2.png | Bin 74431 -> 58616 bytes
doc/images/quickstart/3.png | Bin 80668 -> 68576 bytes
doc/images/quickstart/4.png | Bin 66686 -> 45334 bytes
doc/images/quickstart/5.png | Bin 83038 -> 66066 bytes
doc/images/quickstart/6.png | Bin 98604 -> 83813 bytes
doc/images/quickstart/7.png | Bin 77820 -> 60031 bytes
doc/sdk/cli/subcommands.html.textile.liquid | 7 +-
sdk/go/arvadosclient/arvadosclient.go | 4 +-
sdk/go/arvadosclient/arvadosclient_test.go | 49 +--
sdk/go/arvadostest/run_servers.go | 123 +++++++
sdk/go/keepclient/keepclient_test.go | 48 +--
sdk/go/keepclient/support.go | 9 -
sdk/python/arvados/collection.py | 27 +-
sdk/python/arvados/commands/put.py | 40 ++-
sdk/python/arvados/errors.py | 2 +
sdk/python/arvados/events.py | 66 ++--
sdk/python/arvados/keep.py | 19 +-
sdk/python/gittaggers.py | 20 ++
sdk/python/setup.py | 31 +-
sdk/python/tests/arvados_testutil.py | 35 ++
sdk/python/tests/run_test_server.py | 395 ++++++++++++++-------
sdk/python/tests/test_api.py | 5 +-
sdk/python/tests/test_arv_put.py | 25 +-
sdk/python/tests/test_collections.py | 65 ++--
sdk/python/tests/test_keep_client.py | 68 ++--
sdk/python/tests/test_pipeline_template.py | 9 +-
sdk/python/tests/test_websockets.py | 44 +--
sdk/ruby/lib/arvados/keep.rb | 5 +-
sdk/ruby/test/test_keep_manifest.rb | 66 ++++
services/api/.gitignore | 1 +
services/api/Gemfile | 4 +-
services/api/Gemfile.lock | 16 +-
.../api/app/controllers/application_controller.rb | 19 +-
.../arvados/v1/collections_controller.rb | 7 +-
.../controllers/arvados/v1/groups_controller.rb | 76 ++--
.../app/controllers/arvados/v1/users_controller.rb | 2 +-
services/api/app/models/arvados_model.rb | 25 ++
services/api/app/models/collection.rb | 27 +-
services/api/app/models/database_seeds.rb | 1 +
services/api/config/application.default.yml | 1 -
services/api/config/application.yml.example | 5 -
.../db/migrate/20150123142953_full_text_search.rb | 18 +
...206210804_all_users_can_read_anonymous_group.rb | 12 +
services/api/db/structure.sql | 41 ++-
services/api/lib/current_api_client.rb | 33 +-
services/api/lib/record_filters.rb | 20 +-
services/api/test/fixtures/collections.yml | 29 +-
services/api/test/fixtures/groups.yml | 5 +-
services/api/test/fixtures/links.yml | 14 +
services/api/test/fixtures/pipeline_templates.yml | 19 +
.../arvados/v1/collections_controller_test.rb | 26 +-
.../api/test/functional/arvados/v1/filters_test.rb | 41 +++
.../arvados/v1/groups_controller_test.rb | 17 +-
.../api/test/integration/collections_api_test.rb | 79 ++++-
services/api/test/integration/groups_test.rb | 35 ++
services/api/test/unit/arvados_model_test.rb | 22 ++
services/api/test/unit/collection_test.rb | 47 +++
services/api/test/unit/link_test.rb | 5 +
services/api/test/websocket_runner.rb | 10 +-
services/fuse/arvados_fuse/__init__.py | 4 +-
services/fuse/gittaggers.py | 1 +
services/fuse/setup.py | 33 +-
services/fuse/tests/test_mount.py | 42 +--
services/keepproxy/keepproxy_test.go | 99 ++----
services/{fuse => nodemanager}/MANIFEST.in | 0
services/nodemanager/gittaggers.py | 1 +
services/nodemanager/setup.py | 34 +-
services/nodemanager/tests/test_daemon.py | 16 +-
121 files changed, 1852 insertions(+), 810 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/ajax_error.js
create mode 100644 apps/workbench/app/assets/javascripts/request_shell_access.js
create mode 100644 apps/workbench/app/mailers/request_shell_access_reporter.rb
create mode 100644 apps/workbench/app/views/application/_browser_unsupported.html
create mode 100644 apps/workbench/app/views/request_shell_access_reporter/send_request.text.erb
create mode 100644 apps/workbench/app/views/users/request_shell_access.js
create mode 100644 apps/workbench/public/browser_unsupported.js
create mode 100644 apps/workbench/test/integration/browser_unsupported_test.rb
create mode 100644 apps/workbench/test/support/remove_file_api.js
create mode 100644 sdk/go/arvadostest/run_servers.go
create mode 100644 sdk/python/gittaggers.py
create mode 100644 services/api/db/migrate/20150123142953_full_text_search.rb
create mode 100644 services/api/db/migrate/20150206210804_all_users_can_read_anonymous_group.rb
create mode 120000 services/fuse/gittaggers.py
copy services/{fuse => nodemanager}/MANIFEST.in (100%)
create mode 120000 services/nodemanager/gittaggers.py
via 8ae4b9342772ce4693b5961da5e83f54a36ad0e7 (commit)
via f6b88a9e7e9727f0397f1febbad8c08f6a20463d (commit)
via 857d5a70bcff7d3634d6daf9886fdfe6d46d681c (commit)
via d4b03f2476f0ea6b30baad78672b31938846853e (commit)
via ff3e4c60ffa088479cd0a97b314b4af06b0d67ba (commit)
via fcfc006561b72f76fa4a553aa294a884462e7ca4 (commit)
via cde755bd4d7702c89ebfbd6d2fe8f852509d3786 (commit)
via 54819e3950dfc1adc76159bd4259a70bd531ab2f (commit)
via 5b7ee6cb073457d0cf1f6b7c5ca3ee8965f144ef (commit)
via 31f83b25039677512509120fb385069003bdd4ca (commit)
via 02d728b87a397a7093827de5046e62ce50c4019d (commit)
via 5d3d32d5532d39f1ef85a9e01b9b70cd28cf3579 (commit)
via b17ed3b444abb6c326e69de5ff6a9f8bf019530e (commit)
via e759c71ef7ecbfca3075db4ae94fc7bd0464656e (commit)
via 498a97e6cdb456bf7487f7c62dce08791cb5f453 (commit)
via cd00c7d65d724ea78fe6e59dda333241a7c0775a (commit)
via 26ac1b2f41916d1f4040073a15dfae5f1b294cb5 (commit)
via 1de8e55b47ea46fe1e589fbfe1ff0ae77b9e2cbf (commit)
via b80db28cdd536077e5effe6c08af079532c2059b (commit)
via f16b7abe9b1ae5967ffaab62b9c9ae3f955f44f1 (commit)
via 148ff097b57571dda1b6db063a2eca5a4eb98a35 (commit)
via d3a9326a2c92de950216fb2a88dbbc9de898e4b3 (commit)
via f5a30607d1746b29688363530a3011ee5c2f4f9a (commit)
via 4993b8b44022fd3dc73fcebf20f80d054bdf4370 (commit)
via 6bf9ae122958b25b4a22447f67fb11cf24765d97 (commit)
via 5923d0fa912c73e3725e52c869d72793304ae44a (commit)
via 58bf2ad27c760fb7da0641b239f1871918b84a42 (commit)
via 8676d8d8fe7ea86db75fd9e6f53b07e21437cd6d (commit)
via fb181d9653d80317422e1d979697da908fa804c8 (commit)
via b599ef92fcfc25045eb6a366907555594496bfad (commit)
via 67a4825340187c05cbada61d38c12645a17acb65 (commit)
via 0215bf7b8c61d59462a476d850af999105856177 (commit)
via 34d6dc1f56b59b7c7cc3e6dc7d54053149c49bc6 (commit)
via 045bce46ede1995ed17747c48611f22c478cc82d (commit)
via d9e2de2e142fe1a79bd83064d8d9135ba44fd807 (commit)
via 710b03568da92458279db56608cba84cb5151847 (commit)
via 31e1554c4372d8206618bf7fee48323b08f24ec3 (commit)
via 9b6b5f0bd2ad96deeea2070a4eba56795bb28c1a (commit)
via 07f50aff99bbb837c9419e7a931add36d1611e2d (commit)
via ae7e8221d669b29ff3e098ac9259afb2875e9d3b (commit)
via f6089c82da72f331ba5a44874ce267b18bcaf557 (commit)
via a934fcf84acd4cc3a351fde1b6e21a0bd93757ef (commit)
via fc8e572937f2fd61bdc1e7f34a2e3f9a5cebd7ff (commit)
via d65b683af52e072b3d179b6f32edfbf37e108011 (commit)
via c9f5db97ad5d853cc2f4636d0743037f6048ceeb (commit)
via f7ec673ce72af1e076408f394b6401e4f253e703 (commit)
via b8148b3bcdfc6fe8a8b20e6a4c589b7a50e147a8 (commit)
via c882575c856e01313cf2caf2e4ead1f27bfb33ae (commit)
via 525d5d6351a0610237c52f1564dec5b77cf3af4f (commit)
via 538caa064785b645a2b8f815bf77a30192b20665 (commit)
via 9f1fafa8c7c7f3750d6769d863b82cb826d7ed6e (commit)
via 330a46e91b4ceaefba2bcfc383931eb59c77d461 (commit)
via fefce5e8e133a8fa064bbcdf31d85d41dc4a6729 (commit)
via 204f433a870e2bf1cf7af1fbe076e91f427ef05e (commit)
via 9b61792d905324a98b24224d45347082efbe5205 (commit)
via 16b720950262eb559358cf357f5098a142901665 (commit)
via e73af668c24cd259800c344c3efe8b7d769903da (commit)
via b6a7a62f4f38710f50d08a91a6a9b210700bb011 (commit)
via 0b102fac0e8d2a7d46d088b1bd8f7b27b325dd2a (commit)
via 79aca915815d298d2c20546108284627ee6cb84b (commit)
via b21b81e6623d025da4d93cbf09d523e63d2e07b0 (commit)
via aaffcb23198b4223c48092ccd30ef7152b434187 (commit)
via 20f5b178a850b029ecd501ed49e4ed0a537c1fad (commit)
via fb4921f56d1c13a86add2e59205ec32fa1f6efe4 (commit)
via 1d4a39ab3e97c031683ada9f6c98e4c7365fa414 (commit)
via f6ab9be0046a6f8d760259c1a0eba8ab7c636903 (commit)
via 231242b6378abda494f2c684995519a259cfe174 (commit)
via 1963df31ffb7e95b72e53a0ec5c891f539b6dadb (commit)
via b59b310e23b588c4007af84741d4b94bc9f595f1 (commit)
via e2da84c1fd9052791ed2b684741469570e09ea35 (commit)
via d6ec5672045b29aeaf983a78c5487ae354ccb20d (commit)
via 757212484d9da8bb8d8852bfb6870433d2b4fa97 (commit)
via f85132f1a018179b7127c199932c1f0f3e3f76d5 (commit)
via 2e9f5f1aa841972d1c6d3ff0828d774f60c28307 (commit)
via 71c05eec3e9c8e6f37f14760b04584a8d4c4372c (commit)
via f32690a4a18f85909c0a04de83ecf7819f127df8 (commit)
via df507d6cdebca220ac19dbbc5c16d18498cb852c (commit)
via 4982008e820ed48f362226c61540c18305c6acd6 (commit)
via 77daa60985c94cf4137c8a54681bb89278db8436 (commit)
via bd720586c0152ca4e7d109389bda2c0e463c76bb (commit)
via 2527b9cd7958d89a5ae0dd84856027908c48ae53 (commit)
via dd645c9e973b9b725f310513ce309fa1e1a82421 (commit)
via c9e19eb6c3c6889b55c3b63424b36f1139c9abf0 (commit)
via f8067dd18b72705f3317e85745e87cffc9e25313 (commit)
via cb79358321eff7a49dd4a3fb6e0ea448ead92597 (commit)
via cda964acdb8132d90b881e62db008c574fdd5cc4 (commit)
via 2e5ac62b550f7dd608cf133ae66ef04f801be76b (commit)
via b48e7f0c19f1a7256222c220e938832789492aa3 (commit)
via 7a71d74c538c37437e65f5d22205c224d0fe9207 (commit)
via ff49b1144f5b9f9f7624f3741f5af791073de03c (commit)
via 64c70939c414881de61ac65512701d0ba4068786 (commit)
via d13386351c53a261558052bafd5e2308230cb73b (commit)
via 348801d41f0bd06582675223f07a7ef7f36ac887 (commit)
via afef0760e7281eb7038778a12575dfc32b3162f3 (commit)
via eab43fcf2826f4416a70bef95c3ae04a77b487c9 (commit)
via 9b59cd2f10fa44f4cdbf8986b08e92bdde5a62a7 (commit)
via 1eda4774a59f46296f82231eeb80484aca70a961 (commit)
via 71a556d7d2a9484a4f0bda069f1f7915f548683f (commit)
via f2e686e1c4e117c93d462ec94e315df9e0be02c6 (commit)
via 9f0f926aa45113a50ed1de737b236e9f69f64079 (commit)
via f414dc4c1862c8d471b865dea5c7ac141de6d533 (commit)
via 180498df4db8c43080bb302bc56edea70c940583 (commit)
via 8a8450dcb4dfa4fa222e059091bae03c5e45df3f (commit)
via 7939a927a2b0584210d5e8b2fe73f7625858d6bd (commit)
via 54873fcd103e4887e41e987522e4442b62f682ad (commit)
via 8cd7249b96576285388ef036d04532f72a8f1ee3 (commit)
via c6566ea328710818ab9b65db6187751f1874415c (commit)
via f7cc825b6a7a526a95e9f889ba94f4122f191889 (commit)
via a41baffe4f38019cb5b36875c5e0c838ef9201e5 (commit)
via a94e15cab04a19dcbb02f2e95335e337c8e55036 (commit)
via aa613a590c0b03e90432c25f0190adc99ef4f657 (commit)
via dd72a4de55681da7b5a95ee8b1e659221c48614f (commit)
via 0ea383a2924b37b47f2bc82fbe405fce4a03bd1f (commit)
via 21a824c12633d3775b449ec9b06148546078f1a8 (commit)
via c722a8bba1bd155f3e36ef4402f684e36ea3e5ec (commit)
via e5b8f6826a2034c0c7d7142cff3fb02f64fb8831 (commit)
via 12dee1ebfd6a3fca40c19d751459ba6a071c0a3d (commit)
via 7b9ca2a26b4099e45548d83fc878fe295f2cdc56 (commit)
via 7acac83d2789b36f0e249a3fde9a8d300f15e152 (commit)
via 36d6d1609ef342268cc87fc8bfce51bcf7199929 (commit)
via e20040092b47f79365637dd8e26156d95ab5c6ee (commit)
via a11c56ef66604a9117e3db8c2fa2273c98f88b51 (commit)
via 5b70a11a08dc26b43b3ec4aef178bafe3a801b86 (commit)
via dac304f927000b74d41defcdea26da5d896bfc0c (commit)
via c5fa3f7b2faea363cf73c7e4a2880086c7c9e4f0 (commit)
via 1a78ca155b741a08fcf8e0e284bc4da273f084c2 (commit)
via 826cb14afd19197738ceba11ff382aaaf123a637 (commit)
via 4204a3c2bda7378e8664233d3c2410c5efb95a47 (commit)
via 1147248aa7a30a9a423e7b5b30c6bbb7d4b9bba6 (commit)
via cbf80c08daa5f9099d0821603a128967254709ed (commit)
via 56ad30382d7d4e11cb0160c5f2e30077e1f41c8b (commit)
via b20590222beddb52c8c89294ed3a324c8c7190a2 (commit)
via 13f83b9374e66e4609aff661b467d747067d66c2 (commit)
via 64416e4751edfe6c49c0bed8a7e38071200282d8 (commit)
via dcba3e2c566b2cc3d0575f4b11a1deac808d5cd6 (commit)
via e1523c518fa4bf04ac4c982d0a5dbd681dea279a (commit)
via 9fc99919d72ee495e66ce98584189c651cf994c1 (commit)
via 0c8f599d598f36d67daf0e0e39756ba4d064cbd0 (commit)
via d1957808f6e3ccece499ac2f4048d4ef850b262c (commit)
via 1f8fcb0279a7bb2aa9cf1386ff9516da58216d53 (commit)
via 2cf42c27a7e8b37e29462d0b695e24cb6f3ad5ce (commit)
via bd6f17515de33e6eee9631723730fc65125ebad2 (commit)
via cd3019ce332106ebf80b4323f0f24e71025adcb1 (commit)
via 90af4cd87e9525481bd0bc8120a18ebc27c9a459 (commit)
via 9ae339f1aab32d8473f366c7aaa25633a5a49008 (commit)
via 3d0c0753efb6e2b610d23d16038db218b491a70f (commit)
via 288413d1c5efcf5d207e0556962740a7759891a1 (commit)
via 378c32cf74c1c9d559e3b8e559a9dcdc77ff2017 (commit)
via 4470ba26b332cb92d347af00cdb26c716b1a6953 (commit)
via d01477c59395e6d0895fffe0f60cce6bda9bb083 (commit)
via 5c1cee516b4d41a355fd40a538cf695325e0b712 (commit)
via a1ad1f1d7ce5edf62945340c34ba5e22a36f1157 (commit)
via 13181107ecaeaa92e5d96b05270e56b2d807af39 (commit)
via 48580ba7a6608c89e91afad4b73f2861aafbd7b2 (commit)
via 0d32948a1c78385f9305799c5cb47127492c4320 (commit)
via a2e9008a5fcd7669d5cdd33021212ac754288d19 (commit)
via 0507f73ce83c4a53ce8734f21635928a18d228ad (commit)
via cb6a13cdda9c9b8e2c1dbb151c432ec109e14c82 (commit)
via f9a0cb85f62f1053a9fc1216689b75e6081cf3d7 (commit)
via 655b69e4285476fbd9df4a88ea53e02ea93fa349 (commit)
via 774bc42d958174407d74bc3ef44b2840c2f8a05f (commit)
via bed95fb1306b0708ae61d1977984944430f01bca (commit)
via 7d76a3fbfbe15a6813df5d2d4fa111f1b8e62f9c (commit)
via a533cde870452a69fd7db28806531475aba81486 (commit)
via 981c7264123f24a1873f2692a72f012ac43e726b (commit)
via 9fbf89259b2c4313334a1a2c8f0f30e581cc932e (commit)
via 316eca14d7f7bafb2e0c24b125dee0befe5bdce6 (commit)
via eeebd1e25974beff2455c96100532aaa4dae68fb (commit)
via c430a289af3eeed00e220b5658f9d64191798b1c (commit)
via 6d93d2ecf9c3d75c2b032f0ad9689af6501570e0 (commit)
via 7349451eb29d3bd972f6f051f1cf14459b3fe14e (commit)
via 1b01104f5e4a20a3ede82b6d7250814476e23dc9 (commit)
via 7631b2c9e33a22a9b47fe3396f2f2854745a3ded (commit)
from 8128b451b8e3161ae3db8b7e1858f6eef8105c9f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8ae4b9342772ce4693b5961da5e83f54a36ad0e7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 12 13:41:21 2015 -0500
4823: Revert settings mistakenly added to application.yml.example
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 305ea88..c3e599f 100644
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -32,8 +32,3 @@ common:
# Settings in this section will be used in all environments
# (development, production, test) except when overridden in the
# environment-specific sections above.
- auto_admin_user: peter.amstutz at curoverse.com
- git_repositories_dir: /home/tetron/work/git
- git_internal_dir: /home/tetron/work/internal.git
- secret_token: 5520a2018819190aa4abe42c0807049cf85504a162570592acc9214f4b14914f2e610a40b58e97c57fb5a8e96ec3a00bcef56c1f747766537e6e5022bb3f3764
- blob_signing_key: 5eef1e322b2bfa28f8adb748c6244a66d4eef0cbb5bb45108d7b364e91faf28a73eede086fb0d3670ce2e7822a8d8149af129ab57f9be4fec98bb0be482ee69a
commit f6b88a9e7e9727f0397f1febbad8c08f6a20463d
Merge: 8128b45 857d5a7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 12 13:35:38 2015 -0500
Merge branch 'master' into 4823-python-sdk-writable-collection-api
Conflicts:
sdk/python/arvados/collection.py
sdk/python/tests/test_keep_client.py
diff --cc sdk/python/arvados/collection.py
index 20f5c40,7bfdf78..82708cc
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@@ -268,7 -302,9 +268,7 @@@ class _WriterFile(ArvadosFileBase)
class CollectionWriter(CollectionBase):
- def __init__(self, api_client=None, num_retries=0):
- KEEP_BLOCK_SIZE = 2**26
-
+ def __init__(self, api_client=None, num_retries=0, replication=None):
"""Instantiate a CollectionWriter.
CollectionWriter lets you build a new Arvados Collection from scratch.
@@@ -441,8 -481,10 +445,10 @@@
data_buffer = ''.join(self._data_buffer)
if data_buffer:
self._current_stream_locators.append(
- self._my_keep().put(data_buffer[0:config.KEEP_BLOCK_SIZE]))
+ self._my_keep().put(
- data_buffer[0:self.KEEP_BLOCK_SIZE],
++ data_buffer[0:config.KEEP_BLOCK_SIZE],
+ copies=self.replication))
- self._data_buffer = [data_buffer[self.KEEP_BLOCK_SIZE:]]
+ self._data_buffer = [data_buffer[config.KEEP_BLOCK_SIZE:]]
self._data_buffer_len = len(self._data_buffer[0])
def start_new_file(self, newfilename=None):
diff --cc sdk/python/tests/arvados_testutil.py
index 474c068,124fb35..aa0da98
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@@ -76,11 -64,43 +77,45 @@@ class MockStreamReader(object)
return self._name
def readfrom(self, start, size, num_retries=None):
- return self._data[start:start + size]
+ self._readfrom(start, size, num_retries=num_retries)
+ def _readfrom(self, start, size, num_retries=None):
+ return self._data[start:start + size]
+ class ApiClientMock(object):
+ def api_client_mock(self):
+ return mock.MagicMock(name='api_client_mock')
+
+ def mock_keep_services(self, api_mock=None, status=200, count=12,
+ service_type='disk',
+ service_host=None,
+ service_port=None,
+ service_ssl_flag=False):
+ if api_mock is None:
+ api_mock = self.api_client_mock()
+ body = {
+ 'items_available': count,
+ 'items': [{
+ 'uuid': 'zzzzz-bi6l4-{:015x}'.format(i),
+ 'owner_uuid': 'zzzzz-tpzed-000000000000000',
+ 'service_host': service_host or 'keep0x{:x}'.format(i),
+ 'service_port': service_port or 65535-i,
+ 'service_ssl_flag': service_ssl_flag,
+ 'service_type': service_type,
+ } for i in range(0, count)]
+ }
+ self._mock_api_call(api_mock.keep_services().accessible, status, body)
+ return api_mock
+
+ def _mock_api_call(self, mock_method, code, body):
+ mock_method = mock_method().execute
+ if code == 200:
+ mock_method.return_value = body
+ else:
+ mock_method.side_effect = arvados.errors.ApiError(
+ fake_httplib2_response(code), "{}")
+
+
class ArvadosBaseTestCase(unittest.TestCase):
# This class provides common utility functions for our tests.
diff --cc sdk/python/tests/test_keep_client.py
index 885ae7c,6d4d3cd..1baf135
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@@ -277,58 -257,58 +257,58 @@@ class KeepClientServiceTestCase(unittes
# when connected directly to a Keep server (i.e. non-proxy timeout)
def test_get_timeout(self):
- api_client = self.mock_keep_services(('keep', 10, False, 'disk'))
+ api_client = self.mock_keep_services(count=1)
- keep_client = arvados.KeepClient(api_client=api_client)
force_timeout = [socket.timeout("timed out")]
- with mock.patch('requests.get', side_effect=force_timeout) as mock_request:
+ with tutil.mock_get(force_timeout) as mock_session:
+ keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
- self.assertTrue(mock_request.called)
+ self.assertTrue(mock_session.return_value.get.called)
self.assertEqual(
arvados.KeepClient.DEFAULT_TIMEOUT,
- mock_request.call_args[1]['timeout'])
+ mock_session.return_value.get.call_args[1]['timeout'])
def test_put_timeout(self):
- api_client = self.mock_keep_services(('keep', 10, False, 'disk'))
+ api_client = self.mock_keep_services(count=1)
- keep_client = arvados.KeepClient(api_client=api_client)
force_timeout = [socket.timeout("timed out")]
- with mock.patch('requests.put', side_effect=force_timeout) as mock_request:
+ with tutil.mock_put(force_timeout) as mock_session:
+ keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepWriteError):
keep_client.put('foo')
- self.assertTrue(mock_request.called)
+ self.assertTrue(mock_session.return_value.put.called)
self.assertEqual(
arvados.KeepClient.DEFAULT_TIMEOUT,
- mock_request.call_args[1]['timeout'])
+ mock_session.return_value.put.call_args[1]['timeout'])
def test_proxy_get_timeout(self):
# Force a timeout, verifying that the requests.get or
# requests.put method was called with the proxy_timeout
# setting rather than the default timeout.
- api_client = self.mock_keep_services(('keep', 10, False, 'proxy'))
+ api_client = self.mock_keep_services(service_type='proxy', count=1)
- keep_client = arvados.KeepClient(api_client=api_client)
force_timeout = [socket.timeout("timed out")]
- with mock.patch('requests.get', side_effect=force_timeout) as mock_request:
+ with tutil.mock_get(force_timeout) as mock_session:
+ keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
- self.assertTrue(mock_request.called)
+ self.assertTrue(mock_session.return_value.get.called)
self.assertEqual(
arvados.KeepClient.DEFAULT_PROXY_TIMEOUT,
- mock_request.call_args[1]['timeout'])
+ mock_session.return_value.get.call_args[1]['timeout'])
def test_proxy_put_timeout(self):
# Force a timeout, verifying that the requests.get or
# requests.put method was called with the proxy_timeout
# setting rather than the default timeout.
- api_client = self.mock_keep_services(('keep', 10, False, 'proxy'))
+ api_client = self.mock_keep_services(service_type='proxy', count=1)
- keep_client = arvados.KeepClient(api_client=api_client)
force_timeout = [socket.timeout("timed out")]
- with mock.patch('requests.put', side_effect=force_timeout) as mock_request:
+ with tutil.mock_put(force_timeout) as mock_session:
+ keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepWriteError):
keep_client.put('foo')
- self.assertTrue(mock_request.called)
+ self.assertTrue(mock_session.return_value.put.called)
self.assertEqual(
arvados.KeepClient.DEFAULT_PROXY_TIMEOUT,
- mock_request.call_args[1]['timeout'])
+ mock_session.return_value.put.call_args[1]['timeout'])
def test_probe_order_reference_set(self):
# expected_order[i] is the probe order for
@@@ -431,7 -413,8 +413,7 @@@
self.check_no_services_error('put', arvados.errors.KeepWriteError)
def check_errors_from_last_retry(self, verb, exc_class):
- api_client = self.mock_n_keep_disks(2)
+ api_client = self.mock_keep_services(count=2)
- keep_client = arvados.KeepClient(api_client=api_client)
req_mock = getattr(tutil, 'mock_{}_responses'.format(verb))(
"retry error reporting test", 500, 500, 403, 403)
with req_mock, tutil.skip_sleep, \
@@@ -452,10 -434,10 +434,10 @@@
def test_put_error_does_not_include_successful_puts(self):
data = 'partial failure test'
data_loc = '{}+{}'.format(hashlib.md5(data).hexdigest(), len(data))
- api_client = self.mock_n_keep_disks(3)
+ api_client = self.mock_keep_services(count=3)
- keep_client = arvados.KeepClient(api_client=api_client)
with tutil.mock_put_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
+ keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.service_errors()))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list