[ARVADOS] updated: 682dd5b6cc23a455766a7651e3e841257660b31c
git at public.curoverse.com
git at public.curoverse.com
Tue May 12 13:05:43 EDT 2015
Summary of changes:
apps/workbench/Gemfile | 5 +-
apps/workbench/Gemfile.lock | 9 +-
apps/workbench/app/assets/images/mouse-move.gif | Bin 26082 -> 26082 bytes
.../app/assets/javascripts/infinite_scroll.js | 34 +++
.../app/assets/javascripts/selection.js.erb | 8 +
.../app/assets/stylesheets/application.css.scss | 8 +
.../app/controllers/collections_controller.rb | 4 +-
apps/workbench/app/controllers/jobs_controller.rb | 2 -
.../controllers/pipeline_instances_controller.rb | 4 +-
.../app/controllers/projects_controller.rb | 4 +-
.../app/controllers/repositories_controller.rb | 16 ++
apps/workbench/app/helpers/application_helper.rb | 26 +-
apps/workbench/app/helpers/jobs_helper.rb | 22 --
apps/workbench/app/models/pipeline_instance.rb | 52 ++++
apps/workbench/app/models/repository.rb | 99 +++++++
.../app/views/application/_choose.html.erb | 4 +
.../app/views/application/_content.html.erb | 12 +-
.../app/views/application/_show_sharing.html.erb | 1 +
.../app/views/collections/_show_files.html.erb | 16 +-
.../app/views/collections/hash_matches.html.erb | 8 +-
.../app/views/layouts/application.html.erb | 6 +-
apps/workbench/app/views/layouts/body.html.erb | 11 +-
.../pipeline_instances/_running_component.html.erb | 15 +
.../views/pipeline_instances/_show_log.html.erb | 56 +++-
.../projects/_index_jobs_and_pipelines.html.erb | 2 +-
.../views/projects/_show_contents_rows.html.erb | 2 +-
.../app/views/projects/_show_description.html.erb | 2 +-
.../app/views/projects/_show_tab_contents.html.erb | 6 +-
.../repositories/_repository_breadcrumbs.html.erb | 13 +
.../app/views/repositories/show_blob.html.erb | 13 +
.../app/views/repositories/show_commit.html.erb | 3 +
.../app/views/repositories/show_tree.html.erb | 40 +++
.../app/views/users/_add_ssh_key_popup.html.erb | 2 +-
apps/workbench/config/application.default.yml | 12 +-
apps/workbench/config/database.yml | 40 +--
apps/workbench/config/routes.rb | 5 +
.../test/controllers/projects_controller_test.rb | 27 ++
.../controllers/repositories_controller_test.rb | 59 ++++
.../test/helpers/repository_stub_helper.rb | 33 +++
apps/workbench/test/helpers/share_object_helper.rb | 1 +
.../test/integration/anonymous_access_test.rb | 48 ++--
.../test/integration/application_layout_test.rb | 4 +-
.../workbench/test/integration/collections_test.rb | 31 +-
apps/workbench/test/integration/errors_test.rb | 4 +-
.../test/integration/integration_test_utils.rb | 12 +
apps/workbench/test/integration/jobs_test.rb | 2 +-
.../test/integration/pipeline_instances_test.rb | 41 +++
apps/workbench/test/integration/projects_test.rb | 108 +++++++
.../test/integration/repositories_browse_test.rb | 53 ++++
.../test/integration/user_manage_account_test.rb | 6 +-
.../test/integration/user_profile_test.rb | 4 +-
apps/workbench/test/test_helper.rb | 24 +-
apps/workbench/test/unit/pipeline_instance_test.rb | 66 ++++-
crunch_scripts/crunchutil/vwd.py | 81 ++++--
crunch_scripts/run-command | 30 +-
doc/api/schema/Repository.html.textile.liquid | 6 +-
doc/index.html.liquid | 19 +-
.../create-standard-objects.html.textile.liquid | 2 +-
doc/install/install-api-server.html.textile.liquid | 2 +-
doc/install/install-keepstore.html.textile.liquid | 35 ++-
doc/install/install-sso.html.textile.liquid | 6 +-
.../install-workbench-app.html.textile.liquid | 4 +-
doc/user/topics/run-command.html.textile.liquid | 8 +-
docker/api/application.yml.in | 6 +-
docker/sso/Dockerfile | 1 +
docker/sso/application.yml.in | 26 ++
sdk/go/arvadosclient/arvadosclient.go | 64 +++--
sdk/go/arvadosclient/arvadosclient_test.go | 14 +-
sdk/python/arvados/arvfile.py | 4 +-
sdk/python/arvados/commands/arv_copy.py | 10 +-
sdk/python/arvados/errors.py | 9 +-
sdk/python/arvados/keep.py | 317 +++++++++++++++------
sdk/python/arvados/retry.py | 19 +-
sdk/python/setup.py | 33 +--
sdk/python/tests/arvados_testutil.py | 106 ++++---
sdk/python/tests/keepstub.py | 104 +++++++
sdk/python/tests/nginx.conf | 31 ++
sdk/python/tests/run_test_server.py | 140 ++++++++-
sdk/python/tests/test_arvfile.py | 19 +-
sdk/python/tests/test_collections.py | 23 +-
sdk/python/tests/test_errors.py | 4 +-
sdk/python/tests/test_keep_client.py | 221 ++++++++++----
sdk/python/tests/test_retry.py | 5 +-
sdk/python/tests/test_stream.py | 16 +-
sdk/ruby/lib/arvados/keep.rb | 11 +-
sdk/ruby/test/sdk_fixtures.rb | 5 +
sdk/ruby/test/test_keep_manifest.rb | 37 +++
services/api/Rakefile | 21 ++
.../api/app/controllers/application_controller.rb | 37 ++-
.../arvados/v1/collections_controller.rb | 4 +
.../app/controllers/arvados/v1/jobs_controller.rb | 27 +-
.../controllers/arvados/v1/schema_controller.rb | 2 +-
services/api/app/models/arvados_model.rb | 7 +
services/api/app/models/collection.rb | 4 +-
services/api/app/models/job.rb | 6 +-
services/api/app/models/node.rb | 7 +-
services/api/app/models/repository.rb | 39 ++-
services/api/config/application.default.yml | 70 ++++-
...5759_no_filenames_in_collection_search_index.rb | 11 +
services/api/db/structure.sql | 10 +-
services/api/lib/tasks/delete_old_job_logs.rake | 17 ++
services/api/lib/tasks/test_tasks.rake | 6 +
services/api/test/fixtures/groups.yml | 34 +++
services/api/test/fixtures/jobs.yml | 59 +++-
services/api/test/fixtures/links.yml | 30 ++
services/api/test/fixtures/logs.yml | 38 +++
services/api/test/fixtures/pipeline_instances.yml | 42 ++-
services/api/test/fixtures/pipeline_templates.yml | 40 +++
services/api/test/fixtures/repositories.yml | 7 +
.../arvados/v1/collections_controller_test.rb | 39 +++
.../arvados/v1/job_reuse_controller_test.rb | 108 +++++--
.../arvados/v1/repositories_controller_test.rb | 41 ++-
services/api/test/helpers/git_test_helper.rb | 10 +-
.../api/test/tasks/delete_old_job_logs_test.rb | 50 ++++
services/api/test/test.git.tar | Bin 194560 -> 256000 bytes
services/api/test/unit/arvados_model_test.rb | 2 +-
services/api/test/unit/collection_test.rb | 7 +
services/api/test/unit/commit_test.rb | 60 ++++
services/api/test/unit/job_test.rb | 30 ++
services/api/test/unit/repository_test.rb | 4 +
services/arv-git-httpd/auth_handler.go | 6 +-
services/arv-git-httpd/basic_auth_test.go | 12 +-
services/arv-git-httpd/server_test.go | 67 +++--
services/crunchstat/.gitignore | 1 +
services/crunchstat/crunchstat.go | 26 +-
services/fuse/arvados_fuse/__init__.py | 7 +-
services/fuse/arvados_fuse/fusedir.py | 3 +
services/fuse/tests/test_inodes.py | 71 ++++-
services/keepproxy/.gitignore | 1 +
services/keepproxy/keepproxy.go | 5 +
services/keepproxy/keepproxy_test.go | 16 ++
services/keepstore/.gitignore | 1 +
services/keepstore/bufferpool.go | 44 +++
services/keepstore/bufferpool_test.go | 85 ++++++
services/keepstore/handler_test.go | 26 +-
services/keepstore/handlers.go | 127 ++++-----
services/keepstore/keepstore.go | 119 +++++---
services/keepstore/keepstore_test.go | 10 +-
services/keepstore/logging_router.go | 8 +-
services/keepstore/perms_test.go | 10 +-
services/keepstore/pull_worker_integration_test.go | 19 +-
services/keepstore/pull_worker_test.go | 17 +-
services/keepstore/trash_worker.go | 34 ++-
services/keepstore/trash_worker_test.go | 51 ++--
services/keepstore/volume.go | 8 +-
services/keepstore/volume_test.go | 34 ++-
services/keepstore/volume_unix.go | 219 ++++++--------
services/keepstore/volume_unix_test.go | 15 +-
services/nodemanager/arvnodeman/daemon.py | 8 +-
services/nodemanager/tests/test_daemon.py | 37 +++
150 files changed, 3429 insertions(+), 1030 deletions(-)
delete mode 100644 apps/workbench/app/helpers/jobs_helper.rb
create mode 100644 apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
create mode 100644 apps/workbench/app/views/repositories/show_blob.html.erb
create mode 100644 apps/workbench/app/views/repositories/show_commit.html.erb
create mode 100644 apps/workbench/app/views/repositories/show_tree.html.erb
create mode 100644 apps/workbench/test/helpers/repository_stub_helper.rb
create mode 100644 apps/workbench/test/integration/integration_test_utils.rb
create mode 100644 apps/workbench/test/integration/repositories_browse_test.rb
create mode 100644 docker/sso/application.yml.in
create mode 100644 sdk/python/tests/keepstub.py
create mode 100644 sdk/python/tests/nginx.conf
create mode 100644 services/api/db/migrate/20150423145759_no_filenames_in_collection_search_index.rb
create mode 100644 services/api/lib/tasks/delete_old_job_logs.rake
create mode 100644 services/api/lib/tasks/test_tasks.rake
create mode 100644 services/api/test/tasks/delete_old_job_logs_test.rb
create mode 100644 services/crunchstat/.gitignore
create mode 100644 services/keepproxy/.gitignore
create mode 100644 services/keepstore/.gitignore
create mode 100644 services/keepstore/bufferpool.go
create mode 100644 services/keepstore/bufferpool_test.go
via 682dd5b6cc23a455766a7651e3e841257660b31c (commit)
via dd5deb94678d0a0054cf1c4d7dafe864c86094fd (commit)
via ea8c290061e4ecc80d5d113debe6681090cf6594 (commit)
via 1cea19230b490876f99194ea4d84970d6330ae12 (commit)
via c59093180fd92f0d7c6607a49458446212ebd058 (commit)
via 67f0d86c20139eee996816d44ef75fa52288c515 (commit)
via 184dabe069040d4ab210407de695563400da781e (commit)
via fdd2d805b2edff70e3cb14b4a058183deb69bfa4 (commit)
via 559d26fd819f477fed75201fa952fed19de77f7e (commit)
via a06f90e1dcaa2c4ce2613e066e7c1348ec129ba8 (commit)
via dc0237c289882a058a0d5c91b720a4b37917bc9d (commit)
via 340d6a03a20592e3ba1c69340dbdf70070e43e82 (commit)
via 7ea61e9c890706e1ac020143072ea0a6e17894ab (commit)
via 852325b0c1523a37d780e7e5e0a76c80733eea66 (commit)
via a7af112f0554002ec1ce07312c39e480fc7437f7 (commit)
via 4e26efd8ee2d1896e329dc32488346341557a15b (commit)
via b25f82e548caf7f731f7c4c506b757a6a1a531f0 (commit)
via 14a27997ba2a94c5a7250ddde4519d5f68b6eda0 (commit)
via 31d8ece649c63b008ed79a930a5b237ec795ff22 (commit)
via 81d100b157dee4f24d376cdd8b5d739b385c925e (commit)
via 6c63fcb00c4b15590ec1f2299aeb670a08ae1e52 (commit)
via b8467da4522299faea997eadf2d2171c79c12b27 (commit)
via 58997b4550660886fa4b402a4079267607380eda (commit)
via dc00995b1843cd1decebffe1493e55a16c3c0df9 (commit)
via f820c0ee9a10931fc80c5025a5c92bbd1395cf08 (commit)
via fde21d34d011af2123668983c559632221390fd4 (commit)
via e6c90436e715fd6da88782cbe3f72bd6c2443036 (commit)
via cc303a0ef797c1c752b4fd86e48e2c84fc7d96ca (commit)
via 200f7004f921a68ec40b407dfe31f1db95e98fb9 (commit)
via 02e1074d6020151e9854027df0ca218acb12fa29 (commit)
via b6f54ccb32e05348e9a5c76c7ed3211203cd4e55 (commit)
via d0f98546e68f6f7a714367d0bdbcc468ce217f3e (commit)
via e7a2db3fe3446929a481f20e1358337ada500c72 (commit)
via b9024e06795906fa74925065cef81d094af95fc0 (commit)
via 6bd5a4e01a768df4078ddf77862751ea86435e7b (commit)
via 88b0204247e448ba5f715cb01adba4597bb6aae7 (commit)
via e81a3e9b46d0c7500079df3959e07074068ceb1b (commit)
via 42b7aeb02200285fe3390eefa48be28baee3ca58 (commit)
via e7e61999bf577b3ba5308c747dda4a0011516aa7 (commit)
via b988327f1fe531cc0e1f1cde73d2b18c05850548 (commit)
via 0358074100610abe13cac98b82cbeca4326af986 (commit)
via ae6b514a98d89ce0bdfd47edd4508c42761cb049 (commit)
via 3e48f1c6d32cf1694af48ae2d6f182cfe5a6c73b (commit)
via 4e2319cec48ef5f6fec5789a127ce26fe360d098 (commit)
via cb41817568900d52d3768a91f703f04696f6b702 (commit)
via 8ac0683e65e010d51dfa91edc77000a4f4616646 (commit)
via 6e10831be039fb540fc252d35e0ffe673ea8d398 (commit)
via fe7d9a6cd96c7fadf869482e3bc99b2c18980248 (commit)
via bd2023d19080f6f1b88a3ca94b7d7efc49ac0ef0 (commit)
via 7d12bf89586e96ba827c75fa4bee86cce0fc20cb (commit)
via cb55d2c0309b899c02f4e8dfc161dc7a60bfa9ef (commit)
via 9d4bc458b767e4c05024dfe02207283745e1ba06 (commit)
via e146ad301ac42da3bf5b2d1f0f64ec2b1cd1a5c7 (commit)
via 71ebe5f5239249e9739f570260d9b6aa2d2134ac (commit)
via 73c9a44b85ba72e881c5133422fcabe3f1322142 (commit)
via 83164dbf6265f4d676324b2527ac558d1660663a (commit)
via 47f4f1b26cdebb4ea9b52ac36e6d14f634531ebc (commit)
via ffadd5d842dc27fdc34af809c51a9982f3c4b4fa (commit)
via b02d96279c226836fe73f042a84e15c85bcd744a (commit)
via eb3303340c51d13512397c3d83c955af9f516f47 (commit)
via eb03fb3ad316a637f032f911af7d69950482ead3 (commit)
via 066efa83c8b35216481ef39696d7bb4f5c64bef0 (commit)
via fc16c2ebbb92d9b0e52b6435b717f26b9680c7e6 (commit)
via 6fcd07063276bb57a751fa21bd6ab416b4092b3c (commit)
via 33b41435dd88f58f5bc115f19e84936fef405b91 (commit)
via 49a76facc1c9b64c0e80551942fa76bc8386c2f0 (commit)
via 32254a99680b3dbcff053d7c99a26988e1f839eb (commit)
via 8b689efa29d0db06e0a7c402c5d31107089bba5b (commit)
via 5b970a6c9505527e146e73cb4756a64ecc1679cd (commit)
via e8f91c87ef238ddf6bd96914a196d2d90825e8ee (commit)
via 3f51fa4899fe43c29e1bf49d7911a40eb41a55e8 (commit)
via 13e4bacf309e86428e46385ed94e55915cdc4af2 (commit)
via 0a87aad48d7fccfc4d7d56a8628370cb7370d792 (commit)
via 91c467665ad493108b69c660d5424c77cce5668f (commit)
via 6a7271ad8b9c145c6fb6bbc9e0914f32bcaa225d (commit)
via f6fcf9f51d326fa0a42e9ebdd1343a6a6ef6d8e8 (commit)
via 1281ecab8f2396739ee9232c36796e46cd551426 (commit)
via ee8d5b171e2c30ddc3fe829ba4388778e6a78b27 (commit)
via f81a5a1adb403f9c81159239e92880843dca3c93 (commit)
via da57a99777f022ae6f6a4122c7940d00339f5503 (commit)
via 5bf77ec1bbbd91ee2f1d063d963850c600251ca7 (commit)
via 6fc44a67b911cead1513e6ddb517f56dd509663b (commit)
via bde53a1d5c1e55c42f9c38c5426b9a8a0acb8219 (commit)
via db195a5479485501eb59ce5e914eddc64f72a9f5 (commit)
via 6083e43206e7bfca226ab706a2968899f2eddb07 (commit)
via 11253affeb95b9e15528fce5117ff0ffc83c238d (commit)
via 2826ad75d3ded03fae35522a5d75d45c223d27c1 (commit)
via 7990a7778f70c243b50ea878787ea83689f5b07e (commit)
via dffb8b5eccb3af901512effe562a8b8944b14b89 (commit)
via c42ccd1f88e75b7910623581fbdf8d88c42b55dd (commit)
via 02676af9f5a2580710e9ae062e7028f661740c3a (commit)
via e94da792a8fc562bc35e63b6ff5b2f87822d2755 (commit)
via 4570521e9d639806b1f25c93822d5cf0dbfe42d0 (commit)
via f94d59e86fe1e65c18cf905fa9ea1fcabfd1b1b5 (commit)
via 3da2281b33aa42a6885ee12d930843c3d43a1ffb (commit)
via ec359b66ac1619511757ee78e020536f2ae13581 (commit)
via 624c5c8d13d6c2e21a80379928387944bedae2a3 (commit)
via a4211acb465bd42869bf2a2f9fad6ff2c5e518e0 (commit)
via 6c9dde5e6089efc8a87022ab9771da96932516a7 (commit)
via 8af4c5ffe2d0bf7f009f8821945aa100ddf02279 (commit)
via acc7d4a40ae78e866c983624ec36b7bae3b918a9 (commit)
via ea7c7efb7d2d7429677f6d95eb92128d7574d6a5 (commit)
via 213ace8be1b2279e89abcaf58de004eb8b58632c (commit)
via e24940267012211db40ca0adcedfc33650314b70 (commit)
via 61d399a57fd1e867cc243a0dab0fad73d9689734 (commit)
via e01762a0adcdee209f305cfc2ae902ab4eb94899 (commit)
via 20b988d5b901f459f95043c3702f1f9b104f3fbf (commit)
via 7ab494e42b35b3769a326a16b7e90f1d20147ced (commit)
via 0191b58223f99476cc8553f3ad7c726a4d8b37de (commit)
via de5f63b3abc80783ebe886078c5e4bcfa19ed1da (commit)
via 48c6eca6aa6af517f67d39545b714946676cd2b5 (commit)
via ec0c9f0d12560296d4b678c8ad33e1b82221e626 (commit)
via 19db486cda40c892afb1111bb3d35d482ab1fbb2 (commit)
via b6c7ee612c09a110b6e2a87a12eebc5b81a7a2e8 (commit)
via 040fdf6da00318b5b8e8e83eceeec783680ecb1c (commit)
via 8464bb48ba769c8bf0984c8b1493ca43fd902891 (commit)
via 5b1602a521d3f6689f5988cdc2b25b74ff9fc1e1 (commit)
via 03d89302f14a4a1c448b32d62679bab14e23fa23 (commit)
via 68dfd41d1d5038f7e0a139779ef6ce1c4a7a8351 (commit)
via 5b0d540ad315ad390831ad58f52b02e86686524f (commit)
via 4b17e78749b44772e898619e11aef663810ac6ed (commit)
via d054beda9bc86bc6cd33a127f4d9e40d04f1620d (commit)
via 9bfd0d772abcd643e92fed127112e0ed91c00e65 (commit)
via 11a007fd7bbc3b35b6efd62c84d9ca5a8f45aecb (commit)
via 97403e08475b328115373a2c6a23e82116199aad (commit)
via 45b556d1a204f694b69ca26d115cd6786ca585f3 (commit)
via 123ef960b82540b98c8b68c47ff89bf85da3899c (commit)
via 81fb9f6e13d6e48e23165319ff8f2f9cab4deb3f (commit)
via b1db376a4532ba3484929b3214e59056877d1695 (commit)
via 816683553b8353f18c92f32a6ee9ea1868630067 (commit)
via 41068df20f2a9fe9dbb3e29e55af0c82832359ad (commit)
via 749b87143ebb0bdcbe2d49deee9c66f6de9f86dd (commit)
via 9e0100cee5825584ff7ed6342bf661c026a8bc1d (commit)
via f095b4447de612fd1cd7bee5f9121df76cd65434 (commit)
via 559fdf14486527af646dfab51f2c8b87af8e2eae (commit)
via 2b39eafad562600f7d7beb83f0f7f241a1c47bfc (commit)
via 97ca7222ef893c4914d6425479f05fe32b9df8eb (commit)
via f6f6ed672a240c9b91e78591b4f67a5ed4250758 (commit)
via 1765cd327b15151c113890860dd8818c6fc44962 (commit)
via 2054cdb05d79a3c45e8346661adc81062c383b16 (commit)
via d65e411a33c398c658c8ca4299f64b546511f78b (commit)
via 698b9c0ebf2f12c72718ebde13c63e904fff3332 (commit)
via e5aebef9b94456ded73ccf2ec7378e7ed2f3fb35 (commit)
via 66e68045ed5b7c4ebca49d81e58ef6f821e8bb89 (commit)
via b05e1bb447b7349d99ece17cd7876c08e2ade0ab (commit)
via 9816d2cf5f88d19e4e492c1e965874e5a5b3055c (commit)
via 705174089c9c4f98a721f55187474f7b54dc6d87 (commit)
via da705110826943b04151f959e059b5f4ffcc4c5e (commit)
via 8a42d33f69d5e0889d9f390acb4947c20edff33e (commit)
via 2b42046dcdfff733803e18af140706c9e0adadb6 (commit)
via 983e7c81e1a185eda8f5c71ab78c4b207116edf3 (commit)
via 94b57902775c4aaa02cb7532f07415729ef353df (commit)
via 08717a74dbc29916e4466119d52863b095e271a4 (commit)
via de67953481cbedc480822f97cdfe5eb6dffcf0d3 (commit)
via e24125041ad492b45c97feffb33a037c5adda734 (commit)
via 3c36af3d62af5e4ee98aaafab80858182fb5cab8 (commit)
via 86860e7d65589bc9d93df5b514baee3fc5a5103a (commit)
via 9136a1b1314084e149f86ceec16d1482ccf5d8af (commit)
via 4f99b52f6866deaeb4b614e165b7af0cb3a6adba (commit)
via 3aaf20587881b6640cc27220ad10a23294e61851 (commit)
via 17e59fe02aa59b9fbf3b852d2521714782a6d327 (commit)
via e1eabe76d75d783655c5e3513b9c6e0ffbfd9b52 (commit)
via 910a5f7a8b4c1286beaae0c7c8c45ec092aec28a (commit)
from d2f68bd1e108c3f2dda2322c427050d019b17e04 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 682dd5b6cc23a455766a7651e3e841257660b31c
Merge: d2f68bd dd5deb9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue May 12 13:05:38 2015 -0400
Merge branch 'master' into 3198-writable-fuse
Conflicts:
sdk/python/arvados/arvfile.py
sdk/python/arvados/keep.py
services/fuse/arvados_fuse/__init__.py
diff --cc sdk/python/arvados/arvfile.py
index c0ef581,2e0eaa8..c086e5f
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@@ -734,10 -713,11 +734,10 @@@ class ArvadosFile(object)
self._segments = new_segs
self._modified = True
elif size > self.size():
- raise IOError("truncate() does not support extending the file size")
-
+ raise IOError(errno.EINVAL, "truncate() does not support extending the file size")
def readfrom(self, offset, size, num_retries, exact=False):
- """Read upto `size` bytes from the file starting at `offset`.
+ """Read up to `size` bytes from the file starting at `offset`.
:exact:
If False (default), return less data than requested if the read
@@@ -909,24 -873,11 +909,24 @@@ class ArvadosFileReader(ArvadosFileRead
@_FileLikeObjectBase._before_close
@retry_method
- def read(self, size, num_retries=None):
- """Read up to `size` bytes from the stream, starting at the current file position."""
- data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
- self._filepos += len(data)
- return data
+ def read(self, size=None, num_retries=None):
+ """Read up to `size` bytes from the file and return the result.
+
+ Starts at the current file position. If `size` is None, read the
+ entire remainder of the file.
+ """
+ if size is None:
+ data = []
+ rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+ while rd:
+ data.append(rd)
+ self._filepos += len(rd)
+ rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+ return ''.join(data)
+ else:
- data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
++ data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
+ self._filepos += len(data)
+ return data
@_FileLikeObjectBase._before_close
@retry_method
diff --cc sdk/python/tests/test_arvfile.py
index f89ac72,1701aa4..c5dbb16
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@@ -437,23 -437,28 +437,28 @@@ class ArvadosFileReaderTestCase(StreamF
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
- af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
- return ArvadosFileReader(af, "count.txt")
+ af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
+ return ArvadosFileReader(af)
- def test_read_returns_first_block(self):
- # read() calls will be aligned on block boundaries - see #3663.
+ def test_read_block_crossing_behavior(self):
+ # read() needs to return all the data requested if possible, even if it
+ # crosses uncached blocks: https://arvados.org/issues/5856
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('123', sfile.read(10))
+ self.assertEqual('12345678', sfile.read(8))
def test_successive_reads(self):
+ # Override StreamFileReaderTestCase.test_successive_reads
sfile = self.make_count_reader(nocache=True)
- for expect in ['123', '456', '789', '']:
- self.assertEqual(expect, sfile.read(10))
+ self.assertEqual('1234', sfile.read(4))
+ self.assertEqual('5678', sfile.read(4))
+ self.assertEqual('9', sfile.read(4))
+ self.assertEqual('', sfile.read(4))
def test_tell_after_block_read(self):
+ # Override StreamFileReaderTestCase.test_tell_after_block_read
sfile = self.make_count_reader(nocache=True)
- sfile.read(5)
- self.assertEqual(3, sfile.tell())
+ self.assertEqual('12345678', sfile.read(8))
+ self.assertEqual(8, sfile.tell())
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
diff --cc services/fuse/arvados_fuse/__init__.py
index 1b44eaf,8d04848..b85e65d
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@@ -198,12 -156,11 +198,11 @@@ class Operations(llfuse.Operations)
"""
- def __init__(self, uid, gid,
- encoding="utf-8",
- inode_cache=InodeCache(cap=256*1024*1024),
- num_retries=4):
- def __init__(self, uid, gid, encoding="utf-8", inode_cache=None):
++ def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4):
super(Operations, self).__init__()
+ if not inode_cache:
+ inode_cache = InodeCache(cap=256*1024*1024)
self.inodes = Inodes(inode_cache)
self.uid = uid
self.gid = gid
diff --cc services/fuse/tests/test_inodes.py
index 2c5f3dc,61a365f..cce5b95
--- a/services/fuse/tests/test_inodes.py
+++ b/services/fuse/tests/test_inodes.py
@@@ -79,9 -120,26 +120,27 @@@ class InodeTests(unittest.TestCase)
self.assertTrue(ent3.clear.called)
self.assertEqual(500, cache.total())
+ def test_delete(self):
+ cache = arvados_fuse.InodeCache(1000, 4)
+ inodes = arvados_fuse.Inodes(cache)
+
+ ent1 = mock.MagicMock()
+ ent1.in_use.return_value = False
+ ent1.persisted.return_value = True
+ ent1.clear.return_value = True
+ ent1.objsize.return_value = 500
+ inodes.add_entry(ent1)
+
+ ent3 = mock.MagicMock()
+ ent3.in_use.return_value = False
+ ent3.persisted.return_value = True
+ ent3.objsize.return_value = 600
+ ent3.clear.return_value = True
+
# Delete ent1
+ self.assertEqual(500, cache.total())
ent1.clear.return_value = True
+ ent1.ref_count = 0
inodes.del_entry(ent1)
self.assertEqual(0, cache.total())
cache.touch(ent3)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list