[ARVADOS] updated: f3250432a47c835f4c594348b0d4904a247c3365
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 17 16:21:08 EST 2015
Summary of changes:
.gitignore | 7 +-
COPYING | 2 +-
apps/workbench/.gitignore | 1 +
apps/workbench/Gemfile | 24 +-
apps/workbench/Gemfile.lock | 126 ++++---
.../workbench/app/assets/javascripts/ajax_error.js | 15 +
apps/workbench/app/assets/javascripts/event_log.js | 312 ----------------
.../workbench/app/assets/javascripts/filterable.js | 6 +-
.../javascripts/{event_log.js => job_log_graph.js} | 79 +----
.../app/assets/javascripts/request_shell_access.js | 10 +
.../app/assets/javascripts/selection.js.erb | 50 ++-
.../app/controllers/application_controller.rb | 83 +++--
.../app/controllers/collections_controller.rb | 37 +-
apps/workbench/app/controllers/jobs_controller.rb | 9 +-
.../controllers/pipeline_instances_controller.rb | 5 +
.../controllers/pipeline_templates_controller.rb | 5 +
.../app/controllers/projects_controller.rb | 83 ++---
.../app/controllers/repositories_controller.rb | 14 +
.../workbench/app/controllers/search_controller.rb | 3 +-
apps/workbench/app/controllers/users_controller.rb | 22 +-
apps/workbench/app/helpers/application_helper.rb | 12 +-
.../app/mailers/request_shell_access_reporter.rb | 11 +
apps/workbench/app/models/arvados_api_client.rb | 11 +-
apps/workbench/app/models/arvados_base.rb | 2 +-
apps/workbench/app/models/authorized_key.rb | 4 +
apps/workbench/app/models/collection.rb | 10 +-
apps/workbench/app/models/job.rb | 4 -
apps/workbench/app/models/repository.rb | 7 +
apps/workbench/app/views/application/404.html.erb | 22 +-
.../views/application/_browser_unsupported.html | 24 ++
.../app/views/application/_choose.html.erb | 7 +-
.../application/_create_new_object_button.html.erb | 7 +
.../app/views/application/_show_recent.html.erb | 4 +-
.../_show_sharing.html.erb | 21 +-
.../views/application/_title_and_buttons.html.erb | 6 +-
.../workbench/app/views/application/index.html.erb | 19 +-
.../collections/_create_new_object_button.html.erb | 1 +
.../app/views/collections/_index_tbody.html.erb | 2 +-
.../app/views/collections/_show_files.html.erb | 171 +++++----
apps/workbench/app/views/collections/show.html.erb | 45 ++-
.../views/jobs/_create_new_object_button.html.erb | 1 +
.../jobs/_rerun_job_with_options_popup.html.erb | 48 +++
.../app/views/jobs/_show_job_buttons.html.erb | 32 +-
apps/workbench/app/views/jobs/_show_log.html.erb | 1 +
apps/workbench/app/views/jobs/show.html.erb | 3 +-
apps/workbench/app/views/layouts/body.html.erb | 10 +-
.../pipeline_instances/_running_component.html.erb | 2 +-
.../views/pipeline_instances/_show_inputs.html.erb | 15 +-
.../pipeline_instances/_show_tab_buttons.html.erb | 2 +
.../app/views/pipeline_instances/show.html.erb | 4 +-
.../app/views/pipeline_templates/show.html.erb | 38 +-
.../views/projects/_show_contents_rows.html.erb | 2 +-
.../app/views/projects/_show_dashboard.html.erb | 10 +-
.../app/views/projects/_show_tab_contents.html.erb | 111 +++---
apps/workbench/app/views/projects/show.html.erb | 2 +-
.../send_request.text.erb | 7 +
.../app/views/users/_add_ssh_key_popup.html.erb | 2 +-
.../views/users/_create_new_object_button.html.erb | 6 +
.../app/views/users/_manage_repositories.html.erb | 6 +
.../views/users/_manage_virtual_machines.html.erb | 42 ++-
.../app/views/users/_setup_popup.html.erb | 2 +-
apps/workbench/app/views/users/profile.html.erb | 6 +-
.../app/views/users/request_shell_access.js | 10 +
apps/workbench/app/views/users/welcome.html.erb | 4 +-
apps/workbench/config/application.default.yml | 8 +
.../config/initializers/rack_mini_profile.rb | 5 +
apps/workbench/config/routes.rb | 5 +-
apps/workbench/public/browser_unsupported.js | 15 +
.../controllers/application_controller_test.rb | 37 ++
.../controllers/collections_controller_test.rb | 102 +++++-
.../test/controllers/projects_controller_test.rb | 68 +++-
.../controllers/repositories_controller_test.rb | 61 ++++
.../test/controllers/users_controller_test.rb | 34 ++
apps/workbench/test/diagnostics/pipeline_test.rb | 1 +
apps/workbench/test/helpers/share_object_helper.rb | 78 ++++
.../test/integration/anonymous_access_test.rb | 173 +++++++++
.../test/integration/browser_unsupported_test.rb | 17 +
apps/workbench/test/integration/errors_test.rb | 1 -
.../integration/filterable_infinite_scroll_test.rb | 14 +-
apps/workbench/test/integration/jobs_test.rb | 62 +++-
.../test/integration/pipeline_instances_test.rb | 98 +++--
apps/workbench/test/integration/projects_test.rb | 102 ++----
.../test/integration/repositories_test.rb | 46 +++
.../test/integration/user_manage_account_test.rb | 88 ++++-
.../test/integration/user_profile_test.rb | 10 +-
apps/workbench/test/integration/users_test.rb | 36 ++
apps/workbench/test/integration/websockets_test.rb | 21 +-
apps/workbench/test/integration_helper.rb | 17 +-
apps/workbench/test/performance/browsing_test.rb | 15 +-
apps/workbench/test/performance_test_helper.rb | 2 +-
apps/workbench/test/support/remove_file_api.js | 1 +
apps/workbench/test/test_helper.rb | 127 +++----
apps/workbench/test/unit/repository_test.rb | 18 +
crunch_scripts/crunchutil/subst.py | 7 +-
doc/_config.yml | 12 +
doc/_includes/_arv_run_redirection.liquid | 19 +
doc/_includes/_events_py.liquid | 13 +
doc/_includes/_navbar_top.liquid | 1 +
doc/_layouts/default.html.liquid | 1 +
doc/api/methods/collections.html.textile.liquid | 2 +-
doc/api/methods/groups.html.textile.liquid | 9 +-
doc/api/schema/Collection.html.textile.liquid | 11 +-
doc/css/carousel-override.css | 25 ++
doc/images/download-shared-collection.png | Bin 0 -> 39610 bytes
doc/images/files-uploaded.png | Bin 0 -> 53926 bytes
doc/images/keyfeatures/chooseinputs.png | Bin 0 -> 67586 bytes
doc/images/keyfeatures/collectionpage.png | Bin 0 -> 68735 bytes
doc/images/keyfeatures/dashboard2.png | Bin 0 -> 39651 bytes
doc/images/keyfeatures/graph.png | Bin 0 -> 37727 bytes
doc/images/keyfeatures/log.png | Bin 0 -> 94845 bytes
doc/images/keyfeatures/provenance.png | Bin 0 -> 53567 bytes
doc/images/keyfeatures/rerun.png | Bin 0 -> 56872 bytes
doc/images/keyfeatures/running2.png | Bin 0 -> 40453 bytes
doc/images/keyfeatures/shared.png | Bin 0 -> 46090 bytes
doc/images/keyfeatures/webupload.png | Bin 0 -> 70592 bytes
doc/images/quickstart/1.png | Bin 0 -> 36164 bytes
doc/images/quickstart/2.png | Bin 0 -> 58616 bytes
doc/images/quickstart/3.png | Bin 0 -> 68576 bytes
doc/images/quickstart/4.png | Bin 0 -> 45334 bytes
doc/images/quickstart/5.png | Bin 0 -> 66066 bytes
doc/images/quickstart/6.png | Bin 0 -> 83813 bytes
doc/images/quickstart/7.png | Bin 0 -> 60031 bytes
doc/images/shared-collection.png | Bin 0 -> 31833 bytes
doc/images/upload-tab-in-new-collection.png | Bin 0 -> 77537 bytes
doc/images/upload-using-workbench.png | Bin 0 -> 49690 bytes
doc/images/uses/choosefiles.png | Bin 0 -> 75434 bytes
doc/images/uses/gotohome.png | Bin 0 -> 76218 bytes
doc/images/uses/rename.png | Bin 0 -> 77738 bytes
doc/images/uses/share.png | Bin 0 -> 85695 bytes
doc/images/uses/shared.png | Bin 0 -> 57235 bytes
doc/images/uses/sharedsubdirs.png | Bin 0 -> 67642 bytes
doc/images/uses/uploaddata.png | Bin 0 -> 85535 bytes
doc/images/uses/uploading.png | Bin 0 -> 74365 bytes
doc/index.html.liquid | 12 +-
doc/sdk/cli/index.html.textile.liquid | 44 ++-
doc/sdk/cli/reference.html.textile.liquid | 13 +-
doc/sdk/cli/subcommands.html.textile.liquid | 7 +-
doc/sdk/python/events.html.textile.liquid | 12 +
.../firstpipeline.html.textile.liquid | 89 +++++
.../getting_started/sharedata.html.textile.liquid | 97 +++++
doc/start/index.html.textile.liquid | 128 +++++++
doc/start/index.html.textile.liquid.bkup | 50 +++
doc/user/topics/arv-run.html.textile.liquid | 20 +-
doc/user/topics/arv-web.html.textile.liquid | 98 +++++
.../crunch-tools-overview.html.textile.liquid | 63 ++++
doc/user/topics/run-command.html.textile.liquid | 10 +
.../tutorial-keep-get.html.textile.liquid | 36 +-
.../tutorials/tutorial-keep.html.textile.liquid | 33 +-
docker/arv-web/Dockerfile | 15 +
docker/{api => arv-web}/apache2_foreground.sh | 3 +-
docker/arv-web/apache2_vhost | 19 +
docker/base/Dockerfile | 2 +-
docker/build_tools/Makefile | 6 +
presentations/barcamp/dependencies.go | 20 --
presentations/barcamp/genomes_640.jpg | Bin 90926 -> 0 bytes
presentations/barcamp/goroutine.go | 10 -
presentations/barcamp/keep.slide | 302 ----------------
presentations/barcamp/lolwut.jpg | Bin 58786 -> 0 bytes
presentations/barcamp/server.go | 15 -
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cli/bin/arv | 350 ++++++++----------
sdk/cli/bin/crunch-job | 46 ++-
sdk/go/arvadosclient/arvadosclient.go | 4 +-
sdk/go/arvadosclient/arvadosclient_test.go | 49 +--
sdk/go/arvadostest/run_servers.go | 123 +++++++
sdk/go/keepclient/keepclient_test.go | 48 +--
sdk/go/keepclient/support.go | 9 -
sdk/python/arvados/api.py | 16 +-
sdk/python/arvados/collection.py | 27 +-
sdk/python/arvados/commands/arv_copy.py | 3 +-
sdk/python/arvados/commands/put.py | 48 ++-
sdk/python/arvados/commands/ws.py | 4 +-
sdk/python/arvados/errors.py | 49 ++-
sdk/python/arvados/events.py | 101 ++++--
sdk/python/arvados/keep.py | 61 +++-
sdk/python/gittaggers.py | 20 ++
sdk/python/setup.py | 31 +-
sdk/python/tests/arvados_testutil.py | 64 +++-
sdk/python/tests/run_test_server.py | 395 ++++++++++++++-------
sdk/python/tests/test_api.py | 43 ++-
sdk/python/tests/test_arv_put.py | 39 +-
sdk/python/tests/test_collections.py | 65 ++--
sdk/python/tests/test_errors.py | 68 ++++
sdk/python/tests/test_keep_client.py | 125 +++++--
sdk/python/tests/test_pipeline_template.py | 19 +-
sdk/python/tests/test_websockets.py | 64 ++--
sdk/ruby/arvados.gemspec | 3 +-
sdk/ruby/lib/arvados.rb | 31 +-
sdk/ruby/lib/arvados/google_api_client.rb | 55 +++
sdk/ruby/lib/arvados/keep.rb | 76 ++--
sdk/ruby/test/test_keep_manifest.rb | 66 ++++
services/api/.gitignore | 1 +
services/api/Gemfile | 7 +-
services/api/Gemfile.lock | 22 +-
.../api/app/controllers/application_controller.rb | 32 +-
.../arvados/v1/collections_controller.rb | 13 +-
.../controllers/arvados/v1/groups_controller.rb | 76 ++--
.../arvados/v1/keep_disks_controller.rb | 2 +-
.../app/controllers/arvados/v1/links_controller.rb | 2 +-
.../app/controllers/arvados/v1/nodes_controller.rb | 2 +-
.../arvados/v1/repositories_controller.rb | 8 +-
.../controllers/arvados/v1/schema_controller.rb | 8 +-
.../app/controllers/arvados/v1/users_controller.rb | 13 +-
.../arvados/v1/virtual_machines_controller.rb | 2 +-
.../api/app/controllers/database_controller.rb | 2 +-
.../api/app/models/api_client_authorization.rb | 4 +
services/api/app/models/arvados_model.rb | 40 ++-
services/api/app/models/collection.rb | 115 ++++--
services/api/app/models/database_seeds.rb | 1 +
services/api/app/models/user.rb | 2 +-
services/api/config/application.default.yml | 7 +-
services/api/config/initializers/time_format.rb | 5 +
.../db/migrate/20141208164553_owner_uuid_index.rb | 16 +-
.../20141208174553_descriptions_are_strings.rb | 21 ++
.../20141208174653_collection_file_names.rb | 20 ++
.../api/db/migrate/20141208185217_search_index.rb | 32 ++
...0150122175935_no_description_in_search_index.rb | 30 ++
.../db/migrate/20150123142953_full_text_search.rb | 18 +
...203180223_set_group_class_on_anonymous_group.rb | 14 +
...206210804_all_users_can_read_anonymous_group.rb | 12 +
...20150206230342_rename_replication_attributes.rb | 30 ++
services/api/db/structure.sql | 213 ++++++++++-
services/api/lib/current_api_client.rb | 159 +++++----
services/api/lib/josh_id.rb | 2 +-
services/api/lib/kind_and_etag.rb | 4 +-
services/api/lib/load_param.rb | 12 +-
services/api/lib/record_filters.rb | 20 +-
services/api/test/fixtures/collections.yml | 108 +++++-
services/api/test/fixtures/groups.yml | 5 +-
services/api/test/fixtures/jobs.yml | 54 +++
services/api/test/fixtures/links.yml | 42 +++
services/api/test/fixtures/pipeline_instances.yml | 47 +++
services/api/test/fixtures/pipeline_templates.yml | 39 ++
services/api/test/fixtures/repositories.yml | 5 +
.../arvados/v1/collections_controller_test.rb | 79 ++++-
.../api/test/functional/arvados/v1/filters_test.rb | 79 +++++
.../arvados/v1/groups_controller_test.rb | 17 +-
.../api/test/integration/collections_api_test.rb | 160 +++++++++
services/api/test/integration/groups_test.rb | 56 ++-
services/api/test/performance/browsing_test.rb | 12 -
services/api/test/performance/links_index_test.rb | 14 +
services/api/test/test_helper.rb | 2 +-
services/api/test/unit/arvados_model_test.rb | 58 +++
services/api/test/unit/collection_test.rb | 195 ++++++++++
services/api/test/unit/link_test.rb | 5 +
services/api/test/unit/log_test.rb | 14 +
services/api/test/websocket_runner.rb | 10 +-
services/arv-web/README | 6 +
services/arv-web/arv-web.py | 252 +++++++++++++
services/arv-web/sample-cgi-app/docker_image | 1 +
services/arv-web/sample-cgi-app/public/.htaccess | 3 +
services/arv-web/sample-cgi-app/public/index.cgi | 4 +
.../sample-cgi-app/tmp/.keepkeep} | 0
services/arv-web/sample-rack-app/config.ru | 4 +
services/arv-web/sample-rack-app/docker_image | 1 +
.../sample-rack-app/public/.keepkeep} | 0
.../sample-rack-app/tmp/.keepkeep} | 0
services/arv-web/sample-static-page/docker_image | 1 +
.../arv-web/sample-static-page/public/index.html | 6 +
.../sample-static-page/tmp/.keepkeep} | 0
services/arv-web/sample-wsgi-app/docker_image | 1 +
services/arv-web/sample-wsgi-app/passenger_wsgi.py | 3 +
.../sample-wsgi-app/public/.keepkeep} | 0
.../sample-wsgi-app/tmp/.keepkeep} | 0
services/crunchstat/crunchstat.go | 13 +-
services/crunchstat/crunchstat_test.go | 29 ++
services/fuse/arvados_fuse/__init__.py | 48 ++-
services/fuse/bin/arv-mount | 2 +
services/fuse/gittaggers.py | 1 +
services/fuse/setup.py | 35 +-
services/fuse/tests/test_mount.py | 61 ++--
services/keepproxy/keepproxy_test.go | 99 ++----
services/{fuse => nodemanager}/MANIFEST.in | 0
.../arvnodeman/computenode/dispatch/__init__.py | 2 +-
services/nodemanager/arvnodeman/config.py | 3 +-
services/nodemanager/gittaggers.py | 1 +
services/nodemanager/setup.py | 34 +-
.../nodemanager/tests/test_computenode_dispatch.py | 22 +-
services/nodemanager/tests/test_daemon.py | 16 +-
services/nodemanager/tests/testutil.py | 6 +-
280 files changed, 6140 insertions(+), 2701 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/ajax_error.js
copy apps/workbench/app/assets/javascripts/{event_log.js => job_log_graph.js} (83%)
create mode 100644 apps/workbench/app/assets/javascripts/request_shell_access.js
create mode 100644 apps/workbench/app/mailers/request_shell_access_reporter.rb
create mode 100644 apps/workbench/app/views/application/_browser_unsupported.html
create mode 100644 apps/workbench/app/views/application/_create_new_object_button.html.erb
rename apps/workbench/app/views/{projects => application}/_show_sharing.html.erb (78%)
create mode 100644 apps/workbench/app/views/collections/_create_new_object_button.html.erb
create mode 100644 apps/workbench/app/views/jobs/_create_new_object_button.html.erb
create mode 100644 apps/workbench/app/views/jobs/_rerun_job_with_options_popup.html.erb
create mode 100644 apps/workbench/app/views/request_shell_access_reporter/send_request.text.erb
create mode 100644 apps/workbench/app/views/users/_create_new_object_button.html.erb
create mode 100644 apps/workbench/app/views/users/request_shell_access.js
create mode 100644 apps/workbench/config/initializers/rack_mini_profile.rb
create mode 100644 apps/workbench/public/browser_unsupported.js
create mode 100644 apps/workbench/test/helpers/share_object_helper.rb
create mode 100644 apps/workbench/test/integration/anonymous_access_test.rb
create mode 100644 apps/workbench/test/integration/browser_unsupported_test.rb
create mode 100644 apps/workbench/test/integration/repositories_test.rb
create mode 100644 apps/workbench/test/support/remove_file_api.js
create mode 100644 apps/workbench/test/unit/repository_test.rb
create mode 100644 doc/_includes/_arv_run_redirection.liquid
create mode 100644 doc/_includes/_events_py.liquid
create mode 100644 doc/css/carousel-override.css
create mode 100644 doc/images/download-shared-collection.png
create mode 100644 doc/images/files-uploaded.png
create mode 100644 doc/images/keyfeatures/chooseinputs.png
create mode 100644 doc/images/keyfeatures/collectionpage.png
create mode 100644 doc/images/keyfeatures/dashboard2.png
create mode 100644 doc/images/keyfeatures/graph.png
create mode 100644 doc/images/keyfeatures/log.png
create mode 100644 doc/images/keyfeatures/provenance.png
create mode 100644 doc/images/keyfeatures/rerun.png
create mode 100644 doc/images/keyfeatures/running2.png
create mode 100644 doc/images/keyfeatures/shared.png
create mode 100644 doc/images/keyfeatures/webupload.png
create mode 100644 doc/images/quickstart/1.png
create mode 100644 doc/images/quickstart/2.png
create mode 100644 doc/images/quickstart/3.png
create mode 100644 doc/images/quickstart/4.png
create mode 100644 doc/images/quickstart/5.png
create mode 100644 doc/images/quickstart/6.png
create mode 100644 doc/images/quickstart/7.png
create mode 100644 doc/images/shared-collection.png
create mode 100644 doc/images/upload-tab-in-new-collection.png
create mode 100644 doc/images/upload-using-workbench.png
create mode 100644 doc/images/uses/choosefiles.png
create mode 100644 doc/images/uses/gotohome.png
create mode 100644 doc/images/uses/rename.png
create mode 100644 doc/images/uses/share.png
create mode 100644 doc/images/uses/shared.png
create mode 100644 doc/images/uses/sharedsubdirs.png
create mode 100644 doc/images/uses/uploaddata.png
create mode 100644 doc/images/uses/uploading.png
create mode 100644 doc/sdk/python/events.html.textile.liquid
create mode 100644 doc/start/getting_started/firstpipeline.html.textile.liquid
create mode 100644 doc/start/getting_started/sharedata.html.textile.liquid
create mode 100644 doc/start/index.html.textile.liquid
create mode 100644 doc/start/index.html.textile.liquid.bkup
create mode 100644 doc/user/topics/arv-web.html.textile.liquid
create mode 100644 doc/user/topics/crunch-tools-overview.html.textile.liquid
create mode 100644 docker/arv-web/Dockerfile
copy docker/{api => arv-web}/apache2_foreground.sh (66%)
create mode 100644 docker/arv-web/apache2_vhost
delete mode 100644 presentations/barcamp/dependencies.go
delete mode 100644 presentations/barcamp/genomes_640.jpg
delete mode 100644 presentations/barcamp/goroutine.go
delete mode 100644 presentations/barcamp/keep.slide
delete mode 100644 presentations/barcamp/lolwut.jpg
delete mode 100644 presentations/barcamp/server.go
create mode 100644 sdk/go/arvadostest/run_servers.go
create mode 100644 sdk/python/gittaggers.py
create mode 100644 sdk/python/tests/test_errors.py
create mode 100644 sdk/ruby/lib/arvados/google_api_client.rb
create mode 100644 services/api/config/initializers/time_format.rb
create mode 100644 services/api/db/migrate/20141208174553_descriptions_are_strings.rb
create mode 100644 services/api/db/migrate/20141208174653_collection_file_names.rb
create mode 100644 services/api/db/migrate/20141208185217_search_index.rb
create mode 100644 services/api/db/migrate/20150122175935_no_description_in_search_index.rb
create mode 100644 services/api/db/migrate/20150123142953_full_text_search.rb
create mode 100644 services/api/db/migrate/20150203180223_set_group_class_on_anonymous_group.rb
create mode 100644 services/api/db/migrate/20150206210804_all_users_can_read_anonymous_group.rb
create mode 100644 services/api/db/migrate/20150206230342_rename_replication_attributes.rb
delete mode 100644 services/api/test/performance/browsing_test.rb
create mode 100644 services/api/test/performance/links_index_test.rb
create mode 100644 services/arv-web/README
create mode 100755 services/arv-web/arv-web.py
create mode 100644 services/arv-web/sample-cgi-app/docker_image
create mode 100644 services/arv-web/sample-cgi-app/public/.htaccess
create mode 100755 services/arv-web/sample-cgi-app/public/index.cgi
copy services/{fuse/tests/__init__.py => arv-web/sample-cgi-app/tmp/.keepkeep} (100%)
create mode 100644 services/arv-web/sample-rack-app/config.ru
create mode 100644 services/arv-web/sample-rack-app/docker_image
copy services/{fuse/tests/__init__.py => arv-web/sample-rack-app/public/.keepkeep} (100%)
copy services/{fuse/tests/__init__.py => arv-web/sample-rack-app/tmp/.keepkeep} (100%)
create mode 100644 services/arv-web/sample-static-page/docker_image
create mode 100644 services/arv-web/sample-static-page/public/index.html
copy services/{fuse/tests/__init__.py => arv-web/sample-static-page/tmp/.keepkeep} (100%)
create mode 100644 services/arv-web/sample-wsgi-app/docker_image
create mode 100644 services/arv-web/sample-wsgi-app/passenger_wsgi.py
copy services/{fuse/tests/__init__.py => arv-web/sample-wsgi-app/public/.keepkeep} (100%)
copy services/{fuse/tests/__init__.py => arv-web/sample-wsgi-app/tmp/.keepkeep} (100%)
create mode 120000 services/fuse/gittaggers.py
copy services/{fuse => nodemanager}/MANIFEST.in (100%)
create mode 120000 services/nodemanager/gittaggers.py
discards 8d6532d6f5db2927f353ebe1cd75dcd1e189873a (commit)
discards 40e42c383646edade9b4723dfb8001cb4c873ea5 (commit)
discards bdf093d22ebbdaaadcd822fe32a5fe150fe649f6 (commit)
discards c37ed03a4d05ca49820628d2b43d7bf140668f47 (commit)
discards af0728d3f2a3f738a683459f3ec9f502d027a06f (commit)
discards 7079475cf16df8d85885997c6f9d645e08db6770 (commit)
discards 097d0c20dd8c884682891c50e2b045448948f6a2 (commit)
discards c718d729795f423e4c1254b781a881da7506fe2f (commit)
discards fc696c2f99353a1de472f4620f51d345988d7d27 (commit)
discards f48f9250f4bf1c7326c1379bc7c6cf490ad87d0d (commit)
discards f1491aab5b94ac37410d71eeb40a9479c8a5d952 (commit)
discards 0856c5303ed068c25403eae359c985d5601b6866 (commit)
discards 70a227b4ebc27c64bb64ecb596c1ae2959dd4135 (commit)
discards a31092d05a4c5b72f1a7ca2218a3f45a1fc780a0 (commit)
discards 439a4e1b1a1736083151d10f0365c1b16aea4d0a (commit)
discards 821f806abae252f5d5762416e0766747126bff14 (commit)
discards e4b8d0b51358c88f527e52e55dae9b83a60d9e35 (commit)
discards e921c415ea3c8eb9a2fafa8ca9c9e5cceb53b2c4 (commit)
discards bc50ca6971dc13c84dc518b40887ab30ebe5d941 (commit)
discards aef8ae1e9a50ec6a39141aa91c2b4e8e2da7697c (commit)
discards 89ecce95bc435c32027db8ee9f77d705c089719b (commit)
discards ee278406810bc2522802cd33dcf94d93fb1fc913 (commit)
discards bd001d4c8be314d3905f0b7c7087ba4d82b94e79 (commit)
discards dbeccd1b8575ba96506cb74d04c56f71c127f776 (commit)
discards fb76b0b8e1890ec27cce3ec5cac42fff07a97294 (commit)
discards eb5d5d99b6694423852ec229935fac8376403712 (commit)
discards 0c9832f8d1af07c85ab84b6e42b61aa094543dcf (commit)
discards e94c6eb56982e236daeffc19b9def869f14fbab4 (commit)
discards 9f6c09436da853d56db4922ee1c6e54b3c4b36d6 (commit)
discards a50b8ad5773b13db45070205af0754b88ed2ce25 (commit)
discards 10fda8ea470a9e83bd1d349b33b43d1f684a35cb (commit)
discards 1dd5156a83dc05d959ccffe1567271889954d5a1 (commit)
discards df69d7a558943f25b9cb2fb215e3fad665c8d242 (commit)
discards 9e327b69799585a69c3e6dc997a3d870337e1e5b (commit)
discards 1e7f49ed49f458ab3f9b62145760a14bb86d1bb5 (commit)
discards d35d6189a3dc06ec86ef19927860986a2ef97ac3 (commit)
discards 30412881e854e9255c237f3ca3c4686e48e8e7a3 (commit)
discards 8a2cf36f8254bee55141155a4b9054f4178f5cae (commit)
discards fecfe4c45fb498b22ada515ff4d01544e8dbeb94 (commit)
discards 4b59ecbe561ebd0f41d3c44697cb527fbf993720 (commit)
discards 509065a0f0156cb3c402a37efe20943b4a1fc50e (commit)
discards f048f2dda540064a5c4f0aff16e0e726bf621ef4 (commit)
discards 7136493014aea9787497464269d9aaddf1830e97 (commit)
discards df54f4111b0afaaf073c7387dff5960bf19a53e2 (commit)
discards 593316198c1647b24f66a64e4a0f7a34e75b54ab (commit)
discards debedb222cfea4bfc4519032cad8475858e04034 (commit)
discards 26d74dc0524c87c5dcc0c76040ce413a4848b57a (commit)
via f3250432a47c835f4c594348b0d4904a247c3365 (commit)
via 3598c3003a7987cca5c0536ba8206ec40c1c3649 (commit)
via d54738ad3b2fde0207cf1ebbdc6f4360cd141d90 (commit)
via 96e0115f05e1b9fe1dedbe721cbdcb13c29ee875 (commit)
via 5517f022cdb6233551c9281422c033d18293ec03 (commit)
via 6ee389b798bd7898e16b8ab8bf9394bf97c40f46 (commit)
via 4f8b2d755cb8249cd9118b7d3213a0021b83b0cb (commit)
via 58f8ab40470a8c1db31563e26c66fee9dbdd7477 (commit)
via 1ee6ce5be0c86c1d2e903252ba2a70694be5cf31 (commit)
via c99e40c18e4ba67f529fcd928eed76509dbda130 (commit)
via 1578bb430ce005137f49233ef87fac34ebc51e2e (commit)
via d87717b4ec885059183ef6d7fa6780c343338455 (commit)
via daaeb7a23e8a5baf82e3af0f280856862dbc5aa7 (commit)
via eff37344f3cecd5aed259c9852aca4bdcdfb6922 (commit)
via 5f2d5f96abb8941bed95bd9f47f79f6f1c3ba38d (commit)
via fd172dd6875c8bf1481c6e078590c0ccef934bca (commit)
via 5e003c8f9cf47a5bb716dcb75b8e6b98e4680a4c (commit)
via f842b72b9e2bb50a68ba388922c55f54ad0399ae (commit)
via 6ab5505797c96e3aa5652e0d964eec3787e023ac (commit)
via 8b90f80efca772efd2697ffc70d7809c32564171 (commit)
via 45d8d010e15b0c4860102b78692ec8bc5b2ef158 (commit)
via ef4e4a34213975ed9c1dcd9d4a2efb26f51d027a (commit)
via 7b877a2be249c5e4ab9ab5d73cb68906ca3113d8 (commit)
via 0cefa4c0f3c1b16884b04d6273bd8730166d69ba (commit)
via 4c5352de29ca583c41d9babf795983ee4ea3b78e (commit)
via 11e1ee67236b1dda5dac5e871ecfedd7de8faccf (commit)
via 1e423bd9887adad61999503771b0794fc62efc28 (commit)
via 938338a385a96066552aea6230d773a17cbf3c3e (commit)
via 7ef27804d92e2c38ff6b22aab4b113b3e1817bf5 (commit)
via 1f7a6b50cabab4c8645dd6db92e456c080f2a81c (commit)
via 08c575dc24bbc5732a5fcb1126c23d9a4ca10b73 (commit)
via 23c1bc62ea3644636c0edce8ccee39f2094db190 (commit)
via 175c31a1cc695285c035ca2a54d5b964ab4b1d5f (commit)
via e1999050ade633163524cd9d87d0b77f8b5bdfdc (commit)
via da298b0d96a1e49a1330a4486dcbe22d92d1d743 (commit)
via d27fd3e2648e47f014f7da67056825aca3724004 (commit)
via 8a504ad561c1ffbafee8a7bc8da551f9d4b9a29e (commit)
via b7bd673a45ec3de02fdf846f4a9ebe2638c546cc (commit)
via 43538243995267c417983360d226d6e8eb181139 (commit)
via 3a1face2e3bc02e1fb9c53a2268095811b2e069d (commit)
via 75f4b70625086aaa8ecf8daed23e4d151e54949f (commit)
via e5840cee519c5ff8b88e37e14160f9e4e12908ec (commit)
via e88d1643f64e70e984b2c7943e5ce6569e7e2d37 (commit)
via 6db8f01ae7f8d30f48a88caa351004ea446b33b3 (commit)
via 4df1175e30c21850af394fcd60c9bb7ca3d981a5 (commit)
via c2b8ab7045886b62963feb0cd8f9b9291ce1a8b7 (commit)
via 17800e7d4a9574035dd48b71ec4247f70525d45e (commit)
via 7924fe7d6b4cee88035046005425ce19260c09e9 (commit)
via 3f74a7584760a83539b6c0ba01ffd5078d8858cf (commit)
via 6221a5005318304a2f05f1fa3c9d897ed71d5676 (commit)
via 2415e93fcd6a24b3bfbc319c139737f550835e36 (commit)
via 2d2f3bed79f9504d15503277056feb394c12dd7c (commit)
via 144e23888d46d68c5e32fea9f66a8903e05a3526 (commit)
via e0889a8f6997327fd9b4d826237c8166cf909741 (commit)
via 2ee024868c8152903a43a8c6f5dce601305e99e2 (commit)
via 4879e0e2f75fd387720b4f4b58ca6ae48a798c98 (commit)
via 5bcba288077488791daa43a15d5fd5fb0c6e653c (commit)
via 95cbdf59eb5326c393ae91f243f596984cba7fa7 (commit)
via b14bc80d764b85a6ddfed32198b2144b5adf2637 (commit)
via 4851f29ac730aa09dcb3489a3a6e7479e7b82fb6 (commit)
via 3d63bc278174d245edc4fa06ed88971a2589e080 (commit)
via 970095751e2e836ed296152ae3e9ccb6caa62f62 (commit)
via 1c87e0d76265bb64b717289015181e41e0cbe2f3 (commit)
via e7ef642ff70bb7e6c281f4dd1d353f7fb2b3f5ce (commit)
via 367a6fbc62b4b20af9f5724359fb0d0e423dc718 (commit)
via 1a68b00bcb3dd92f597de72274a56ed4a1144c2b (commit)
via 0cf4805d979615190f38ce1d01f1b2d0e8927988 (commit)
via 3594ad790c998c1b1711ea65057fcae9477986d4 (commit)
via 08fbba821251a58cc99921cb477fbbc076a1bfee (commit)
via 9cf25951c6b64449fe24ff9965e7dabe85c8ff4e (commit)
via fbfd3b4c049d4b3d24b22bfb5462f098c73596aa (commit)
via 2ccc3eda37cd728d6526804c228d7383dc8960a9 (commit)
via d7f6013f1e728a3a7b42d6736b78cc81ac7de127 (commit)
via dab166a0e63d256b2ccfd209493a35696f88726f (commit)
via 25d58aaf041ac109ff76db5168c193272958d454 (commit)
via fb62ab318be2202b9d403e65d6dc86a9d7e72a3a (commit)
via af550e54c034136e5fcb187e7f81e3d82170f9c8 (commit)
via 9684e729ae3ebf438fd2c1c440bb0d8c45ca25af (commit)
via a224de262c6e94c592eb8c9a2f909954d24b7c9c (commit)
via 63cb5c235ccacdc1665a89560bc8c16fcbefd8d6 (commit)
via 4106786a571e8d919e474d8ae205c3b2c9042b26 (commit)
via cc5699578c16dfb96911b8abcd1b35b8ec0ed7c0 (commit)
via 1128f6e0d62f71f4ee91ab609c918ae5bb291edd (commit)
via 1bcfe8651af341c6e7cd01a19443c7c288efa932 (commit)
via d5809a1e62e1b1a3984fff88118e036b1f174ff1 (commit)
via 50df4956c5b0e93efd781fbb070d9d5d30d39eda (commit)
via 606375516f678222465f2643b8162c6973bb28d2 (commit)
via 398a5ef03226f2f3ba06b6ad05a61f3a4b403cf7 (commit)
via 820ce7ad92cba95587800a275e14bbf24670898f (commit)
via 91abe2648d8ca1a3a5185e94beb505ad33db9e2c (commit)
via a67bb34f6f19662f0a30e4aa670774c4595cb7a4 (commit)
via 7a14311a666f471fe34c95759b40ba10b1813ab7 (commit)
via 2917d8c1e42d2c4569f131778febb18db7445ac4 (commit)
via 61fd9276456c112a4e22273227b0feafed35530f (commit)
via 857d5a70bcff7d3634d6daf9886fdfe6d46d681c (commit)
via d4b03f2476f0ea6b30baad78672b31938846853e (commit)
via ff3e4c60ffa088479cd0a97b314b4af06b0d67ba (commit)
via fcfc006561b72f76fa4a553aa294a884462e7ca4 (commit)
via cde755bd4d7702c89ebfbd6d2fe8f852509d3786 (commit)
via 54819e3950dfc1adc76159bd4259a70bd531ab2f (commit)
via 5b7ee6cb073457d0cf1f6b7c5ca3ee8965f144ef (commit)
via 31f83b25039677512509120fb385069003bdd4ca (commit)
via 02d728b87a397a7093827de5046e62ce50c4019d (commit)
via 5d3d32d5532d39f1ef85a9e01b9b70cd28cf3579 (commit)
via d302307a4a66867419722034228823d1fc3910a6 (commit)
via d62b73382398808a440f15fdda2eea2e15e44282 (commit)
via 966bd97704f635315ab7ba50f23590a5fc9a97be (commit)
via ec0c0f54da513b2b8221d65d9a2c621a7d95d79e (commit)
via 353a72e637532f2641e55c79edc0de52e2dd3508 (commit)
via b17ed3b444abb6c326e69de5ff6a9f8bf019530e (commit)
via e759c71ef7ecbfca3075db4ae94fc7bd0464656e (commit)
via 498a97e6cdb456bf7487f7c62dce08791cb5f453 (commit)
via cd00c7d65d724ea78fe6e59dda333241a7c0775a (commit)
via 26ac1b2f41916d1f4040073a15dfae5f1b294cb5 (commit)
via 5dbf5c8ea2d9eb2bc8e10a03ca625f12ed71f12c (commit)
via c9ff74363edc2f3271c117184be2d1ad7fad633a (commit)
via 1de8e55b47ea46fe1e589fbfe1ff0ae77b9e2cbf (commit)
via b80db28cdd536077e5effe6c08af079532c2059b (commit)
via f16b7abe9b1ae5967ffaab62b9c9ae3f955f44f1 (commit)
via 148ff097b57571dda1b6db063a2eca5a4eb98a35 (commit)
via d3a9326a2c92de950216fb2a88dbbc9de898e4b3 (commit)
via da2492bfc43032c3374b6509a7208127ec48093a (commit)
via f5a30607d1746b29688363530a3011ee5c2f4f9a (commit)
via 4993b8b44022fd3dc73fcebf20f80d054bdf4370 (commit)
via 6bf9ae122958b25b4a22447f67fb11cf24765d97 (commit)
via 5923d0fa912c73e3725e52c869d72793304ae44a (commit)
via 58bf2ad27c760fb7da0641b239f1871918b84a42 (commit)
via 8676d8d8fe7ea86db75fd9e6f53b07e21437cd6d (commit)
via fb181d9653d80317422e1d979697da908fa804c8 (commit)
via b599ef92fcfc25045eb6a366907555594496bfad (commit)
via 67a4825340187c05cbada61d38c12645a17acb65 (commit)
via 0215bf7b8c61d59462a476d850af999105856177 (commit)
via 34d6dc1f56b59b7c7cc3e6dc7d54053149c49bc6 (commit)
via 045bce46ede1995ed17747c48611f22c478cc82d (commit)
via d9e2de2e142fe1a79bd83064d8d9135ba44fd807 (commit)
via 710b03568da92458279db56608cba84cb5151847 (commit)
via 31e1554c4372d8206618bf7fee48323b08f24ec3 (commit)
via 9b6b5f0bd2ad96deeea2070a4eba56795bb28c1a (commit)
via 07f50aff99bbb837c9419e7a931add36d1611e2d (commit)
via ae7e8221d669b29ff3e098ac9259afb2875e9d3b (commit)
via f6089c82da72f331ba5a44874ce267b18bcaf557 (commit)
via a934fcf84acd4cc3a351fde1b6e21a0bd93757ef (commit)
via 26d0d60ce820b3b30d6645839c9c1af354e7498d (commit)
via 46efb1e7f49d08095a6a49a4c6aff045c5eb6f16 (commit)
via fc8e572937f2fd61bdc1e7f34a2e3f9a5cebd7ff (commit)
via d65b683af52e072b3d179b6f32edfbf37e108011 (commit)
via c9f5db97ad5d853cc2f4636d0743037f6048ceeb (commit)
via f7ec673ce72af1e076408f394b6401e4f253e703 (commit)
via b8148b3bcdfc6fe8a8b20e6a4c589b7a50e147a8 (commit)
via c882575c856e01313cf2caf2e4ead1f27bfb33ae (commit)
via 525d5d6351a0610237c52f1564dec5b77cf3af4f (commit)
via 538caa064785b645a2b8f815bf77a30192b20665 (commit)
via 9f1fafa8c7c7f3750d6769d863b82cb826d7ed6e (commit)
via 330a46e91b4ceaefba2bcfc383931eb59c77d461 (commit)
via fefce5e8e133a8fa064bbcdf31d85d41dc4a6729 (commit)
via 204f433a870e2bf1cf7af1fbe076e91f427ef05e (commit)
via 9b61792d905324a98b24224d45347082efbe5205 (commit)
via 16b720950262eb559358cf357f5098a142901665 (commit)
via e73af668c24cd259800c344c3efe8b7d769903da (commit)
via b6a7a62f4f38710f50d08a91a6a9b210700bb011 (commit)
via 0b102fac0e8d2a7d46d088b1bd8f7b27b325dd2a (commit)
via 79aca915815d298d2c20546108284627ee6cb84b (commit)
via b21b81e6623d025da4d93cbf09d523e63d2e07b0 (commit)
via aaffcb23198b4223c48092ccd30ef7152b434187 (commit)
via 20f5b178a850b029ecd501ed49e4ed0a537c1fad (commit)
via fb4921f56d1c13a86add2e59205ec32fa1f6efe4 (commit)
via 1d4a39ab3e97c031683ada9f6c98e4c7365fa414 (commit)
via f6ab9be0046a6f8d760259c1a0eba8ab7c636903 (commit)
via 231242b6378abda494f2c684995519a259cfe174 (commit)
via 1963df31ffb7e95b72e53a0ec5c891f539b6dadb (commit)
via b59b310e23b588c4007af84741d4b94bc9f595f1 (commit)
via e2da84c1fd9052791ed2b684741469570e09ea35 (commit)
via d6ec5672045b29aeaf983a78c5487ae354ccb20d (commit)
via 757212484d9da8bb8d8852bfb6870433d2b4fa97 (commit)
via f85132f1a018179b7127c199932c1f0f3e3f76d5 (commit)
via 2e9f5f1aa841972d1c6d3ff0828d774f60c28307 (commit)
via 71c05eec3e9c8e6f37f14760b04584a8d4c4372c (commit)
via f32690a4a18f85909c0a04de83ecf7819f127df8 (commit)
via df507d6cdebca220ac19dbbc5c16d18498cb852c (commit)
via 4982008e820ed48f362226c61540c18305c6acd6 (commit)
via 77daa60985c94cf4137c8a54681bb89278db8436 (commit)
via bd720586c0152ca4e7d109389bda2c0e463c76bb (commit)
via 2527b9cd7958d89a5ae0dd84856027908c48ae53 (commit)
via dd645c9e973b9b725f310513ce309fa1e1a82421 (commit)
via c9e19eb6c3c6889b55c3b63424b36f1139c9abf0 (commit)
via f8067dd18b72705f3317e85745e87cffc9e25313 (commit)
via cb79358321eff7a49dd4a3fb6e0ea448ead92597 (commit)
via cda964acdb8132d90b881e62db008c574fdd5cc4 (commit)
via 2e5ac62b550f7dd608cf133ae66ef04f801be76b (commit)
via b48e7f0c19f1a7256222c220e938832789492aa3 (commit)
via 7a71d74c538c37437e65f5d22205c224d0fe9207 (commit)
via ff49b1144f5b9f9f7624f3741f5af791073de03c (commit)
via f5a0a6337d620ac12f7fffea65d0803d46dcb87e (commit)
via 78596140bd879feb2eacd899060e5ab75a5c94f3 (commit)
via 2d7883822203e66afee1a36c7e86a844bc23719c (commit)
via 64c70939c414881de61ac65512701d0ba4068786 (commit)
via d13386351c53a261558052bafd5e2308230cb73b (commit)
via 348801d41f0bd06582675223f07a7ef7f36ac887 (commit)
via afef0760e7281eb7038778a12575dfc32b3162f3 (commit)
via eab43fcf2826f4416a70bef95c3ae04a77b487c9 (commit)
via 9b59cd2f10fa44f4cdbf8986b08e92bdde5a62a7 (commit)
via 1eda4774a59f46296f82231eeb80484aca70a961 (commit)
via 71a556d7d2a9484a4f0bda069f1f7915f548683f (commit)
via f2e686e1c4e117c93d462ec94e315df9e0be02c6 (commit)
via 9f0f926aa45113a50ed1de737b236e9f69f64079 (commit)
via f414dc4c1862c8d471b865dea5c7ac141de6d533 (commit)
via 180498df4db8c43080bb302bc56edea70c940583 (commit)
via 8a8450dcb4dfa4fa222e059091bae03c5e45df3f (commit)
via 7939a927a2b0584210d5e8b2fe73f7625858d6bd (commit)
via 54873fcd103e4887e41e987522e4442b62f682ad (commit)
via 8cd7249b96576285388ef036d04532f72a8f1ee3 (commit)
via c6566ea328710818ab9b65db6187751f1874415c (commit)
via f7cc825b6a7a526a95e9f889ba94f4122f191889 (commit)
via a41baffe4f38019cb5b36875c5e0c838ef9201e5 (commit)
via a94e15cab04a19dcbb02f2e95335e337c8e55036 (commit)
via aa613a590c0b03e90432c25f0190adc99ef4f657 (commit)
via dd72a4de55681da7b5a95ee8b1e659221c48614f (commit)
via 0ea383a2924b37b47f2bc82fbe405fce4a03bd1f (commit)
via 21a824c12633d3775b449ec9b06148546078f1a8 (commit)
via c722a8bba1bd155f3e36ef4402f684e36ea3e5ec (commit)
via e5b8f6826a2034c0c7d7142cff3fb02f64fb8831 (commit)
via 12dee1ebfd6a3fca40c19d751459ba6a071c0a3d (commit)
via 7b9ca2a26b4099e45548d83fc878fe295f2cdc56 (commit)
via 7acac83d2789b36f0e249a3fde9a8d300f15e152 (commit)
via 36d6d1609ef342268cc87fc8bfce51bcf7199929 (commit)
via e20040092b47f79365637dd8e26156d95ab5c6ee (commit)
via a11c56ef66604a9117e3db8c2fa2273c98f88b51 (commit)
via 5b70a11a08dc26b43b3ec4aef178bafe3a801b86 (commit)
via dac304f927000b74d41defcdea26da5d896bfc0c (commit)
via c5fa3f7b2faea363cf73c7e4a2880086c7c9e4f0 (commit)
via 1a78ca155b741a08fcf8e0e284bc4da273f084c2 (commit)
via 826cb14afd19197738ceba11ff382aaaf123a637 (commit)
via 4204a3c2bda7378e8664233d3c2410c5efb95a47 (commit)
via 1147248aa7a30a9a423e7b5b30c6bbb7d4b9bba6 (commit)
via 1e2e0e4f14a730ba1bbea6a4ced5d87ea2766c35 (commit)
via cbf80c08daa5f9099d0821603a128967254709ed (commit)
via 56ad30382d7d4e11cb0160c5f2e30077e1f41c8b (commit)
via b20590222beddb52c8c89294ed3a324c8c7190a2 (commit)
via 13f83b9374e66e4609aff661b467d747067d66c2 (commit)
via 64416e4751edfe6c49c0bed8a7e38071200282d8 (commit)
via dcba3e2c566b2cc3d0575f4b11a1deac808d5cd6 (commit)
via e1523c518fa4bf04ac4c982d0a5dbd681dea279a (commit)
via 9fc99919d72ee495e66ce98584189c651cf994c1 (commit)
via 0c8f599d598f36d67daf0e0e39756ba4d064cbd0 (commit)
via d1957808f6e3ccece499ac2f4048d4ef850b262c (commit)
via 1f8fcb0279a7bb2aa9cf1386ff9516da58216d53 (commit)
via 2cf42c27a7e8b37e29462d0b695e24cb6f3ad5ce (commit)
via bd6f17515de33e6eee9631723730fc65125ebad2 (commit)
via cd3019ce332106ebf80b4323f0f24e71025adcb1 (commit)
via 90af4cd87e9525481bd0bc8120a18ebc27c9a459 (commit)
via c58a2c83fa6338358962b8161c576e5391d7bf2b (commit)
via 9ae339f1aab32d8473f366c7aaa25633a5a49008 (commit)
via 3d0c0753efb6e2b610d23d16038db218b491a70f (commit)
via 288413d1c5efcf5d207e0556962740a7759891a1 (commit)
via 378c32cf74c1c9d559e3b8e559a9dcdc77ff2017 (commit)
via 35c50eb231e3ea0f469db136c92ce9bb9d853ac1 (commit)
via 08b3d5b95216643081c7749bc84a09659d554b7d (commit)
via 12abd300828412255248e98754b767df5deeba3d (commit)
via cad21664e646dd103996dcc36839e77bfd17cdd5 (commit)
via fefdb915c4cdf2c2d92061d9221eacd7cac4682f (commit)
via de6150af4a477390eb8ba73c2f67c2c46c91a3ef (commit)
via bf9d2be7afcc640e18606fa8b9b2e0bc3f2190d1 (commit)
via e31b4ee6fa3da212239dfb5a2e4761ff51ed0928 (commit)
via e98e77b844dc5a4d2dcfa0752f3bd6b74822d88c (commit)
via 549f0a0d6f686d1472b6d5bacc3eb85927c915d5 (commit)
via fa166d52969bb6f002fb62b554ef227194e0febe (commit)
via 74b859d9429c1f048f8e71a799d6a44ff3e870d7 (commit)
via 970766e3167be72c1fe6abdc1609831721dbf62d (commit)
via 18b5b6fa43e380549c6698b9998990a748a9d3e8 (commit)
via 56714c8df2f49a5b28ded29402d1af0cb3e45ba1 (commit)
via 60789154369c7a882561dbecff466787acfef6d5 (commit)
via 063c5461ff2a709455536c759d849d2f393bda68 (commit)
via e0619201d96eb7f9cb8229d1c883f0665d1488ba (commit)
via 906aa5e3427c1e89e5f426191e33af4b1c27fc7e (commit)
via b495d8bd000cc60e2288b93a788aa71cce7f6dec (commit)
via 4d8209df92198e6207d3d38cbb7a189cb319bf3c (commit)
via afabee438a48de83632f0eec51f78a0529b71a0e (commit)
via 5b7495879ac4e9b34fc989f58aae4773f56bb191 (commit)
via 1b39a6e15bb68a088f25bb31c31298a1155dc26c (commit)
via 416c543625bb6a7195a48988dc8f32643ca10aa1 (commit)
via 4cd97b2cf2035c44762865f10b0f51e3ac807566 (commit)
via 0d2e6cf379f33188fa19aed9c0c246a2514e9e81 (commit)
via 9fcabe977798468f2ee896b5f5c1ba6d80703341 (commit)
via b9a8de2241fdbc6c69511efc4af318209e4b4942 (commit)
via 9ea36303cc851a5ffa8d61695c8b4ed14e8954d5 (commit)
via 89a8208e6f88de78991c654ce001b26519b99f0c (commit)
via e1104f98771283a7659eadf881f006e3a3acb4d5 (commit)
via 17c5cc48844053d6aec318fa3fc8fe95b2cf1b4c (commit)
via 42fc0557db4117d736b7511a0f785bdfa1d5111e (commit)
via 7edde4785f7cc325a8c2b109d6fcda176af2650c (commit)
via ec27ba2a576189755d443d54213318741f73d125 (commit)
via 50ab35e7c81a9c1363da289bf76424d669bdf80c (commit)
via 4bc67f80590ebbfe530d55b9109542f2b404e7b7 (commit)
via b23240bded4d76bf953a4f8c499f58d9066c34dc (commit)
via 33e6d3356af3f7eaa484dbaa7a671aa25f5042e4 (commit)
via 03428065459077f94f44c53573944f22bda63779 (commit)
via 9a6559f936fb93f63e163690be4ed670a76dd135 (commit)
via 64e9180b32bd158e157d3a11325d70b90ce1aeac (commit)
via 89d4aec69e1b6d8d1f687951f39d8e11f0e66ecf (commit)
via 96326e988977cd5147dcd4962d6c02bb43e7b330 (commit)
via d487907328f3581cd7c93f73729b1e089430523d (commit)
via 16e6df55c039d4986cdf789b94c62840819beba7 (commit)
via 7f2c45b1312014831e2efd8aa0fb8a116085b036 (commit)
via 89fb910b523686fdf725691c44cb4c63ba464487 (commit)
via 4470ba26b332cb92d347af00cdb26c716b1a6953 (commit)
via 411ce5bc5bf433c976f55fe001fe5979456207f2 (commit)
via 876ca8678991e6fdd96872226cf50c92a2e58229 (commit)
via 332015d1131801b0280aa37aa00eefa5c3c00bd4 (commit)
via d01477c59395e6d0895fffe0f60cce6bda9bb083 (commit)
via 5c1cee516b4d41a355fd40a538cf695325e0b712 (commit)
via 73d8ff765611e02a44525a75e1b97348ea3dc185 (commit)
via a1ad1f1d7ce5edf62945340c34ba5e22a36f1157 (commit)
via 13181107ecaeaa92e5d96b05270e56b2d807af39 (commit)
via 62eaf39454cf7d3874691fc59174c546767de89c (commit)
via 2c447c25f810b52e986649a3f4138c671d066092 (commit)
via 48580ba7a6608c89e91afad4b73f2861aafbd7b2 (commit)
via 0d32948a1c78385f9305799c5cb47127492c4320 (commit)
via 1a462bc4d29fc17cef377d232fc2bf0fd0e72358 (commit)
via 6b972e1753c0606eade16e98c32f8da2f7678652 (commit)
via 9f931ff8477f5ba02b53dcc7a8f65032f0a6d013 (commit)
via 4d1e5c0506cf21824acec1bd8bbf7901b8f4ae6c (commit)
via 7f6d2dbe9ce1a294b4be897e79d612d8c1db71d0 (commit)
via c8f0f0c517af5462c68bcadd72e557e390f1ee19 (commit)
via 487dc40959c4a7a8838624c6e108236320e79c8d (commit)
via b6ca3da4f529dc300f750891022847b071813e57 (commit)
via a2e9008a5fcd7669d5cdd33021212ac754288d19 (commit)
via 0507f73ce83c4a53ce8734f21635928a18d228ad (commit)
via cb6a13cdda9c9b8e2c1dbb151c432ec109e14c82 (commit)
via 934fa5ac0e4da50705739303b55f694a60665529 (commit)
via 71d9ea52ae12dc80d7062657fc5c059a128106d7 (commit)
via 94a3addbee3e4a293ba23483dc34ae316d93fbdd (commit)
via 5dbde597e597b4360584d70eb0dccfc6b48c07d0 (commit)
via f9a0cb85f62f1053a9fc1216689b75e6081cf3d7 (commit)
via 655b69e4285476fbd9df4a88ea53e02ea93fa349 (commit)
via 774bc42d958174407d74bc3ef44b2840c2f8a05f (commit)
via bed95fb1306b0708ae61d1977984944430f01bca (commit)
via 9af0808e182e297b9c957c93f629016e1517eba1 (commit)
via 9fb82043d0b69e7d7262b617a4c76ae42b636dab (commit)
via 1eeb6688285b7415f48b016af27790bf116aa968 (commit)
via 51f9e6cb1d83249133fa9981cd7b1a882d11964b (commit)
via 44cbe230042e6fef66f7e05514927f8b51c77657 (commit)
via 088bc7b980536ee2b27c8abf4bfc09c348000589 (commit)
via 4215621e0a148252e988088e5836689ca63920a9 (commit)
via ae7a6c9ad19ff6936c35ae8c29c08b7adeb835f6 (commit)
via 8233babd1d979b545e0b8f15455787af66307d9a (commit)
via 29e2e288979dc11d690189532ca24af949d66c8a (commit)
via 63ea99a3f7dbc8cd484d32a0f42d766d2b0c3743 (commit)
via 5ae2677e5ba023d23d10b2a81c800a007aa80646 (commit)
via 04a8e038125c6ffa3c912ba6b1fd0b31af01e7ad (commit)
via 7d76a3fbfbe15a6813df5d2d4fa111f1b8e62f9c (commit)
via 1060cff7480331bcad2b4564e270922b2b3b1f94 (commit)
via a533cde870452a69fd7db28806531475aba81486 (commit)
via 981c7264123f24a1873f2692a72f012ac43e726b (commit)
via 9fbf89259b2c4313334a1a2c8f0f30e581cc932e (commit)
via 2717e707ce048dd9b47754d620663a76256958cf (commit)
via 390d19356d2b6f16bebf918103a83a0ce6ede9d2 (commit)
via c67f050829438c41b994c29f689db0ffdf28ee82 (commit)
via a51da9f0338017a7c193dcb36fc957b9ab71e4b1 (commit)
via 0fb26747fa229d6b19ec911b907259a8e84acd83 (commit)
via 14b6221c34638d4f63c023ad806dbd1bdb22e7b7 (commit)
via d2299e0187a0b8621a17c9819215489b979da150 (commit)
via 1ceddc7165cfaa14ab5fb56953e8b0094b791d07 (commit)
via 805ad01a254753e2543123e67530fde79ecb00bd (commit)
via e2999ea6cca6ce034ba0fc921f91314776b76200 (commit)
via 7ee4b2b591bba8425a9c18dd1bf63d48911a686b (commit)
via 687f94f0e17dca2639ce1704b18f483f2e6bedb3 (commit)
via 4884240effa1cbbcc74774a8b14d39d47b483ce7 (commit)
via c6c4e8a6e0acb0dad79bebc40ca09ba2390da6ed (commit)
via 620283796c61a1d09ab8fa9c6ac9842c438e625f (commit)
via a29c6fb9134cd81a2b14426dc5863c68cf8eb2f1 (commit)
via 4926ecdc0ab989d329fbb75e447469ed4efee184 (commit)
via 829551057b61b57d1d08aae39c9095da30166f2a (commit)
via ddc4a3cdfec39a1b2cc37bbd1e1920ed30d17c49 (commit)
via 77d1908a811b84008cac3834fac542c7b301e1a3 (commit)
via 6e43023a9bae34280b2ce364a0adf67f16305615 (commit)
via 643bf50185e8cc9abe4abd0d441b5012d1760cac (commit)
via 194eaad976697799f0e7dcccb581b52dc77feee3 (commit)
via 1e133a2abc1827f210eaba7db39a92d20f8b5876 (commit)
via 3957f3dbb398a1039c8ebb12a008bf9a7f17a4f0 (commit)
via ef2d368867bfae2ca702184c57a8cd873f1c4e43 (commit)
via 99a25735aeed6d5a3cd895a6a91f269d8bc8b3f1 (commit)
via 4a7010560e08d8c34656ccf2a7a9fa56fe21d40d (commit)
via 2cdf164495a95e2c424652b421973b04aa597572 (commit)
via fcd5763d2e36f2d9b87c0e77d2133ef078755b89 (commit)
via 0ba8a5a067e22b09568da468b306ec26607c011b (commit)
via f5d682412c2b8ed0508fb55c2e8b7437b9ef142f (commit)
via 71a9cb71e4cabe1444f68222c0bd7ad4324d6a5e (commit)
via e89251c160ae409f8af2f9ecae5ffb210ccd0a8d (commit)
via ffa928f7e9752b3bfab87cbca14392909416bf91 (commit)
via 16c5d87acf21a189b4f7aacd50a3231a795153bd (commit)
via 22ded15b4bd32f1de6e1b97f39ea869708d02085 (commit)
via 2b9204f0eb626f6fa4159752b76e8864508afa08 (commit)
via e9025ef3cad5e1da15e29145ce637d7097971148 (commit)
via 4242f1bafd192503419347cd02dfe508fe26135f (commit)
via 0dfab8a6e4c9e10361de36a4c34d38e19c690035 (commit)
via 5799ec682138f41da615c253ad38af834c120497 (commit)
via 32002eb6e87a391d1fad2b1b206dbc74d6659fc5 (commit)
via 58e26628d80daff38e286d4bcf4efa73c3b293ef (commit)
via 8cd6bbad65698c707313ebd66c4fc7530aec6415 (commit)
via 6f3e7bf1bdfc66cfa6c6091566425c870154943d (commit)
via 292689282fd68d09dbd424d66aec7f664cadc6e6 (commit)
via e5025b6b54a9f9b630fe69954da362fc2b5eb62f (commit)
via 8caa085f488017b7152a55c7ce0aed166e860d6b (commit)
via f8164fe91b06263aacebfad42c2c0df26037d162 (commit)
via 38e27663cf656f0c9c443a2715f249afe39a8bfb (commit)
via 08671179720e2e8a96ef740c1b9c60c931477d20 (commit)
via d446d49dd75f14f3b454fa89190068b4e475c946 (commit)
via 045fc7238eecb1cc1b005c2fbe1325c9703986b9 (commit)
via 4c731f7597f251957b4ded726a0c33d8b8bf0ad7 (commit)
via 68bc991c90b76cb367840c636cf7e065d26ff79d (commit)
via 3298dd77cd065e9fab95177fe5480d4ec2524038 (commit)
via 1fae5d56406b1f6f713899392eb073c19b6e8023 (commit)
via 5c1db683ce8a1192508fb6e84993b139252b7d60 (commit)
via b4fd2ab4e409e36f894c1fb27144e4fa2854f389 (commit)
via 2ab024a39d41977cb56c699cbc0bca19b8ca3ac9 (commit)
via 316eca14d7f7bafb2e0c24b125dee0befe5bdce6 (commit)
via eeebd1e25974beff2455c96100532aaa4dae68fb (commit)
via c430a289af3eeed00e220b5658f9d64191798b1c (commit)
via 89fbb8bf806dde53fd7cfc761cf3d0719fbebf59 (commit)
via 27205826b53fee3d76f1e0128c4ad4c26bb0a3ca (commit)
via 224701a0e17975957d3fcc0262c50806e2a7698d (commit)
via 261192400cbf11f2f21b200cf1ceb7779856f65f (commit)
via 7c34347eed35a0713154cb19ff2dbb8b41f8e729 (commit)
via ef969ca8dabe571a9866a7b3b7c39098785022fa (commit)
via 575443545936e6b8a1a4783e3d0e17d0246c6e99 (commit)
via dc32d1900e2070d43f0321b1a7df420ff3f601be (commit)
via ffcfb73e6a5dfd626b1d523accc42e7a6af5e8ed (commit)
via 6d93d2ecf9c3d75c2b032f0ad9689af6501570e0 (commit)
via 7349451eb29d3bd972f6f051f1cf14459b3fe14e (commit)
via 030e58ed7a775922a6914cec63c7a6b0c7ede5ae (commit)
via 279c6f8bc69a3edfcf7213e0c1b7621f60a46c06 (commit)
via e6c0c0562df1c133ec58720b7bf580692ea696e1 (commit)
via aba427922c856a496418a422c9d64aea8401810f (commit)
via 9c569a32645164e99fa44867626bb5a11887c338 (commit)
via f53484e4d86933cefdca5ca967658f8ffba200d6 (commit)
via 7eec2f46099c6bf913149a7d19c4903feb2bf835 (commit)
via 9707b0c9304596799d01cf5c2d1d630026a49d79 (commit)
via 47585fc714a92d26e008f2fe9a46357f5ad9070d (commit)
via 692ecbf5b376381275b19bcb3984e74dd72e9b22 (commit)
via 570a38bc06cca2a2902f7da3d0a2b63709480845 (commit)
via faf89f7e200e69dca019cf530e121d08ac06f105 (commit)
via ac8b77cf3ed0d979e7e6b73c09ce552a51aeb3e9 (commit)
via 420a4a7230e85ef816b39e73f7192348a30181c6 (commit)
via 6b27e59da1d5790ed53de12ffbf82c83de50088c (commit)
via 003f0f21e2a7052b29f375b725f90668e46d49b8 (commit)
via 1b01104f5e4a20a3ede82b6d7250814476e23dc9 (commit)
via 594a1e9eb632ab9deaee7d596e3004b48277e2cc (commit)
via 691d642124e94bb1bc8f662497b00e6c39597fd8 (commit)
via 962ca37032fa5170b55e2661f3b577d8497d1fe9 (commit)
via d2dd87596f3034fd6250e8c957ee4e97a2939470 (commit)
via 7631b2c9e33a22a9b47fe3396f2f2854745a3ded (commit)
via 204c462041de0b063ed1169c0f600f082400815f (commit)
via effc4bd7d29bc7af805e576391061d9d68f2db31 (commit)
via 9175278147b9aaaa9dc09f73e34e20f07a791c33 (commit)
via 1ec89c1c31669eb89bd1997cfa9d3c50f0204dbe (commit)
via 59963e1ee432a51a3f7ae3d98068a541b2738879 (commit)
via 904e7471a5f5c00ba8e81e878e6d2371926b29e6 (commit)
via 964ab3dd90ff1508efc0c77378cde2b3a4da1029 (commit)
via f9732ad8460d013c2f28363655d0d1b91894dca5 (commit)
via 9d6e2ff4e27718938a72455917f2e7b62a8c66dc (commit)
via b97ac7f96234cbbb491bdbaade840ab50802f357 (commit)
via 8d61b3420d4918010cd39ac3e2bc8687b440bc9f (commit)
via 7d4e2529d04c3649ffecdce7a9e1f9835387454d (commit)
via 3fec99e70492e163763ed185aafbf022bf5e16b7 (commit)
via f117f0a6cde3866b2a0bef7c45f30d71f33c6786 (commit)
via 5933c83688fcb9cfd7b45b52c171fab6a1e894c5 (commit)
via 432f7af8ccddacb7941d94a833cc24551c2ac5df (commit)
via 2ce797b91ad36d8b682f19b99732d2465b938019 (commit)
via eb7dde727b22ff92f9c63f4e4ac704a7fba97249 (commit)
via c66671061105e9569e4ddcc9fe50efc171951ee7 (commit)
via 952bfa87465a27f83dca7feca7d369fda4200eb5 (commit)
via 51835046f9dce64c06329c43ca2042b6a7574b5e (commit)
via 1acdfd97cb761eed9357b8c0e59ab80dc56c2652 (commit)
via 079a14ebb221f2f0b4aa9eca266cc3efb8eb0150 (commit)
via 7ced789760c7e7b4df4fe3ddecf0611ff1fe12e5 (commit)
via cc13e1f6081cf51216f137a2bb6dbd85d0c15d1b (commit)
via 9ca2424764925abd825a58be9f1ed1f54bf24e1d (commit)
via b83c44a8537de8ade4f08efb48cc1a68785a9f8f (commit)
via 212e68e9df1ee2bf2b9642d87452e4520532f84d (commit)
via 013c95756d5578e596fbe3c21e1e0a3e7a1b0cc3 (commit)
via 306bd74955208294cc15b790b189f5f2656a949f (commit)
via ea84b1dc3c09fbb0ab72d15767e4cacb8f37b12a (commit)
via 0d2745ce5adc885161377b046cd1005a3822f48d (commit)
via cadb785fc63280862d71376def0128e4c70951f0 (commit)
via e360b9a4cbdc3d1fd8a93326349517754e7314ef (commit)
via 9a894a1fb3409eb50216de9186fe37704a3cf46f (commit)
via a91c9255e07d97ab23e77228ef17b8c0416c31d7 (commit)
via 93e77cdfaec21c3f53bf9efa3cfddb75971b12b7 (commit)
via ddb2032b5396a02ce63bddbf55bbc624540a14f6 (commit)
via f5588e5c01baf3084ad42d8be506c4698b1affce (commit)
via dde9d2845e69c4e2b50964e6a00f3fdb9053dbf6 (commit)
via 3e1af54110b9168cccb9ea624da72e202d275b5f (commit)
via 57f1fd52bb98ac58751e8c9a8f874eca1668b65b (commit)
via f880a22da46a0af8606284a344c7bc986f65be28 (commit)
via c4f571a3111819e1b6fb9aa53be5469ed05b8c84 (commit)
via 927f967049c054e41ef4ad51b7eb3dfd631fe2ce (commit)
via 9c2e673ba96ba5d11e56a4aba67157e64dfe01e2 (commit)
via 3fa179311344ed08b9bb8df113fb7901eac13ee7 (commit)
via ae23b3770a1df1e952265c2675c6b17e6c3060ba (commit)
via 1371663d86021d17a218f7103871df0f80a3d356 (commit)
via 553276ad4d4c000604f019956c8af86397d86b33 (commit)
via cb833bd3216cb09a36b171d1805f3c0ad6d03d64 (commit)
via 752b9167416a5c8b64bef4946fbc15939b4bfe75 (commit)
via 1974e0e333395ee3eccedc2410441a7b018187da (commit)
via f4cd3a9893e2f4a0b6fffcd6824a422e3d603b96 (commit)
via eec57e32699568c315ca1134e9186a04c76d2359 (commit)
via f2834f4447a9b9b601c0b4f49434a77f9f054e50 (commit)
via 2d095685cb582285e7ed89c7a984e5277aae0104 (commit)
via cd4f50633b256fdf38ae749e6f00ace10c7d45c0 (commit)
via 351a497c96770f379acdfe58ebeff34262e1308b (commit)
via fa82df2ec3cab51eaaadd2a31b457d0e2ceafbcb (commit)
via 554728c26640c8ac26c3e97b5c3e06b6e4be3e15 (commit)
via e5ed756501133df3fbcce69399da46439a8d3e6c (commit)
via 3e4845620b14d478d088de0ca73ce227353623b7 (commit)
via 288d22d8a7ff1f9a441d2b8058382e807873d7d5 (commit)
via 5f4ca24ce0d52857901664c3bb5e228316af4d74 (commit)
via 2185f76a9aeb9fbde23e189cda1db4dcc0949177 (commit)
via 0af4622373325785f76f09aea8fdd71aa3c03339 (commit)
via 138a4b0a5de177faede5255841a5c6fca06b31f4 (commit)
via cb15699af6eb0cad3560280b741a135b9af57a80 (commit)
via 9fea1c7774bd256788ee76385c0eab05d4508796 (commit)
via 5fa38c36644bd122b8c31601aa864f05bf4fba73 (commit)
via 251cc49d99c14d28942636fbcef9515bd39c9575 (commit)
via 220be2d936dc16190bbfa40f44f3be69a36432e3 (commit)
via 591446b97754292031a4214f08039adebd56cfe8 (commit)
via 77d377e3bbe37ee7dd36691d5ad04ae0def08d4b (commit)
via a66a265608dbde7f3dd02dcd64ffba55f6a772fd (commit)
via e95e66b8be5d92fe2890f9e33e682cc3c8c6cc73 (commit)
via 5aacaabf6190d644bc957903f0f712959234fa07 (commit)
via 3c81026770b7d9251116679b549f116533f4f469 (commit)
via 5810ac278a9abac129369dafcdb42436452c74c8 (commit)
via feb8948b4c8096be5ed0956eac6f8fa629fef8ba (commit)
via 198963abf16d86d809a6a9fde36bc9ffd74685fd (commit)
via ec7cd27f399097d76822eb05867102e2b7d19be7 (commit)
via 226d028a3493a2b2c539449133f6742650a19d04 (commit)
via 42fec323465b515e85a8bd460b2cee7972abd658 (commit)
via 11c04627d0fd81782d6fff890412c7a820cb370a (commit)
via a2806c0a3f7cc1a2c38539a6e208dff7a3172a88 (commit)
via 8295ba721133c341fd72350f7ca9438d1a99cdbe (commit)
via cf670551c6832408f3f24cfad892cbfdd94d913a (commit)
via 874f973f3af6b5dee65bfa3c55658430dadb5451 (commit)
via 3566686444d84498031b8fdef9cbe78bcbf54b9a (commit)
via ea28328480051317963e31c9be155c2c28babb09 (commit)
via cac35f0e43192a790eee669f83c26c7d47ccbed4 (commit)
via 40bbccf9acf872cd610adeb317f06f5ba57504a0 (commit)
via 9b318db813552ab4d3c8e4c904ed32bf03779ef7 (commit)
via 840592d4a4e02072c982aeaeab5b38daecfcd840 (commit)
via 942cee2087227f066286e77985a3a7af21fd47a1 (commit)
via 7d6b0f37bf50907bc0664e177c1c8c8cc697b4f8 (commit)
via 34971328ff74e671777e80fd11d79b8a912e4e0b (commit)
via b910bb3156993b8c6c8cdae2f562e77c6a540a6e (commit)
via 164ebe21b6dfa0de9db6cc450dd65ce734c43e21 (commit)
via 46c6882969833a640c877a958921d25b36bf4dfd (commit)
via 8714664ec145da92155e0554025491f57b393f28 (commit)
via 67414a0a471a8046dac310c9a924ae6c0ce674c8 (commit)
via b6f73f7acffa9b2f37c78826ff6cee89987a7edc (commit)
via 49030c3161523bc427e0aaa9d3edb517fc4c296c (commit)
via 74eaca91f86f352ddd4ea40ab49a4aaa484e9610 (commit)
via 9468edd5242e5372ce2ed36f2a7448b63a5b45b1 (commit)
via 9df51438367e129d0d306801eb20215cc2a0fafc (commit)
via ceb2ed3a4f6c68a1c95d92271ba77b5e3d8b6a58 (commit)
via db8213fdb447e701467787f631beba495d0c7fbb (commit)
via 48c12769b32edce2b2d7592c58dcc7ed9ea63369 (commit)
via 011cdc12d3d3a664e8a8ef09bc0e7f5223afa6df (commit)
via 4edb153ec9217aff3f833e9d8c645f9b525f41c6 (commit)
via ba0977d3755ada267f60b1f0a3299e267f013ddd (commit)
via 9bec80bcc55f2985f694b8172e755e94e11f59ae (commit)
via f7e3001b53e9f26fd1d413308093bb9700bb9287 (commit)
via e11eaaf4620e9d5da4e575f8139c4c4c372604ec (commit)
via 9a385c7c5a99dae7a365f84a1536ed324a52f85b (commit)
via c0c8ae6e8e64f268346c52b02f293a3e4a18b529 (commit)
via cf09cb606af8c08d07e99149734e903c11f497f6 (commit)
via a63c26327c01ff188451365efd72d69d231653c0 (commit)
via 8f9055ed305180509af7e2527a5bfee84d417a58 (commit)
via c48d1065413543feff55260bb46b29f984b228df (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (8d6532d6f5db2927f353ebe1cd75dcd1e189873a)
\
N -- N -- N (f3250432a47c835f4c594348b0d4904a247c3365)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f3250432a47c835f4c594348b0d4904a247c3365
Merge: 8a504ad 3598c30
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 17 13:46:32 2015 -0500
3408: Merge branch 'master' into 3408-production-datamanager
commit 8a504ad561c1ffbafee8a7bc8da551f9d4b9a29e
Merge: b7bd673 1128f6e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 13 17:02:16 2015 -0500
Merge branch 'master' into 3408-production-datamanager
commit b7bd673a45ec3de02fdf846f4a9ebe2638c546cc
Author: mishaz <misha at curoverse.com>
Date: Tue Feb 10 02:39:39 2015 +0000
Added different event types for started, partially complete and final log entries.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index e6e1ed6..e7bc8d1 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -1,9 +1,8 @@
// Logger periodically writes a log to the Arvados SDK.
//
// This package is useful for maintaining a log object that is updated
-// over time. Every time the object is updated, it will be written to
-// the log. Writes will be throttled to no more frequent than
-// WriteInterval.
+// over time. This log object will be periodically written to the log,
+// as specified by WriteInterval in the Params.
//
// This package is safe for concurrent use as long as:
// The maps passed to a LogMutator are not accessed outside of the
@@ -26,10 +25,16 @@ import (
"time"
)
+const (
+ startSuffix = "-start"
+ partialSuffix = "-partial"
+ finalSuffix = "-final"
+)
+
type LoggerParams struct {
- Client arvadosclient.ArvadosClient // The client we use to write log entries
- EventType string // The event type to assign to the log entry.
- WriteInterval time.Duration // Wait at least this long between log writes
+ Client arvadosclient.ArvadosClient // The client we use to write log entries
+ EventTypePrefix string // The prefix we use for the event type in the log entry
+ WriteInterval time.Duration // Wait at least this long between log writes
}
// A LogMutator is a function which modifies the log entry.
@@ -57,6 +62,7 @@ type Logger struct {
modified bool // Has this data been modified since the last write?
workToDo chan LogMutator // Work to do in the worker thread.
writeTicker *time.Ticker // On each tick we write the log data to arvados, if it has been modified.
+ hasWritten bool // Whether we've written at all yet.
writeHooks []LogMutator // Mutators we call before each write.
}
@@ -67,8 +73,8 @@ func NewLogger(params LoggerParams) *Logger {
if ¶ms.Client == nil {
log.Fatal("Nil arvados client in LoggerParams passed in to NewLogger()")
}
- if params.EventType == "" {
- log.Fatal("Empty event type in LoggerParams passed in to NewLogger()")
+ if params.EventTypePrefix == "" {
+ log.Fatal("Empty event type prefix in LoggerParams passed in to NewLogger()")
}
l := &Logger{data: make(map[string]interface{}),
@@ -119,11 +125,9 @@ func (l *Logger) FinalUpdate(mutator LogMutator) {
// Apply the final update
l.workToDo <- mutator
- // Perform the write and signal that we can return.
+ // Perform the final write and signal that we can return.
l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
- // TODO(misha): Add a boolean arg to write() to indicate that it's
- // final so that we can set the appropriate event type.
- l.write()
+ l.write(true)
done <- true
}
@@ -146,7 +150,7 @@ func (l *Logger) work() {
select {
case <-l.writeTicker.C:
if l.modified {
- l.write()
+ l.write(false)
l.modified = false
}
case mutator := <-l.workToDo:
@@ -157,16 +161,22 @@ func (l *Logger) work() {
}
// Actually writes the log entry.
-func (l *Logger) write() {
+func (l *Logger) write(isFinal bool) {
// Run all our hooks
for _, hook := range l.writeHooks {
hook(l.properties, l.entry)
}
- // Update the event type in case it was modified or is missing.
- // TODO(misha): Fix this to write different event types.
- l.entry["event_type"] = l.params.EventType
+ // Update the event type.
+ if isFinal {
+ l.entry["event_type"] = l.params.EventTypePrefix + finalSuffix
+ } else if l.hasWritten {
+ l.entry["event_type"] = l.params.EventTypePrefix + partialSuffix
+ } else {
+ l.entry["event_type"] = l.params.EventTypePrefix + startSuffix
+ }
+ l.hasWritten = true
// Write the log entry.
// This is a network write and will take a while, which is bad
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index bd68db1..2c5c36d 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -15,15 +15,15 @@ import (
)
var (
- logEventType string
+ logEventTypePrefix string
logFrequencySeconds int
)
func init() {
- flag.StringVar(&logEventType,
- "log-event-type",
- "experimental-data-manager-report",
- "event_type to use in our arvados log entries. Set to empty to turn off logging")
+ flag.StringVar(&logEventTypePrefix,
+ "log-event-type-prefix",
+ "experimental-data-manager",
+ "Prefix to use in the event_type of our arvados log entries. Set to empty to turn off logging")
flag.IntVar(&logFrequencySeconds,
"log-frequency-seconds",
20,
@@ -45,9 +45,9 @@ func main() {
}
var arvLogger *logger.Logger
- if logEventType != "" {
+ if logEventTypePrefix != "" {
arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: logEventType,
+ EventTypePrefix: logEventTypePrefix,
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
commit 43538243995267c417983360d226d6e8eb181139
Author: mishaz <misha at curoverse.com>
Date: Tue Feb 10 02:11:34 2015 +0000
Moved some logging code from datamananager to loggerutil.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index f63f462..bd68db1 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -9,9 +9,8 @@ import (
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
"git.curoverse.com/arvados.git/services/datamanager/keep"
+ "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
- "os"
- "runtime"
"time"
)
@@ -52,23 +51,9 @@ func main() {
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
+ loggerutil.LogRunInfo(arvLogger)
if arvLogger != nil {
- now := time.Now()
- arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
- runInfo := make(map[string]interface{})
- runInfo["time_started"] = now
- runInfo["args"] = os.Args
- hostname, err := os.Hostname()
- if err != nil {
- runInfo["hostname_error"] = err.Error()
- } else {
- runInfo["hostname"] = hostname
- }
- runInfo["pid"] = os.Getpid()
- p["run_info"] = runInfo
- })
-
- arvLogger.AddWriteHook(LogMemoryAlloc)
+ arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
}
collectionChannel := make(chan collection.ReadCollections)
@@ -96,11 +81,3 @@ func main() {
})
}
}
-
-// TODO(misha): Consider moving this to loggerutil
-func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
- runInfo := properties["run_info"].(map[string]interface{})
- var memStats runtime.MemStats
- runtime.ReadMemStats(&memStats)
- runInfo["alloc_bytes_in_use"] = memStats.Alloc
-}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index c19a7ab..1514922 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -5,9 +5,42 @@ package loggerutil
import (
"git.curoverse.com/arvados.git/sdk/go/logger"
"log"
+ "os"
+ "runtime"
"time"
)
+// Useful to call at the begining of execution to log info about the
+// current run.
+func LogRunInfo(arvLogger *logger.Logger) {
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ runInfo := make(map[string]interface{})
+ runInfo["time_started"] = now
+ runInfo["args"] = os.Args
+ hostname, err := os.Hostname()
+ if err != nil {
+ runInfo["hostname_error"] = err.Error()
+ } else {
+ runInfo["hostname"] = hostname
+ }
+ runInfo["pid"] = os.Getpid()
+ p["run_info"] = runInfo
+ })
+ }
+}
+
+// A LogMutator that records the current memory usage. This is most useful as a logger write hook.
+//
+// Assumes we already have a map named "run_info" in properties. LogRunInfo() can create such a map for you if you call it.
+func LogMemoryAlloc(p map[string]interface{}, e map[string]interface{}) {
+ runInfo := p["run_info"].(map[string]interface{})
+ var memStats runtime.MemStats
+ runtime.ReadMemStats(&memStats)
+ runInfo["alloc_bytes_in_use"] = memStats.Alloc
+}
+
func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
commit 3a1face2e3bc02e1fb9c53a2268095811b2e069d
Author: mishaz <misha at curoverse.com>
Date: Tue Feb 10 01:55:37 2015 +0000
Updated logger to do all work in a dedicated goroutine, so we don't need to worry about locking. Small changes to calling code.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index a53ab3c..e6e1ed6 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -2,8 +2,8 @@
//
// This package is useful for maintaining a log object that is updated
// over time. Every time the object is updated, it will be written to
-// the log. Writes will be throttled to no more than one every
-// WriteFrequencySeconds
+// the log. Writes will be throttled to no more frequent than
+// WriteInterval.
//
// This package is safe for concurrent use as long as:
// The maps passed to a LogMutator are not accessed outside of the
@@ -23,14 +23,13 @@ package logger
import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"log"
- "sync"
"time"
)
type LoggerParams struct {
- Client arvadosclient.ArvadosClient // The client we use to write log entries
- EventType string // The event type to assign to the log entry.
- MinimumWriteInterval time.Duration // Wait at least this long between log writes
+ Client arvadosclient.ArvadosClient // The client we use to write log entries
+ EventType string // The event type to assign to the log entry.
+ WriteInterval time.Duration // Wait at least this long between log writes
}
// A LogMutator is a function which modifies the log entry.
@@ -52,13 +51,12 @@ type Logger struct {
entry map[string]interface{} // Convenience shortcut into data
properties map[string]interface{} // Convenience shortcut into data
- lock sync.Locker // Synchronizes access to this struct
params LoggerParams // Parameters we were given
- // Variables used to determine when and if we write to the log.
- nextWriteAllowed time.Time // The next time we can write, respecting MinimumWriteInterval
- modified bool // Has this data been modified since the last write?
- writeScheduled bool // Is a write been scheduled for the future?
+ // Variables to coordinate updating and writing.
+ modified bool // Has this data been modified since the last write?
+ workToDo chan LogMutator // Work to do in the worker thread.
+ writeTicker *time.Ticker // On each tick we write the log data to arvados, if it has been modified.
writeHooks []LogMutator // Mutators we call before each write.
}
@@ -74,91 +72,88 @@ func NewLogger(params LoggerParams) *Logger {
}
l := &Logger{data: make(map[string]interface{}),
- lock: &sync.Mutex{},
params: params}
l.entry = make(map[string]interface{})
l.data["log"] = l.entry
l.properties = make(map[string]interface{})
l.entry["properties"] = l.properties
- return l
-}
-// Updates the log data and then writes it to the api server. If the
-// log has been recently written then the write will be postponed to
-// respect MinimumWriteInterval and this function will return before
-// the write occurs.
-func (l *Logger) Update(mutator LogMutator) {
- l.lock.Lock()
+ l.workToDo = make(chan LogMutator, 10)
+ l.writeTicker = time.NewTicker(params.WriteInterval)
- mutator(l.properties, l.entry)
- l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
+ // Start the worker goroutine.
+ go l.work()
- l.considerWriting()
-
- l.lock.Unlock()
+ return l
}
-// Similar to Update(), but forces a write without respecting the
-// MinimumWriteInterval. This is useful if you know that you're about
-// to quit (e.g. if you discovered a fatal error, or you're finished),
-// since go will not wait for timers (including the pending write
-// timer) to go off before exiting.
-func (l *Logger) ForceUpdate(mutator LogMutator) {
- l.lock.Lock()
+// Exported functions will be called from other goroutines, therefore
+// all they are allowed to do is enqueue work to be done in the worker
+// goroutine.
- mutator(l.properties, l.entry)
- l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
-
- l.write()
- l.lock.Unlock()
-}
-
-// Adds a hook which will be called every time this logger writes an entry.
-func (l *Logger) AddWriteHook(hook LogMutator) {
- l.lock.Lock()
- l.writeHooks = append(l.writeHooks, hook)
- // TODO(misha): Consider setting modified and attempting a write.
- l.lock.Unlock()
+// Enqueues an update. This will happen in another goroutine after
+// this method returns.
+func (l *Logger) Update(mutator LogMutator) {
+ l.workToDo <- mutator
}
-// This function is called on a timer when we have something to write,
-// but need to schedule the write for the future to respect
-// MinimumWriteInterval.
-func (l *Logger) acquireLockConsiderWriting() {
- l.lock.Lock()
+// Similar to Update(), but writes the log entry as soon as possible
+// (ignoring MinimumWriteInterval) and blocks until the entry has been
+// written. This is useful if you know that you're about to quit
+// (e.g. if you discovered a fatal error, or you're finished), since
+// go will not wait for timers (including the pending write timer) to
+// go off before exiting.
+func (l *Logger) FinalUpdate(mutator LogMutator) {
+ // Block on this channel until everything finishes
+ done := make(chan bool)
+
+ // TODO(misha): Consider not accepting any future updates somehow,
+ // since they won't get written if they come in after this.
+
+ // Stop the periodic write ticker. We'll perform the final write
+ // before returning from this function.
+ l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+ l.writeTicker.Stop()
+ }
- // We are the scheduled write, so there are no longer future writes
- // scheduled.
- l.writeScheduled = false
+ // Apply the final update
+ l.workToDo <- mutator
- l.considerWriting()
+ // Perform the write and signal that we can return.
+ l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+ // TODO(misha): Add a boolean arg to write() to indicate that it's
+ // final so that we can set the appropriate event type.
+ l.write()
+ done <- true
+ }
- l.lock.Unlock()
+ // Wait until we've performed the write.
+ <-done
}
-// The above methods each acquire the lock and release it.
-// =======================================================
-// The below methods all assume we're holding a lock.
-
-// Check whether we have anything to write. If we do, then either
-// write it now or later, based on what we're allowed.
-func (l *Logger) considerWriting() {
- if !l.modified {
- // Nothing to write
- } else if l.writeAllowedNow() {
- l.write()
- } else if l.writeScheduled {
- // A future write is already scheduled, we don't need to do anything.
- } else {
- writeAfter := l.nextWriteAllowed.Sub(time.Now())
- time.AfterFunc(writeAfter, l.acquireLockConsiderWriting)
- l.writeScheduled = true
+// Adds a hook which will be called every time this logger writes an entry.
+func (l *Logger) AddWriteHook(hook LogMutator) {
+ // We do the work in a LogMutator so that it happens in the worker
+ // goroutine.
+ l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+ l.writeHooks = append(l.writeHooks, hook)
}
}
-// Whether writing now would respect MinimumWriteInterval
-func (l *Logger) writeAllowedNow() bool {
- return l.nextWriteAllowed.Before(time.Now())
+// The worker loop
+func (l *Logger) work() {
+ for {
+ select {
+ case <-l.writeTicker.C:
+ if l.modified {
+ l.write()
+ l.modified = false
+ }
+ case mutator := <-l.workToDo:
+ mutator(l.properties, l.entry)
+ l.modified = true
+ }
+ }
}
// Actually writes the log entry.
@@ -170,24 +165,20 @@ func (l *Logger) write() {
}
// Update the event type in case it was modified or is missing.
+ // TODO(misha): Fix this to write different event types.
l.entry["event_type"] = l.params.EventType
// Write the log entry.
// This is a network write and will take a while, which is bad
- // because we're holding a lock and all other goroutines will back
- // up behind it.
+ // because we're blocking all the other work on this goroutine.
//
// TODO(misha): Consider rewriting this so that we can encode l.data
- // into a string, release the lock, write the string, and then
- // acquire the lock again to note that we succeeded in writing. This
- // will be tricky and will require support in the client.
+ // into a string, and then perform the actual write in another
+ // routine. This will be tricky and will require support in the
+ // client.
err := l.params.Client.Create("logs", l.data, nil)
if err != nil {
log.Printf("Attempted to log: %v", l.data)
log.Fatalf("Received error writing log: %v", err)
}
-
- // Update stats.
- l.nextWriteAllowed = time.Now().Add(l.params.MinimumWriteInterval)
- l.modified = false
}
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 424db83..9a7a838 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -65,15 +65,6 @@ func init() {
"File to write the heap profiles to. Leave blank to skip profiling.")
}
-// // Methods to implement util.SdkListResponse Interface
-// func (s SdkCollectionList) NumItemsAvailable() (numAvailable int, err error) {
-// return s.ItemsAvailable, nil
-// }
-
-// func (s SdkCollectionList) NumItemsContained() (numContained int, err error) {
-// return len(s.Items), nil
-// }
-
// Write the heap profile to a file for later review.
// Since a file is expected to only contain a single heap profile this
// function overwrites the previously written profile, so it is safe
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 398c877..f63f462 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -48,14 +48,15 @@ func main() {
var arvLogger *logger.Logger
if logEventType != "" {
arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: logEventType,
- MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+ EventType: logEventType,
+ WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
runInfo := make(map[string]interface{})
- runInfo["time_started"] = time.Now()
+ runInfo["time_started"] = now
runInfo["args"] = os.Args
hostname, err := os.Hostname()
if err != nil {
@@ -90,7 +91,7 @@ func main() {
// Log that we're finished. We force the recording, since go will
// not wait for the timer before exiting.
if arvLogger != nil {
- arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
})
}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 20a5931..dcd6c49 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -69,15 +69,6 @@ type KeepServiceList struct {
KeepServers []ServerAddress `json:"items"`
}
-// Methods to implement util.SdkListResponse Interface
-func (k KeepServiceList) NumItemsAvailable() (numAvailable int, err error) {
- return k.ItemsAvailable, nil
-}
-
-func (k KeepServiceList) NumItemsContained() (numContained int, err error) {
- return len(k.KeepServers), nil
-}
-
var (
// Don't access the token directly, use getDataManagerToken() to
// make sure it's been read.
@@ -244,10 +235,11 @@ func GetServerStatus(arvLogger *logger.Logger,
keepServer.Port)
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := make(map[string]interface{})
- serverInfo["time_status_request_sent"] = time.Now()
+ serverInfo["time_status_request_sent"] = now
keepInfo[keepServer.String()] = serverInfo
})
@@ -274,10 +266,11 @@ func GetServerStatus(arvLogger *logger.Logger,
}
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- serverInfo["time_status_response_processed"] = time.Now()
+ serverInfo["time_status_response_processed"] = now
serverInfo["status"] = keepStatus
})
}
@@ -289,10 +282,11 @@ func CreateIndexRequest(arvLogger *logger.Logger,
log.Println("About to fetch keep server contents from " + url)
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- serverInfo["time_index_request_sent"] = time.Now()
+ serverInfo["time_index_request_sent"] = now
})
}
@@ -319,11 +313,11 @@ func ReadServerResponse(arvLogger *logger.Logger,
}
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-
- serverInfo["time_index_response_received"] = time.Now()
+ serverInfo["time_index_response_received"] = now
})
}
@@ -393,11 +387,12 @@ func ReadServerResponse(arvLogger *logger.Logger,
numSizeDisagreements)
if arvLogger != nil {
+ now := time.Now()
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- serverInfo["time_processing_finished"] = time.Now()
+ serverInfo["time_processing_finished"] = now
serverInfo["lines_received"] = numLines
serverInfo["duplicates_seen"] = numDuplicates
serverInfo["size_disagreements_seen"] = numSizeDisagreements
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index fa876d4..c19a7ab 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -8,12 +8,9 @@ import (
"time"
)
-// Assumes you haven't already called arvLogger.Edit()!
-// If you have called arvLogger.Edit() this method will hang waiting
-// for the lock you're already holding.
func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
- arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
p["FATAL"] = message
p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
})
commit 75f4b70625086aaa8ecf8daed23e4d151e54949f
Author: mishaz <misha at curoverse.com>
Date: Fri Jan 30 01:32:31 2015 +0000
Renamed timestamp fields to begin with "time_"
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index cbdae6e..398c877 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -55,7 +55,7 @@ func main() {
if arvLogger != nil {
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
runInfo := make(map[string]interface{})
- runInfo["start_time"] = time.Now()
+ runInfo["time_started"] = time.Now()
runInfo["args"] = os.Args
hostname, err := os.Hostname()
if err != nil {
@@ -91,7 +91,7 @@ func main() {
// not wait for the timer before exiting.
if arvLogger != nil {
arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
- p["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
})
}
}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index 20b7f18..fa876d4 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -15,7 +15,7 @@ func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
p["FATAL"] = message
- p["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
})
}
commit e5840cee519c5ff8b88e37e14160f9e4e12908ec
Author: mishaz <misha at curoverse.com>
Date: Fri Jan 30 01:25:11 2015 +0000
Now fetch Keep Server Status and record it to the log. Renamed some fields and added a comment for a potential improvement to decrease lock contention.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index ca344be..a53ab3c 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -173,6 +173,14 @@ func (l *Logger) write() {
l.entry["event_type"] = l.params.EventType
// Write the log entry.
+ // This is a network write and will take a while, which is bad
+ // because we're holding a lock and all other goroutines will back
+ // up behind it.
+ //
+ // TODO(misha): Consider rewriting this so that we can encode l.data
+ // into a string, release the lock, write the string, and then
+ // acquire the lock again to note that we succeeded in writing. This
+ // will be tricky and will require support in the client.
err := l.params.Client.Create("logs", l.data, nil)
if err != nil {
log.Printf("Attempted to log: %v", l.data)
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 39ac30a..20a5931 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -4,6 +4,7 @@ package keep
import (
"bufio"
+ "encoding/json"
"flag"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -223,6 +224,8 @@ func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
client http.Client) (response ServerResponse) {
+ GetServerStatus(arvLogger, keepServer, client)
+
req := CreateIndexRequest(arvLogger, keepServer)
resp, err := client.Do(req)
if err != nil {
@@ -233,6 +236,53 @@ func GetServerContents(arvLogger *logger.Logger,
return ReadServerResponse(arvLogger, keepServer, resp)
}
+func GetServerStatus(arvLogger *logger.Logger,
+ keepServer ServerAddress,
+ client http.Client) {
+ url := fmt.Sprintf("http://%s:%d/status.json",
+ keepServer.Host,
+ keepServer.Port)
+
+ if arvLogger != nil {
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := make(map[string]interface{})
+ serverInfo["time_status_request_sent"] = time.Now()
+
+ keepInfo[keepServer.String()] = serverInfo
+ })
+ }
+
+ resp, err := client.Get(url)
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error getting keep status from %s: %v", url, err))
+ } else if resp.StatusCode != 200 {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error code %d in response to request "+
+ "for %s status: %s",
+ resp.StatusCode, url, resp.Status))
+ }
+
+ var keepStatus map[string]interface{}
+ decoder := json.NewDecoder(resp.Body)
+ decoder.UseNumber()
+ err = decoder.Decode(&keepStatus)
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error decoding keep status from %s: %v", url, err))
+ }
+
+ if arvLogger != nil {
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+ serverInfo["time_status_response_processed"] = time.Now()
+ serverInfo["status"] = keepStatus
+ })
+ }
+}
+
func CreateIndexRequest(arvLogger *logger.Logger,
keepServer ServerAddress) (req *http.Request) {
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
@@ -241,10 +291,8 @@ func CreateIndexRequest(arvLogger *logger.Logger,
if arvLogger != nil {
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := p["keep_info"].(map[string]interface{})
- serverInfo := make(map[string]interface{})
- serverInfo["request_sent"] = time.Now()
-
- keepInfo[keepServer.String()] = serverInfo
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+ serverInfo["time_index_request_sent"] = time.Now()
})
}
@@ -275,7 +323,7 @@ func ReadServerResponse(arvLogger *logger.Logger,
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- serverInfo["response_received"] = time.Now()
+ serverInfo["time_index_response_received"] = time.Now()
})
}
@@ -349,7 +397,7 @@ func ReadServerResponse(arvLogger *logger.Logger,
keepInfo := p["keep_info"].(map[string]interface{})
serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- serverInfo["processing_finished"] = time.Now()
+ serverInfo["time_processing_finished"] = time.Now()
serverInfo["lines_received"] = numLines
serverInfo["duplicates_seen"] = numDuplicates
serverInfo["size_disagreements_seen"] = numSizeDisagreements
commit e88d1643f64e70e984b2c7943e5ce6569e7e2d37
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 27 01:27:37 2015 +0000
Improved erorr message to make it clear what's a size and what's a timestamp.
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index cf6803a..39ac30a 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -302,7 +302,7 @@ func ReadServerResponse(arvLogger *logger.Logger,
numSizeDisagreements += 1
// TODO(misha): Consider failing here.
message := fmt.Sprintf("Saw different sizes for the same block "+
- "on %s: %v %v",
+ "on %s: %+v %+v",
keepServer.String(),
storedBlock,
blockInfo)
commit 6db8f01ae7f8d30f48a88caa351004ea446b33b3
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 27 01:06:21 2015 +0000
Renamed BlockDigest's ToString() to String() to implement fmt.Stringer() interface so that we get more readable error messages when structs contain BlockDigests.
diff --git a/sdk/go/blockdigest/blockdigest.go b/sdk/go/blockdigest/blockdigest.go
index 0742839..9b818d3 100644
--- a/sdk/go/blockdigest/blockdigest.go
+++ b/sdk/go/blockdigest/blockdigest.go
@@ -15,7 +15,7 @@ type BlockDigest struct {
l uint64
}
-func (d *BlockDigest) ToString() (s string) {
+func (d BlockDigest) String() string {
return fmt.Sprintf("%016x%016x", d.h, d.l)
}
diff --git a/sdk/go/blockdigest/blockdigest_test.go b/sdk/go/blockdigest/blockdigest_test.go
index 9081bb8..068a138 100644
--- a/sdk/go/blockdigest/blockdigest_test.go
+++ b/sdk/go/blockdigest/blockdigest_test.go
@@ -1,6 +1,7 @@
package blockdigest
import (
+ "fmt"
"strings"
"testing"
)
@@ -13,8 +14,8 @@ func expectValidDigestString(t *testing.T, s string) {
expected := strings.ToLower(s)
- if expected != bd.ToString() {
- t.Fatalf("Expected %s to be returned by FromString(%s).ToString() but instead we received %s", expected, s, bd.ToString())
+ if expected != bd.String() {
+ t.Fatalf("Expected %s to be returned by FromString(%s).String() but instead we received %s", expected, s, bd.String())
}
}
@@ -43,3 +44,36 @@ func TestBlockDigestWorksAsMapKey(t *testing.T) {
bd := AssertFromString("01234567890123456789abcdefabcdef")
m[bd] = 5
}
+
+func TestBlockDigestGetsPrettyPrintedByPrintf(t *testing.T) {
+ input := "01234567890123456789abcdefabcdef"
+ prettyPrinted := fmt.Sprintf("%v", AssertFromString(input))
+ if prettyPrinted != input {
+ t.Fatalf("Expected blockDigest produced from \"%s\" to be printed as " +
+ "\"%s\", but instead it was printed as %s",
+ input, input, prettyPrinted)
+ }
+}
+
+func TestBlockDigestGetsPrettyPrintedByPrintfInNestedStructs(t *testing.T) {
+ input := "01234567890123456789abcdefabcdef"
+ value := 42
+ nested := struct{
+ // Fun trivia fact: If this field was called "digest" instead of
+ // "Digest", then it would not be exported and String() would
+ // never get called on it and our output would look very
+ // different.
+ Digest BlockDigest
+ value int
+ }{
+ AssertFromString(input),
+ value,
+ }
+ prettyPrinted := fmt.Sprintf("%+v", nested)
+ expected := fmt.Sprintf("{Digest:%s value:%d}", input, value)
+ if prettyPrinted != expected {
+ t.Fatalf("Expected blockDigest produced from \"%s\" to be printed as " +
+ "\"%s\", but instead it was printed as %s",
+ input, expected, prettyPrinted)
+ }
+}
commit 4df1175e30c21850af394fcd60c9bb7ca3d981a5
Author: mishaz <misha at curoverse.com>
Date: Sat Jan 24 02:22:01 2015 +0000
A bunch of changes, most in response to Peter's review.
Logger:
Edit() and Record() have been replaced with the single Update() method which takes a function as input (suggested by Tom).
lastWrite replaced by nextWriteAllowed, for cleaner logic
Added writeScheduled to reduce the number of writes scheduled and attempted, thereby reducing lock contention
Added sanity-checking of params
A bunch of overdue cleanup
Update documentation to reflect the above changes
Manifest:
Renamed ManifestLine to ManifestStream
Util:
Deleted a lot of crap that proved less useful than I thought.
Moved collection.NumberCollectionsAvailable() to util.NumberItemsAvailable() and made it more generic.
collection:
Just cleanup in response to changes in above packages.
keep:
Switched Mtime from int to int64 to avoid y2038 problems.
Switched approach for avoiding keep proxy from using "accessible" to filtering on service_type = disk.
Cleanup in response to changes in above packages.
loggerutil:
Cleanup in response to changes in logger.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 11c6b53..ca344be 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -1,27 +1,23 @@
// Logger periodically writes a log to the Arvados SDK.
//
-// This package is useful for maintaining a log object that is built
-// up over time. Every time the object is modified, it will be written
-// to the log. Writes will be throttled to no more than one every
+// This package is useful for maintaining a log object that is updated
+// over time. Every time the object is updated, it will be written to
+// the log. Writes will be throttled to no more than one every
// WriteFrequencySeconds
//
// This package is safe for concurrent use as long as:
-// 1. The maps returned by Edit() are only edited in the same routine
-// that called Edit()
-// 2. Those maps not edited after calling Record()
-// An easy way to assure this is true is to place the call to Edit()
-// within a short block as shown below in the Usage Example:
+// The maps passed to a LogMutator are not accessed outside of the
+// LogMutator
//
// Usage:
// arvLogger := logger.NewLogger(params)
-// {
-// properties, entry := arvLogger.Edit() // This will block if others are using the logger
+// arvLogger.Update(func(properties map[string]interface{},
+// entry map[string]interface{}) {
// // Modifiy properties and entry however you want
// // properties is a shortcut for entry["properties"].(map[string]interface{})
// // properties can take any values you want to give it,
// // entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
-// }
-// arvLogger.Record() // This triggers the actual log write
+// })
package logger
import (
@@ -51,24 +47,32 @@ type LogMutator func(map[string]interface{}, map[string]interface{})
// A Logger is used to build up a log entry over time and write every
// version of it.
type Logger struct {
- // The Data we write
+ // The data we write
data map[string]interface{} // The entire map that we give to the api
entry map[string]interface{} // Convenience shortcut into data
properties map[string]interface{} // Convenience shortcut into data
- lock sync.Locker // Synchronizes editing and writing
+ lock sync.Locker // Synchronizes access to this struct
params LoggerParams // Parameters we were given
- // TODO(misha): replace lastWrite with nextWriteAllowed
- lastWrite time.Time // The last time we wrote a log entry
- modified bool // Has this data been modified since the last write
+ // Variables used to determine when and if we write to the log.
+ nextWriteAllowed time.Time // The next time we can write, respecting MinimumWriteInterval
+ modified bool // Has this data been modified since the last write?
+ writeScheduled bool // Is a write been scheduled for the future?
- writeHooks []LogMutator
+ writeHooks []LogMutator // Mutators we call before each write.
}
// Create a new logger based on the specified parameters.
func NewLogger(params LoggerParams) *Logger {
- // TODO(misha): Add some params checking here.
+ // sanity check parameters
+ if ¶ms.Client == nil {
+ log.Fatal("Nil arvados client in LoggerParams passed in to NewLogger()")
+ }
+ if params.EventType == "" {
+ log.Fatal("Empty event type in LoggerParams passed in to NewLogger()")
+ }
+
l := &Logger{data: make(map[string]interface{}),
lock: &sync.Mutex{},
params: params}
@@ -79,68 +83,85 @@ func NewLogger(params LoggerParams) *Logger {
return l
}
-// Get access to the maps you can edit. This will hold a lock until
-// you call Record. Do not edit the maps in any other goroutines or
-// after calling Record.
-// You don't need to edit both maps,
-// properties can take any values you want to give it,
-// entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
-// properties is a shortcut for entry["properties"].(map[string]interface{})
-func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
+// Updates the log data and then writes it to the api server. If the
+// log has been recently written then the write will be postponed to
+// respect MinimumWriteInterval and this function will return before
+// the write occurs.
+func (l *Logger) Update(mutator LogMutator) {
l.lock.Lock()
- l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
- return l.properties, l.entry
+ mutator(l.properties, l.entry)
+ l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
+
+ l.considerWriting()
+
+ l.lock.Unlock()
}
-// function to test new api, replacing Edit() and Record()
-func (l *Logger) MutateLog(mutator LogMutator) {
- mutator(l.Edit())
- l.Record()
+// Similar to Update(), but forces a write without respecting the
+// MinimumWriteInterval. This is useful if you know that you're about
+// to quit (e.g. if you discovered a fatal error, or you're finished),
+// since go will not wait for timers (including the pending write
+// timer) to go off before exiting.
+func (l *Logger) ForceUpdate(mutator LogMutator) {
+ l.lock.Lock()
+
+ mutator(l.properties, l.entry)
+ l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
+
+ l.write()
+ l.lock.Unlock()
}
// Adds a hook which will be called every time this logger writes an entry.
-// The hook takes properties and entry as arguments, in that order.
-// This is useful for stuff like memory profiling.
-// This must be called between Edit() and Record() (e.g. while holding the lock)
func (l *Logger) AddWriteHook(hook LogMutator) {
- // TODO(misha): Acquire lock here! and erase comment about edit.
+ l.lock.Lock()
l.writeHooks = append(l.writeHooks, hook)
- // TODO(misha): consider flipping the dirty bit here.
+ // TODO(misha): Consider setting modified and attempting a write.
+ l.lock.Unlock()
+}
+
+// This function is called on a timer when we have something to write,
+// but need to schedule the write for the future to respect
+// MinimumWriteInterval.
+func (l *Logger) acquireLockConsiderWriting() {
+ l.lock.Lock()
+
+ // We are the scheduled write, so there are no longer future writes
+ // scheduled.
+ l.writeScheduled = false
+
+ l.considerWriting()
+
+ l.lock.Unlock()
}
-// Write the log entry you've built up so far. Do not edit the maps
-// returned by Edit() after calling this method.
-// If you have already written within MinimumWriteInterval, then this
-// will schedule a future write instead.
-// In either case, the lock will be released before Record() returns.
-func (l *Logger) Record() {
- if l.writeAllowedNow() {
- // We haven't written in the allowed interval yet, try to write.
+// The above methods each acquire the lock and release it.
+// =======================================================
+// The below methods all assume we're holding a lock.
+
+// Check whether we have anything to write. If we do, then either
+// write it now or later, based on what we're allowed.
+func (l *Logger) considerWriting() {
+ if !l.modified {
+ // Nothing to write
+ } else if l.writeAllowedNow() {
l.write()
+ } else if l.writeScheduled {
+ // A future write is already scheduled, we don't need to do anything.
} else {
- // TODO(misha): Only allow one outstanding write to be scheduled.
- nextTimeToWrite := l.lastWrite.Add(l.params.MinimumWriteInterval)
- writeAfter := nextTimeToWrite.Sub(time.Now())
+ writeAfter := l.nextWriteAllowed.Sub(time.Now())
time.AfterFunc(writeAfter, l.acquireLockConsiderWriting)
+ l.writeScheduled = true
}
- l.lock.Unlock()
}
-// Similar to Record, but forces a write without respecting the
-// MinimumWriteInterval. This is useful if you know that you're about
-// to quit (e.g. if you discovered a fatal error).
-func (l *Logger) ForceRecord() {
- l.write()
- l.lock.Unlock()
-}
-
-// Whether enough time has elapsed since the last write.
+// Whether writing now would respect MinimumWriteInterval
func (l *Logger) writeAllowedNow() bool {
- return l.lastWrite.Add(l.params.MinimumWriteInterval).Before(time.Now())
+ return l.nextWriteAllowed.Before(time.Now())
}
-// Actually writes the log entry. This method assumes we're holding the lock.
+// Actually writes the log entry.
func (l *Logger) write() {
// Run all our hooks
@@ -159,15 +180,6 @@ func (l *Logger) write() {
}
// Update stats.
- l.lastWrite = time.Now()
+ l.nextWriteAllowed = time.Now().Add(l.params.MinimumWriteInterval)
l.modified = false
}
-
-func (l *Logger) acquireLockConsiderWriting() {
- l.lock.Lock()
- if l.modified && l.writeAllowedNow() {
- // We have something new to write and we're allowed to write.
- l.write()
- }
- l.lock.Unlock()
-}
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index c9f9018..f6698c6 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -26,7 +26,8 @@ type BlockLocator struct {
Hints []string
}
-type ManifestLine struct {
+// Represents a single line from a manifest.
+type ManifestStream struct {
StreamName string
Blocks []string
Files []string
@@ -59,7 +60,7 @@ func ParseBlockLocator(s string) (b BlockLocator, err error) {
return
}
-func parseManifestLine(s string) (m ManifestLine) {
+func parseManifestStream(s string) (m ManifestStream) {
tokens := strings.Split(s, " ")
m.StreamName = tokens[0]
tokens = tokens[1:]
@@ -74,8 +75,8 @@ func parseManifestLine(s string) (m ManifestLine) {
return
}
-func (m *Manifest) LineIter() <-chan ManifestLine {
- ch := make(chan ManifestLine)
+func (m *Manifest) StreamIter() <-chan ManifestStream {
+ ch := make(chan ManifestStream)
go func(input string) {
// This slice holds the current line and the remainder of the
// manifest. We parse one line at a time, to save effort if we
@@ -85,7 +86,7 @@ func (m *Manifest) LineIter() <-chan ManifestLine {
lines = strings.SplitN(lines[1], "\n", 2)
if len(lines[0]) > 0 {
// Only parse non-blank lines
- ch <- parseManifestLine(lines[0])
+ ch <- parseManifestStream(lines[0])
}
if len(lines) == 1 {
break
@@ -101,8 +102,8 @@ func (m *Manifest) LineIter() <-chan ManifestLine {
// the same block multiple times.
func (m *Manifest) BlockIterWithDuplicates() <-chan BlockLocator {
blockChannel := make(chan BlockLocator)
- go func(lineChannel <-chan ManifestLine) {
- for m := range lineChannel {
+ go func(streamChannel <-chan ManifestStream) {
+ for m := range streamChannel {
for _, block := range m.Blocks {
if b, err := ParseBlockLocator(block); err == nil {
blockChannel <- b
@@ -112,6 +113,6 @@ func (m *Manifest) BlockIterWithDuplicates() <-chan BlockLocator {
}
}
close(blockChannel)
- }(m.LineIter())
+ }(m.StreamIter())
return blockChannel
}
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index f8641ce..c1bfb14 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -57,7 +57,7 @@ func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
}
}
-func expectManifestLine(t *testing.T, actual ManifestLine, expected ManifestLine) {
+func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
expectEqual(t, actual.StreamName, expected.StreamName)
expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
expectStringSlicesEqual(t, actual.Files, expected.Files)
@@ -108,9 +108,9 @@ func TestLocatorPatternBasic(t *testing.T) {
expectLocatorPatternFail(t, "12345678901234567890123456789012+12345+A+B2")
}
-func TestParseManifestLineSimple(t *testing.T) {
- m := parseManifestLine(". 365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
- expectManifestLine(t, m, ManifestLine{StreamName: ".",
+func TestParseManifestStreamSimple(t *testing.T) {
+ m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
+ expectManifestStream(t, m, ManifestStream{StreamName: ".",
Blocks: []string{"365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
Files: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
}
@@ -126,24 +126,24 @@ func TestParseBlockLocatorSimple(t *testing.T) {
"Af0c9a66381f3b028677411926f0be1c6282fe67c at 542b5ddf"}})
}
-func TestLineIterShortManifestWithBlankLines(t *testing.T) {
+func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
content, err := ioutil.ReadFile("testdata/short_manifest")
if err != nil {
t.Fatalf("Unexpected error reading manifest from file: %v", err)
}
manifest := Manifest{string(content)}
- lineIter := manifest.LineIter()
+ streamIter := manifest.StreamIter()
- firstLine := <-lineIter
- expectManifestLine(t,
- firstLine,
- ManifestLine{StreamName: ".",
+ firstStream := <-streamIter
+ expectManifestStream(t,
+ firstStream,
+ ManifestStream{StreamName: ".",
Blocks: []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475 at 5441920c"},
Files: []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
- received, ok := <- lineIter
+ received, ok := <- streamIter
if ok {
- t.Fatalf("Expected lineIter to be closed, but received %v instead.",
+ t.Fatalf("Expected streamIter to be closed, but received %v instead.",
received)
}
}
diff --git a/sdk/go/util/util.go b/sdk/go/util/util.go
index 4505db6..6bc8625 100644
--- a/sdk/go/util/util.go
+++ b/sdk/go/util/util.go
@@ -3,40 +3,9 @@
package util
import (
- "errors"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "log"
)
-type SdkListResponse interface {
- NumItemsAvailable() (int, error)
- NumItemsContained() (int, error)
-}
-
-type UnstructuredSdkListResponse map[string]interface{}
-
-func (m UnstructuredSdkListResponse) NumItemsAvailable() (numAvailable int, err error) {
- if itemsAvailable, ok := m["items_available"]; !ok {
- err = errors.New("Could not find \"items_available\" field in " +
- "UnstructuredSdkListResponse that NumItemsAvailable was called on.")
- } else {
- // TODO(misha): Check whether this assertion will work before casting
- numAvailable = int(itemsAvailable.(float64))
- }
- return
-}
-
-func (m UnstructuredSdkListResponse) NumItemsContained() (numContained int, err error) {
- if value, ok := m["items"]; ok {
- // TODO(misha): check whether this assertion will work before casting
- numContained = len(value.([]interface{}))
- } else {
- err = errors.New(`Could not find "items" field in ` +
- "UnstructuredSdkListResponse that NumItemsContained was called on.")
- }
- return
-}
-
func UserIsAdmin(arv arvadosclient.ArvadosClient) (is_admin bool, err error) {
type user struct {
IsAdmin bool `json:"is_admin"`
@@ -46,38 +15,20 @@ func UserIsAdmin(arv arvadosclient.ArvadosClient) (is_admin bool, err error) {
return u.IsAdmin, err
}
-// TODO(misha): Consider returning an error here instead of fatal'ing
-func ContainsAllAvailableItems(response SdkListResponse) (containsAll bool, numContained int, numAvailable int) {
- var err error
- numContained, err = response.NumItemsContained()
- if err != nil {
- log.Fatalf("Error retrieving number of items contained in SDK response: %v",
- err)
- }
- numAvailable, err = response.NumItemsAvailable()
- if err != nil {
- log.Fatalf("Error retrieving number of items available from "+
- "SDK response: %v",
- err)
+// Returns the total count of a particular type of resource
+//
+// resource - the arvados resource to count
+// return
+// count - the number of items of type resource the api server reports, if no error
+// err - error accessing the resource, or nil if no error
+func NumberItemsAvailable(client arvadosclient.ArvadosClient, resource string) (count int, err error) {
+ var response struct {
+ ItemsAvailable int `json:"items_available"`
}
- containsAll = numContained == numAvailable
- return
-}
-
-func IterateSdkListItems(response map[string]interface{}) (c <-chan map[string]interface{}, err error) {
- if value, ok := response["items"]; ok {
- ch := make(chan map[string]interface{})
- c = ch
- items := value.([]interface{})
- go func() {
- for _, item := range items {
- ch <- item.(map[string]interface{})
- }
- close(ch)
- }()
- } else {
- err = errors.New("Could not find \"items\" field in response " +
- "passed to IterateSdkListItems()")
+ sdkParams := arvadosclient.Dict{"limit": 0}
+ err = client.List(resource, sdkParams, &response)
+ if err == nil {
+ count = response.ItemsAvailable
}
return
}
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 73115f5..424db83 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -9,6 +9,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
+ "git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
"os"
@@ -101,10 +102,13 @@ func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollec
ComputeSizeOfOwnedCollections(&results)
if params.Logger != nil {
- properties, _ := params.Logger.Edit()
- collectionInfo := properties["collection_info"].(map[string]interface{})
- collectionInfo["owner_to_collection_size"] = results.OwnerToCollectionSize
- params.Logger.Record()
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ collectionInfo := p["collection_info"].(map[string]interface{})
+ // Since maps are shallow copied, we run a risk of concurrent
+ // updates here. By copying results.OwnerToCollectionSize into
+ // the log, we're assuming that it won't be updated.
+ collectionInfo["owner_to_collection_size"] = results.OwnerToCollectionSize
+ })
}
log.Printf("Uuid to Size used: %v", results.OwnerToCollectionSize)
@@ -143,7 +147,12 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
sdkParams["limit"] = params.BatchSize
}
- initialNumberOfCollectionsAvailable := NumberCollectionsAvailable(params.Client)
+ initialNumberOfCollectionsAvailable, err :=
+ util.NumberItemsAvailable(params.Client, "collections")
+ if err != nil {
+ loggerutil.FatalWithMessage(params.Logger,
+ fmt.Sprintf("Error querying collection count: %v", err))
+ }
// Include a 1% margin for collections added while we're reading so
// that we don't have to grow the map in most cases.
maxExpectedCollections := int(
@@ -151,12 +160,12 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
if params.Logger != nil {
- properties, _ := params.Logger.Edit()
- collectionInfo := make(map[string]interface{})
- collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
- collectionInfo["batch_size"] = params.BatchSize
- properties["collection_info"] = collectionInfo
- params.Logger.Record()
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ collectionInfo := make(map[string]interface{})
+ collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ collectionInfo["batch_size"] = params.BatchSize
+ p["collection_info"] = collectionInfo
+ })
}
// These values are just for getting the loop to run the first time,
@@ -196,13 +205,13 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
maxManifestSize, totalManifestSize)
if params.Logger != nil {
- properties, _ := params.Logger.Edit()
- collectionInfo := properties["collection_info"].(map[string]interface{})
- collectionInfo["collections_read"] = totalCollections
- collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
- collectionInfo["total_manifest_size"] = totalManifestSize
- collectionInfo["max_manifest_size"] = maxManifestSize
- params.Logger.Record()
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ collectionInfo := p["collection_info"].(map[string]interface{})
+ collectionInfo["collections_read"] = totalCollections
+ collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
+ collectionInfo["total_manifest_size"] = totalManifestSize
+ collectionInfo["max_manifest_size"] = maxManifestSize
+ })
}
}
@@ -282,17 +291,6 @@ func ProcessCollections(arvLogger *logger.Logger,
return
}
-func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) int {
- var collections SdkCollectionList
- sdkParams := arvadosclient.Dict{"limit": 0}
- err := client.List("collections", sdkParams, &collections)
- if err != nil {
- log.Fatalf("error querying collections for items available: %v", err)
- }
-
- return collections.ItemsAvailable
-}
-
func ComputeSizeOfOwnedCollections(readCollections *ReadCollections) {
readCollections.OwnerToCollectionSize = make(map[string]int)
for _, coll := range readCollections.UuidToCollection {
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 97c911d..cbdae6e 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -53,8 +53,7 @@ func main() {
}
if arvLogger != nil {
- arvLogger.MutateLog(func(properties map[string]interface{},
- entry map[string]interface{}) {
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
runInfo := make(map[string]interface{})
runInfo["start_time"] = time.Now()
runInfo["args"] = os.Args
@@ -65,12 +64,10 @@ func main() {
runInfo["hostname"] = hostname
}
runInfo["pid"] = os.Getpid()
- properties["run_info"] = runInfo
+ p["run_info"] = runInfo
})
- arvLogger.Edit()
arvLogger.AddWriteHook(LogMemoryAlloc)
- arvLogger.Record()
}
collectionChannel := make(chan collection.ReadCollections)
@@ -86,19 +83,20 @@ func main() {
readCollections := <-collectionChannel
- // Make compiler happy.
+ // TODO(misha): Use these together to verify replication.
_ = readCollections
_ = keepServerInfo
- // Log that we're finished
+ // Log that we're finished. We force the recording, since go will
+ // not wait for the timer before exiting.
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
- // Force the recording, since go will not wait for the timer before exiting.
- arvLogger.ForceRecord()
+ arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ p["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ })
}
}
+// TODO(misha): Consider moving this to loggerutil
func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
runInfo := properties["run_info"].(map[string]interface{})
var memStats runtime.MemStats
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 97bbf9d..cf6803a 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -6,12 +6,10 @@ import (
"bufio"
"flag"
"fmt"
- //"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- "git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"io/ioutil"
"log"
@@ -31,14 +29,14 @@ type ServerAddress struct {
type BlockInfo struct {
Digest blockdigest.BlockDigest
Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Mtime int64 // TODO(misha): Replace this with a timestamp.
}
// Info about a specified block given by a server
type BlockServerInfo struct {
ServerIndex int
Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Mtime int64 // TODO(misha): Replace this with a timestamp.
}
type ServerContents struct {
@@ -137,47 +135,39 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
"contain a valid ArvadosClient, but instead it is nil.")
}
- sdkParams := arvadosclient.Dict{}
+ sdkParams := arvadosclient.Dict{
+ "filters": [][]string{[]string{"service_type", "=", "disk"}},
+ }
if params.Limit > 0 {
sdkParams["limit"] = params.Limit
}
var sdkResponse KeepServiceList
- err := params.Client.Call("GET", "keep_services", "", "accessible", sdkParams, &sdkResponse)
+ err := params.Client.List("keep_services", sdkParams, &sdkResponse)
if err != nil {
loggerutil.FatalWithMessage(params.Logger,
fmt.Sprintf("Error requesting keep disks from API server: %v", err))
}
- // TODO(misha): Rewrite this block, stop using ContainsAllAvailableItems()
- {
- var numReceived, numAvailable int
- results.ReadAllServers, numReceived, numAvailable =
- util.ContainsAllAvailableItems(sdkResponse)
-
- if !results.ReadAllServers {
- log.Printf("ERROR: Did not receive all keep server addresses.")
- }
- log.Printf("Received %d of %d available keep server addresses.",
- numReceived,
- numAvailable)
- }
-
if params.Logger != nil {
- properties, _ := params.Logger.Edit()
- keepInfo := make(map[string]interface{})
-
- keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
- keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
- keepInfo["keep_servers"] = sdkResponse.KeepServers
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := make(map[string]interface{})
- properties["keep_info"] = keepInfo
+ keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
+ keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
+ keepInfo["keep_servers"] = sdkResponse.KeepServers
- params.Logger.Record()
+ p["keep_info"] = keepInfo
+ })
}
- log.Printf("Received keep services list: %v", sdkResponse)
+ log.Printf("Received keep services list: %+v", sdkResponse)
+
+ if len(sdkResponse.KeepServers) < sdkResponse.ItemsAvailable {
+ loggerutil.FatalWithMessage(params.Logger,
+ fmt.Sprintf("Did not receive all available keep servers: %+v", sdkResponse))
+ }
results.KeepServerIndexToAddress = sdkResponse.KeepServers
results.KeepServerAddressToIndex = make(map[ServerAddress]int)
@@ -249,14 +239,13 @@ func CreateIndexRequest(arvLogger *logger.Logger,
log.Println("About to fetch keep server contents from " + url)
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- keepInfo := properties["keep_info"].(map[string]interface{})
- serverInfo := make(map[string]interface{})
- serverInfo["request_sent"] = time.Now()
-
- keepInfo[keepServer.String()] = serverInfo
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := make(map[string]interface{})
+ serverInfo["request_sent"] = time.Now()
- arvLogger.Record()
+ keepInfo[keepServer.String()] = serverInfo
+ })
}
req, err := http.NewRequest("GET", url, nil)
@@ -282,13 +271,12 @@ func ReadServerResponse(arvLogger *logger.Logger,
}
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- keepInfo := properties["keep_info"].(map[string]interface{})
- serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-
- serverInfo["response_received"] = time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- arvLogger.Record()
+ serverInfo["response_received"] = time.Now()
+ })
}
response.Address = keepServer
@@ -320,16 +308,16 @@ func ReadServerResponse(arvLogger *logger.Logger,
blockInfo)
log.Println(message)
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- keepInfo := properties["keep_info"].(map[string]interface{})
- serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
- var error_list []string
- read_error_list, has_list := serverInfo["error_list"]
- if has_list {
- error_list = read_error_list.([]string)
- } // If we didn't have the list, error_list is already an empty list
- serverInfo["error_list"] = append(error_list, message)
- arvLogger.Record()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+ var error_list []string
+ read_error_list, has_list := serverInfo["error_list"]
+ if has_list {
+ error_list = read_error_list.([]string)
+ } // If we didn't have the list, error_list is already an empty list
+ serverInfo["error_list"] = append(error_list, message)
+ })
}
}
// Keep the block that is bigger, or the block that's newer in
@@ -357,16 +345,15 @@ func ReadServerResponse(arvLogger *logger.Logger,
numSizeDisagreements)
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- keepInfo := properties["keep_info"].(map[string]interface{})
- serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-
- serverInfo["processing_finished"] = time.Now()
- serverInfo["lines_received"] = numLines
- serverInfo["dupicates_seen"] = numDuplicates
- serverInfo["size_disagreements_seen"] = numSizeDisagreements
-
- arvLogger.Record()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+
+ serverInfo["processing_finished"] = time.Now()
+ serverInfo["lines_received"] = numLines
+ serverInfo["duplicates_seen"] = numDuplicates
+ serverInfo["size_disagreements_seen"] = numSizeDisagreements
+ })
}
}
resp.Body.Close()
@@ -392,7 +379,7 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
return
}
- blockInfo.Mtime, err = strconv.Atoi(tokens[1])
+ blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
if err != nil {
return
}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index f97f7c1..20b7f18 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -13,10 +13,10 @@ import (
// for the lock you're already holding.
func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- properties["FATAL"] = message
- properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
- arvLogger.ForceRecord()
+ arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+ p["FATAL"] = message
+ p["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ })
}
log.Fatalf(message)
commit c2b8ab7045886b62963feb0cd8f9b9291ce1a8b7
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 21 01:34:29 2015 +0000
Added comment, ran gofmt.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 2a4a962..11c6b53 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -59,6 +59,7 @@ type Logger struct {
lock sync.Locker // Synchronizes editing and writing
params LoggerParams // Parameters we were given
+ // TODO(misha): replace lastWrite with nextWriteAllowed
lastWrite time.Time // The last time we wrote a log entry
modified bool // Has this data been modified since the last write
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index deeea5d..97c911d 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -55,18 +55,18 @@ func main() {
if arvLogger != nil {
arvLogger.MutateLog(func(properties map[string]interface{},
entry map[string]interface{}) {
- runInfo := make(map[string]interface{})
- runInfo["start_time"] = time.Now()
- runInfo["args"] = os.Args
- hostname, err := os.Hostname()
- if err != nil {
- runInfo["hostname_error"] = err.Error()
- } else {
- runInfo["hostname"] = hostname
- }
- runInfo["pid"] = os.Getpid()
- properties["run_info"] = runInfo
- })
+ runInfo := make(map[string]interface{})
+ runInfo["start_time"] = time.Now()
+ runInfo["args"] = os.Args
+ hostname, err := os.Hostname()
+ if err != nil {
+ runInfo["hostname_error"] = err.Error()
+ } else {
+ runInfo["hostname"] = hostname
+ }
+ runInfo["pid"] = os.Getpid()
+ properties["run_info"] = runInfo
+ })
arvLogger.Edit()
arvLogger.AddWriteHook(LogMemoryAlloc)
commit 17800e7d4a9574035dd48b71ec4247f70525d45e
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 21 01:31:17 2015 +0000
Added Logger.MutateLog() on Tom's suggestion. Tried it out in one instance to make sure it works.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index ea0be33..2a4a962 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -37,6 +37,17 @@ type LoggerParams struct {
MinimumWriteInterval time.Duration // Wait at least this long between log writes
}
+// A LogMutator is a function which modifies the log entry.
+// It takes two maps as arguments, properties is the first and entry
+// is the second
+// properties is a shortcut for entry["properties"].(map[string]interface{})
+// properties can take any values you want to give it.
+// entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
+// properties and entry are only safe to access inside the LogMutator,
+// they should not be stored anywhere, otherwise you'll risk
+// concurrent access.
+type LogMutator func(map[string]interface{}, map[string]interface{})
+
// A Logger is used to build up a log entry over time and write every
// version of it.
type Logger struct {
@@ -51,7 +62,7 @@ type Logger struct {
lastWrite time.Time // The last time we wrote a log entry
modified bool // Has this data been modified since the last write
- writeHooks []func(map[string]interface{}, map[string]interface{})
+ writeHooks []LogMutator
}
// Create a new logger based on the specified parameters.
@@ -81,13 +92,20 @@ func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]int
return l.properties, l.entry
}
+// function to test new api, replacing Edit() and Record()
+func (l *Logger) MutateLog(mutator LogMutator) {
+ mutator(l.Edit())
+ l.Record()
+}
+
// Adds a hook which will be called every time this logger writes an entry.
// The hook takes properties and entry as arguments, in that order.
// This is useful for stuff like memory profiling.
// This must be called between Edit() and Record() (e.g. while holding the lock)
-func (l *Logger) AddWriteHook(hook func(map[string]interface{},
- map[string]interface{})) {
+func (l *Logger) AddWriteHook(hook LogMutator) {
+ // TODO(misha): Acquire lock here! and erase comment about edit.
l.writeHooks = append(l.writeHooks, hook)
+ // TODO(misha): consider flipping the dirty bit here.
}
// Write the log entry you've built up so far. Do not edit the maps
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index e73bdb9..deeea5d 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -53,21 +53,23 @@ func main() {
}
if arvLogger != nil {
- properties, _ := arvLogger.Edit()
- runInfo := make(map[string]interface{})
- runInfo["start_time"] = time.Now()
- runInfo["args"] = os.Args
- hostname, err := os.Hostname()
- if err != nil {
- runInfo["hostname_error"] = err.Error()
- } else {
- runInfo["hostname"] = hostname
- }
- runInfo["pid"] = os.Getpid()
- properties["run_info"] = runInfo
-
+ arvLogger.MutateLog(func(properties map[string]interface{},
+ entry map[string]interface{}) {
+ runInfo := make(map[string]interface{})
+ runInfo["start_time"] = time.Now()
+ runInfo["args"] = os.Args
+ hostname, err := os.Hostname()
+ if err != nil {
+ runInfo["hostname_error"] = err.Error()
+ } else {
+ runInfo["hostname"] = hostname
+ }
+ runInfo["pid"] = os.Getpid()
+ properties["run_info"] = runInfo
+ })
+
+ arvLogger.Edit()
arvLogger.AddWriteHook(LogMemoryAlloc)
-
arvLogger.Record()
}
@@ -98,7 +100,6 @@ func main() {
}
func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
- _ = entry // keep the compiler from complaining
runInfo := properties["run_info"].(map[string]interface{})
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
commit 7924fe7d6b4cee88035046005425ce19260c09e9
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 14 01:40:50 2015 +0000
Finished adding logging to keep.GetServerContents but have not tested fully yet.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index ff57127..ea0be33 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -100,6 +100,7 @@ func (l *Logger) Record() {
// We haven't written in the allowed interval yet, try to write.
l.write()
} else {
+ // TODO(misha): Only allow one outstanding write to be scheduled.
nextTimeToWrite := l.lastWrite.Add(l.params.MinimumWriteInterval)
writeAfter := nextTimeToWrite.Sub(time.Now())
time.AfterFunc(writeAfter, l.acquireLockConsiderWriting)
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 6e97f57..97bbf9d 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -19,6 +19,7 @@ import (
"strconv"
"strings"
"sync"
+ "time"
)
type ServerAddress struct {
@@ -235,7 +236,8 @@ func GetServerContents(arvLogger *logger.Logger,
req := CreateIndexRequest(arvLogger, keepServer)
resp, err := client.Do(req)
if err != nil {
- log.Fatalf("Error fetching %s: %v", req.URL.String(), err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error fetching %s: %v", req.URL.String(), err))
}
return ReadServerResponse(arvLogger, keepServer, resp)
@@ -243,13 +245,24 @@ func GetServerContents(arvLogger *logger.Logger,
func CreateIndexRequest(arvLogger *logger.Logger,
keepServer ServerAddress) (req *http.Request) {
- // Create and send request.
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
+ if arvLogger != nil {
+ properties, _ := arvLogger.Edit()
+ keepInfo := properties["keep_info"].(map[string]interface{})
+ serverInfo := make(map[string]interface{})
+ serverInfo["request_sent"] = time.Now()
+
+ keepInfo[keepServer.String()] = serverInfo
+
+ arvLogger.Record()
+ }
+
req, err := http.NewRequest("GET", url, nil)
if err != nil {
- log.Fatalf("Error building http request for %s: %v", url, err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error building http request for %s: %v", url, err))
}
req.Header.Add("Authorization",
@@ -262,8 +275,20 @@ func ReadServerResponse(arvLogger *logger.Logger,
resp *http.Response) (response ServerResponse) {
if resp.StatusCode != 200 {
- log.Fatalf("Received error code %d in response to request for %s index: %s",
- resp.StatusCode, keepServer.String(), resp.Status)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error code %d in response to request "+
+ "for %s index: %s",
+ resp.StatusCode, keepServer.String(), resp.Status))
+ }
+
+ if arvLogger != nil {
+ properties, _ := arvLogger.Edit()
+ keepInfo := properties["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+
+ serverInfo["response_received"] = time.Now()
+
+ arvLogger.Record()
}
response.Address = keepServer
@@ -275,9 +300,11 @@ func ReadServerResponse(arvLogger *logger.Logger,
numLines++
blockInfo, err := parseBlockInfoFromIndexLine(scanner.Text())
if err != nil {
- log.Fatalf("Error parsing BlockInfo from index line received from %s: %v",
- keepServer.String(),
- err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error parsing BlockInfo from index line "+
+ "received from %s: %v",
+ keepServer.String(),
+ err))
}
if storedBlock, ok := response.Contents.BlockDigestToInfo[blockInfo.Digest]; ok {
@@ -286,10 +313,24 @@ func ReadServerResponse(arvLogger *logger.Logger,
if storedBlock.Size != blockInfo.Size {
numSizeDisagreements += 1
// TODO(misha): Consider failing here.
- log.Printf("Saw different sizes for the same block on %s: %v %v",
+ message := fmt.Sprintf("Saw different sizes for the same block "+
+ "on %s: %v %v",
keepServer.String(),
storedBlock,
blockInfo)
+ log.Println(message)
+ if arvLogger != nil {
+ properties, _ := arvLogger.Edit()
+ keepInfo := properties["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+ var error_list []string
+ read_error_list, has_list := serverInfo["error_list"]
+ if has_list {
+ error_list = read_error_list.([]string)
+ } // If we didn't have the list, error_list is already an empty list
+ serverInfo["error_list"] = append(error_list, message)
+ arvLogger.Record()
+ }
}
// Keep the block that is bigger, or the block that's newer in
// the case of a size tie.
@@ -303,9 +344,10 @@ func ReadServerResponse(arvLogger *logger.Logger,
}
}
if err := scanner.Err(); err != nil {
- log.Fatalf("Received error scanning index response from %s: %v",
- keepServer.String(),
- err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error scanning index response from %s: %v",
+ keepServer.String(),
+ err))
} else {
log.Printf("%s index contained %d lines with %d duplicates with "+
"%d size disagreements",
@@ -313,6 +355,19 @@ func ReadServerResponse(arvLogger *logger.Logger,
numLines,
numDuplicates,
numSizeDisagreements)
+
+ if arvLogger != nil {
+ properties, _ := arvLogger.Edit()
+ keepInfo := properties["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
+
+ serverInfo["processing_finished"] = time.Now()
+ serverInfo["lines_received"] = numLines
+ serverInfo["dupicates_seen"] = numDuplicates
+ serverInfo["size_disagreements_seen"] = numSizeDisagreements
+
+ arvLogger.Record()
+ }
}
resp.Body.Close()
return
commit 3f74a7584760a83539b6c0ba01ffd5078d8858cf
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 14 00:50:21 2015 +0000
ran gofmt
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 87ec861..6e97f57 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -93,7 +93,7 @@ func init() {
"File with the API token we should use to contact keep servers.")
}
-func (s ServerAddress) String() (string) {
+func (s ServerAddress) String() string {
return fmt.Sprintf("%s:%d", s.Host, s.Port)
}
commit 6221a5005318304a2f05f1fa3c9d897ed71d5676
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 14 00:49:31 2015 +0000
Broke keep.GetServerContents() into smaller functions.
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index de4cbd4..87ec861 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -93,6 +93,10 @@ func init() {
"File with the API token we should use to contact keep servers.")
}
+func (s ServerAddress) String() (string) {
+ return fmt.Sprintf("%s:%d", s.Host, s.Port)
+}
+
func getDataManagerToken(arvLogger *logger.Logger) string {
readDataManagerToken := func() {
if dataManagerTokenFile == "" {
@@ -224,11 +228,21 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
return
}
-// TODO(misha): Break this function apart into smaller, easier to
-// understand functions.
func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
client http.Client) (response ServerResponse) {
+
+ req := CreateIndexRequest(arvLogger, keepServer)
+ resp, err := client.Do(req)
+ if err != nil {
+ log.Fatalf("Error fetching %s: %v", req.URL.String(), err)
+ }
+
+ return ReadServerResponse(arvLogger, keepServer, resp)
+}
+
+func CreateIndexRequest(arvLogger *logger.Logger,
+ keepServer ServerAddress) (req *http.Request) {
// Create and send request.
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
@@ -240,16 +254,16 @@ func GetServerContents(arvLogger *logger.Logger,
req.Header.Add("Authorization",
fmt.Sprintf("OAuth2 %s", getDataManagerToken(arvLogger)))
+ return
+}
- resp, err := client.Do(req)
- if err != nil {
- log.Fatalf("Error fetching %s: %v", url, err)
- }
+func ReadServerResponse(arvLogger *logger.Logger,
+ keepServer ServerAddress,
+ resp *http.Response) (response ServerResponse) {
- // Process response.
if resp.StatusCode != 200 {
- log.Fatalf("Received error code %d in response to request for %s: %s",
- resp.StatusCode, url, resp.Status)
+ log.Fatalf("Received error code %d in response to request for %s index: %s",
+ resp.StatusCode, keepServer.String(), resp.Status)
}
response.Address = keepServer
@@ -262,7 +276,7 @@ func GetServerContents(arvLogger *logger.Logger,
blockInfo, err := parseBlockInfoFromIndexLine(scanner.Text())
if err != nil {
log.Fatalf("Error parsing BlockInfo from index line received from %s: %v",
- url,
+ keepServer.String(),
err)
}
@@ -273,7 +287,7 @@ func GetServerContents(arvLogger *logger.Logger,
numSizeDisagreements += 1
// TODO(misha): Consider failing here.
log.Printf("Saw different sizes for the same block on %s: %v %v",
- url,
+ keepServer.String(),
storedBlock,
blockInfo)
}
@@ -289,11 +303,13 @@ func GetServerContents(arvLogger *logger.Logger,
}
}
if err := scanner.Err(); err != nil {
- log.Fatalf("Received error scanning response from %s: %v", url, err)
+ log.Fatalf("Received error scanning index response from %s: %v",
+ keepServer.String(),
+ err)
} else {
- log.Printf("%s contained %d lines with %d duplicates with "+
+ log.Printf("%s index contained %d lines with %d duplicates with "+
"%d size disagreements",
- url,
+ keepServer.String(),
numLines,
numDuplicates,
numSizeDisagreements)
commit 2415e93fcd6a24b3bfbc319c139737f550835e36
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 13 23:56:05 2015 +0000
Removed channel awareness from keep.GetServerContents().
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index b9e6df0..de4cbd4 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -188,7 +188,17 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// Send off all the index requests concurrently
responseChan := make(chan ServerResponse)
for _, keepServer := range sdkResponse.KeepServers {
- go GetServerContents(params.Logger, keepServer, client, responseChan)
+ // The above keepsServer variable is reused for each iteration, so
+ // it would be shared across all goroutines. This would result in
+ // us querying one server n times instead of n different servers
+ // as we intended. To avoid this we add it as an explicit
+ // parameter which gets copied. This bug and solution is described
+ // in https://golang.org/doc/effective_go.html#channels
+ go func(keepServer ServerAddress) {
+ responseChan <- GetServerContents(params.Logger,
+ keepServer,
+ client)
+ }(keepServer)
}
results.ServerToContents = make(map[ServerAddress]ServerContents)
@@ -218,8 +228,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// understand functions.
func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
- client http.Client,
- responseChan chan<- ServerResponse) {
+ client http.Client) (response ServerResponse) {
// Create and send request.
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
@@ -243,7 +252,6 @@ func GetServerContents(arvLogger *logger.Logger,
resp.StatusCode, url, resp.Status)
}
- response := ServerResponse{}
response.Address = keepServer
response.Contents.BlockDigestToInfo =
make(map[blockdigest.BlockDigest]BlockInfo)
@@ -291,7 +299,7 @@ func GetServerContents(arvLogger *logger.Logger,
numSizeDisagreements)
}
resp.Body.Close()
- responseChan <- response
+ return
}
func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) {
commit 2d2f3bed79f9504d15503277056feb394c12dd7c
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 13 23:27:05 2015 +0000
gofmt'd all my source code. No other changes.
diff --git a/sdk/go/blockdigest/blockdigest.go b/sdk/go/blockdigest/blockdigest.go
index 5225af6..0742839 100644
--- a/sdk/go/blockdigest/blockdigest.go
+++ b/sdk/go/blockdigest/blockdigest.go
@@ -28,9 +28,13 @@ func FromString(s string) (dig BlockDigest, err error) {
var d BlockDigest
d.h, err = strconv.ParseUint(s[:16], 16, 64)
- if err != nil {return}
+ if err != nil {
+ return
+ }
d.l, err = strconv.ParseUint(s[16:], 16, 64)
- if err != nil {return}
+ if err != nil {
+ return
+ }
dig = d
return
}
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 294ba92..ff57127 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -32,33 +32,33 @@ import (
)
type LoggerParams struct {
- Client arvadosclient.ArvadosClient // The client we use to write log entries
- EventType string // The event type to assign to the log entry.
- MinimumWriteInterval time.Duration // Wait at least this long between log writes
+ Client arvadosclient.ArvadosClient // The client we use to write log entries
+ EventType string // The event type to assign to the log entry.
+ MinimumWriteInterval time.Duration // Wait at least this long between log writes
}
// A Logger is used to build up a log entry over time and write every
// version of it.
type Logger struct {
// The Data we write
- data map[string]interface{} // The entire map that we give to the api
- entry map[string]interface{} // Convenience shortcut into data
- properties map[string]interface{} // Convenience shortcut into data
+ data map[string]interface{} // The entire map that we give to the api
+ entry map[string]interface{} // Convenience shortcut into data
+ properties map[string]interface{} // Convenience shortcut into data
- lock sync.Locker // Synchronizes editing and writing
- params LoggerParams // Parameters we were given
+ lock sync.Locker // Synchronizes editing and writing
+ params LoggerParams // Parameters we were given
- lastWrite time.Time // The last time we wrote a log entry
- modified bool // Has this data been modified since the last write
+ lastWrite time.Time // The last time we wrote a log entry
+ modified bool // Has this data been modified since the last write
- writeHooks []func(map[string]interface{},map[string]interface{})
+ writeHooks []func(map[string]interface{}, map[string]interface{})
}
// Create a new logger based on the specified parameters.
func NewLogger(params LoggerParams) *Logger {
// TODO(misha): Add some params checking here.
l := &Logger{data: make(map[string]interface{}),
- lock: &sync.Mutex{},
+ lock: &sync.Mutex{},
params: params}
l.entry = make(map[string]interface{})
l.data["log"] = l.entry
@@ -70,13 +70,13 @@ func NewLogger(params LoggerParams) *Logger {
// Get access to the maps you can edit. This will hold a lock until
// you call Record. Do not edit the maps in any other goroutines or
// after calling Record.
-// You don't need to edit both maps,
+// You don't need to edit both maps,
// properties can take any values you want to give it,
// entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
// properties is a shortcut for entry["properties"].(map[string]interface{})
func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
l.lock.Lock()
- l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
+ l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
return l.properties, l.entry
}
@@ -120,7 +120,6 @@ func (l *Logger) writeAllowedNow() bool {
return l.lastWrite.Add(l.params.MinimumWriteInterval).Before(time.Now())
}
-
// Actually writes the log entry. This method assumes we're holding the lock.
func (l *Logger) write() {
@@ -144,7 +143,6 @@ func (l *Logger) write() {
l.modified = false
}
-
func (l *Logger) acquireLockConsiderWriting() {
l.lock.Lock()
if l.modified && l.writeAllowedNow() {
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 1227f49..c9f9018 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -21,20 +21,20 @@ type Manifest struct {
}
type BlockLocator struct {
- Digest blockdigest.BlockDigest
- Size int
- Hints []string
+ Digest blockdigest.BlockDigest
+ Size int
+ Hints []string
}
type ManifestLine struct {
- StreamName string
- Blocks []string
- Files []string
+ StreamName string
+ Blocks []string
+ Files []string
}
func ParseBlockLocator(s string) (b BlockLocator, err error) {
if !LocatorPattern.MatchString(s) {
- err = fmt.Errorf("String \"%s\" does not match BlockLocator pattern " +
+ err = fmt.Errorf("String \"%s\" does not match BlockLocator pattern "+
"\"%s\".",
s,
LocatorPattern.String())
@@ -45,9 +45,13 @@ func ParseBlockLocator(s string) (b BlockLocator, err error) {
// We expect both of the following to succeed since LocatorPattern
// restricts the strings appropriately.
blockDigest, err = blockdigest.FromString(tokens[0])
- if err != nil {return}
+ if err != nil {
+ return
+ }
blockSize, err = strconv.ParseInt(tokens[1], 10, 0)
- if err != nil {return}
+ if err != nil {
+ return
+ }
b.Digest = blockDigest
b.Size = int(blockSize)
b.Hints = tokens[2:]
@@ -92,7 +96,6 @@ func (m *Manifest) LineIter() <-chan ManifestLine {
return ch
}
-
// Blocks may appear mulitple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
diff --git a/sdk/go/util/util.go b/sdk/go/util/util.go
index 9b0fe21..4505db6 100644
--- a/sdk/go/util/util.go
+++ b/sdk/go/util/util.go
@@ -56,7 +56,7 @@ func ContainsAllAvailableItems(response SdkListResponse) (containsAll bool, numC
}
numAvailable, err = response.NumItemsAvailable()
if err != nil {
- log.Fatalf("Error retrieving number of items available from " +
+ log.Fatalf("Error retrieving number of items available from "+
"SDK response: %v",
err)
}
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index fbdec15..73115f5 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -21,44 +21,44 @@ var (
heap_profile_filename string
// globals for debugging
totalManifestSize uint64
- maxManifestSize uint64
+ maxManifestSize uint64
)
type Collection struct {
- Uuid string
- OwnerUuid string
- ReplicationLevel int
+ Uuid string
+ OwnerUuid string
+ ReplicationLevel int
BlockDigestToSize map[blockdigest.BlockDigest]int
- TotalSize int
+ TotalSize int
}
type ReadCollections struct {
- ReadAllCollections bool
- UuidToCollection map[string]Collection
+ ReadAllCollections bool
+ UuidToCollection map[string]Collection
OwnerToCollectionSize map[string]int
}
type GetCollectionsParams struct {
- Client arvadosclient.ArvadosClient
- Logger *logger.Logger
+ Client arvadosclient.ArvadosClient
+ Logger *logger.Logger
BatchSize int
}
type SdkCollectionInfo struct {
- Uuid string `json:"uuid"`
- OwnerUuid string `json:"owner_uuid"`
- Redundancy int `json:"redundancy"`
- ModifiedAt time.Time `json:"modified_at"`
- ManifestText string `json:"manifest_text"`
+ Uuid string `json:"uuid"`
+ OwnerUuid string `json:"owner_uuid"`
+ Redundancy int `json:"redundancy"`
+ ModifiedAt time.Time `json:"modified_at"`
+ ManifestText string `json:"manifest_text"`
}
type SdkCollectionList struct {
- ItemsAvailable int `json:"items_available"`
- Items []SdkCollectionInfo `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Items []SdkCollectionInfo `json:"items"`
}
func init() {
- flag.StringVar(&heap_profile_filename,
+ flag.StringVar(&heap_profile_filename,
"heap-profile",
"",
"File to write the heap profiles to. Leave blank to skip profiling.")
@@ -96,13 +96,12 @@ func WriteHeapProfile() {
}
}
-
func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
results = GetCollections(params)
ComputeSizeOfOwnedCollections(&results)
if params.Logger != nil {
- properties,_ := params.Logger.Edit()
+ properties, _ := params.Logger.Edit()
collectionInfo := properties["collection_info"].(map[string]interface{})
collectionInfo["owner_to_collection_size"] = results.OwnerToCollectionSize
params.Logger.Record()
@@ -136,8 +135,8 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
"modified_at"}
sdkParams := arvadosclient.Dict{
- "select": fieldsWanted,
- "order": []string{"modified_at ASC"},
+ "select": fieldsWanted,
+ "order": []string{"modified_at ASC"},
"filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}}}
if params.BatchSize > 0 {
@@ -152,7 +151,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
if params.Logger != nil {
- properties,_ := params.Logger.Edit()
+ properties, _ := params.Logger.Edit()
collectionInfo := make(map[string]interface{})
collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
collectionInfo["batch_size"] = params.BatchSize
@@ -181,23 +180,23 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
// Process collection and update our date filter.
sdkParams["filters"].([][]string)[0][2] =
ProcessCollections(params.Logger,
- collections.Items,
- results.UuidToCollection).Format(time.RFC3339)
+ collections.Items,
+ results.UuidToCollection).Format(time.RFC3339)
// update counts
previousTotalCollections = totalCollections
totalCollections = len(results.UuidToCollection)
- log.Printf("%d collections read, %d new in last batch, " +
+ log.Printf("%d collections read, %d new in last batch, "+
"%s latest modified date, %.0f %d %d avg,max,total manifest size",
totalCollections,
- totalCollections - previousTotalCollections,
+ totalCollections-previousTotalCollections,
sdkParams["filters"].([][]string)[0][2],
float32(totalManifestSize)/float32(totalCollections),
maxManifestSize, totalManifestSize)
if params.Logger != nil {
- properties,_ := params.Logger.Edit()
+ properties, _ := params.Logger.Edit()
collectionInfo := properties["collection_info"].(map[string]interface{})
collectionInfo["collections_read"] = totalCollections
collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
@@ -216,7 +215,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
return
}
-
// StrCopy returns a newly allocated string.
// It is useful to copy slices so that the garbage collector can reuse
// the memory of the longer strings they came from.
@@ -224,22 +222,21 @@ func StrCopy(s string) string {
return string([]byte(s))
}
-
func ProcessCollections(arvLogger *logger.Logger,
receivedCollections []SdkCollectionInfo,
uuidToCollection map[string]Collection) (latestModificationDate time.Time) {
for _, sdkCollection := range receivedCollections {
collection := Collection{Uuid: StrCopy(sdkCollection.Uuid),
- OwnerUuid: StrCopy(sdkCollection.OwnerUuid),
- ReplicationLevel: sdkCollection.Redundancy,
+ OwnerUuid: StrCopy(sdkCollection.OwnerUuid),
+ ReplicationLevel: sdkCollection.Redundancy,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
if sdkCollection.ModifiedAt.IsZero() {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf(
- "Arvados SDK collection returned with unexpected zero " +
- "modifcation date. This probably means that either we failed to " +
- "parse the modification date or the API server has changed how " +
+ "Arvados SDK collection returned with unexpected zero "+
+ "modifcation date. This probably means that either we failed to "+
+ "parse the modification date or the API server has changed how "+
"it returns modification dates: %v",
collection))
}
@@ -256,11 +253,10 @@ func ProcessCollections(arvLogger *logger.Logger,
if manifestSize > maxManifestSize {
maxManifestSize = manifestSize
}
-
+
blockChannel := manifest.BlockIterWithDuplicates()
for block := range blockChannel {
- if stored_size, stored := collection.BlockDigestToSize[block.Digest];
- stored && stored_size != block.Size {
+ if stored_size, stored := collection.BlockDigestToSize[block.Digest]; stored && stored_size != block.Size {
message := fmt.Sprintf(
"Collection %s contains multiple sizes (%d and %d) for block %s",
collection.Uuid,
@@ -286,8 +282,7 @@ func ProcessCollections(arvLogger *logger.Logger,
return
}
-
-func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) (int) {
+func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) int {
var collections SdkCollectionList
sdkParams := arvadosclient.Dict{"limit": 0}
err := client.List("collections", sdkParams, &collections)
@@ -298,7 +293,6 @@ func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) (int) {
return collections.ItemsAvailable
}
-
func ComputeSizeOfOwnedCollections(readCollections *ReadCollections) {
readCollections.OwnerToCollectionSize = make(map[string]int)
for _, coll := range readCollections.UuidToCollection {
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index d7d926e..e73bdb9 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -16,16 +16,16 @@ import (
)
var (
- logEventType string
+ logEventType string
logFrequencySeconds int
)
func init() {
- flag.StringVar(&logEventType,
+ flag.StringVar(&logEventType,
"log-event-type",
"experimental-data-manager-report",
"event_type to use in our arvados log entries. Set to empty to turn off logging")
- flag.IntVar(&logFrequencySeconds,
+ flag.IntVar(&logFrequencySeconds,
"log-frequency-seconds",
20,
"How frequently we'll write log entries in seconds.")
@@ -48,7 +48,7 @@ func main() {
var arvLogger *logger.Logger
if logEventType != "" {
arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: logEventType,
+ EventType: logEventType,
MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
@@ -73,22 +73,24 @@ func main() {
collectionChannel := make(chan collection.ReadCollections)
- go func() { collectionChannel <- collection.GetCollectionsAndSummarize(
- collection.GetCollectionsParams{
- Client: arv, Logger: arvLogger, BatchSize: 50}) }()
+ go func() {
+ collectionChannel <- collection.GetCollectionsAndSummarize(
+ collection.GetCollectionsParams{
+ Client: arv, Logger: arvLogger, BatchSize: 50})
+ }()
keepServerInfo := keep.GetKeepServersAndSummarize(
keep.GetKeepServersParams{Client: arv, Logger: arvLogger, Limit: 1000})
readCollections := <-collectionChannel
- // Make compiler happy.
+ // Make compiler happy.
_ = readCollections
_ = keepServerInfo
// Log that we're finished
if arvLogger != nil {
- properties,_ := arvLogger.Edit()
+ properties, _ := arvLogger.Edit()
properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
// Force the recording, since go will not wait for the timer before exiting.
arvLogger.ForceRecord()
@@ -96,7 +98,7 @@ func main() {
}
func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
- _ = entry // keep the compiler from complaining
+ _ = entry // keep the compiler from complaining
runInfo := properties["run_info"].(map[string]interface{})
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 8b46066..b9e6df0 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -13,8 +13,8 @@ import (
"git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
- "log"
"io/ioutil"
+ "log"
"net/http"
"strconv"
"strings"
@@ -23,21 +23,21 @@ import (
type ServerAddress struct {
Host string `json:"service_host"`
- Port int `json:"service_port"`
+ Port int `json:"service_port"`
}
// Info about a particular block returned by the server
type BlockInfo struct {
- Digest blockdigest.BlockDigest
- Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Digest blockdigest.BlockDigest
+ Size int
+ Mtime int // TODO(misha): Replace this with a timestamp.
}
// Info about a specified block given by a server
type BlockServerInfo struct {
ServerIndex int
Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Mtime int // TODO(misha): Replace this with a timestamp.
}
type ServerContents struct {
@@ -45,28 +45,28 @@ type ServerContents struct {
}
type ServerResponse struct {
- Address ServerAddress
+ Address ServerAddress
Contents ServerContents
}
type ReadServers struct {
- ReadAllServers bool
- KeepServerIndexToAddress []ServerAddress
- KeepServerAddressToIndex map[ServerAddress]int
- ServerToContents map[ServerAddress]ServerContents
- BlockToServers map[blockdigest.BlockDigest][]BlockServerInfo
- BlockReplicationCounts map[int]int
+ ReadAllServers bool
+ KeepServerIndexToAddress []ServerAddress
+ KeepServerAddressToIndex map[ServerAddress]int
+ ServerToContents map[ServerAddress]ServerContents
+ BlockToServers map[blockdigest.BlockDigest][]BlockServerInfo
+ BlockReplicationCounts map[int]int
}
type GetKeepServersParams struct {
Client arvadosclient.ArvadosClient
Logger *logger.Logger
- Limit int
+ Limit int
}
type KeepServiceList struct {
- ItemsAvailable int `json:"items_available"`
- KeepServers []ServerAddress `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ KeepServers []ServerAddress `json:"items"`
}
// Methods to implement util.SdkListResponse Interface
@@ -81,20 +81,20 @@ func (k KeepServiceList) NumItemsContained() (numContained int, err error) {
var (
// Don't access the token directly, use getDataManagerToken() to
// make sure it's been read.
- dataManagerToken string
- dataManagerTokenFile string
- dataManagerTokenFileReadOnce sync.Once
+ dataManagerToken string
+ dataManagerTokenFile string
+ dataManagerTokenFileReadOnce sync.Once
)
func init() {
- flag.StringVar(&dataManagerTokenFile,
+ flag.StringVar(&dataManagerTokenFile,
"data-manager-token-file",
"",
"File with the API token we should use to contact keep servers.")
}
-func getDataManagerToken(arvLogger *logger.Logger) (string) {
- readDataManagerToken := func () {
+func getDataManagerToken(arvLogger *logger.Logger) string {
+ readDataManagerToken := func() {
if dataManagerTokenFile == "" {
flag.Usage()
loggerutil.FatalWithMessage(arvLogger,
@@ -151,7 +151,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
results.ReadAllServers, numReceived, numAvailable =
util.ContainsAllAvailableItems(sdkResponse)
- if (!results.ReadAllServers) {
+ if !results.ReadAllServers {
log.Printf("ERROR: Did not receive all keep server addresses.")
}
log.Printf("Received %d of %d available keep server addresses.",
@@ -160,7 +160,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
}
if params.Logger != nil {
- properties,_ := params.Logger.Edit()
+ properties, _ := params.Logger.Edit()
keepInfo := make(map[string]interface{})
keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
@@ -196,8 +196,8 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// Read all the responses
for i := range sdkResponse.KeepServers {
- _ = i // Here to prevent go from complaining.
- response := <- responseChan
+ _ = i // Here to prevent go from complaining.
+ response := <-responseChan
log.Printf("Received channel response from %v containing %d files",
response.Address,
len(response.Contents.BlockDigestToInfo))
@@ -207,7 +207,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
results.BlockToServers[blockInfo.Digest] = append(
results.BlockToServers[blockInfo.Digest],
BlockServerInfo{ServerIndex: serverIndex,
- Size: blockInfo.Size,
+ Size: blockInfo.Size,
Mtime: blockInfo.Mtime})
}
}
@@ -219,7 +219,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
client http.Client,
- responseChan chan<- ServerResponse) () {
+ responseChan chan<- ServerResponse) {
// Create and send request.
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
@@ -273,7 +273,7 @@ func GetServerContents(arvLogger *logger.Logger,
// the case of a size tie.
if storedBlock.Size < blockInfo.Size ||
(storedBlock.Size == blockInfo.Size &&
- storedBlock.Mtime < blockInfo.Mtime) {
+ storedBlock.Mtime < blockInfo.Mtime) {
response.Contents.BlockDigestToInfo[blockInfo.Digest] = blockInfo
}
} else {
@@ -283,7 +283,7 @@ func GetServerContents(arvLogger *logger.Logger,
if err := scanner.Err(); err != nil {
log.Fatalf("Received error scanning response from %s: %v", url, err)
} else {
- log.Printf("%s contained %d lines with %d duplicates with " +
+ log.Printf("%s contained %d lines with %d duplicates with "+
"%d size disagreements",
url,
numLines,
@@ -297,7 +297,7 @@ func GetServerContents(arvLogger *logger.Logger,
func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) {
tokens := strings.Fields(indexLine)
if len(tokens) != 2 {
- err = fmt.Errorf("Expected 2 tokens per line but received a " +
+ err = fmt.Errorf("Expected 2 tokens per line but received a "+
"line containing %v instead.",
tokens)
}
@@ -307,7 +307,7 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
return
}
if len(locator.Hints) > 0 {
- err = fmt.Errorf("Block locator in index line should not contain hints " +
+ err = fmt.Errorf("Block locator in index line should not contain hints "+
"but it does: %v",
locator)
return
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index e4a53c2..f97f7c1 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -13,7 +13,7 @@ import (
// for the lock you're already holding.
func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
- properties,_ := arvLogger.Edit()
+ properties, _ := arvLogger.Edit()
properties["FATAL"] = message
properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
arvLogger.ForceRecord()
@@ -21,4 +21,3 @@ func FatalWithMessage(arvLogger *logger.Logger, message string) {
log.Fatalf(message)
}
-
commit 144e23888d46d68c5e32fea9f66a8903e05a3526
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 13 01:15:34 2015 +0000
Started logging in keep.go. More work to be done.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 9b1f6d6..d7d926e 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -78,7 +78,7 @@ func main() {
Client: arv, Logger: arvLogger, BatchSize: 50}) }()
keepServerInfo := keep.GetKeepServersAndSummarize(
- keep.GetKeepServersParams{Client: arv, Limit: 1000})
+ keep.GetKeepServersParams{Client: arv, Logger: arvLogger, Limit: 1000})
readCollections := <-collectionChannel
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 413e1be..8b46066 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -9,8 +9,10 @@ import (
//"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
+ "git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/sdk/go/util"
+ "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
"io/ioutil"
"net/http"
@@ -58,6 +60,7 @@ type ReadServers struct {
type GetKeepServersParams struct {
Client arvadosclient.ArvadosClient
+ Logger *logger.Logger
Limit int
}
@@ -90,17 +93,19 @@ func init() {
"File with the API token we should use to contact keep servers.")
}
-func getDataManagerToken() (string) {
+func getDataManagerToken(arvLogger *logger.Logger) (string) {
readDataManagerToken := func () {
if dataManagerTokenFile == "" {
flag.Usage()
- log.Fatalf("Data Manager Token needed, but data manager token file not specified.")
+ loggerutil.FatalWithMessage(arvLogger,
+ "Data Manager Token needed, but data manager token file not specified.")
} else {
rawRead, err := ioutil.ReadFile(dataManagerTokenFile)
if err != nil {
- log.Fatalf("Unexpected error reading token file %s: %v",
- dataManagerTokenFile,
- err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Unexpected error reading token file %s: %v",
+ dataManagerTokenFile,
+ err))
}
dataManagerToken = strings.TrimSpace(string(rawRead))
}
@@ -136,11 +141,11 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
err := params.Client.Call("GET", "keep_services", "", "accessible", sdkParams, &sdkResponse)
if err != nil {
- log.Fatalf("Error requesting keep disks from API server: %v", err)
+ loggerutil.FatalWithMessage(params.Logger,
+ fmt.Sprintf("Error requesting keep disks from API server: %v", err))
}
- log.Printf("Received keep services list: %v", sdkResponse)
-
+ // TODO(misha): Rewrite this block, stop using ContainsAllAvailableItems()
{
var numReceived, numAvailable int
results.ReadAllServers, numReceived, numAvailable =
@@ -154,6 +159,21 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
numAvailable)
}
+ if params.Logger != nil {
+ properties,_ := params.Logger.Edit()
+ keepInfo := make(map[string]interface{})
+
+ keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
+ keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
+ keepInfo["keep_servers"] = sdkResponse.KeepServers
+
+ properties["keep_info"] = keepInfo
+
+ params.Logger.Record()
+ }
+
+ log.Printf("Received keep services list: %v", sdkResponse)
+
results.KeepServerIndexToAddress = sdkResponse.KeepServers
results.KeepServerAddressToIndex = make(map[ServerAddress]int)
for i, address := range results.KeepServerIndexToAddress {
@@ -168,7 +188,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// Send off all the index requests concurrently
responseChan := make(chan ServerResponse)
for _, keepServer := range sdkResponse.KeepServers {
- go GetServerContents(keepServer, client, responseChan)
+ go GetServerContents(params.Logger, keepServer, client, responseChan)
}
results.ServerToContents = make(map[ServerAddress]ServerContents)
@@ -196,7 +216,8 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// TODO(misha): Break this function apart into smaller, easier to
// understand functions.
-func GetServerContents(keepServer ServerAddress,
+func GetServerContents(arvLogger *logger.Logger,
+ keepServer ServerAddress,
client http.Client,
responseChan chan<- ServerResponse) () {
// Create and send request.
@@ -209,7 +230,7 @@ func GetServerContents(keepServer ServerAddress,
}
req.Header.Add("Authorization",
- fmt.Sprintf("OAuth2 %s", getDataManagerToken()))
+ fmt.Sprintf("OAuth2 %s", getDataManagerToken(arvLogger)))
resp, err := client.Do(req)
if err != nil {
commit e0889a8f6997327fd9b4d826237c8166cf909741
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 13 00:49:53 2015 +0000
Created loggerutil to hold common datamanager logger code. Moved FatalWithMessage to it.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index f9e2a3d..fbdec15 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -9,6 +9,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
+ "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
"os"
"runtime"
@@ -173,7 +174,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
var collections SdkCollectionList
err := params.Client.List("collections", sdkParams, &collections)
if err != nil {
- fatalWithMessage(params.Logger,
+ loggerutil.FatalWithMessage(params.Logger,
fmt.Sprintf("Error querying collections: %v", err))
}
@@ -234,7 +235,7 @@ func ProcessCollections(arvLogger *logger.Logger,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
if sdkCollection.ModifiedAt.IsZero() {
- fatalWithMessage(arvLogger,
+ loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf(
"Arvados SDK collection returned with unexpected zero " +
"modifcation date. This probably means that either we failed to " +
@@ -266,7 +267,7 @@ func ProcessCollections(arvLogger *logger.Logger,
stored_size,
block.Size,
block.Digest)
- fatalWithMessage(arvLogger, message)
+ loggerutil.FatalWithMessage(arvLogger, message)
}
collection.BlockDigestToSize[block.Digest] = block.Size
}
@@ -307,18 +308,3 @@ func ComputeSizeOfOwnedCollections(readCollections *ReadCollections) {
return
}
-
-
-// Assumes you haven't already called arvLogger.Edit()!
-// If you have called arvLogger.Edit() this method will hang waiting
-// for the lock you're already holding.
-func fatalWithMessage(arvLogger *logger.Logger, message string) {
- if arvLogger != nil {
- properties,_ := arvLogger.Edit()
- properties["FATAL"] = message
- properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
- arvLogger.ForceRecord()
- }
-
- log.Fatalf(message)
-}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
new file mode 100644
index 0000000..e4a53c2
--- /dev/null
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -0,0 +1,24 @@
+/* Datamanager-specific logging methods. */
+
+package loggerutil
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/logger"
+ "log"
+ "time"
+)
+
+// Assumes you haven't already called arvLogger.Edit()!
+// If you have called arvLogger.Edit() this method will hang waiting
+// for the lock you're already holding.
+func FatalWithMessage(arvLogger *logger.Logger, message string) {
+ if arvLogger != nil {
+ properties,_ := arvLogger.Edit()
+ properties["FATAL"] = message
+ properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ arvLogger.ForceRecord()
+ }
+
+ log.Fatalf(message)
+}
+
commit 2ee024868c8152903a43a8c6f5dce601305e99e2
Author: mishaz <misha at curoverse.com>
Date: Tue Jan 13 00:23:17 2015 +0000
Moved some logic from datamanager.go to keep.go.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 6bd9ee8..9b1f6d6 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -77,10 +77,14 @@ func main() {
collection.GetCollectionsParams{
Client: arv, Logger: arvLogger, BatchSize: 50}) }()
- RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
+ keepServerInfo := keep.GetKeepServersAndSummarize(
+ keep.GetKeepServersParams{Client: arv, Limit: 1000})
readCollections := <-collectionChannel
- _ = readCollections // Make compiler happy.
+
+ // Make compiler happy.
+ _ = readCollections
+ _ = keepServerInfo
// Log that we're finished
if arvLogger != nil {
@@ -91,20 +95,6 @@ func main() {
}
}
-func RunKeep(params keep.GetKeepServersParams) {
- readServers := keep.GetKeepServers(params)
-
- log.Printf("Returned %d keep disks", len(readServers.ServerToContents))
-
- blockReplicationCounts := make(map[int]int)
- for _, infos := range readServers.BlockToServers {
- replication := len(infos)
- blockReplicationCounts[replication] += 1
- }
-
- log.Printf("Replication level distribution: %v", blockReplicationCounts)
-}
-
func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
_ = entry // keep the compiler from complaining
runInfo := properties["run_info"].(map[string]interface{})
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 91af201..413e1be 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -48,11 +48,12 @@ type ServerResponse struct {
}
type ReadServers struct {
- ReadAllServers bool
- KeepServerIndexToAddress []ServerAddress
- KeepServerAddressToIndex map[ServerAddress]int
- ServerToContents map[ServerAddress]ServerContents
- BlockToServers map[blockdigest.BlockDigest][]BlockServerInfo
+ ReadAllServers bool
+ KeepServerIndexToAddress []ServerAddress
+ KeepServerAddressToIndex map[ServerAddress]int
+ ServerToContents map[ServerAddress]ServerContents
+ BlockToServers map[blockdigest.BlockDigest][]BlockServerInfo
+ BlockReplicationCounts map[int]int
}
type GetKeepServersParams struct {
@@ -109,6 +110,17 @@ func getDataManagerToken() (string) {
return dataManagerToken
}
+func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServers) {
+ results = GetKeepServers(params)
+ log.Printf("Returned %d keep disks", len(results.ServerToContents))
+
+ ComputeBlockReplicationCounts(&results)
+ log.Printf("Replication level distribution: %v",
+ results.BlockReplicationCounts)
+
+ return
+}
+
func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
if ¶ms.Client == nil {
log.Fatalf("params.Client passed to GetKeepServers() should " +
@@ -288,3 +300,11 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
blockInfo.Size = locator.Size
return
}
+
+func ComputeBlockReplicationCounts(readServers *ReadServers) {
+ readServers.BlockReplicationCounts = make(map[int]int)
+ for _, infos := range readServers.BlockToServers {
+ replication := len(infos)
+ readServers.BlockReplicationCounts[replication] += 1
+ }
+}
commit 4879e0e2f75fd387720b4f4b58ca6ae48a798c98
Author: mishaz <misha at curoverse.com>
Date: Mon Jan 12 23:20:50 2015 +0000
Started reading collections and keep data in parallel. Moved some logic from datamanager.go to collections.go. Added logging to end of run.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index d8352db..f9e2a3d 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -34,6 +34,7 @@ type Collection struct {
type ReadCollections struct {
ReadAllCollections bool
UuidToCollection map[string]Collection
+ OwnerToCollectionSize map[string]int
}
type GetCollectionsParams struct {
@@ -95,6 +96,31 @@ func WriteHeapProfile() {
}
+func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
+ results = GetCollections(params)
+ ComputeSizeOfOwnedCollections(&results)
+
+ if params.Logger != nil {
+ properties,_ := params.Logger.Edit()
+ collectionInfo := properties["collection_info"].(map[string]interface{})
+ collectionInfo["owner_to_collection_size"] = results.OwnerToCollectionSize
+ params.Logger.Record()
+ }
+
+ log.Printf("Uuid to Size used: %v", results.OwnerToCollectionSize)
+ log.Printf("Read and processed %d collections",
+ len(results.UuidToCollection))
+
+ // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
+ // lots of behaviors can become warnings (and obviously we can't
+ // write anything).
+ // if !readCollections.ReadAllCollections {
+ // log.Fatalf("Did not read all collections")
+ // }
+
+ return
+}
+
func GetCollections(params GetCollectionsParams) (results ReadCollections) {
if ¶ms.Client == nil {
log.Fatalf("params.Client passed to GetCollections() should " +
@@ -272,6 +298,17 @@ func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) (int) {
}
+func ComputeSizeOfOwnedCollections(readCollections *ReadCollections) {
+ readCollections.OwnerToCollectionSize = make(map[string]int)
+ for _, coll := range readCollections.UuidToCollection {
+ readCollections.OwnerToCollectionSize[coll.OwnerUuid] =
+ readCollections.OwnerToCollectionSize[coll.OwnerUuid] + coll.TotalSize
+ }
+
+ return
+}
+
+
// Assumes you haven't already called arvLogger.Edit()!
// If you have called arvLogger.Edit() this method will hang waiting
// for the lock you're already holding.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 87a71a9..6bd9ee8 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -71,30 +71,24 @@ func main() {
arvLogger.Record()
}
- // TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
- // This requires waiting on them to finish before you let main() exit.
+ collectionChannel := make(chan collection.ReadCollections)
- RunCollections(collection.GetCollectionsParams{
- Client: arv, Logger: arvLogger, BatchSize: 50})
+ go func() { collectionChannel <- collection.GetCollectionsAndSummarize(
+ collection.GetCollectionsParams{
+ Client: arv, Logger: arvLogger, BatchSize: 50}) }()
RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
-}
-
-func RunCollections(params collection.GetCollectionsParams) {
- readCollections := collection.GetCollections(params)
- UserUsage := ComputeSizeOfOwnedCollections(readCollections)
- log.Printf("Uuid to Size used: %v", UserUsage)
+ readCollections := <-collectionChannel
+ _ = readCollections // Make compiler happy.
- // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
- // lots of behaviors can become warnings (and obviously we can't
- // write anything).
- // if !readCollections.ReadAllCollections {
- // log.Fatalf("Did not read all collections")
- // }
-
- log.Printf("Read and processed %d collections",
- len(readCollections.UuidToCollection))
+ // Log that we're finished
+ if arvLogger != nil {
+ properties,_ := arvLogger.Edit()
+ properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ // Force the recording, since go will not wait for the timer before exiting.
+ arvLogger.ForceRecord()
+ }
}
func RunKeep(params keep.GetKeepServersParams) {
@@ -111,15 +105,6 @@ func RunKeep(params keep.GetKeepServersParams) {
log.Printf("Replication level distribution: %v", blockReplicationCounts)
}
-func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
- results map[string]int) {
- results = make(map[string]int)
- for _, coll := range readCollections.UuidToCollection {
- results[coll.OwnerUuid] = results[coll.OwnerUuid] + coll.TotalSize
- }
- return
-}
-
func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
_ = entry // keep the compiler from complaining
runInfo := properties["run_info"].(map[string]interface{})
commit 5bcba288077488791daa43a15d5fd5fb0c6e653c
Merge: 95cbdf5 61fdce2
Author: mishaz <misha at curoverse.com>
Date: Sun Jan 11 18:29:13 2015 +0000
Merge branch 'master' of git.curoverse.com:arvados into 3408-production-datamanager
Conflicts resolved:
services/api/Gemfile
services/api/Gemfile.lock
commit 95cbdf59eb5326c393ae91f243f596984cba7fa7
Author: mishaz <misha at curoverse.com>
Date: Fri Jan 9 04:53:42 2015 +0000
Added recording of fatal errors to logger.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index dd37377..d8352db 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -4,6 +4,7 @@ package collection
import (
"flag"
+ "fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
@@ -146,17 +147,15 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
var collections SdkCollectionList
err := params.Client.List("collections", sdkParams, &collections)
if err != nil {
- if params.Logger != nil {
- properties,_ := params.Logger.Edit()
- properties["FATAL"] = err.Error()
- params.Logger.Record()
- }
- log.Fatalf("error querying collections: %+v", err)
+ fatalWithMessage(params.Logger,
+ fmt.Sprintf("Error querying collections: %v", err))
}
// Process collection and update our date filter.
sdkParams["filters"].([][]string)[0][2] =
- ProcessCollections(collections.Items, results.UuidToCollection).Format(time.RFC3339)
+ ProcessCollections(params.Logger,
+ collections.Items,
+ results.UuidToCollection).Format(time.RFC3339)
// update counts
previousTotalCollections = totalCollections
@@ -199,7 +198,8 @@ func StrCopy(s string) string {
}
-func ProcessCollections(receivedCollections []SdkCollectionInfo,
+func ProcessCollections(arvLogger *logger.Logger,
+ receivedCollections []SdkCollectionInfo,
uuidToCollection map[string]Collection) (latestModificationDate time.Time) {
for _, sdkCollection := range receivedCollections {
collection := Collection{Uuid: StrCopy(sdkCollection.Uuid),
@@ -208,13 +208,15 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
if sdkCollection.ModifiedAt.IsZero() {
- log.Fatalf(
- "Arvados SDK collection returned with unexpected zero modifcation " +
- "date. This probably means that either we failed to parse the " +
- "modification date or the API server has changed how it returns " +
- "modification dates: %v",
- collection)
+ fatalWithMessage(arvLogger,
+ fmt.Sprintf(
+ "Arvados SDK collection returned with unexpected zero " +
+ "modifcation date. This probably means that either we failed to " +
+ "parse the modification date or the API server has changed how " +
+ "it returns modification dates: %v",
+ collection))
}
+
if sdkCollection.ModifiedAt.After(latestModificationDate) {
latestModificationDate = sdkCollection.ModifiedAt
}
@@ -232,12 +234,13 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
for block := range blockChannel {
if stored_size, stored := collection.BlockDigestToSize[block.Digest];
stored && stored_size != block.Size {
- log.Fatalf(
+ message := fmt.Sprintf(
"Collection %s contains multiple sizes (%d and %d) for block %s",
collection.Uuid,
stored_size,
block.Size,
block.Digest)
+ fatalWithMessage(arvLogger, message)
}
collection.BlockDigestToSize[block.Digest] = block.Size
}
@@ -267,3 +270,18 @@ func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) (int) {
return collections.ItemsAvailable
}
+
+
+// Assumes you haven't already called arvLogger.Edit()!
+// If you have called arvLogger.Edit() this method will hang waiting
+// for the lock you're already holding.
+func fatalWithMessage(arvLogger *logger.Logger, message string) {
+ if arvLogger != nil {
+ properties,_ := arvLogger.Edit()
+ properties["FATAL"] = message
+ properties["run_info"].(map[string]interface{})["end_time"] = time.Now()
+ arvLogger.ForceRecord()
+ }
+
+ log.Fatalf(message)
+}
commit b14bc80d764b85a6ddfed32198b2144b5adf2637
Author: mishaz <misha at curoverse.com>
Date: Fri Jan 9 04:11:29 2015 +0000
Added ForceRecord() method to enable writing remaining log changes before exiting.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 038482d..294ba92 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -107,6 +107,13 @@ func (l *Logger) Record() {
l.lock.Unlock()
}
+// Similar to Record, but forces a write without respecting the
+// MinimumWriteInterval. This is useful if you know that you're about
+// to quit (e.g. if you discovered a fatal error).
+func (l *Logger) ForceRecord() {
+ l.write()
+ l.lock.Unlock()
+}
// Whether enough time has elapsed since the last write.
func (l *Logger) writeAllowedNow() bool {
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 7e94a11..dd37377 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -58,7 +58,7 @@ func init() {
flag.StringVar(&heap_profile_filename,
"heap-profile",
"",
- "File to write the heap profiles to.")
+ "File to write the heap profiles to. Leave blank to skip profiling.")
}
// // Methods to implement util.SdkListResponse Interface
@@ -146,6 +146,11 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
var collections SdkCollectionList
err := params.Client.List("collections", sdkParams, &collections)
if err != nil {
+ if params.Logger != nil {
+ properties,_ := params.Logger.Edit()
+ properties["FATAL"] = err.Error()
+ params.Logger.Record()
+ }
log.Fatalf("error querying collections: %+v", err)
}
commit 4851f29ac730aa09dcb3489a3a6e7479e7b82fb6
Author: mishaz <misha at curoverse.com>
Date: Fri Jan 9 04:00:40 2015 +0000
Switched Logger edit hooks to write hooks so they'll trigger less often.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index fd40cce..038482d 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -51,7 +51,7 @@ type Logger struct {
lastWrite time.Time // The last time we wrote a log entry
modified bool // Has this data been modified since the last write
- editHooks []func(map[string]interface{},map[string]interface{})
+ writeHooks []func(map[string]interface{},map[string]interface{})
}
// Create a new logger based on the specified parameters.
@@ -78,23 +78,16 @@ func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]int
l.lock.Lock()
l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
- // Run all our hooks
- for _, hook := range l.editHooks {
- hook(l.properties, l.entry)
- }
-
return l.properties, l.entry
}
-// Adds a hook which will be called everytime Edit() is called.
+// Adds a hook which will be called every time this logger writes an entry.
// The hook takes properties and entry as arguments, in that order.
// This is useful for stuff like memory profiling.
-// This must be called between Edit() and Record().
-// For convenience AddEditHook will call hook when it is added as well.
-func (l *Logger) AddEditHook(hook func(map[string]interface{},
+// This must be called between Edit() and Record() (e.g. while holding the lock)
+func (l *Logger) AddWriteHook(hook func(map[string]interface{},
map[string]interface{})) {
- l.editHooks = append(l.editHooks, hook)
- hook(l.properties, l.entry)
+ l.writeHooks = append(l.writeHooks, hook)
}
// Write the log entry you've built up so far. Do not edit the maps
@@ -123,13 +116,23 @@ func (l *Logger) writeAllowedNow() bool {
// Actually writes the log entry. This method assumes we're holding the lock.
func (l *Logger) write() {
+
+ // Run all our hooks
+ for _, hook := range l.writeHooks {
+ hook(l.properties, l.entry)
+ }
+
// Update the event type in case it was modified or is missing.
l.entry["event_type"] = l.params.EventType
+
+ // Write the log entry.
err := l.params.Client.Create("logs", l.data, nil)
if err != nil {
log.Printf("Attempted to log: %v", l.data)
log.Fatalf("Received error writing log: %v", err)
}
+
+ // Update stats.
l.lastWrite = time.Now()
l.modified = false
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 92dd9c5..87a71a9 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -66,7 +66,7 @@ func main() {
runInfo["pid"] = os.Getpid()
properties["run_info"] = runInfo
- arvLogger.AddEditHook(LogMemoryAlloc)
+ arvLogger.AddWriteHook(LogMemoryAlloc)
arvLogger.Record()
}
commit 3d63bc278174d245edc4fa06ed88971a2589e080
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 23:22:09 2015 +0000
Switched batch size to 50. Added logging of batch size.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 212e33f..7e94a11 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -127,6 +127,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
properties,_ := params.Logger.Edit()
collectionInfo := make(map[string]interface{})
collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ collectionInfo["batch_size"] = params.BatchSize
properties["collection_info"] = collectionInfo
params.Logger.Record()
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index bf98902..92dd9c5 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -75,7 +75,7 @@ func main() {
// This requires waiting on them to finish before you let main() exit.
RunCollections(collection.GetCollectionsParams{
- Client: arv, Logger: arvLogger, BatchSize: 100})
+ Client: arv, Logger: arvLogger, BatchSize: 50})
RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
}
commit 970095751e2e836ed296152ae3e9ccb6caa62f62
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 23:17:52 2015 +0000
Added memory alloc in use to stats exported to log. Also added EditHooks to Logger, enabling users to add functions to get called on each Edit() call.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 80cb627..fd40cce 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -50,6 +50,8 @@ type Logger struct {
lastWrite time.Time // The last time we wrote a log entry
modified bool // Has this data been modified since the last write
+
+ editHooks []func(map[string]interface{},map[string]interface{})
}
// Create a new logger based on the specified parameters.
@@ -75,9 +77,26 @@ func NewLogger(params LoggerParams) *Logger {
func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
l.lock.Lock()
l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
+
+ // Run all our hooks
+ for _, hook := range l.editHooks {
+ hook(l.properties, l.entry)
+ }
+
return l.properties, l.entry
}
+// Adds a hook which will be called everytime Edit() is called.
+// The hook takes properties and entry as arguments, in that order.
+// This is useful for stuff like memory profiling.
+// This must be called between Edit() and Record().
+// For convenience AddEditHook will call hook when it is added as well.
+func (l *Logger) AddEditHook(hook func(map[string]interface{},
+ map[string]interface{})) {
+ l.editHooks = append(l.editHooks, hook)
+ hook(l.properties, l.entry)
+}
+
// Write the log entry you've built up so far. Do not edit the maps
// returned by Edit() after calling this method.
// If you have already written within MinimumWriteInterval, then this
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index c64160c..212e33f 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -168,7 +168,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
properties,_ := params.Logger.Edit()
collectionInfo := properties["collection_info"].(map[string]interface{})
collectionInfo["collections_read"] = totalCollections
- collectionInfo["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
+ collectionInfo["latest_modified_date_seen"] = sdkParams["filters"].([][]string)[0][2]
collectionInfo["total_manifest_size"] = totalManifestSize
collectionInfo["max_manifest_size"] = maxManifestSize
params.Logger.Record()
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index dc6a431..bf98902 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -11,6 +11,7 @@ import (
"git.curoverse.com/arvados.git/services/datamanager/keep"
"log"
"os"
+ "runtime"
"time"
)
@@ -64,6 +65,9 @@ func main() {
}
runInfo["pid"] = os.Getpid()
properties["run_info"] = runInfo
+
+ arvLogger.AddEditHook(LogMemoryAlloc)
+
arvLogger.Record()
}
@@ -71,7 +75,7 @@ func main() {
// This requires waiting on them to finish before you let main() exit.
RunCollections(collection.GetCollectionsParams{
- Client: arv, Logger: arvLogger, BatchSize: 500})
+ Client: arv, Logger: arvLogger, BatchSize: 100})
RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
}
@@ -115,3 +119,11 @@ func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
}
return
}
+
+func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
+ _ = entry // keep the compiler from complaining
+ runInfo := properties["run_info"].(map[string]interface{})
+ var memStats runtime.MemStats
+ runtime.ReadMemStats(&memStats)
+ runInfo["alloc_bytes_in_use"] = memStats.Alloc
+}
commit 1c87e0d76265bb64b717289015181e41e0cbe2f3
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 22:35:37 2015 +0000
Added structure to data manager log entries, grouping similar fields.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index b2fcd5c..c64160c 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -125,7 +125,9 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
if params.Logger != nil {
properties,_ := params.Logger.Edit()
- properties["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ collectionInfo := make(map[string]interface{})
+ collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ properties["collection_info"] = collectionInfo
params.Logger.Record()
}
@@ -164,10 +166,11 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
if params.Logger != nil {
properties,_ := params.Logger.Edit()
- properties["collections_read"] = totalCollections
- properties["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
- properties["total_manifest_size"] = totalManifestSize
- properties["max_manifest_size"] = maxManifestSize
+ collectionInfo := properties["collection_info"].(map[string]interface{})
+ collectionInfo["collections_read"] = totalCollections
+ collectionInfo["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
+ collectionInfo["total_manifest_size"] = totalManifestSize
+ collectionInfo["max_manifest_size"] = maxManifestSize
params.Logger.Record()
}
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index b0ce1a7..dc6a431 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -53,14 +53,17 @@ func main() {
if arvLogger != nil {
properties, _ := arvLogger.Edit()
- properties["start_time"] = time.Now()
- properties["args"] = os.Args
+ runInfo := make(map[string]interface{})
+ runInfo["start_time"] = time.Now()
+ runInfo["args"] = os.Args
hostname, err := os.Hostname()
if err != nil {
- properties["hostname_error"] = err.Error()
+ runInfo["hostname_error"] = err.Error()
} else {
- properties["hostname"] = hostname
+ runInfo["hostname"] = hostname
}
+ runInfo["pid"] = os.Getpid()
+ properties["run_info"] = runInfo
arvLogger.Record()
}
commit e7ef642ff70bb7e6c281f4dd1d353f7fb2b3f5ce
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 22:24:00 2015 +0000
Added ability to turn off logging by passing an empty string as the event type.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 1c0c1e6..b2fcd5c 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -123,11 +123,11 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
float64(initialNumberOfCollectionsAvailable) * 1.01)
results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
- {
+ if params.Logger != nil {
properties,_ := params.Logger.Edit()
properties["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ params.Logger.Record()
}
- params.Logger.Record()
// These values are just for getting the loop to run the first time,
// afterwards they'll be set to real values.
@@ -162,14 +162,14 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
float32(totalManifestSize)/float32(totalCollections),
maxManifestSize, totalManifestSize)
- {
+ if params.Logger != nil {
properties,_ := params.Logger.Edit()
properties["collections_read"] = totalCollections
properties["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
properties["total_manifest_size"] = totalManifestSize
properties["max_manifest_size"] = maxManifestSize
+ params.Logger.Record()
}
- params.Logger.Record()
}
// Just in case this lowers the numbers reported in the heap profile.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 4c12c45..b0ce1a7 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -23,7 +23,7 @@ func init() {
flag.StringVar(&logEventType,
"log-event-type",
"experimental-data-manager-report",
- "event_type to use in our arvados log entries.")
+ "event_type to use in our arvados log entries. Set to empty to turn off logging")
flag.IntVar(&logFrequencySeconds,
"log-frequency-seconds",
20,
@@ -44,11 +44,14 @@ func main() {
log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
}
- arvLogger := logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: logEventType,
- MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+ var arvLogger *logger.Logger
+ if logEventType != "" {
+ arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
+ EventType: logEventType,
+ MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+ }
- {
+ if arvLogger != nil {
properties, _ := arvLogger.Edit()
properties["start_time"] = time.Now()
properties["args"] = os.Args
@@ -58,8 +61,8 @@ func main() {
} else {
properties["hostname"] = hostname
}
+ arvLogger.Record()
}
- arvLogger.Record()
// TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
// This requires waiting on them to finish before you let main() exit.
commit 367a6fbc62b4b20af9f5724359fb0d0e423dc718
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 22:16:49 2015 +0000
Started using Logger in data manager and collections.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index e9a36f2..1c0c1e6 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -6,8 +6,8 @@ import (
"flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
+ "git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- //"git.curoverse.com/arvados.git/sdk/go/util"
"log"
"os"
"runtime"
@@ -37,6 +37,7 @@ type ReadCollections struct {
type GetCollectionsParams struct {
Client arvadosclient.ArvadosClient
+ Logger *logger.Logger
BatchSize int
}
@@ -115,9 +116,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
sdkParams["limit"] = params.BatchSize
}
- // MISHA UNDO THIS TEMPORARY HACK TO FIND BUG!
- sdkParams["limit"] = 50
-
initialNumberOfCollectionsAvailable := NumberCollectionsAvailable(params.Client)
// Include a 1% margin for collections added while we're reading so
// that we don't have to grow the map in most cases.
@@ -125,6 +123,12 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
float64(initialNumberOfCollectionsAvailable) * 1.01)
results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
+ {
+ properties,_ := params.Logger.Edit()
+ properties["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ }
+ params.Logger.Record()
+
// These values are just for getting the loop to run the first time,
// afterwards they'll be set to real values.
previousTotalCollections := -1
@@ -157,6 +161,15 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
sdkParams["filters"].([][]string)[0][2],
float32(totalManifestSize)/float32(totalCollections),
maxManifestSize, totalManifestSize)
+
+ {
+ properties,_ := params.Logger.Edit()
+ properties["collections_read"] = totalCollections
+ properties["latest_modified_date"] = sdkParams["filters"].([][]string)[0][2]
+ properties["total_manifest_size"] = totalManifestSize
+ properties["max_manifest_size"] = maxManifestSize
+ }
+ params.Logger.Record()
}
// Just in case this lowers the numbers reported in the heap profile.
@@ -199,7 +212,9 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
manifest := manifest.Manifest{sdkCollection.ManifestText}
manifestSize := uint64(len(sdkCollection.ManifestText))
- totalManifestSize += manifestSize
+ if _, alreadySeen := uuidToCollection[collection.Uuid]; !alreadySeen {
+ totalManifestSize += manifestSize
+ }
if manifestSize > maxManifestSize {
maxManifestSize = manifestSize
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 7b79f0f..4c12c45 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -5,12 +5,31 @@ package main
import (
"flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
"git.curoverse.com/arvados.git/services/datamanager/keep"
"log"
+ "os"
+ "time"
)
+var (
+ logEventType string
+ logFrequencySeconds int
+)
+
+func init() {
+ flag.StringVar(&logEventType,
+ "log-event-type",
+ "experimental-data-manager-report",
+ "event_type to use in our arvados log entries.")
+ flag.IntVar(&logFrequencySeconds,
+ "log-frequency-seconds",
+ 20,
+ "How frequently we'll write log entries in seconds.")
+}
+
func main() {
flag.Parse()
@@ -25,27 +44,51 @@ func main() {
log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
}
+ arvLogger := logger.NewLogger(logger.LoggerParams{Client: arv,
+ EventType: logEventType,
+ MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+
+ {
+ properties, _ := arvLogger.Edit()
+ properties["start_time"] = time.Now()
+ properties["args"] = os.Args
+ hostname, err := os.Hostname()
+ if err != nil {
+ properties["hostname_error"] = err.Error()
+ } else {
+ properties["hostname"] = hostname
+ }
+ }
+ arvLogger.Record()
+
// TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
+ // This requires waiting on them to finish before you let main() exit.
+
+ RunCollections(collection.GetCollectionsParams{
+ Client: arv, Logger: arvLogger, BatchSize: 500})
- // readCollections := collection.GetCollections(
- // collection.GetCollectionsParams{
- // Client: arv, BatchSize: 500})
+ RunKeep(keep.GetKeepServersParams{Client: arv, Limit: 1000})
+}
- // UserUsage := ComputeSizeOfOwnedCollections(readCollections)
- // log.Printf("Uuid to Size used: %v", UserUsage)
+func RunCollections(params collection.GetCollectionsParams) {
+ readCollections := collection.GetCollections(params)
- // // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
- // // lots of behaviors can become warnings (and obviously we can't
- // // write anything).
- // // if !readCollections.ReadAllCollections {
- // // log.Fatalf("Did not read all collections")
- // // }
+ UserUsage := ComputeSizeOfOwnedCollections(readCollections)
+ log.Printf("Uuid to Size used: %v", UserUsage)
- // log.Printf("Read and processed %d collections",
- // len(readCollections.UuidToCollection))
+ // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
+ // lots of behaviors can become warnings (and obviously we can't
+ // write anything).
+ // if !readCollections.ReadAllCollections {
+ // log.Fatalf("Did not read all collections")
+ // }
+
+ log.Printf("Read and processed %d collections",
+ len(readCollections.UuidToCollection))
+}
- readServers := keep.GetKeepServers(
- keep.GetKeepServersParams{Client: arv, Limit: 1000})
+func RunKeep(params keep.GetKeepServersParams) {
+ readServers := keep.GetKeepServers(params)
log.Printf("Returned %d keep disks", len(readServers.ServerToContents))
commit 1a68b00bcb3dd92f597de72274a56ed4a1144c2b
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 21:06:49 2015 +0000
Added support for MinimumWriteInterval.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index a3d2f30..80cb627 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -1,4 +1,4 @@
-// Periodically writes a log to the Arvados SDK.
+// Logger periodically writes a log to the Arvados SDK.
//
// This package is useful for maintaining a log object that is built
// up over time. Every time the object is modified, it will be written
@@ -22,7 +22,6 @@
// // entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
// }
// arvLogger.Record() // This triggers the actual log write
-
package logger
import (
@@ -41,11 +40,16 @@ type LoggerParams struct {
// A Logger is used to build up a log entry over time and write every
// version of it.
type Logger struct {
+ // The Data we write
data map[string]interface{} // The entire map that we give to the api
- entry map[string]interface{} // convenience shortcut into data
- properties map[string]interface{} // convenience shortcut into data
- lock sync.Locker // Synchronizes editing and writing
- params LoggerParams // parameters we were given
+ entry map[string]interface{} // Convenience shortcut into data
+ properties map[string]interface{} // Convenience shortcut into data
+
+ lock sync.Locker // Synchronizes editing and writing
+ params LoggerParams // Parameters we were given
+
+ lastWrite time.Time // The last time we wrote a log entry
+ modified bool // Has this data been modified since the last write
}
// Create a new logger based on the specified parameters.
@@ -53,8 +57,6 @@ func NewLogger(params LoggerParams) *Logger {
// TODO(misha): Add some params checking here.
l := &Logger{data: make(map[string]interface{}),
lock: &sync.Mutex{},
- // TODO(misha): Consider copying the params so they're not
- // modified after creation.
params: params}
l.entry = make(map[string]interface{})
l.data["log"] = l.entry
@@ -72,17 +74,35 @@ func NewLogger(params LoggerParams) *Logger {
// properties is a shortcut for entry["properties"].(map[string]interface{})
func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
l.lock.Lock()
+ l.modified = true // We don't actually know the caller will modifiy the data, but we assume they will.
return l.properties, l.entry
}
-// Write the log entry you've built up so far. Do not edit the maps returned by Edit() after calling this method.
+// Write the log entry you've built up so far. Do not edit the maps
+// returned by Edit() after calling this method.
+// If you have already written within MinimumWriteInterval, then this
+// will schedule a future write instead.
+// In either case, the lock will be released before Record() returns.
func (l *Logger) Record() {
- // TODO(misha): Add a check (and storage) to make sure we respect MinimumWriteInterval
- l.write()
+ if l.writeAllowedNow() {
+ // We haven't written in the allowed interval yet, try to write.
+ l.write()
+ } else {
+ nextTimeToWrite := l.lastWrite.Add(l.params.MinimumWriteInterval)
+ writeAfter := nextTimeToWrite.Sub(time.Now())
+ time.AfterFunc(writeAfter, l.acquireLockConsiderWriting)
+ }
l.lock.Unlock()
}
-// Actually writes the log entry.
+
+// Whether enough time has elapsed since the last write.
+func (l *Logger) writeAllowedNow() bool {
+ return l.lastWrite.Add(l.params.MinimumWriteInterval).Before(time.Now())
+}
+
+
+// Actually writes the log entry. This method assumes we're holding the lock.
func (l *Logger) write() {
// Update the event type in case it was modified or is missing.
l.entry["event_type"] = l.params.EventType
@@ -91,4 +111,16 @@ func (l *Logger) write() {
log.Printf("Attempted to log: %v", l.data)
log.Fatalf("Received error writing log: %v", err)
}
+ l.lastWrite = time.Now()
+ l.modified = false
+}
+
+
+func (l *Logger) acquireLockConsiderWriting() {
+ l.lock.Lock()
+ if l.modified && l.writeAllowedNow() {
+ // We have something new to write and we're allowed to write.
+ l.write()
+ }
+ l.lock.Unlock()
}
commit 0cf4805d979615190f38ce1d01f1b2d0e8927988
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 20:14:10 2015 +0000
Fixed bugs in logger, changed interface some, added documentation.
Still need to add support for MinimunWriteInterval.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index 6835750..a3d2f30 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -5,13 +5,23 @@
// to the log. Writes will be throttled to no more than one every
// WriteFrequencySeconds
//
-// This package is safe for concurrent use.
+// This package is safe for concurrent use as long as:
+// 1. The maps returned by Edit() are only edited in the same routine
+// that called Edit()
+// 2. Those maps not edited after calling Record()
+// An easy way to assure this is true is to place the call to Edit()
+// within a short block as shown below in the Usage Example:
//
// Usage:
// arvLogger := logger.NewLogger(params)
-// logData := arvLogger.Acquire() // This will block if others are using the logger
-// // Modify the logObject however you want here ..
-// logData = arvLogger.Release() // This triggers the actual write, and replaces logObject with a nil pointer so you don't try to modify it when you're no longer holding the lock
+// {
+// properties, entry := arvLogger.Edit() // This will block if others are using the logger
+// // Modifiy properties and entry however you want
+// // properties is a shortcut for entry["properties"].(map[string]interface{})
+// // properties can take any values you want to give it,
+// // entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
+// }
+// arvLogger.Record() // This triggers the actual log write
package logger
@@ -22,58 +32,63 @@ import (
"time"
)
-const (
- eventTypeLabel string = "event-type"
- propertiesLabel string = "properties"
-)
-
type LoggerParams struct {
Client arvadosclient.ArvadosClient // The client we use to write log entries
EventType string // The event type to assign to the log entry.
MinimumWriteInterval time.Duration // Wait at least this long between log writes
}
+// A Logger is used to build up a log entry over time and write every
+// version of it.
type Logger struct {
- data map[string]interface{}
- lock sync.Locker
- params LoggerParams
+ data map[string]interface{} // The entire map that we give to the api
+ entry map[string]interface{} // convenience shortcut into data
+ properties map[string]interface{} // convenience shortcut into data
+ lock sync.Locker // Synchronizes editing and writing
+ params LoggerParams // parameters we were given
}
+// Create a new logger based on the specified parameters.
func NewLogger(params LoggerParams) *Logger {
+ // TODO(misha): Add some params checking here.
l := &Logger{data: make(map[string]interface{}),
lock: &sync.Mutex{},
// TODO(misha): Consider copying the params so they're not
// modified after creation.
params: params}
- l.data[propertiesLabel] = make(map[string]interface{})
+ l.entry = make(map[string]interface{})
+ l.data["log"] = l.entry
+ l.properties = make(map[string]interface{})
+ l.entry["properties"] = l.properties
return l
}
-func (l *Logger) Acquire() map[string]interface{} {
+// Get access to the maps you can edit. This will hold a lock until
+// you call Record. Do not edit the maps in any other goroutines or
+// after calling Record.
+// You don't need to edit both maps,
+// properties can take any values you want to give it,
+// entry will only take the fields listed at http://doc.arvados.org/api/schema/Log.html
+// properties is a shortcut for entry["properties"].(map[string]interface{})
+func (l *Logger) Edit() (properties map[string]interface{}, entry map[string]interface{}) {
l.lock.Lock()
- return l.data[propertiesLabel].(map[string]interface{})
+ return l.properties, l.entry
}
-func (l *Logger) Release() map[string]interface{} {
+// Write the log entry you've built up so far. Do not edit the maps returned by Edit() after calling this method.
+func (l *Logger) Record() {
// TODO(misha): Add a check (and storage) to make sure we respect MinimumWriteInterval
l.write()
l.lock.Unlock()
- return nil
}
+// Actually writes the log entry.
func (l *Logger) write() {
// Update the event type in case it was modified or is missing.
- // l.data[eventTypeLabel] = l.params.EventType
- // m := make(map[string]interface{})
- // m["body"] = l.data
- // //err := l.params.Client.Create("logs", l.data, nil)
- // //err := l.params.Client.Create("logs", m, nil)
- // var results map[string]interface{}
- err := l.params.Client.Create("logs",
- arvadosclient.Dict{"log": arvadosclient.Dict{
- eventTypeLabel: l.params.EventType,
- propertiesLabel: l.data}}, nil)
+ l.entry["event_type"] = l.params.EventType
+ err := l.params.Client.Create("logs", l.data, nil)
if err != nil {
+ log.Printf("Attempted to log: %v", l.data)
log.Fatalf("Received error writing log: %v", err)
}
}
diff --git a/sdk/go/logger/main/testlogger.go b/sdk/go/logger/main/testlogger.go
index 5a12352..6cd7dfb 100644
--- a/sdk/go/logger/main/testlogger.go
+++ b/sdk/go/logger/main/testlogger.go
@@ -10,11 +10,6 @@ import (
"log"
)
-const (
- eventType string = "experimental-logger-testing"
-)
-
-
func main() {
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
@@ -22,11 +17,13 @@ func main() {
}
l := logger.NewLogger(logger.LoggerParams{Client: arv,
- EventType: eventType,
+ EventType: "experimental-logger-testing",
// No minimum write interval
})
- logData := l.Acquire()
- logData["Ninja"] = "Misha"
- logData = l.Release()
+ {
+ properties, _ := l.Edit()
+ properties["Ninja"] = "Misha"
+ }
+ l.Record()
}
commit 3594ad790c998c1b1711ea65057fcae9477986d4
Author: mishaz <misha at curoverse.com>
Date: Thu Jan 8 01:47:51 2015 +0000
Added logger to write log messages that grow over time. Not working yet.
diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
new file mode 100644
index 0000000..6835750
--- /dev/null
+++ b/sdk/go/logger/logger.go
@@ -0,0 +1,79 @@
+// Periodically writes a log to the Arvados SDK.
+//
+// This package is useful for maintaining a log object that is built
+// up over time. Every time the object is modified, it will be written
+// to the log. Writes will be throttled to no more than one every
+// WriteFrequencySeconds
+//
+// This package is safe for concurrent use.
+//
+// Usage:
+// arvLogger := logger.NewLogger(params)
+// logData := arvLogger.Acquire() // This will block if others are using the logger
+// // Modify the logObject however you want here ..
+// logData = arvLogger.Release() // This triggers the actual write, and replaces logObject with a nil pointer so you don't try to modify it when you're no longer holding the lock
+
+package logger
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "log"
+ "sync"
+ "time"
+)
+
+const (
+ eventTypeLabel string = "event-type"
+ propertiesLabel string = "properties"
+)
+
+type LoggerParams struct {
+ Client arvadosclient.ArvadosClient // The client we use to write log entries
+ EventType string // The event type to assign to the log entry.
+ MinimumWriteInterval time.Duration // Wait at least this long between log writes
+}
+
+type Logger struct {
+ data map[string]interface{}
+ lock sync.Locker
+ params LoggerParams
+}
+
+func NewLogger(params LoggerParams) *Logger {
+ l := &Logger{data: make(map[string]interface{}),
+ lock: &sync.Mutex{},
+ // TODO(misha): Consider copying the params so they're not
+ // modified after creation.
+ params: params}
+ l.data[propertiesLabel] = make(map[string]interface{})
+ return l
+}
+
+func (l *Logger) Acquire() map[string]interface{} {
+ l.lock.Lock()
+ return l.data[propertiesLabel].(map[string]interface{})
+}
+
+func (l *Logger) Release() map[string]interface{} {
+ // TODO(misha): Add a check (and storage) to make sure we respect MinimumWriteInterval
+ l.write()
+ l.lock.Unlock()
+ return nil
+}
+
+func (l *Logger) write() {
+ // Update the event type in case it was modified or is missing.
+ // l.data[eventTypeLabel] = l.params.EventType
+ // m := make(map[string]interface{})
+ // m["body"] = l.data
+ // //err := l.params.Client.Create("logs", l.data, nil)
+ // //err := l.params.Client.Create("logs", m, nil)
+ // var results map[string]interface{}
+ err := l.params.Client.Create("logs",
+ arvadosclient.Dict{"log": arvadosclient.Dict{
+ eventTypeLabel: l.params.EventType,
+ propertiesLabel: l.data}}, nil)
+ if err != nil {
+ log.Fatalf("Received error writing log: %v", err)
+ }
+}
diff --git a/sdk/go/logger/main/testlogger.go b/sdk/go/logger/main/testlogger.go
new file mode 100644
index 0000000..5a12352
--- /dev/null
+++ b/sdk/go/logger/main/testlogger.go
@@ -0,0 +1,32 @@
+// This binary tests the logger package.
+// It's not a standard unit test. Instead it writes to the actual log
+// and you have to clean up after it.
+
+package main
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/logger"
+ "log"
+)
+
+const (
+ eventType string = "experimental-logger-testing"
+)
+
+
+func main() {
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %v", err)
+ }
+
+ l := logger.NewLogger(logger.LoggerParams{Client: arv,
+ EventType: eventType,
+ // No minimum write interval
+ })
+
+ logData := l.Acquire()
+ logData["Ninja"] = "Misha"
+ logData = l.Release()
+}
commit 08fbba821251a58cc99921cb477fbbc076a1bfee
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 7 04:16:40 2015 +0000
Started focusing on Keep Server responses again. Switched to using blockdigest instead of strings. Added per block info so that we can track block replication across servers.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 6393787..7b79f0f 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -7,7 +7,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
-// "git.curoverse.com/arvados.git/services/datamanager/keep"
+ "git.curoverse.com/arvados.git/services/datamanager/keep"
"log"
)
@@ -27,29 +27,35 @@ func main() {
// TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
- readCollections := collection.GetCollections(
- collection.GetCollectionsParams{
- Client: arv, BatchSize: 500})
+ // readCollections := collection.GetCollections(
+ // collection.GetCollectionsParams{
+ // Client: arv, BatchSize: 500})
- //log.Printf("Read Collections: %v", readCollections)
+ // UserUsage := ComputeSizeOfOwnedCollections(readCollections)
+ // log.Printf("Uuid to Size used: %v", UserUsage)
- UserUsage := ComputeSizeOfOwnedCollections(readCollections)
- log.Printf("Uuid to Size used: %v", UserUsage)
+ // // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
+ // // lots of behaviors can become warnings (and obviously we can't
+ // // write anything).
+ // // if !readCollections.ReadAllCollections {
+ // // log.Fatalf("Did not read all collections")
+ // // }
- // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
- // lots of behaviors can become warnings (and obviously we can't
- // write anything).
- // if !readCollections.ReadAllCollections {
- // log.Fatalf("Did not read all collections")
- // }
+ // log.Printf("Read and processed %d collections",
+ // len(readCollections.UuidToCollection))
- log.Printf("Read and processed %d collections",
- len(readCollections.UuidToCollection))
+ readServers := keep.GetKeepServers(
+ keep.GetKeepServersParams{Client: arv, Limit: 1000})
- // readServers := keep.GetKeepServers(
- // keep.GetKeepServersParams{Client: arv, Limit: 1000})
+ log.Printf("Returned %d keep disks", len(readServers.ServerToContents))
- // log.Printf("Returned %d keep disks", len(readServers.AddressToContents))
+ blockReplicationCounts := make(map[int]int)
+ for _, infos := range readServers.BlockToServers {
+ replication := len(infos)
+ blockReplicationCounts[replication] += 1
+ }
+
+ log.Printf("Replication level distribution: %v", blockReplicationCounts)
}
func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index e600c4a..91af201 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -8,6 +8,7 @@ import (
"fmt"
//"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/sdk/go/util"
"log"
@@ -23,14 +24,22 @@ type ServerAddress struct {
Port int `json:"service_port"`
}
+// Info about a particular block returned by the server
type BlockInfo struct {
- Digest string
+ Digest blockdigest.BlockDigest
Size int
Mtime int // TODO(misha): Replace this with a timestamp.
}
+// Info about a specified block given by a server
+type BlockServerInfo struct {
+ ServerIndex int
+ Size int
+ Mtime int // TODO(misha): Replace this with a timestamp.
+}
+
type ServerContents struct {
- BlockDigestToInfo map[string]BlockInfo
+ BlockDigestToInfo map[blockdigest.BlockDigest]BlockInfo
}
type ServerResponse struct {
@@ -40,7 +49,10 @@ type ServerResponse struct {
type ReadServers struct {
ReadAllServers bool
- AddressToContents map[ServerAddress]ServerContents
+ KeepServerIndexToAddress []ServerAddress
+ KeepServerAddressToIndex map[ServerAddress]int
+ ServerToContents map[ServerAddress]ServerContents
+ BlockToServers map[blockdigest.BlockDigest][]BlockServerInfo
}
type GetKeepServersParams struct {
@@ -130,6 +142,14 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
numAvailable)
}
+ results.KeepServerIndexToAddress = sdkResponse.KeepServers
+ results.KeepServerAddressToIndex = make(map[ServerAddress]int)
+ for i, address := range results.KeepServerIndexToAddress {
+ results.KeepServerAddressToIndex[address] = i
+ }
+
+ log.Printf("Got Server Addresses: %v", results)
+
// This is safe for concurrent use
client := http.Client{}
@@ -139,7 +159,8 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
go GetServerContents(keepServer, client, responseChan)
}
- results.AddressToContents = make(map[ServerAddress]ServerContents)
+ results.ServerToContents = make(map[ServerAddress]ServerContents)
+ results.BlockToServers = make(map[blockdigest.BlockDigest][]BlockServerInfo)
// Read all the responses
for i := range sdkResponse.KeepServers {
@@ -148,7 +169,15 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
log.Printf("Received channel response from %v containing %d files",
response.Address,
len(response.Contents.BlockDigestToInfo))
- results.AddressToContents[response.Address] = response.Contents
+ results.ServerToContents[response.Address] = response.Contents
+ serverIndex := results.KeepServerAddressToIndex[response.Address]
+ for _, blockInfo := range response.Contents.BlockDigestToInfo {
+ results.BlockToServers[blockInfo.Digest] = append(
+ results.BlockToServers[blockInfo.Digest],
+ BlockServerInfo{ServerIndex: serverIndex,
+ Size: blockInfo.Size,
+ Mtime: blockInfo.Mtime})
+ }
}
return
}
@@ -183,7 +212,8 @@ func GetServerContents(keepServer ServerAddress,
response := ServerResponse{}
response.Address = keepServer
- response.Contents.BlockDigestToInfo = make(map[string]BlockInfo)
+ response.Contents.BlockDigestToInfo =
+ make(map[blockdigest.BlockDigest]BlockInfo)
scanner := bufio.NewScanner(resp.Body)
numLines, numDuplicates, numSizeDisagreements := 0, 0, 0
for scanner.Scan() {
commit 9cf25951c6b64449fe24ff9965e7dabe85c8ff4e
Author: mishaz <misha at curoverse.com>
Date: Wed Jan 7 01:45:55 2015 +0000
Fixed heap profile writing so that we overwrite previous heap profiles rather than adding to them. Minor cleanup too.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 48121cd..e9a36f2 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -10,13 +10,16 @@ import (
//"git.curoverse.com/arvados.git/sdk/go/util"
"log"
"os"
+ "runtime"
"runtime/pprof"
"time"
)
var (
heap_profile_filename string
- heap_profile *os.File
+ // globals for debugging
+ totalManifestSize uint64
+ maxManifestSize uint64
)
type Collection struct {
@@ -66,21 +69,35 @@ func init() {
// return len(s.Items), nil
// }
-func GetCollections(params GetCollectionsParams) (results ReadCollections) {
- if ¶ms.Client == nil {
- log.Fatalf("params.Client passed to GetCollections() should " +
- "contain a valid ArvadosClient, but instead it is nil.")
- }
-
- // TODO(misha): move this code somewhere better and make sure it's
- // only run once
+// Write the heap profile to a file for later review.
+// Since a file is expected to only contain a single heap profile this
+// function overwrites the previously written profile, so it is safe
+// to call multiple times in a single run.
+// Otherwise we would see cumulative numbers as explained here:
+// https://groups.google.com/d/msg/golang-nuts/ZyHciRglQYc/2nh4Ndu2fZcJ
+func WriteHeapProfile() {
if heap_profile_filename != "" {
- var err error
- heap_profile, err = os.Create(heap_profile_filename)
+
+ heap_profile, err := os.Create(heap_profile_filename)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ defer heap_profile.Close()
+
+ err = pprof.WriteHeapProfile(heap_profile)
if err != nil {
log.Fatal(err)
}
}
+}
+
+
+func GetCollections(params GetCollectionsParams) (results ReadCollections) {
+ if ¶ms.Client == nil {
+ log.Fatalf("params.Client passed to GetCollections() should " +
+ "contain a valid ArvadosClient, but instead it is nil.")
+ }
fieldsWanted := []string{"manifest_text",
"owner_uuid",
@@ -101,19 +118,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
// MISHA UNDO THIS TEMPORARY HACK TO FIND BUG!
sdkParams["limit"] = 50
- // {
- // var numReceived, numAvailable int
- // results.ReadAllCollections, numReceived, numAvailable =
- // util.ContainsAllAvailableItems(collections)
-
- // if (!results.ReadAllCollections) {
- // log.Printf("ERROR: Did not receive all collections.")
- // }
- // log.Printf("Received %d of %d available collections.",
- // numReceived,
- // numAvailable)
- // }
-
initialNumberOfCollectionsAvailable := NumberCollectionsAvailable(params.Client)
// Include a 1% margin for collections added while we're reading so
// that we don't have to grow the map in most cases.
@@ -129,12 +133,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
// We're still finding new collections
// Write the heap profile for examining memory usage
- if heap_profile != nil {
- err := pprof.WriteHeapProfile(heap_profile)
- if err != nil {
- log.Fatal(err)
- }
- }
+ WriteHeapProfile()
// Get next batch of collections.
var collections SdkCollectionList
@@ -152,19 +151,19 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
totalCollections = len(results.UuidToCollection)
log.Printf("%d collections read, %d new in last batch, " +
- "%s latest modified date",
+ "%s latest modified date, %.0f %d %d avg,max,total manifest size",
totalCollections,
totalCollections - previousTotalCollections,
- sdkParams["filters"].([][]string)[0][2])
+ sdkParams["filters"].([][]string)[0][2],
+ float32(totalManifestSize)/float32(totalCollections),
+ maxManifestSize, totalManifestSize)
}
+ // Just in case this lowers the numbers reported in the heap profile.
+ runtime.GC()
+
// Write the heap profile for examining memory usage
- if heap_profile != nil {
- err := pprof.WriteHeapProfile(heap_profile)
- if err != nil {
- log.Fatal(err)
- }
- }
+ WriteHeapProfile()
return
}
@@ -198,6 +197,13 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
latestModificationDate = sdkCollection.ModifiedAt
}
manifest := manifest.Manifest{sdkCollection.ManifestText}
+ manifestSize := uint64(len(sdkCollection.ManifestText))
+
+ totalManifestSize += manifestSize
+ if manifestSize > maxManifestSize {
+ maxManifestSize = manifestSize
+ }
+
blockChannel := manifest.BlockIterWithDuplicates()
for block := range blockChannel {
if stored_size, stored := collection.BlockDigestToSize[block.Digest];
commit fbfd3b4c049d4b3d24b22bfb5462f098c73596aa
Author: mishaz <misha at curoverse.com>
Date: Wed Dec 24 20:26:38 2014 +0000
Added string copying to try to reduce memory usage, didn't seem to work. Cleaned up logging (and logging logic) so that we only see one line per batch.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index fac4d1a..48121cd 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -93,8 +93,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
"select": fieldsWanted,
"order": []string{"modified_at ASC"},
"filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}}}
- // MISHA UNDO THIS TEMPORARY HACK TO FIND BUG!
- //"filters": [][]string{[]string{"modified_at", ">=", "2014-11-05T20:44:50Z"}}}
if params.BatchSize > 0 {
sdkParams["limit"] = params.BatchSize
@@ -123,13 +121,12 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
float64(initialNumberOfCollectionsAvailable) * 1.01)
results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
+ // These values are just for getting the loop to run the first time,
+ // afterwards they'll be set to real values.
previousTotalCollections := -1
- for len(results.UuidToCollection) > previousTotalCollections {
+ totalCollections := 0
+ for totalCollections > previousTotalCollections {
// We're still finding new collections
- log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
-
- // update count
- previousTotalCollections = len(results.UuidToCollection)
// Write the heap profile for examining memory usage
if heap_profile != nil {
@@ -141,7 +138,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
// Get next batch of collections.
var collections SdkCollectionList
- log.Printf("Running with SDK Params: %v", sdkParams)
err := params.Client.List("collections", sdkParams, &collections)
if err != nil {
log.Fatalf("error querying collections: %+v", err)
@@ -150,9 +146,17 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
// Process collection and update our date filter.
sdkParams["filters"].([][]string)[0][2] =
ProcessCollections(collections.Items, results.UuidToCollection).Format(time.RFC3339)
- log.Printf("Latest date seen %s", sdkParams["filters"].([][]string)[0][2])
+
+ // update counts
+ previousTotalCollections = totalCollections
+ totalCollections = len(results.UuidToCollection)
+
+ log.Printf("%d collections read, %d new in last batch, " +
+ "%s latest modified date",
+ totalCollections,
+ totalCollections - previousTotalCollections,
+ sdkParams["filters"].([][]string)[0][2])
}
- log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
// Write the heap profile for examining memory usage
if heap_profile != nil {
@@ -166,11 +170,19 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
}
+// StrCopy returns a newly allocated string.
+// It is useful to copy slices so that the garbage collector can reuse
+// the memory of the longer strings they came from.
+func StrCopy(s string) string {
+ return string([]byte(s))
+}
+
+
func ProcessCollections(receivedCollections []SdkCollectionInfo,
uuidToCollection map[string]Collection) (latestModificationDate time.Time) {
for _, sdkCollection := range receivedCollections {
- collection := Collection{Uuid: sdkCollection.Uuid,
- OwnerUuid: sdkCollection.OwnerUuid,
+ collection := Collection{Uuid: StrCopy(sdkCollection.Uuid),
+ OwnerUuid: StrCopy(sdkCollection.OwnerUuid),
ReplicationLevel: sdkCollection.Redundancy,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
commit 2ccc3eda37cd728d6526804c228d7383dc8960a9
Author: mishaz <misha at curoverse.com>
Date: Wed Dec 24 19:29:08 2014 +0000
Started parsing modification date as a timestamp instead of leaving it as a string.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 927b888..fac4d1a 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -11,6 +11,7 @@ import (
"log"
"os"
"runtime/pprof"
+ "time"
)
var (
@@ -37,11 +38,11 @@ type GetCollectionsParams struct {
}
type SdkCollectionInfo struct {
- Uuid string `json:"uuid"`
- OwnerUuid string `json:"owner_uuid"`
- Redundancy int `json:"redundancy"`
- ModifiedAt string `json:"modified_at"`
- ManifestText string `json:"manifest_text"`
+ Uuid string `json:"uuid"`
+ OwnerUuid string `json:"owner_uuid"`
+ Redundancy int `json:"redundancy"`
+ ModifiedAt time.Time `json:"modified_at"`
+ ManifestText string `json:"manifest_text"`
}
type SdkCollectionList struct {
@@ -147,7 +148,8 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
}
// Process collection and update our date filter.
- sdkParams["filters"].([][]string)[0][2] = ProcessCollections(collections.Items, results.UuidToCollection)
+ sdkParams["filters"].([][]string)[0][2] =
+ ProcessCollections(collections.Items, results.UuidToCollection).Format(time.RFC3339)
log.Printf("Latest date seen %s", sdkParams["filters"].([][]string)[0][2])
}
log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
@@ -165,16 +167,22 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
func ProcessCollections(receivedCollections []SdkCollectionInfo,
- uuidToCollection map[string]Collection) (latestModificationDate string) {
+ uuidToCollection map[string]Collection) (latestModificationDate time.Time) {
for _, sdkCollection := range receivedCollections {
collection := Collection{Uuid: sdkCollection.Uuid,
OwnerUuid: sdkCollection.OwnerUuid,
ReplicationLevel: sdkCollection.Redundancy,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
- // log.Printf("Seeing modification date, owner_uuid: %s %s",
- // sdkCollection.ModifiedAt,
- // sdkCollection.OwnerUuid)
- if sdkCollection.ModifiedAt > latestModificationDate {
+
+ if sdkCollection.ModifiedAt.IsZero() {
+ log.Fatalf(
+ "Arvados SDK collection returned with unexpected zero modifcation " +
+ "date. This probably means that either we failed to parse the " +
+ "modification date or the API server has changed how it returns " +
+ "modification dates: %v",
+ collection)
+ }
+ if sdkCollection.ModifiedAt.After(latestModificationDate) {
latestModificationDate = sdkCollection.ModifiedAt
}
manifest := manifest.Manifest{sdkCollection.ManifestText}
commit d7f6013f1e728a3a7b42d6736b78cc81ac7de127
Author: mishaz <misha at curoverse.com>
Date: Wed Dec 24 01:36:43 2014 +0000
Switched from strings to BlockDigests to hold block digests more efficiently. Started clearing out manifest text once we finished with it. Made profiling conitional on flag (before it crashed if not provided). Added final heap profile once collections were finished.
Runs to completion!
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 159b568..927b888 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -5,6 +5,7 @@ package collection
import (
"flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/manifest"
//"git.curoverse.com/arvados.git/sdk/go/util"
"log"
@@ -21,7 +22,7 @@ type Collection struct {
Uuid string
OwnerUuid string
ReplicationLevel int
- BlockDigestToSize map[string]int
+ BlockDigestToSize map[blockdigest.BlockDigest]int
TotalSize int
}
@@ -80,10 +81,6 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
}
}
-
-
-
-
fieldsWanted := []string{"manifest_text",
"owner_uuid",
"uuid",
@@ -134,7 +131,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
previousTotalCollections = len(results.UuidToCollection)
// Write the heap profile for examining memory usage
- {
+ if heap_profile != nil {
err := pprof.WriteHeapProfile(heap_profile)
if err != nil {
log.Fatal(err)
@@ -155,6 +152,14 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
}
log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
+ // Write the heap profile for examining memory usage
+ if heap_profile != nil {
+ err := pprof.WriteHeapProfile(heap_profile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
return
}
@@ -165,7 +170,7 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
collection := Collection{Uuid: sdkCollection.Uuid,
OwnerUuid: sdkCollection.OwnerUuid,
ReplicationLevel: sdkCollection.Redundancy,
- BlockDigestToSize: make(map[string]int)}
+ BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
// log.Printf("Seeing modification date, owner_uuid: %s %s",
// sdkCollection.ModifiedAt,
// sdkCollection.OwnerUuid)
@@ -191,6 +196,11 @@ func ProcessCollections(receivedCollections []SdkCollectionInfo,
collection.TotalSize += size
}
uuidToCollection[collection.Uuid] = collection
+
+ // Clear out all the manifest strings that we don't need anymore.
+ // These hopefully form the bulk of our memory usage.
+ manifest.Text = ""
+ sdkCollection.ManifestText = ""
}
return
commit dab166a0e63d256b2ccfd209493a35696f88726f
Author: mishaz <misha at curoverse.com>
Date: Wed Dec 24 00:24:07 2014 +0000
Changes to manifest that I forgot to add to previous checking.
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index d7b5eb5..1227f49 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -6,6 +6,7 @@ package manifest
import (
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"log"
"regexp"
"strconv"
@@ -20,7 +21,7 @@ type Manifest struct {
}
type BlockLocator struct {
- Digest string
+ Digest blockdigest.BlockDigest
Size int
Hints []string
}
@@ -40,14 +41,16 @@ func ParseBlockLocator(s string) (b BlockLocator, err error) {
} else {
tokens := strings.Split(s, "+")
var blockSize int64
- // We expect ParseInt to succeed since LocatorPattern restricts
- // tokens[1] to contain exclusively digits.
+ var blockDigest blockdigest.BlockDigest
+ // We expect both of the following to succeed since LocatorPattern
+ // restricts the strings appropriately.
+ blockDigest, err = blockdigest.FromString(tokens[0])
+ if err != nil {return}
blockSize, err = strconv.ParseInt(tokens[1], 10, 0)
- if err == nil {
- b.Digest = tokens[0]
- b.Size = int(blockSize)
- b.Hints = tokens[2:]
- }
+ if err != nil {return}
+ b.Digest = blockDigest
+ b.Size = int(blockSize)
+ b.Hints = tokens[2:]
}
return
}
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 7a0a641..f8641ce 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -1,6 +1,7 @@
package manifest
import (
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"io/ioutil"
"runtime"
"testing"
@@ -119,7 +120,7 @@ func TestParseBlockLocatorSimple(t *testing.T) {
if err != nil {
t.Fatalf("Unexpected error parsing block locator: %v", err)
}
- expectBlockLocator(t, b, BlockLocator{Digest: "365f83f5f808896ec834c8b595288735",
+ expectBlockLocator(t, b, BlockLocator{Digest: blockdigest.AssertFromString("365f83f5f808896ec834c8b595288735"),
Size: 2310,
Hints: []string{"K at qr1hi",
"Af0c9a66381f3b028677411926f0be1c6282fe67c at 542b5ddf"}})
@@ -158,7 +159,7 @@ func TestBlockIterLongManifest(t *testing.T) {
firstBlock := <-blockChannel
expectBlockLocator(t,
firstBlock,
- BlockLocator{Digest: "b746e3d2104645f2f64cd3cc69dd895d",
+ BlockLocator{Digest: blockdigest.AssertFromString("b746e3d2104645f2f64cd3cc69dd895d"),
Size: 15693477,
Hints: []string{"E2866e643690156651c03d876e638e674dcd79475 at 5441920c"}})
blocksRead := 1
@@ -171,7 +172,7 @@ func TestBlockIterLongManifest(t *testing.T) {
expectBlockLocator(t,
lastBlock,
- BlockLocator{Digest: "f9ce82f59e5908d2d70e18df9679b469",
+ BlockLocator{Digest: blockdigest.AssertFromString("f9ce82f59e5908d2d70e18df9679b469"),
Size: 31367794,
Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db at 5441920c"}})
}
commit 25d58aaf041ac109ff76db5168c193272958d454
Author: mishaz <misha at curoverse.com>
Date: Tue Dec 23 23:55:12 2014 +0000
Added blockdigest class to store digests more efficiently. This has the nice side effect of reducing how many string slices we use from the SDK, so the large string can get garbage collected once we remove other usages.
diff --git a/sdk/go/blockdigest/blockdigest.go b/sdk/go/blockdigest/blockdigest.go
new file mode 100644
index 0000000..5225af6
--- /dev/null
+++ b/sdk/go/blockdigest/blockdigest.go
@@ -0,0 +1,45 @@
+/* Stores a Block Locator Digest compactly. Can be used as a map key. */
+
+package blockdigest
+
+import (
+ "fmt"
+ "log"
+ "strconv"
+)
+
+// Stores a Block Locator Digest compactly, up to 128 bits.
+// Can be used as a map key.
+type BlockDigest struct {
+ h uint64
+ l uint64
+}
+
+func (d *BlockDigest) ToString() (s string) {
+ return fmt.Sprintf("%016x%016x", d.h, d.l)
+}
+
+// Will create a new BlockDigest unless an error is encountered.
+func FromString(s string) (dig BlockDigest, err error) {
+ if len(s) != 32 {
+ err = fmt.Errorf("Block digest should be exactly 32 characters but this one is %d: %s", len(s), s)
+ return
+ }
+
+ var d BlockDigest
+ d.h, err = strconv.ParseUint(s[:16], 16, 64)
+ if err != nil {return}
+ d.l, err = strconv.ParseUint(s[16:], 16, 64)
+ if err != nil {return}
+ dig = d
+ return
+}
+
+// Will fatal with the error if an error is encountered
+func AssertFromString(s string) BlockDigest {
+ d, err := FromString(s)
+ if err != nil {
+ log.Fatalf("Error creating BlockDigest from %s: %v", s, err)
+ }
+ return d
+}
diff --git a/sdk/go/blockdigest/blockdigest_test.go b/sdk/go/blockdigest/blockdigest_test.go
new file mode 100644
index 0000000..9081bb8
--- /dev/null
+++ b/sdk/go/blockdigest/blockdigest_test.go
@@ -0,0 +1,45 @@
+package blockdigest
+
+import (
+ "strings"
+ "testing"
+)
+
+func expectValidDigestString(t *testing.T, s string) {
+ bd, err := FromString(s)
+ if err != nil {
+ t.Fatalf("Expected %s to produce a valid BlockDigest but instead got error: %v", s, err)
+ }
+
+ expected := strings.ToLower(s)
+
+ if expected != bd.ToString() {
+ t.Fatalf("Expected %s to be returned by FromString(%s).ToString() but instead we received %s", expected, s, bd.ToString())
+ }
+}
+
+func expectInvalidDigestString(t *testing.T, s string) {
+ _, err := FromString(s)
+ if err == nil {
+ t.Fatalf("Expected %s to be an invalid BlockDigest, but did not receive an error", s)
+ }
+}
+
+func TestValidDigestStrings(t *testing.T) {
+ expectValidDigestString(t, "01234567890123456789abcdefabcdef")
+ expectValidDigestString(t, "01234567890123456789ABCDEFABCDEF")
+ expectValidDigestString(t, "01234567890123456789AbCdEfaBcDeF")
+}
+
+func TestInvalidDigestStrings(t *testing.T) {
+ expectInvalidDigestString(t, "01234567890123456789abcdefabcdeg")
+ expectInvalidDigestString(t, "01234567890123456789abcdefabcde")
+ expectInvalidDigestString(t, "01234567890123456789abcdefabcdefa")
+ expectInvalidDigestString(t, "g1234567890123456789abcdefabcdef")
+}
+
+func TestBlockDigestWorksAsMapKey(t *testing.T) {
+ m := make(map[BlockDigest]int)
+ bd := AssertFromString("01234567890123456789abcdefabcdef")
+ m[bd] = 5
+}
commit fb62ab318be2202b9d403e65d6dc86a9d7e72a3a
Author: mishaz <misha at curoverse.com>
Date: Tue Dec 23 19:33:07 2014 +0000
Long overdue checkin of data manager. Current code runs, but uses way too much memory and eventually crashes. This checkin includes heap profiling to track down memory usage.
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index b108870..7a0a641 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -115,7 +115,7 @@ func TestParseManifestLineSimple(t *testing.T) {
}
func TestParseBlockLocatorSimple(t *testing.T) {
- b, err := parseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
+ b, err := ParseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
if err != nil {
t.Fatalf("Unexpected error parsing block locator: %v", err)
}
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 07e8b7e..159b568 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -3,10 +3,18 @@
package collection
import (
+ "flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- "git.curoverse.com/arvados.git/sdk/go/util"
+ //"git.curoverse.com/arvados.git/sdk/go/util"
"log"
+ "os"
+ "runtime/pprof"
+)
+
+var (
+ heap_profile_filename string
+ heap_profile *os.File
)
type Collection struct {
@@ -24,14 +32,14 @@ type ReadCollections struct {
type GetCollectionsParams struct {
Client arvadosclient.ArvadosClient
- Limit int
- LogEveryNthCollectionProcessed int // 0 means don't report any
+ BatchSize int
}
type SdkCollectionInfo struct {
Uuid string `json:"uuid"`
OwnerUuid string `json:"owner_uuid"`
Redundancy int `json:"redundancy"`
+ ModifiedAt string `json:"modified_at"`
ManifestText string `json:"manifest_text"`
}
@@ -40,14 +48,21 @@ type SdkCollectionList struct {
Items []SdkCollectionInfo `json:"items"`
}
-// Methods to implement util.SdkListResponse Interface
-func (s SdkCollectionList) NumItemsAvailable() (numAvailable int, err error) {
- return s.ItemsAvailable, nil
+func init() {
+ flag.StringVar(&heap_profile_filename,
+ "heap-profile",
+ "",
+ "File to write the heap profiles to.")
}
-func (s SdkCollectionList) NumItemsContained() (numContained int, err error) {
- return len(s.Items), nil
-}
+// // Methods to implement util.SdkListResponse Interface
+// func (s SdkCollectionList) NumItemsAvailable() (numAvailable int, err error) {
+// return s.ItemsAvailable, nil
+// }
+
+// func (s SdkCollectionList) NumItemsContained() (numContained int, err error) {
+// return len(s.Items), nil
+// }
func GetCollections(params GetCollectionsParams) (results ReadCollections) {
if ¶ms.Client == nil {
@@ -55,46 +70,108 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
"contain a valid ArvadosClient, but instead it is nil.")
}
+ // TODO(misha): move this code somewhere better and make sure it's
+ // only run once
+ if heap_profile_filename != "" {
+ var err error
+ heap_profile, err = os.Create(heap_profile_filename)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+
+
+
+
fieldsWanted := []string{"manifest_text",
"owner_uuid",
"uuid",
// TODO(misha): Start using the redundancy field.
- "redundancy"}
-
- sdkParams := arvadosclient.Dict{"select": fieldsWanted}
- if params.Limit > 0 {
- sdkParams["limit"] = params.Limit
- }
-
- var collections SdkCollectionList
- err := params.Client.List("collections", sdkParams, &collections)
- if err != nil {
- log.Fatalf("error querying collections: %v", err)
+ "redundancy",
+ "modified_at"}
+
+ sdkParams := arvadosclient.Dict{
+ "select": fieldsWanted,
+ "order": []string{"modified_at ASC"},
+ "filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}}}
+ // MISHA UNDO THIS TEMPORARY HACK TO FIND BUG!
+ //"filters": [][]string{[]string{"modified_at", ">=", "2014-11-05T20:44:50Z"}}}
+
+ if params.BatchSize > 0 {
+ sdkParams["limit"] = params.BatchSize
}
- {
- var numReceived, numAvailable int
- results.ReadAllCollections, numReceived, numAvailable =
- util.ContainsAllAvailableItems(collections)
+ // MISHA UNDO THIS TEMPORARY HACK TO FIND BUG!
+ sdkParams["limit"] = 50
+
+ // {
+ // var numReceived, numAvailable int
+ // results.ReadAllCollections, numReceived, numAvailable =
+ // util.ContainsAllAvailableItems(collections)
+
+ // if (!results.ReadAllCollections) {
+ // log.Printf("ERROR: Did not receive all collections.")
+ // }
+ // log.Printf("Received %d of %d available collections.",
+ // numReceived,
+ // numAvailable)
+ // }
+
+ initialNumberOfCollectionsAvailable := NumberCollectionsAvailable(params.Client)
+ // Include a 1% margin for collections added while we're reading so
+ // that we don't have to grow the map in most cases.
+ maxExpectedCollections := int(
+ float64(initialNumberOfCollectionsAvailable) * 1.01)
+ results.UuidToCollection = make(map[string]Collection, maxExpectedCollections)
+
+ previousTotalCollections := -1
+ for len(results.UuidToCollection) > previousTotalCollections {
+ // We're still finding new collections
+ log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
+
+ // update count
+ previousTotalCollections = len(results.UuidToCollection)
+
+ // Write the heap profile for examining memory usage
+ {
+ err := pprof.WriteHeapProfile(heap_profile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
- if (!results.ReadAllCollections) {
- log.Printf("ERROR: Did not receive all collections.")
+ // Get next batch of collections.
+ var collections SdkCollectionList
+ log.Printf("Running with SDK Params: %v", sdkParams)
+ err := params.Client.List("collections", sdkParams, &collections)
+ if err != nil {
+ log.Fatalf("error querying collections: %+v", err)
}
- log.Printf("Received %d of %d available collections.",
- numReceived,
- numAvailable)
+
+ // Process collection and update our date filter.
+ sdkParams["filters"].([][]string)[0][2] = ProcessCollections(collections.Items, results.UuidToCollection)
+ log.Printf("Latest date seen %s", sdkParams["filters"].([][]string)[0][2])
}
+ log.Printf("previous, current: %d %d", previousTotalCollections, len(results.UuidToCollection))
- results.UuidToCollection = make(map[string]Collection)
- for i, sdkCollection := range collections.Items {
- count := i + 1
- if m := params.LogEveryNthCollectionProcessed; m >0 && (count % m) == 0 {
- log.Printf("Processing collection #%d", count)
- }
+ return
+}
+
+
+func ProcessCollections(receivedCollections []SdkCollectionInfo,
+ uuidToCollection map[string]Collection) (latestModificationDate string) {
+ for _, sdkCollection := range receivedCollections {
collection := Collection{Uuid: sdkCollection.Uuid,
OwnerUuid: sdkCollection.OwnerUuid,
ReplicationLevel: sdkCollection.Redundancy,
BlockDigestToSize: make(map[string]int)}
+ // log.Printf("Seeing modification date, owner_uuid: %s %s",
+ // sdkCollection.ModifiedAt,
+ // sdkCollection.OwnerUuid)
+ if sdkCollection.ModifiedAt > latestModificationDate {
+ latestModificationDate = sdkCollection.ModifiedAt
+ }
manifest := manifest.Manifest{sdkCollection.ManifestText}
blockChannel := manifest.BlockIterWithDuplicates()
for block := range blockChannel {
@@ -113,8 +190,20 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
for _, size := range collection.BlockDigestToSize {
collection.TotalSize += size
}
- results.UuidToCollection[collection.Uuid] = collection
+ uuidToCollection[collection.Uuid] = collection
}
return
}
+
+
+func NumberCollectionsAvailable(client arvadosclient.ArvadosClient) (int) {
+ var collections SdkCollectionList
+ sdkParams := arvadosclient.Dict{"limit": 0}
+ err := client.List("collections", sdkParams, &collections)
+ if err != nil {
+ log.Fatalf("error querying collections for items available: %v", err)
+ }
+
+ return collections.ItemsAvailable
+}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index cd3e010..6393787 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -7,7 +7,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/collection"
- "git.curoverse.com/arvados.git/services/datamanager/keep"
+// "git.curoverse.com/arvados.git/services/datamanager/keep"
"log"
)
@@ -29,9 +29,9 @@ func main() {
readCollections := collection.GetCollections(
collection.GetCollectionsParams{
- Client: arv, Limit: 50, LogEveryNthCollectionProcessed: 10})
+ Client: arv, BatchSize: 500})
- log.Printf("Read Collections: %v", readCollections)
+ //log.Printf("Read Collections: %v", readCollections)
UserUsage := ComputeSizeOfOwnedCollections(readCollections)
log.Printf("Uuid to Size used: %v", UserUsage)
@@ -46,10 +46,10 @@ func main() {
log.Printf("Read and processed %d collections",
len(readCollections.UuidToCollection))
- readServers := keep.GetKeepServers(
- keep.GetKeepServersParams{Client: arv, Limit: 1000})
+ // readServers := keep.GetKeepServers(
+ // keep.GetKeepServersParams{Client: arv, Limit: 1000})
- log.Printf("Returned %d keep disks", len(readServers.AddressToContents))
+ // log.Printf("Returned %d keep disks", len(readServers.AddressToContents))
}
func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
commit af550e54c034136e5fcb187e7f81e3d82170f9c8
Author: mishaz <misha at curoverse.com>
Date: Sat Nov 22 00:57:40 2014 +0000
Added reporting of disk usage. This is the Collection Storage of each user as described here: https://arvados.org/projects/arvados/wiki/Data_Manager_Design_Doc#Reports-Produced
But it does not include the size of projects owned by the user (projects and subprojects are each reported as their own users)
Report is just logged to screen for now.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index bf76dd8..07e8b7e 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -14,6 +14,7 @@ type Collection struct {
OwnerUuid string
ReplicationLevel int
BlockDigestToSize map[string]int
+ TotalSize int
}
type ReadCollections struct {
@@ -108,6 +109,10 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections) {
}
collection.BlockDigestToSize[block.Digest] = block.Size
}
+ collection.TotalSize = 0
+ for _, size := range collection.BlockDigestToSize {
+ collection.TotalSize += size
+ }
results.UuidToCollection[collection.Uuid] = collection
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 64073af..cd3e010 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -25,12 +25,17 @@ func main() {
log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
}
+ // TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
+
readCollections := collection.GetCollections(
collection.GetCollectionsParams{
Client: arv, Limit: 50, LogEveryNthCollectionProcessed: 10})
log.Printf("Read Collections: %v", readCollections)
+ UserUsage := ComputeSizeOfOwnedCollections(readCollections)
+ log.Printf("Uuid to Size used: %v", UserUsage)
+
// TODO(misha): Add a "readonly" flag. If we're in readonly mode,
// lots of behaviors can become warnings (and obviously we can't
// write anything).
@@ -46,3 +51,12 @@ func main() {
log.Printf("Returned %d keep disks", len(readServers.AddressToContents))
}
+
+func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
+ results map[string]int) {
+ results = make(map[string]int)
+ for _, coll := range readCollections.UuidToCollection {
+ results[coll.OwnerUuid] = results[coll.OwnerUuid] + coll.TotalSize
+ }
+ return
+}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index e3d3387..e600c4a 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -196,7 +196,7 @@ func GetServerContents(keepServer ServerAddress,
}
if storedBlock, ok := response.Contents.BlockDigestToInfo[blockInfo.Digest]; ok {
- // This server is reporting multiple copies of the same block.
+ // This server returned multiple lines containing the same block digest.
numDuplicates += 1
if storedBlock.Size != blockInfo.Size {
numSizeDisagreements += 1
commit 9684e729ae3ebf438fd2c1c440bb0d8c45ca25af
Author: mishaz <misha at curoverse.com>
Date: Thu Oct 16 20:57:06 2014 +0000
Started reading index from keep servers.
Added lots of code to handle unexpected results from keep server.
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index d3316ff..d7b5eb5 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -31,7 +31,7 @@ type ManifestLine struct {
Files []string
}
-func parseBlockLocator(s string) (b BlockLocator, err error) {
+func ParseBlockLocator(s string) (b BlockLocator, err error) {
if !LocatorPattern.MatchString(s) {
err = fmt.Errorf("String \"%s\" does not match BlockLocator pattern " +
"\"%s\".",
@@ -98,7 +98,7 @@ func (m *Manifest) BlockIterWithDuplicates() <-chan BlockLocator {
go func(lineChannel <-chan ManifestLine) {
for m := range lineChannel {
for _, block := range m.Blocks {
- if b, err := parseBlockLocator(block); err == nil {
+ if b, err := ParseBlockLocator(block); err == nil {
blockChannel <- b
} else {
log.Printf("ERROR: Failed to parse block: %v", err)
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index f3706d8..e3d3387 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -8,10 +8,12 @@ import (
"fmt"
//"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/sdk/go/util"
"log"
"io/ioutil"
"net/http"
+ "strconv"
"strings"
"sync"
)
@@ -21,8 +23,19 @@ type ServerAddress struct {
Port int `json:"service_port"`
}
+type BlockInfo struct {
+ Digest string
+ Size int
+ Mtime int // TODO(misha): Replace this with a timestamp.
+}
+
type ServerContents struct {
- BlockDigestToSize map[string]int
+ BlockDigestToInfo map[string]BlockInfo
+}
+
+type ServerResponse struct {
+ Address ServerAddress
+ Contents ServerContents
}
type ReadServers struct {
@@ -96,7 +109,8 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
}
var sdkResponse KeepServiceList
- err := params.Client.List("keep_services", sdkParams, &sdkResponse)
+ err := params.Client.Call("GET", "keep_services", "", "accessible", sdkParams, &sdkResponse)
+
if err != nil {
log.Fatalf("Error requesting keep disks from API server: %v", err)
}
@@ -118,45 +132,129 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
// This is safe for concurrent use
client := http.Client{}
-
- // TODO(misha): Do these in parallel
+
+ // Send off all the index requests concurrently
+ responseChan := make(chan ServerResponse)
for _, keepServer := range sdkResponse.KeepServers {
- url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
- log.Println("About to fetch keep server contents from " + url)
+ go GetServerContents(keepServer, client, responseChan)
+ }
- req, err := http.NewRequest("GET", url, nil)
- if err != nil {
- log.Fatalf("Error building http request for %s: %v", url, err)
- }
+ results.AddressToContents = make(map[ServerAddress]ServerContents)
+
+ // Read all the responses
+ for i := range sdkResponse.KeepServers {
+ _ = i // Here to prevent go from complaining.
+ response := <- responseChan
+ log.Printf("Received channel response from %v containing %d files",
+ response.Address,
+ len(response.Contents.BlockDigestToInfo))
+ results.AddressToContents[response.Address] = response.Contents
+ }
+ return
+}
+
+// TODO(misha): Break this function apart into smaller, easier to
+// understand functions.
+func GetServerContents(keepServer ServerAddress,
+ client http.Client,
+ responseChan chan<- ServerResponse) () {
+ // Create and send request.
+ url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
+ log.Println("About to fetch keep server contents from " + url)
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Fatalf("Error building http request for %s: %v", url, err)
+ }
- req.Header.Add("Authorization",
- fmt.Sprintf("OAuth2 %s", getDataManagerToken()))
+ req.Header.Add("Authorization",
+ fmt.Sprintf("OAuth2 %s", getDataManagerToken()))
- resp, err := client.Do(req)
+ resp, err := client.Do(req)
+ if err != nil {
+ log.Fatalf("Error fetching %s: %v", url, err)
+ }
+
+ // Process response.
+ if resp.StatusCode != 200 {
+ log.Fatalf("Received error code %d in response to request for %s: %s",
+ resp.StatusCode, url, resp.Status)
+ }
+
+ response := ServerResponse{}
+ response.Address = keepServer
+ response.Contents.BlockDigestToInfo = make(map[string]BlockInfo)
+ scanner := bufio.NewScanner(resp.Body)
+ numLines, numDuplicates, numSizeDisagreements := 0, 0, 0
+ for scanner.Scan() {
+ numLines++
+ blockInfo, err := parseBlockInfoFromIndexLine(scanner.Text())
if err != nil {
- log.Fatalf("Error fetching %s: %v", url, err)
+ log.Fatalf("Error parsing BlockInfo from index line received from %s: %v",
+ url,
+ err)
}
- if resp.StatusCode != 200 {
- log.Printf("%v", req)
- log.Fatalf("Received error code %d in response to request for %s: %s",
- resp.StatusCode, url, resp.Status)
- }
- scanner := bufio.NewScanner(resp.Body)
- numLines := 0
- for scanner.Scan() {
- numLines++
- if numLines == 1 {
- log.Printf("First line from %s is %s", url, scanner.Text())
+ if storedBlock, ok := response.Contents.BlockDigestToInfo[blockInfo.Digest]; ok {
+ // This server is reporting multiple copies of the same block.
+ numDuplicates += 1
+ if storedBlock.Size != blockInfo.Size {
+ numSizeDisagreements += 1
+ // TODO(misha): Consider failing here.
+ log.Printf("Saw different sizes for the same block on %s: %v %v",
+ url,
+ storedBlock,
+ blockInfo)
+ }
+ // Keep the block that is bigger, or the block that's newer in
+ // the case of a size tie.
+ if storedBlock.Size < blockInfo.Size ||
+ (storedBlock.Size == blockInfo.Size &&
+ storedBlock.Mtime < blockInfo.Mtime) {
+ response.Contents.BlockDigestToInfo[blockInfo.Digest] = blockInfo
}
- }
- if err := scanner.Err(); err != nil {
- log.Fatalf("Received error scanning response from %s: %v", url, err)
} else {
- log.Printf("Read %d lines from %s.", numLines, url)
+ response.Contents.BlockDigestToInfo[blockInfo.Digest] = blockInfo
}
- resp.Body.Close()
+ }
+ if err := scanner.Err(); err != nil {
+ log.Fatalf("Received error scanning response from %s: %v", url, err)
+ } else {
+ log.Printf("%s contained %d lines with %d duplicates with " +
+ "%d size disagreements",
+ url,
+ numLines,
+ numDuplicates,
+ numSizeDisagreements)
+ }
+ resp.Body.Close()
+ responseChan <- response
+}
+
+func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) {
+ tokens := strings.Fields(indexLine)
+ if len(tokens) != 2 {
+ err = fmt.Errorf("Expected 2 tokens per line but received a " +
+ "line containing %v instead.",
+ tokens)
+ }
+
+ var locator manifest.BlockLocator
+ if locator, err = manifest.ParseBlockLocator(tokens[0]); err != nil {
+ return
+ }
+ if len(locator.Hints) > 0 {
+ err = fmt.Errorf("Block locator in index line should not contain hints " +
+ "but it does: %v",
+ locator)
+ return
}
+ blockInfo.Mtime, err = strconv.Atoi(tokens[1])
+ if err != nil {
+ return
+ }
+ blockInfo.Digest = locator.Digest
+ blockInfo.Size = locator.Size
return
}
commit a224de262c6e94c592eb8c9a2f909954d24b7c9c
Author: mishaz <misha at curoverse.com>
Date: Wed Oct 15 20:53:53 2014 +0000
Started reading response from keep server.
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 507e5f6..f3706d8 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -3,6 +3,7 @@
package keep
import (
+ "bufio"
"flag"
"fmt"
//"git.curoverse.com/arvados.git/sdk/go/keepclient"
@@ -141,6 +142,20 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
log.Fatalf("Received error code %d in response to request for %s: %s",
resp.StatusCode, url, resp.Status)
}
+ scanner := bufio.NewScanner(resp.Body)
+ numLines := 0
+ for scanner.Scan() {
+ numLines++
+ if numLines == 1 {
+ log.Printf("First line from %s is %s", url, scanner.Text())
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ log.Fatalf("Received error scanning response from %s: %v", url, err)
+ } else {
+ log.Printf("Read %d lines from %s.", numLines, url)
+ }
+ resp.Body.Close()
}
return
commit 63cb5c235ccacdc1665a89560bc8c16fcbefd8d6
Merge: 74dba22 61790a8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Feb 13 16:22:55 2015 -0500
Merge branch 'master' of git.curoverse.com:arvados into 3408-production-datamanager
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list