[ARVADOS] updated: 2a62b82e96a6f56357e913117e04c36b1ccf7585
Git user
git at public.curoverse.com
Tue Dec 6 13:29:39 EST 2016
Summary of changes:
apps/workbench/Gemfile | 6 +-
apps/workbench/Gemfile.lock | 16 +-
.../app/controllers/application_controller.rb | 10 +-
.../app/controllers/projects_controller.rb | 4 +-
apps/workbench/app/models/container_work_unit.rb | 13 +-
.../app/views/projects/_show_dashboard.html.erb | 16 +-
.../app/views/projects/_show_processes.html.erb | 5 +
apps/workbench/app/views/projects/show.html.erb | 16 +-
.../test/controllers/disabled_api_test.rb | 3 +
.../test/controllers/projects_controller_test.rb | 21 +-
.../test/integration/application_layout_test.rb | 6 +-
.../test/integration/pipeline_instances_test.rb | 2 +-
apps/workbench/test/integration/projects_test.rb | 2 +-
apps/workbench/test/integration/work_units_test.rb | 2 +-
apps/workbench/test/performance/browsing_test.rb | 2 +-
build/README | 6 +-
build/build-dev-docker-jobs-image.sh | 63 ++++++
build/run-build-packages-one-target.sh | 11 +-
build/run-build-packages.sh | 37 +++-
build/run-library.sh | 17 ++
build/run-tests.sh | 8 +
.../_container_runtime_constraints.liquid | 1 -
.../_container_scheduling_parameters.liquid | 7 +
doc/_includes/_crunch1only_begin.liquid | 2 +
doc/_includes/_crunch1only_end.liquid | 1 +
doc/_includes/_notebox_begin_warning.liquid | 2 +
doc/_includes/_pipeline_deprecation_notice.liquid | 2 +-
.../methods/container_requests.html.textile.liquid | 9 +-
doc/api/methods/containers.html.textile.liquid | 5 +
doc/images/upload-using-workbench.png | Bin 49690 -> 23979 bytes
doc/images/workbench-dashboard.png | Bin 94257 -> 57930 bytes
doc/images/workbench-move-selected.png | Bin 47833 -> 15450 bytes
doc/install/install-keep-web.html.textile.liquid | 2 +-
doc/install/install-keepproxy.html.textile.liquid | 2 +-
doc/install/install-ws.html.textile.liquid | 205 ++++++++++++++++++
doc/user/cwl/cwl-runner.html.textile.liquid | 8 +-
.../getting_started/workbench.html.textile.liquid | 2 +-
doc/user/topics/arv-run.html.textile.liquid | 4 +
...nning-pipeline-command-line.html.textile.liquid | 4 +
...tutorial-pipeline-workbench.html.textile.liquid | 6 +-
sdk/cli/bin/crunch-job | 6 +-
sdk/cwl/arvados_cwl/__init__.py | 143 ++++++++----
sdk/cwl/arvados_cwl/arvcontainer.py | 58 ++---
sdk/cwl/arvados_cwl/arvdocker.py | 5 +-
sdk/cwl/arvados_cwl/arvjob.py | 46 ++--
sdk/cwl/arvados_cwl/arvworkflow.py | 14 +-
sdk/cwl/arvados_cwl/done.py | 3 +
sdk/cwl/arvados_cwl/runner.py | 23 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_container.py | 71 ++----
sdk/cwl/tests/test_submit.py | 148 ++++++++++---
sdk/dev-jobs.dockerfile | 38 ++++
sdk/go/httpserver/id_generator.go | 31 +++
sdk/go/httpserver/request_limiter.go | 26 ++-
sdk/go/stats/duration.go | 35 +++
sdk/go/stats/duration_test.go | 23 ++
sdk/python/arvados/arvfile.py | 2 +-
sdk/python/arvados/keep.py | 134 +++++++-----
sdk/python/tests/test_events.py | 57 +++--
sdk/python/tests/test_keep_client.py | 76 ++++---
services/api/Gemfile | 19 +-
services/api/Gemfile.lock | 239 +++++++++++----------
.../api/app/controllers/application_controller.rb | 18 +-
.../arvados/v1/collections_controller.rb | 4 +-
.../arvados/v1/container_requests_controller.rb | 1 +
.../arvados/v1/containers_controller.rb | 1 +
.../controllers/arvados/v1/groups_controller.rb | 9 +-
.../controllers/arvados/v1/schema_controller.rb | 22 +-
.../arvados/v1/user_agreements_controller.rb | 3 +-
.../app/controllers/arvados/v1/users_controller.rb | 2 +-
.../arvados/v1/virtual_machines_controller.rb | 2 +-
.../api/app/controllers/database_controller.rb | 2 +-
services/api/app/middlewares/arvados_api_token.rb | 2 +-
services/api/app/models/arvados_model.rb | 9 +-
services/api/app/models/blob.rb | 4 +-
services/api/app/models/collection.rb | 15 +-
services/api/app/models/commit_ancestor.rb | 6 +-
services/api/app/models/container_request.rb | 19 +-
services/api/app/models/job.rb | 42 ++--
services/api/app/models/link.rb | 1 -
services/api/app/models/log.rb | 1 -
services/api/app/models/node.rb | 6 +-
services/api/app/models/repository.rb | 2 +-
services/api/app/models/user.rb | 4 +-
services/api/config/boot.rb | 2 +-
services/api/config/initializers/inflections.rb | 8 +-
services/api/config/initializers/load_config.rb | 4 +-
.../api/config/initializers/preload_all_models.rb | 2 +-
...add_output_and_log_uuid_to_container_request.rb | 22 ++
..._log_uuids_to_container_request_search_index.rb | 21 ++
services/api/db/structure.sql | 12 +-
services/api/lib/crunch_dispatch.rb | 10 +-
services/api/lib/current_api_client.rb | 10 +-
services/api/lib/eventbus.rb | 35 +--
services/api/lib/load_param.rb | 4 +-
services/api/lib/salvage_collection.rb | 2 +-
services/api/script/arvados-git-sync.rb | 4 +-
.../api/script/migrate-gitolite-to-uuid-storage.rb | 4 +-
services/api/test/factories/user.rb | 2 +-
services/api/test/fixtures/container_requests.yml | 2 +
.../api_client_authorizations_controller_test.rb | 2 +-
.../arvados/v1/collections_controller_test.rb | 8 +-
.../v1/container_requests_controller_test.rb | 22 ++
.../api/test/functional/arvados/v1/filters_test.rb | 6 +-
.../arvados/v1/groups_controller_test.rb | 19 +-
.../functional/arvados/v1/jobs_controller_test.rb | 2 +-
.../functional/arvados/v1/links_controller_test.rb | 4 +-
.../arvados/v1/repositories_controller_test.rb | 13 +-
.../arvados/v1/schema_controller_test.rb | 2 +-
.../functional/arvados/v1/users_controller_test.rb | 4 +-
.../arvados/v1/virtual_machines_controller_test.rb | 1 -
services/api/test/helpers/users_test_helper.rb | 2 +-
.../api/test/integration/collections_api_test.rb | 14 +-
.../integration/collections_performance_test.rb | 6 +-
services/api/test/integration/cross_origin_test.rb | 2 +-
.../api/test/integration/database_reset_test.rb | 9 +-
.../api/test/integration/user_sessions_test.rb | 2 +-
services/api/test/integration/websocket_test.rb | 136 ++++++++----
services/api/test/test_helper.rb | 32 ++-
services/api/test/unit/app_version_test.rb | 7 +-
services/api/test/unit/authorized_key_test.rb | 2 +-
.../api/test/unit/collection_performance_test.rb | 5 +-
services/api/test/unit/collection_test.rb | 16 +-
services/api/test/unit/commit_test.rb | 18 +-
services/api/test/unit/container_request_test.rb | 12 +-
services/api/test/unit/container_test.rb | 55 +++--
services/api/test/unit/fail_jobs_test.rb | 4 +-
services/api/test/unit/job_test.rb | 6 +-
services/api/test/unit/log_test.rb | 2 +-
services/api/test/unit/node_test.rb | 2 +-
services/api/test/unit/owner_test.rb | 3 +-
services/api/test/unit/permission_test.rb | 14 +-
services/api/test/unit/pipeline_instance_test.rb | 1 -
services/api/test/unit/salvage_collection_test.rb | 12 +-
services/api/test/unit/user_test.rb | 1 -
services/api/test/websocket_runner.rb | 53 -----
services/keepstore/azure_blob_volume.go | 2 +-
services/keepstore/azure_blob_volume_test.go | 2 +-
services/keepstore/bufferpool.go | 3 +-
services/keepstore/config.go | 22 +-
services/keepstore/config_test.go | 2 +-
services/keepstore/count.go | 44 ++++
services/keepstore/handler_test.go | 2 +-
services/keepstore/handlers.go | 91 +++++---
services/keepstore/keepstore.go | 13 +-
services/keepstore/logging_router.go | 46 +++-
services/keepstore/pull_worker.go | 3 +-
services/keepstore/s3_volume.go | 127 ++++++++++-
services/keepstore/s3_volume_test.go | 32 ++-
services/keepstore/trash_worker.go | 2 +-
services/keepstore/usage.go | 4 +
services/keepstore/volume.go | 45 +++-
services/keepstore/volume_unix.go | 10 +-
.../keepproxy.service => ws/arvados-ws.service} | 6 +-
services/ws/config.go | 6 +-
services/ws/doc.go | 49 +++++
services/ws/event.go | 1 +
services/ws/{pg.go => event_source.go} | 18 +-
services/ws/handler.go | 47 ++--
services/ws/log.go | 29 +++
services/ws/main.go | 32 +--
services/ws/permission.go | 2 +-
services/ws/router.go | 58 +++--
services/ws/session_v0.go | 6 +-
services/ws/session_v1.go | 2 +-
tools/arvbash/arvbash.sh | 124 +++++++++++
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +
tools/arvbox/lib/arvbox/docker/common.sh | 4 +
.../lib/arvbox/docker/service/api/run-service | 4 +-
169 files changed, 2409 insertions(+), 976 deletions(-)
create mode 100644 apps/workbench/app/views/projects/_show_processes.html.erb
create mode 100755 build/build-dev-docker-jobs-image.sh
create mode 100644 doc/_includes/_container_scheduling_parameters.liquid
create mode 100644 doc/_includes/_crunch1only_begin.liquid
create mode 100644 doc/_includes/_crunch1only_end.liquid
create mode 100644 doc/_includes/_notebox_begin_warning.liquid
create mode 100644 doc/install/install-ws.html.textile.liquid
create mode 100644 sdk/dev-jobs.dockerfile
create mode 100644 sdk/go/httpserver/id_generator.go
create mode 100644 sdk/go/stats/duration.go
create mode 100644 sdk/go/stats/duration_test.go
create mode 100644 services/api/db/migrate/20161115171221_add_output_and_log_uuid_to_container_request.rb
create mode 100644 services/api/db/migrate/20161115174218_add_output_and_log_uuids_to_container_request_search_index.rb
create mode 100644 services/api/test/functional/arvados/v1/container_requests_controller_test.rb
delete mode 100644 services/api/test/websocket_runner.rb
create mode 100644 services/keepstore/count.go
copy services/{keepproxy/keepproxy.service => ws/arvados-ws.service} (55%)
create mode 100644 services/ws/doc.go
rename services/ws/{pg.go => event_source.go} (89%)
create mode 100755 tools/arvbash/arvbash.sh
via 2a62b82e96a6f56357e913117e04c36b1ccf7585 (commit)
via 2ec950c960fa8e95de9593fdf09f40833861c36e (commit)
via 324ffadbac85457cba901c7c195b8a20a32a8c4d (commit)
via 83ec78f608d67622ccc87b67cc16276a48a0430a (commit)
via 7ef471d9d0513b0b06cf0007e575f16eda93ec29 (commit)
via 45dd8543ac140a3326e6a8063bd43bd6c4ad05c8 (commit)
via d63601c63f651ab9fe4fefb5a7e8d76bf0495da3 (commit)
via 8595030d0314e6f88a245e66f90cce0a306b6867 (commit)
via dd2320c6877939365caf79767b5244d86a288437 (commit)
via fd717a88835be62ae1b5c4b1ecd74c21cab0e744 (commit)
via b9381df7b730762ee7e32a9ab58c6cbf3dcaed89 (commit)
via 21c1d7b918eb4019a8341f433979333da876218d (commit)
via beeab71df7bda32524c6afe5bf19f435ee3e0ade (commit)
via 252047244ba1c285a689e13ea0d59effa9d837ee (commit)
via 85f965a266837c9a2637c1180d8789c7d6dd0944 (commit)
via cb6baf8dc12be24ae664335dce29529252ac35df (commit)
via 49d18afe093301bbb48892e05b2c9c732e038abe (commit)
via 9f1ce358ac564abf62965cccf2bf3afa3bcfef95 (commit)
via d25dedaec8ea386c18b7f61c08a3097ba3c4f26c (commit)
via bd6b334c80cea328a51a8612d40ef16bdd6ab2e2 (commit)
via 84861876fd077f1892c6419fecfd348d7588747c (commit)
via 03f812007d8a28d7023dddb6f1fafb72e65ec525 (commit)
via 9e9190af52e4fc448f623aa2f4dd1fee803d99cd (commit)
via 2993bc9aea20c2295d1b2b3cbecf3b1f7e06bb17 (commit)
via 9be8a468534acaf324e9c18b831677f0ae067e60 (commit)
via b0cbc39917df391c0f0e7f31d9d46c5cc9070520 (commit)
via 0250713cc4e6d7fdf41fd7c0a99c6307e2eac72e (commit)
via 0d2bed2452f0840612be0b1bc792ffeff576d065 (commit)
via 4d164d025cd1e8c5c719b1e0f670e6cb10a6fe5e (commit)
via 66622dea0bb8725d0cbec0976175d11162b17815 (commit)
via 87143451d4c6535576fe7232729ae9ce93a26d30 (commit)
via e1b133b13f9ac50a87051d07c36a3904d6f01028 (commit)
via 0f1fca27f3b4b40ff8f6be729e12f2feeba05f8f (commit)
via cc0cd658d3db6820fba9daf380da4fa177b38f5c (commit)
via 44702d2cdc6f6f76c16d9d5da9ae3225ab07de2f (commit)
via 15aca78766bda903480e327340d67d3d882ee69f (commit)
via 5f38e62616773fcc97d795d1dd707c7fce801f2a (commit)
via 1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7 (commit)
via e64e03d71d1ad9c4ed9354fce55be7af17b4a56e (commit)
via 2b3af1807efa6bc4b3aaa298c1e24998d8ddd1c5 (commit)
via 79e63a733c48b04d7c9b6bcc6120af72b3f14641 (commit)
via 2eaa77dc327c024f2faa3fbd322e7054454b6442 (commit)
via 277ad30ef4824ab6363ba24f10f62d9fb6544ad7 (commit)
via 515e3f5c3dac3076217bd59545d01604008c01e6 (commit)
via 5646a899f667ee14efc32e9db84c72c0938ac6dc (commit)
via b9e5c8b32858338850da3e12ce27570b828898b3 (commit)
via bde488e7eff0e9a94cbf9709d28de13603857a3d (commit)
via dfd9492ee371ce86f7b4543106eb5060bc98928c (commit)
via dcc80dc5b02e46170400d42ca72672cceff03ba2 (commit)
via b5d3273b5aa503b22fdbeab7f8979e720e30c119 (commit)
via 7a76d3b2fc40c41e2d028daa0ee150b47421f0c5 (commit)
via 8735f2aa77b9e3c51449738e429399e4501ec94e (commit)
via 03406083ed63e5a0c118216e3e33fc8823c808a7 (commit)
via 7bf6588ce6589a194df2c7f45b9a443025c2ff67 (commit)
via d71a3ed8a91b7491df1f675389e1038d1a781e8e (commit)
via a12d68dd2e2646beb4ae68e4dd0825272e279508 (commit)
via b738c7e7357a143dade94dc5a1bad2b69bac2b27 (commit)
via 8f39c027e4895a8d872093c3dd16aa51b26a3731 (commit)
via e4664336c420836bf26f423faf2af9316302da93 (commit)
via 16942303133cb4e6be76d4d33a211d1e0ce2ea38 (commit)
via 52f9f7b3cf35efb7e8d1189b80f9b0f7afe0e111 (commit)
via 1bb30a981efa038bf07f139f996eb9d77749fced (commit)
via 1d45a45d1a1ebb0c02b4a9dd7ece73a55a6b24db (commit)
via 92fbda579013213b6f0d101f4fee71d149f3cb02 (commit)
via f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a (commit)
via 86f04235021d84afa0d28d105111422e0dd15738 (commit)
via 7c2b38632ae2af8a2f0f5c8fa2e55523220a3335 (commit)
via 8c014223e42683d308798475c021bad7a794e998 (commit)
via 13341931986ffc0cef740f0422fe482e14437b48 (commit)
via 4f678d4c1a5aa2aec73e0145e81c2629dee6689f (commit)
via 239115f5ff0bc2c82fe7d7be78cab1752cfba372 (commit)
via d13a369065b4e72047c8c267d6ceaa5c84d50c3d (commit)
via 064d34b61f3c3102ec7bc66d7d40867acfc6464c (commit)
via a983bbeec3db12156cc96741034ffe8d4053866b (commit)
via adda93976055fde37e92fa3fde7c08f529687132 (commit)
via 65ab86d8af08548d0f264dfc7f462fba0880ea7e (commit)
via 15635ff9fa2c964fb45467c9846ed92f7f6388b9 (commit)
via e2598332f9ecd40ab0dfd025bd1e5eba02ad8673 (commit)
via 0e1522f8e0cbe5d1626a7f66dc4c28b7b1bf0efc (commit)
via 2f6b00cc2a8765e9f8e07f98239f8f1f06887e2e (commit)
via e4a51416586f73593ac68bf0d2a74c53a4875f7e (commit)
via 81ccd9264d0742ba1bf0b9dfbc31ff4f15929f3e (commit)
via 6f461f4d0a996da85140982846a5d5c10ccfaae4 (commit)
via 9c3cc0f61751720cfdea62717934746d1aa32b72 (commit)
via 4ed4b6554535849341673efb7f80392dd5fba946 (commit)
via 09a2e88c51e5432e607f2a38466e55b4ba15e887 (commit)
via 7588bb6abd3886af9e1f3078db573a691a974771 (commit)
via 55e693dc29608f9b9975bb40c2fe4cf9c51df3f1 (commit)
via 726becee14c0f4aaa1b0f72fe33a16ee1de7ccd4 (commit)
via e41abf0cefe61a69a5dac27647066116eba5af96 (commit)
via 5debe2b77df18d381d7d59976e76f4ea6c9d5ce0 (commit)
via f3c31e7c71f076f5feafbb3f14e210f4d0de9012 (commit)
via 372a67d4364d4776aaa8a5ae9a4dd0ac16a0c524 (commit)
via 6e38822705235d01fbb7d51626b073174a65e46e (commit)
via 08d5c8073523b4c5d72413d8e2095f3b68b58190 (commit)
via 7692a12d05d54f0e209dcc2ff9ae9152560c7d52 (commit)
via 3afec6824121eea6aed8c2d25567a5e8974d3100 (commit)
via 185bc33b57a164463a7b67fca84b7596f6f79ad3 (commit)
via e5fd39ad76ac8bcb5e7bfe0c6938fdeb7c8ab4e3 (commit)
via 29356d0c02566aa33b3dbb9513dc701bad8fd16b (commit)
via b1e2f45d0a926617c991410feda842a5056ff5f0 (commit)
via bf08ad601e8c69e812dcfd5fd88cd711d35647ae (commit)
via 8c716ec575b5f7679a2ab95ebea944a46ff756c1 (commit)
via d542de20d8617f5823ab8f675c114f78aaf4a924 (commit)
via 515a58c0ef8634fca2397a8609f868524a42132c (commit)
via 4568673894b4a752503ad403bd391767ac1805e5 (commit)
via b45d7c92a23390c8be246219a1c84b8736854581 (commit)
via 78889e115e6fffd5eb82e54a541bd4858f804f91 (commit)
via 161a519439dcad4d77ca400ec48bb58cb685b54a (commit)
via c1ebef70f3b66080b51ef700383f44d70736f495 (commit)
via 5977b70a38e7102a6a369074897af990944c8934 (commit)
via 1071e1163f894c2a73df76cd400d102748e5281d (commit)
via 322d784513c37abc8fa1d1c05dd5f41019868735 (commit)
via 6f2ac70b70d4b64b728b815d5be429d5a165e2f0 (commit)
via c654baedb04251a741c840860041768ec661d3e7 (commit)
via 329b27f1407d900f8de7872077e6c91ebb32107c (commit)
via 9f53e085c98249ba79d85ba59e6f1ca624fede10 (commit)
via c40265a873d73f03a9ca077f18fe305d883fb4a5 (commit)
via a07c8bfac95524f3074d11cda0d6689b5e7bf9cc (commit)
via d53262e8e07785a9d2566966ee41ab8949ef1962 (commit)
via ce924ad9cdbab7ede0df7409d43b2660c1329979 (commit)
via 2d8150ed20d97491930d99a52d923872bca73939 (commit)
via 47f987da576d7dac80e5a03ad6613b5eb1f58660 (commit)
via ad825d1c73655d3f62fd485a1bc32fc3e76531f6 (commit)
via a4e81f561cce966074a72ff6219826a89152451a (commit)
via a4762b52738e12c39d93c14501a2f62463b05d07 (commit)
via 5fe0d5e7496fad7bd1c4bab0e5ca5f348c0eec63 (commit)
via 4bccbaed84c6b398f4cb4dbc7a9bc345e79d6550 (commit)
via b6bc9bbe45a68a07b1ea8139e1f4d698873739a7 (commit)
via a9a677e1655c461e742e46cc3c239f8605f4fc6b (commit)
via 8668135855b400e7f6047ecacd9cfa27fed723f7 (commit)
via 20d3523229805ee800ec11bf4ab6e41c4e18eea6 (commit)
via ddce4f6de6f0a77b45e9f3358eb6f0c1f1870fba (commit)
via e7a865275a832420b9d63c0ab3ebf87eaca57d26 (commit)
via d8db04f0165e57ab8021f2c1e4ff7061e494ceba (commit)
via a078c965c1ed1e54f678ad93305b91b6c7dcb1a9 (commit)
via 53a9aa7651a9fed6d88f4ece0f8d4cd10a77a63f (commit)
via 5a553b7b1fc75a84f8784eeb812361616911accd (commit)
via 6361d996c7a2c7d7648abfcf1699aa989e552f22 (commit)
via 15488bac7de5fa73c2695589c6436a6848615e84 (commit)
via b21e623902cb32b6d5a2fcf2c6ac9d92d472cb58 (commit)
via 5cd2757a79e7be7ed00156a69191893c3bb7e1c6 (commit)
via 42f433ac4486c18fa6408d5f942dc394e5ff149e (commit)
via 9cc813f25cb1d0c912c90ad8ed58166cbaaeff1c (commit)
via 72912c6b25fcb10c0acc540daf4f8a25d802784b (commit)
via 8886788e82b7e45c9211f6bf3a23c601ed0b88c1 (commit)
via 279db7d614a25c5e81c6efda5c8950988682974a (commit)
via 0b228d13b944b4b94165bf22a8f796070bd711bf (commit)
via 014f72fcdac2b26f9f2bbf257707eb59c676eebb (commit)
via 1671eb078bc4d254ebdfe5ac8dd40c83b371e8ac (commit)
via 554350245276ee082417cdcfcb8fd3f4f593e00f (commit)
via c2f6922a1997ca73d46866f03474a23c76c7afa5 (commit)
via 450cf2524ff41b2a0fcea09e2e6ccbf3bdbf78ed (commit)
via c632b87902785c4bbc5242b095e53bcf657f6aae (commit)
via 61925eaac13ef601879178d6343de9f9714f3da1 (commit)
via c6b3743bae761b8802cf7e016e54833d833ba9f7 (commit)
via 722670d1113e1af551d8470841a38d8d9218b4e8 (commit)
via 2bb3fb59369abd48211a872ca1a8e49cb70be5f3 (commit)
via e34688e4a78595aaabe7307f0184df84ff703c6f (commit)
via b8a539e3a0803083b1fcbb23755fc12cffff6f31 (commit)
via 2e9514e54e31bb37c438cf5d611d49a7c15235b4 (commit)
via fd308c814745d9c4eed5c3f79b44b9f9142df28a (commit)
from b0ba939812720869fca0a75b07d42518d4953345 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2a62b82e96a6f56357e913117e04c36b1ccf7585
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Dec 6 13:29:26 2016 -0500
8460: Add install doc page.
diff --git a/doc/install/install-ws.html.textile.liquid b/doc/install/install-ws.html.textile.liquid
new file mode 100644
index 0000000..ffbaf5e
--- /dev/null
+++ b/doc/install/install-ws.html.textile.liquid
@@ -0,0 +1,205 @@
+---
+layout: default
+navsection: installguide
+title: Install the websocket server
+...
+
+{% include 'notebox_begin_warning' %}
+
+This websocket server is an alternative to the puma server that comes with the API server. It is available as an *experimental pre-release* and is not recommended for production sites.
+
+{% include 'notebox_end' %}
+
+The arvados-ws server provides event notifications to websocket clients. It can be installed anywhere with access to Postgres database and the Arvados API server, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for additional information.
+
+By convention, we use the following hostname for the websocket service.
+
+<notextile>
+<pre><code>ws.<span class="userinput">uuid_prefix.your.domain</span></code></pre>
+</notextile>
+
+The above hostname should resolve from anywhere on the internet.
+
+h2. Install arvados-ws
+
+Typically arvados-ws runs on the same host as the API server.
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-ws</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-ws</span>
+</code></pre>
+</notextile>
+
+Verify that @arvados-ws@ is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-ws -h</span>
+Usage of arvados-ws:
+ -config path
+ path to config file (default "/etc/arvados/ws/ws.yml")
+ -dump-config
+ show current configuration and exit
+</code></pre>
+</notextile>
+
+h3. Create a configuration file
+
+Create @/etc/arvados/ws/ws.yml@ using the following template. Replace @xxxxxxxx@ with the "password you generated during database setup":install-postgresql.html#api.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-ws -h</span>
+Client:
+ APIHost: <span class="userinput">uuid_prefix.your.domain</span>:443
+Listen: ":<span class="userinput">9003</span>"
+Postgres:
+ dbname: arvados_production
+ host: localhost
+ password: <span class="userinput">xxxxxxxx</span>
+ user: arvados
+</code></pre>
+</notextile>
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the arvados-ws service should already be set up. Start it and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-ws</span>
+~$ <span class="userinput">sudo systemctl status arvados-ws</span>
+● arvados-ws.service - Arvados websocket server
+ Loaded: loaded (/lib/systemd/system/arvados-ws.service; enabled)
+ Active: active (running) since Tue 2016-12-06 11:20:48 EST; 10s ago
+ Docs: https://doc.arvados.org/
+ Main PID: 9421 (arvados-ws)
+ CGroup: /system.slice/arvados-ws.service
+ └─9421 /usr/bin/arvados-ws
+
+Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"level":"info","msg":"started","time":"2016-12-06T11:20:48.207617188-05:00"}
+Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:20:48.244956506-05:00"}
+Dec 06 11:20:48 zzzzz systemd[1]: Started Arvados websocket server.
+</code></pre>
+</notextile>
+
+If it is not running, use @journalctl@ to check logs for errors:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo journalctl -n10 -u arvados-ws</span>
+...
+Dec 06 11:12:48 zzzzz systemd[1]: Starting Arvados websocket server...
+Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"level":"info","msg":"started","time":"2016-12-06T11:12:48.030496636-05:00"}
+Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"error":"pq: password authentication failed for user \"arvados\"","level":"fatal","msg":"db.Ping failed","time":"2016-12-06T11:12:48.058206400-05:00"}
+</code></pre>
+</notextile>
+
+Skip ahead to "confirm the service is working":#confirm.
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the arvados-ws daemon. {% include 'install_runit' %}
+
+Create a supervised service.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-ws</span>
+~$ <span class="userinput">cd /etc/service/arvados-ws</span>
+~$ <span class="userinput">sudo mkdir log log/main</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec arvados-ws 2>&1\n' | sudo tee run</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
+~$ <span class="userinput">sudo chmod +x run log/run</span>
+~$ <span class="userinput">sudo sv exit .</span>
+~$ <span class="userinput">cd -</span>
+</code></pre>
+</notextile>
+
+Use @sv stat@ and check the log file to verify the service is running.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-ws</span>
+run: /etc/service/arvados-ws: (pid 12520) 2s; run: log: (pid 12519) 2s
+~$ <span class="userinput">tail /etc/service/arvados-ws/log/main/current</span>
+{"level":"info","msg":"started","time":"2016-12-06T11:56:20.669171449-05:00"}
+{"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:56:20.708847627-05:00"}
+</code></pre>
+</notextile>
+
+h3(#confirm). Confirm the service is working
+
+Confirm the service is listening on its assigned port and responding to requests.
+
+<notextile>
+<pre><code>~$ <span class="userinput">curl http://0.0.0.0:<b>9003</b>/status.json</span>
+{"Clients":1}
+</code></pre>
+</notextile>
+
+h3. Set up a reverse proxy with SSL support
+
+The arvados-ws service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
+
+This is best achieved by putting a reverse proxy with SSL support in front of arvados-ws, running on port 443 and passing requests to arvados-ws on port 9003 (or whatever port you chose in your configuration file).
+
+For example, using Nginx:
+
+<notextile><pre>
+upstream arvados-ws {
+ server 127.0.0.1:<span class="userinput">9003</span>;
+}
+
+server {
+ listen <span class="userinput">[your public IP address]</span>:443 ssl;
+ server_name ws.<span class="userinput">uuid_prefix.your.domain</span>;
+
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate <span class="userinput"/>YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput"/>YOUR/PATH/TO/cert.key</span>;
+
+ location / {
+ proxy_pass http://arvados-ws;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+}
+</pre></notextile>
+
+If Nginx is already configured to proxy @ws@ requests to puma, move that configuration out of the way or change its @server_name@ so it doesn't conflict.
+
+h3. Update API server configuration
+
+Ensure the websocket server address is correct in the API server configuration file @/etc/arvados/api/application.yml at .
+
+<notextile>
+<pre><code>websocket_address: wss://ws.<span class="userinput">uuid_prefix.your.domain</span>/websocket
+</code></pre>
+</notextile>
+
+Restart the Nginx to reload the API server configuration.
+
+<notextile>
+<pre><code>$ sudo nginx -s reload</span>
+</code></pre>
+</notextile>
+
+h3. Verify DNS and proxy setup
+
+Use a host elsewhere on the Internet to confirm that your DNS, proxy, and SSL are configured correctly.
+
+<notextile>
+<pre><code>$ <span class="userinput">curl https://ws.<b>uuid_prefix.your.domain</b>/status.json</span>
+{"Clients":1}
+</code></pre>
+</notextile>
diff --git a/services/ws/doc.go b/services/ws/doc.go
index 1f01b68..c2d3187 100644
--- a/services/ws/doc.go
+++ b/services/ws/doc.go
@@ -2,7 +2,7 @@
// cache-invalidation event feed at "ws://.../websocket") to
// websocket clients.
//
-// See https://doc.arvados.org/install/install-arvados-ws.html.
+// See https://doc.arvados.org/install/install-ws.html.
//
// Usage
//
commit 2ec950c960fa8e95de9593fdf09f40833861c36e
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Dec 6 11:07:00 2016 -0500
8460: Add json mime type.
diff --git a/services/ws/router.go b/services/ws/router.go
index 13c5ac3..974459f 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -130,6 +130,7 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
func jsonHandler(fn func() interface{}) http.HandlerFunc {
return func(resp http.ResponseWriter, req *http.Request) {
logger := logger(req.Context())
+ resp.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(resp)
err := enc.Encode(fn())
if err != nil {
commit 324ffadbac85457cba901c7c195b8a20a32a8c4d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Dec 6 11:06:44 2016 -0500
8460: Fix binary path.
diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
index 6de6848..ebccf0c 100644
--- a/services/ws/arvados-ws.service
+++ b/services/ws/arvados-ws.service
@@ -6,7 +6,7 @@ AssertPathExists=/etc/arvados/ws/ws.yml
[Service]
Type=notify
-ExecStart=/usr/bin/ws
+ExecStart=/usr/bin/arvados-ws
Restart=always
[Install]
commit 83ec78f608d67622ccc87b67cc16276a48a0430a
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Dec 5 15:40:25 2016 -0500
8460: golint fixes.
diff --git a/services/ws/config.go b/services/ws/config.go
index e2d69d0..0faa863 100644
--- a/services/ws/config.go
+++ b/services/ws/config.go
@@ -6,7 +6,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
)
-type Config struct {
+type wsConfig struct {
Client arvados.Client
Postgres pgConfig
Listen string
@@ -18,8 +18,8 @@ type Config struct {
ServerEventQueue int
}
-func DefaultConfig() Config {
- return Config{
+func defaultConfig() wsConfig {
+ return wsConfig{
Client: arvados.Client{
APIHost: "localhost:443",
},
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index bb32374..ea90ec7 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -44,7 +44,7 @@ type pgEventSource struct {
eventsOut uint64
}
-var _ DebugStatuser = (*pgEventSource)(nil)
+var _ debugStatuser = (*pgEventSource)(nil)
func (ps *pgEventSource) setup() {
ps.shutdown = make(chan error, 1)
diff --git a/services/ws/log.go b/services/ws/log.go
index ddfbb9e..1fbfbad 100644
--- a/services/ws/log.go
+++ b/services/ws/log.go
@@ -32,7 +32,7 @@ func logger(ctx context.Context) *logrus.Entry {
}
// loggerConfig sets up logging to behave as configured.
-func loggerConfig(cfg Config) {
+func loggerConfig(cfg wsConfig) {
lvl, err := logrus.ParseLevel(cfg.LogLevel)
if err != nil {
logrus.Fatal(err)
diff --git a/services/ws/main.go b/services/ws/main.go
index 650a249..77ebf9e 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -15,7 +15,7 @@ func main() {
configPath := flag.String("config", "/etc/arvados/ws/ws.yml", "`path` to config file")
dumpConfig := flag.Bool("dump-config", false, "show current configuration and exit")
- cfg := DefaultConfig()
+ cfg := defaultConfig()
flag.Parse()
err := config.LoadFile(&cfg, *configPath)
@@ -47,7 +47,7 @@ func main() {
Handler: &router{
Config: &cfg,
eventSource: eventSource,
- newPermChecker: func() permChecker { return NewPermChecker(cfg.Client) },
+ newPermChecker: func() permChecker { return newPermChecker(cfg.Client) },
},
}
// Bootstrap the eventSource by attaching a dummy subscriber
diff --git a/services/ws/permission.go b/services/ws/permission.go
index 42f7b37..e467e06 100644
--- a/services/ws/permission.go
+++ b/services/ws/permission.go
@@ -18,7 +18,7 @@ type permChecker interface {
Check(uuid string) (bool, error)
}
-func NewPermChecker(ac arvados.Client) permChecker {
+func newPermChecker(ac arvados.Client) permChecker {
ac.AuthToken = ""
return &cachingPermChecker{
Client: &ac,
diff --git a/services/ws/router.go b/services/ws/router.go
index 3f9800f..13c5ac3 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -22,7 +22,7 @@ type wsConn interface {
}
type router struct {
- Config *Config
+ Config *wsConfig
eventSource eventSource
newPermChecker func() permChecker
@@ -41,7 +41,7 @@ type routerDebugStatus struct {
ReqsActive int64
}
-type DebugStatuser interface {
+type debugStatuser interface {
DebugStatus() interface{}
}
@@ -53,8 +53,8 @@ func (rtr *router) setup() {
QueueSize: rtr.Config.ClientEventQueue,
}
rtr.mux = http.NewServeMux()
- rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
- rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
+ rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0))
+ rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
}
@@ -98,7 +98,7 @@ func (rtr *router) DebugStatus() interface{} {
"HTTP": rtr.status,
"Outgoing": rtr.handler.DebugStatus(),
}
- if es, ok := rtr.eventSource.(DebugStatuser); ok {
+ if es, ok := rtr.eventSource.(debugStatuser); ok {
s["EventSource"] = es.DebugStatus()
}
return s
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index 9a9707b..1cb3021 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -33,7 +33,7 @@ type v0session struct {
setupOnce sync.Once
}
-func NewSessionV0(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
+func newSessionV0(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
sess := &v0session{
sendq: sendq,
ws: ws,
diff --git a/services/ws/session_v1.go b/services/ws/session_v1.go
index 763fe59..701dce0 100644
--- a/services/ws/session_v1.go
+++ b/services/ws/session_v1.go
@@ -5,6 +5,6 @@ import (
"errors"
)
-func NewSessionV1(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
+func newSessionV1(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
return nil, errors.New("Not implemented")
}
commit 7ef471d9d0513b0b06cf0007e575f16eda93ec29
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Dec 5 15:20:16 2016 -0500
8460: Log entry at startup.
diff --git a/services/ws/main.go b/services/ws/main.go
index c4e1078..650a249 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -34,6 +34,7 @@ func main() {
return
}
+ log.Info("started")
eventSource := &pgEventSource{
DataSource: cfg.Postgres.ConnectionString(),
QueueSize: cfg.ServerEventQueue,
commit 45dd8543ac140a3326e6a8063bd43bd6c4ad05c8
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Dec 5 15:20:03 2016 -0500
8460: Cancel context before returning from handler.
diff --git a/services/ws/handler.go b/services/ws/handler.go
index ca9231c..b07b78c 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -31,6 +31,7 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
h.setupOnce.Do(h.setup)
ctx, cancel := context.WithCancel(ws.Request().Context())
+ defer cancel()
log := logger(ctx)
incoming := eventSource.NewSink()
commit d63601c63f651ab9fe4fefb5a7e8d76bf0495da3
Merge: 8595030 b9381df
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Dec 5 14:36:15 2016 -0500
8460: Merge branch 'master' into 8460-websocket-go
commit 8595030d0314e6f88a245e66f90cce0a306b6867
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Dec 5 14:35:25 2016 -0500
8460: Add simple /status.json
diff --git a/services/ws/router.go b/services/ws/router.go
index 78cedce..3f9800f 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -55,7 +55,8 @@ func (rtr *router) setup() {
rtr.mux = http.NewServeMux()
rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
- rtr.mux.HandleFunc("/debug.json", rtr.serveDebugStatus)
+ rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
+ rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
}
func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -103,14 +104,9 @@ func (rtr *router) DebugStatus() interface{} {
return s
}
-func (rtr *router) serveDebugStatus(resp http.ResponseWriter, req *http.Request) {
- rtr.setupOnce.Do(rtr.setup)
- logger := logger(req.Context())
- logger.Debug("status")
- enc := json.NewEncoder(resp)
- err := enc.Encode(rtr.DebugStatus())
- if err != nil {
- logger.WithError(err).Error("status encode failed")
+func (rtr *router) Status() interface{} {
+ return map[string]interface{}{
+ "Clients": atomic.LoadInt64(&rtr.status.ReqsActive),
}
}
@@ -130,3 +126,16 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
}).Info("accept request")
rtr.mux.ServeHTTP(resp, req)
}
+
+func jsonHandler(fn func() interface{}) http.HandlerFunc {
+ return func(resp http.ResponseWriter, req *http.Request) {
+ logger := logger(req.Context())
+ enc := json.NewEncoder(resp)
+ err := enc.Encode(fn())
+ if err != nil {
+ msg := "encode failed"
+ logger.WithError(err).Error(msg)
+ http.Error(resp, msg, http.StatusInternalServerError)
+ }
+ }
+}
commit dd2320c6877939365caf79767b5244d86a288437
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Dec 3 03:52:43 2016 -0500
8460: status.json -> debug.json
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 1653879..bb32374 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -44,6 +44,8 @@ type pgEventSource struct {
eventsOut uint64
}
+var _ DebugStatuser = (*pgEventSource)(nil)
+
func (ps *pgEventSource) setup() {
ps.shutdown = make(chan error, 1)
ps.sinks = make(map[*pgEventSink]bool)
@@ -172,7 +174,7 @@ func (ps *pgEventSource) DB() *sql.DB {
return ps.db
}
-func (ps *pgEventSource) Status() interface{} {
+func (ps *pgEventSource) DebugStatus() interface{} {
ps.mtx.Lock()
defer ps.mtx.Unlock()
blocked := 0
diff --git a/services/ws/handler.go b/services/ws/handler.go
index 8d8edca..ca9231c 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -192,7 +192,7 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
return
}
-func (h *handler) Status() interface{} {
+func (h *handler) DebugStatus() interface{} {
h.mtx.Lock()
defer h.mtx.Unlock()
diff --git a/services/ws/router.go b/services/ws/router.go
index 073a398..78cedce 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -33,16 +33,16 @@ type router struct {
lastReqID int64
lastReqMtx sync.Mutex
- status routerStatus
+ status routerDebugStatus
}
-type routerStatus struct {
+type routerDebugStatus struct {
ReqsReceived int64
ReqsActive int64
}
-type Statuser interface {
- Status() interface{}
+type DebugStatuser interface {
+ DebugStatus() interface{}
}
type sessionFactory func(wsConn, chan<- interface{}, *sql.DB, permChecker) (session, error)
@@ -55,7 +55,7 @@ func (rtr *router) setup() {
rtr.mux = http.NewServeMux()
rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
- rtr.mux.HandleFunc("/status.json", rtr.serveStatus)
+ rtr.mux.HandleFunc("/debug.json", rtr.serveDebugStatus)
}
func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -92,23 +92,23 @@ func (rtr *router) newReqID() string {
return strconv.FormatInt(id, 36)
}
-func (rtr *router) Status() interface{} {
+func (rtr *router) DebugStatus() interface{} {
s := map[string]interface{}{
"HTTP": rtr.status,
- "Outgoing": rtr.handler.Status(),
+ "Outgoing": rtr.handler.DebugStatus(),
}
- if es, ok := rtr.eventSource.(Statuser); ok {
- s["EventSource"] = es.Status()
+ if es, ok := rtr.eventSource.(DebugStatuser); ok {
+ s["EventSource"] = es.DebugStatus()
}
return s
}
-func (rtr *router) serveStatus(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) serveDebugStatus(resp http.ResponseWriter, req *http.Request) {
rtr.setupOnce.Do(rtr.setup)
logger := logger(req.Context())
logger.Debug("status")
enc := json.NewEncoder(resp)
- err := enc.Encode(rtr.Status())
+ err := enc.Encode(rtr.DebugStatus())
if err != nil {
logger.WithError(err).Error("status encode failed")
}
commit fd717a88835be62ae1b5c4b1ecd74c21cab0e744
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Dec 3 03:44:45 2016 -0500
8460: Rename pg -> event_source.go
diff --git a/services/ws/pg.go b/services/ws/event_source.go
similarity index 100%
rename from services/ws/pg.go
rename to services/ws/event_source.go
commit 0f1fca27f3b4b40ff8f6be729e12f2feeba05f8f
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Dec 1 10:35:50 2016 -0500
8460: fixup 15aca78 stats.Duration
diff --git a/services/ws/doc.go b/services/ws/doc.go
index d280568..1f01b68 100644
--- a/services/ws/doc.go
+++ b/services/ws/doc.go
@@ -1,4 +1,6 @@
-// Arvados-ws is an Arvados event feed for Websocket clients.
+// Arvados-ws exposes Arvados APIs (currently just one, the
+// cache-invalidation event feed at "ws://.../websocket") to
+// websocket clients.
//
// See https://doc.arvados.org/install/install-arvados-ws.html.
//
@@ -34,6 +36,10 @@
//
// A log is printed each time a client connects or disconnects.
//
+// Enable additional logs by configuring:
+//
+// LogLevel: debug
+//
// Runtime status
//
// GET /debug.json responds with debug stats.
diff --git a/services/ws/handler.go b/services/ws/handler.go
index dace39b..8d8edca 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -2,12 +2,12 @@ package main
import (
"context"
- "fmt"
"io"
"sync"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/stats"
)
type handler struct {
@@ -16,7 +16,7 @@ type handler struct {
QueueSize int
mtx sync.Mutex
- lastDelay map[chan interface{}]time.Duration
+ lastDelay map[chan interface{}]stats.Duration
setupOnce sync.Once
}
@@ -27,7 +27,7 @@ type handlerStats struct {
EventCount uint64
}
-func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsConn, chan<- interface{}) (session, error)) (stats handlerStats) {
+func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsConn, chan<- interface{}) (session, error)) (hStats handlerStats) {
h.setupOnce.Do(h.setup)
ctx, cancel := context.WithCancel(ws.Request().Context())
@@ -132,14 +132,14 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
log.Debug("sent")
if e != nil {
- stats.QueueDelayNs += t0.Sub(e.Ready)
+ hStats.QueueDelayNs += t0.Sub(e.Ready)
h.mtx.Lock()
- h.lastDelay[queue] = time.Since(e.Ready)
+ h.lastDelay[queue] = stats.Duration(time.Since(e.Ready))
h.mtx.Unlock()
}
- stats.WriteDelayNs += time.Since(t0)
- stats.EventBytes += uint64(len(buf))
- stats.EventCount++
+ hStats.WriteDelayNs += time.Since(t0)
+ hStats.EventBytes += uint64(len(buf))
+ hStats.EventCount++
}
}()
@@ -201,10 +201,8 @@ func (h *handler) Status() interface{} {
QueueMin int
QueueMax int
QueueTotal uint64
- queueDelayMin time.Duration
- QueueDelayMin string
- queueDelayMax time.Duration
- QueueDelayMax string
+ QueueDelayMin stats.Duration
+ QueueDelayMax stats.Duration
}
for q, lastDelay := range h.lastDelay {
s.QueueCount++
@@ -216,18 +214,16 @@ func (h *handler) Status() interface{} {
if s.QueueMin > n || s.QueueCount == 1 {
s.QueueMin = n
}
- if (s.queueDelayMin > lastDelay || s.queueDelayMin == 0) && lastDelay > 0 {
- s.queueDelayMin = lastDelay
+ if (s.QueueDelayMin > lastDelay || s.QueueDelayMin == 0) && lastDelay > 0 {
+ s.QueueDelayMin = lastDelay
}
- if s.queueDelayMax < lastDelay {
- s.queueDelayMax = lastDelay
+ if s.QueueDelayMax < lastDelay {
+ s.QueueDelayMax = lastDelay
}
}
- s.QueueDelayMin = fmt.Sprintf("%.06f", s.queueDelayMin.Seconds())
- s.QueueDelayMax = fmt.Sprintf("%.06f", s.queueDelayMax.Seconds())
return &s
}
func (h *handler) setup() {
- h.lastDelay = make(map[chan interface{}]time.Duration)
+ h.lastDelay = make(map[chan interface{}]stats.Duration)
}
diff --git a/services/ws/pg.go b/services/ws/pg.go
index dc899c5..1653879 100644
--- a/services/ws/pg.go
+++ b/services/ws/pg.go
@@ -2,13 +2,13 @@ package main
import (
"database/sql"
- "fmt"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/stats"
"github.com/lib/pq"
)
@@ -184,7 +184,7 @@ func (ps *pgEventSource) Status() interface{} {
"EventsOut": atomic.LoadUint64(&ps.eventsOut),
"Queue": len(ps.queue),
"QueueLimit": cap(ps.queue),
- "QueueDelay": fmt.Sprintf("%.06f", ps.lastQDelay.Seconds()),
+ "QueueDelay": stats.Duration(ps.lastQDelay),
"Sinks": len(ps.sinks),
"SinksBlocked": blocked,
}
commit cc0cd658d3db6820fba9daf380da4fa177b38f5c
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Dec 1 02:29:11 2016 -0500
8460: Add godoc page.
diff --git a/services/ws/doc.go b/services/ws/doc.go
new file mode 100644
index 0000000..d280568
--- /dev/null
+++ b/services/ws/doc.go
@@ -0,0 +1,43 @@
+// Arvados-ws is an Arvados event feed for Websocket clients.
+//
+// See https://doc.arvados.org/install/install-arvados-ws.html.
+//
+// Usage
+//
+// arvados-ws [-config /etc/arvados/ws/ws.yml] [-dump-config]
+//
+// Minimal configuration
+//
+// Client:
+// APIHost: localhost:443
+// Listen: ":1234"
+// Postgres:
+// dbname: arvados_production
+// host: localhost
+// password: xyzzy
+// user: arvados
+//
+// Options
+//
+// -config path
+//
+// Load configuration from the given file instead of the default
+// /etc/arvados/ws/ws.yml
+//
+// -dump-config
+//
+// Print the loaded configuration to stdout and exit.
+//
+// Logs
+//
+// Logs are printed to stderr, formatted as JSON.
+//
+// A log is printed each time a client connects or disconnects.
+//
+// Runtime status
+//
+// GET /debug.json responds with debug stats.
+//
+// GET /status.json responds with health check results and
+// activity/usage metrics.
+package main
commit 44702d2cdc6f6f76c16d9d5da9ae3225ab07de2f
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Dec 1 02:28:38 2016 -0500
8460: Add systemd unit file.
diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
new file mode 100644
index 0000000..6de6848
--- /dev/null
+++ b/services/ws/arvados-ws.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Arvados websocket server
+Documentation=https://doc.arvados.org/
+After=network.target
+AssertPathExists=/etc/arvados/ws/ws.yml
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/ws
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/ws/main.go b/services/ws/main.go
index 0031dc0..c4e1078 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -7,6 +7,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/config"
+ "github.com/coreos/go-systemd/daemon"
)
func main() {
@@ -48,8 +49,14 @@ func main() {
newPermChecker: func() permChecker { return NewPermChecker(cfg.Client) },
},
}
+ // Bootstrap the eventSource by attaching a dummy subscriber
+ // and hanging up.
eventSource.NewSink().Stop()
+ if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
+ log.WithError(err).Warn("error notifying init daemon")
+ }
+
log.WithField("Listen", srv.Addr).Info("listening")
log.Fatal(srv.ListenAndServe())
}
commit 15aca78766bda903480e327340d67d3d882ee69f
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Dec 1 01:33:55 2016 -0500
8460: Move loggedDuration from keepstore to sdk pkg as stats.Duration.
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 2c28817..379c616 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -91,6 +91,7 @@ sdk/go/httpserver
sdk/go/manifest
sdk/go/blockdigest
sdk/go/streamer
+sdk/go/stats
sdk/go/crunchrunner
sdk/cwl
tools/crunchstat-summary
@@ -757,6 +758,7 @@ gostuff=(
sdk/go/manifest
sdk/go/streamer
sdk/go/crunchrunner
+ sdk/go/stats
lib/crunchstat
services/arv-git-httpd
services/crunchstat
diff --git a/sdk/go/stats/duration.go b/sdk/go/stats/duration.go
new file mode 100644
index 0000000..103dea0
--- /dev/null
+++ b/sdk/go/stats/duration.go
@@ -0,0 +1,35 @@
+package stats
+
+import (
+ "fmt"
+ "strconv"
+ "time"
+)
+
+// Duration is a duration that is displayed as a number of seconds in
+// fixed-point notation.
+type Duration time.Duration
+
+// MarshalJSON implements json.Marshaler.
+func (d Duration) MarshalJSON() ([]byte, error) {
+ return []byte(d.String()), nil
+}
+
+// String implements fmt.Stringer.
+func (d Duration) String() string {
+ return fmt.Sprintf("%.6f", time.Duration(d).Seconds())
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (d *Duration) UnmarshalJSON(data []byte) error {
+ return d.Set(string(data))
+}
+
+// Value implements flag.Value
+func (d *Duration) Set(s string) error {
+ sec, err := strconv.ParseFloat(s, 64)
+ if err == nil {
+ *d = Duration(sec * float64(time.Second))
+ }
+ return err
+}
diff --git a/sdk/go/stats/duration_test.go b/sdk/go/stats/duration_test.go
new file mode 100644
index 0000000..730e646
--- /dev/null
+++ b/sdk/go/stats/duration_test.go
@@ -0,0 +1,23 @@
+package stats
+
+import (
+ "testing"
+ "time"
+)
+
+func TestString(t *testing.T) {
+ d := Duration(123123123123 * time.Nanosecond)
+ if s, expect := d.String(), "123.123123"; s != expect {
+ t.Errorf("got %s, expect %s", s, expect)
+ }
+}
+
+func TestSet(t *testing.T) {
+ var d Duration
+ if err := d.Set("123.456"); err != nil {
+ t.Fatal(err)
+ }
+ if got, expect := time.Duration(d).Nanoseconds(), int64(123456000000); got != expect {
+ t.Errorf("got %d, expect %d", got, expect)
+ }
+}
diff --git a/services/keepstore/logging_router.go b/services/keepstore/logging_router.go
index bfd006e..e34f858 100644
--- a/services/keepstore/logging_router.go
+++ b/services/keepstore/logging_router.go
@@ -5,12 +5,12 @@ package main
import (
"context"
- "fmt"
"net/http"
"strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "git.curoverse.com/arvados.git/sdk/go/stats"
log "github.com/Sirupsen/logrus"
)
@@ -97,25 +97,11 @@ func (loggingRouter *LoggingRESTRouter) ServeHTTP(wrappedResp http.ResponseWrite
}
lgr.WithFields(log.Fields{
- "timeTotal": loggedDuration(tDone.Sub(tStart)),
- "timeToStatus": loggedDuration(resp.sentHdr.Sub(tStart)),
- "timeWriteBody": loggedDuration(tDone.Sub(resp.sentHdr)),
+ "timeTotal": stats.Duration(tDone.Sub(tStart)),
+ "timeToStatus": stats.Duration(resp.sentHdr.Sub(tStart)),
+ "timeWriteBody": stats.Duration(tDone.Sub(resp.sentHdr)),
"respStatusCode": resp.Status,
"respStatus": statusText,
"respBytes": resp.Length,
}).Info("response")
}
-
-type loggedDuration time.Duration
-
-// MarshalJSON formats a duration as a number of seconds, using
-// fixed-point notation with no more than 6 decimal places.
-func (d loggedDuration) MarshalJSON() ([]byte, error) {
- return []byte(d.String()), nil
-}
-
-// String formats a duration as a number of seconds, using
-// fixed-point notation with no more than 6 decimal places.
-func (d loggedDuration) String() string {
- return fmt.Sprintf("%.6f", time.Duration(d).Seconds())
-}
commit 2eaa77dc327c024f2faa3fbd322e7054454b6442
Merge: 8c01422 bde488e
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 30 15:09:13 2016 -0500
8460: Merge branch 'master' into 8460-websocket-go
commit 8c014223e42683d308798475c021bad7a794e998
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 24 14:18:59 2016 -0500
8460: More statistics in status.json: events in/out, reqs received, lowest client queue delay.
diff --git a/services/ws/event.go b/services/ws/event.go
index 280035b..fa2a5df 100644
--- a/services/ws/event.go
+++ b/services/ws/event.go
@@ -22,6 +22,7 @@ type eventSource interface {
type event struct {
LogID uint64
Received time.Time
+ Ready time.Time
Serial uint64
db *sql.DB
diff --git a/services/ws/handler.go b/services/ws/handler.go
index d2c119a..dace39b 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -2,6 +2,7 @@ package main
import (
"context"
+ "fmt"
"io"
"sync"
"time"
@@ -15,7 +16,7 @@ type handler struct {
QueueSize int
mtx sync.Mutex
- queues map[chan interface{}]struct{}
+ lastDelay map[chan interface{}]time.Duration
setupOnce sync.Once
}
@@ -37,11 +38,11 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
queue := make(chan interface{}, h.QueueSize)
h.mtx.Lock()
- h.queues[queue] = struct{}{}
+ h.lastDelay[queue] = 0
h.mtx.Unlock()
defer func() {
h.mtx.Lock()
- delete(h.queues, queue)
+ delete(h.lastDelay, queue)
h.mtx.Unlock()
}()
@@ -131,7 +132,10 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
log.Debug("sent")
if e != nil {
- stats.QueueDelayNs += t0.Sub(e.Received)
+ stats.QueueDelayNs += t0.Sub(e.Ready)
+ h.mtx.Lock()
+ h.lastDelay[queue] = time.Since(e.Ready)
+ h.mtx.Unlock()
}
stats.WriteDelayNs += time.Since(t0)
stats.EventBytes += uint64(len(buf))
@@ -193,21 +197,37 @@ func (h *handler) Status() interface{} {
defer h.mtx.Unlock()
var s struct {
- QueueCount int
- QueueMax int
- QueueTotal uint64
+ QueueCount int
+ QueueMin int
+ QueueMax int
+ QueueTotal uint64
+ queueDelayMin time.Duration
+ QueueDelayMin string
+ queueDelayMax time.Duration
+ QueueDelayMax string
}
- for q := range h.queues {
+ for q, lastDelay := range h.lastDelay {
+ s.QueueCount++
n := len(q)
s.QueueTotal += uint64(n)
if s.QueueMax < n {
s.QueueMax = n
}
+ if s.QueueMin > n || s.QueueCount == 1 {
+ s.QueueMin = n
+ }
+ if (s.queueDelayMin > lastDelay || s.queueDelayMin == 0) && lastDelay > 0 {
+ s.queueDelayMin = lastDelay
+ }
+ if s.queueDelayMax < lastDelay {
+ s.queueDelayMax = lastDelay
+ }
}
- s.QueueCount = len(h.queues)
+ s.QueueDelayMin = fmt.Sprintf("%.06f", s.queueDelayMin.Seconds())
+ s.QueueDelayMax = fmt.Sprintf("%.06f", s.queueDelayMax.Seconds())
return &s
}
func (h *handler) setup() {
- h.queues = make(map[chan interface{}]struct{})
+ h.lastDelay = make(map[chan interface{}]time.Duration)
}
diff --git a/services/ws/pg.go b/services/ws/pg.go
index b6b064e..dc899c5 100644
--- a/services/ws/pg.go
+++ b/services/ws/pg.go
@@ -6,6 +6,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/lib/pq"
@@ -39,6 +40,8 @@ type pgEventSource struct {
shutdown chan error
lastQDelay time.Duration
+ eventsIn uint64
+ eventsOut uint64
}
func (ps *pgEventSource) setup() {
@@ -89,9 +92,11 @@ func (ps *pgEventSource) run() {
WithField("serial", e.Serial).
WithField("detail", e.Detail()).
Debug("event ready")
- ps.lastQDelay = time.Now().Sub(e.Received)
+ e.Ready = time.Now()
+ ps.lastQDelay = e.Ready.Sub(e.Received)
ps.mtx.Lock()
+ atomic.AddUint64(&ps.eventsOut, uint64(len(ps.sinks)))
for sink := range ps.sinks {
sink.channel <- e
}
@@ -136,6 +141,7 @@ func (ps *pgEventSource) run() {
db: ps.db,
}
logger(nil).WithField("event", e).Debug("incoming")
+ atomic.AddUint64(&ps.eventsIn, 1)
ps.queue <- e
go e.Detail()
}
@@ -174,6 +180,8 @@ func (ps *pgEventSource) Status() interface{} {
blocked += len(sink.channel)
}
return map[string]interface{}{
+ "EventsIn": atomic.LoadUint64(&ps.eventsIn),
+ "EventsOut": atomic.LoadUint64(&ps.eventsOut),
"Queue": len(ps.queue),
"QueueLimit": cap(ps.queue),
"QueueDelay": fmt.Sprintf("%.06f", ps.lastQDelay.Seconds()),
diff --git a/services/ws/router.go b/services/ws/router.go
index 18eaf73..073a398 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -37,7 +37,8 @@ type router struct {
}
type routerStatus struct {
- Connections int64
+ ReqsReceived int64
+ ReqsActive int64
}
type Statuser interface {
@@ -97,7 +98,7 @@ func (rtr *router) Status() interface{} {
"Outgoing": rtr.handler.Status(),
}
if es, ok := rtr.eventSource.(Statuser); ok {
- s["Incoming"] = es.Status()
+ s["EventSource"] = es.Status()
}
return s
}
@@ -115,8 +116,9 @@ func (rtr *router) serveStatus(resp http.ResponseWriter, req *http.Request) {
func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
rtr.setupOnce.Do(rtr.setup)
- atomic.AddInt64(&rtr.status.Connections, 1)
- defer atomic.AddInt64(&rtr.status.Connections, -1)
+ atomic.AddInt64(&rtr.status.ReqsReceived, 1)
+ atomic.AddInt64(&rtr.status.ReqsActive, 1)
+ defer atomic.AddInt64(&rtr.status.ReqsActive, -1)
logger := logger(req.Context()).
WithField("RequestID", rtr.newReqID())
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index a60a4a3..9a9707b 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -164,9 +164,11 @@ func (sub *v0subscribe) sendOldEvents(sess *v0session) {
// same thing all over again.
time.Sleep(100 * time.Millisecond)
}
+ now := time.Now()
e := &event{
LogID: id,
- Received: time.Now(),
+ Received: now,
+ Ready: now,
db: sess.db,
}
if sub.match(sess, e) {
commit 161a519439dcad4d77ca400ec48bb58cb685b54a
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 20 00:48:46 2016 -0500
8460: Move logging setup to log.go, use fixed-width timestamps.
diff --git a/services/ws/log.go b/services/ws/log.go
index fbf21b9..ddfbb9e 100644
--- a/services/ws/log.go
+++ b/services/ws/log.go
@@ -11,10 +11,17 @@ var (
rootLogger = logrus.New()
)
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
+// contextWithLogger returns a new child context such that
+// logger(child) returns the given logger.
func contextWithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
return context.WithValue(ctx, loggerCtxKey, logger)
}
+// logger returns the logger suitable for the given context -- the one
+// attached by contextWithLogger() if applicable, otherwise the
+// top-level logger with no fields/values.
func logger(ctx context.Context) *logrus.Entry {
if ctx != nil {
if logger, ok := ctx.Value(loggerCtxKey).(*logrus.Entry); ok {
@@ -23,3 +30,25 @@ func logger(ctx context.Context) *logrus.Entry {
}
return rootLogger.WithFields(nil)
}
+
+// loggerConfig sets up logging to behave as configured.
+func loggerConfig(cfg Config) {
+ lvl, err := logrus.ParseLevel(cfg.LogLevel)
+ if err != nil {
+ logrus.Fatal(err)
+ }
+ rootLogger.Level = lvl
+ switch cfg.LogFormat {
+ case "text":
+ rootLogger.Formatter = &logrus.TextFormatter{
+ FullTimestamp: true,
+ TimestampFormat: rfc3339NanoFixed,
+ }
+ case "json":
+ rootLogger.Formatter = &logrus.JSONFormatter{
+ TimestampFormat: rfc3339NanoFixed,
+ }
+ default:
+ logrus.WithField("LogFormat", cfg.LogFormat).Fatal("unknown log format")
+ }
+}
diff --git a/services/ws/main.go b/services/ws/main.go
index 33728dc..0031dc0 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -7,7 +7,6 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/config"
- "github.com/Sirupsen/logrus"
)
func main() {
@@ -23,24 +22,7 @@ func main() {
log.Fatal(err)
}
- lvl, err := logrus.ParseLevel(cfg.LogLevel)
- if err != nil {
- log.Fatal(err)
- }
- rootLogger.Level = lvl
- switch cfg.LogFormat {
- case "text":
- rootLogger.Formatter = &logrus.TextFormatter{
- FullTimestamp: true,
- TimestampFormat: time.RFC3339Nano,
- }
- case "json":
- rootLogger.Formatter = &logrus.JSONFormatter{
- TimestampFormat: time.RFC3339Nano,
- }
- default:
- log.WithField("LogFormat", cfg.LogFormat).Fatal("unknown log format")
- }
+ loggerConfig(cfg)
if *dumpConfig {
txt, err := config.Dump(&cfg)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list