[ARVADOS] updated: 2a62b82e96a6f56357e913117e04c36b1ccf7585

Git user git at public.curoverse.com
Tue Dec 6 13:29:39 EST 2016


Summary of changes:
 apps/workbench/Gemfile                             |   6 +-
 apps/workbench/Gemfile.lock                        |  16 +-
 .../app/controllers/application_controller.rb      |  10 +-
 .../app/controllers/projects_controller.rb         |   4 +-
 apps/workbench/app/models/container_work_unit.rb   |  13 +-
 .../app/views/projects/_show_dashboard.html.erb    |  16 +-
 .../app/views/projects/_show_processes.html.erb    |   5 +
 apps/workbench/app/views/projects/show.html.erb    |  16 +-
 .../test/controllers/disabled_api_test.rb          |   3 +
 .../test/controllers/projects_controller_test.rb   |  21 +-
 .../test/integration/application_layout_test.rb    |   6 +-
 .../test/integration/pipeline_instances_test.rb    |   2 +-
 apps/workbench/test/integration/projects_test.rb   |   2 +-
 apps/workbench/test/integration/work_units_test.rb |   2 +-
 apps/workbench/test/performance/browsing_test.rb   |   2 +-
 build/README                                       |   6 +-
 build/build-dev-docker-jobs-image.sh               |  63 ++++++
 build/run-build-packages-one-target.sh             |  11 +-
 build/run-build-packages.sh                        |  37 +++-
 build/run-library.sh                               |  17 ++
 build/run-tests.sh                                 |   8 +
 .../_container_runtime_constraints.liquid          |   1 -
 .../_container_scheduling_parameters.liquid        |   7 +
 doc/_includes/_crunch1only_begin.liquid            |   2 +
 doc/_includes/_crunch1only_end.liquid              |   1 +
 doc/_includes/_notebox_begin_warning.liquid        |   2 +
 doc/_includes/_pipeline_deprecation_notice.liquid  |   2 +-
 .../methods/container_requests.html.textile.liquid |   9 +-
 doc/api/methods/containers.html.textile.liquid     |   5 +
 doc/images/upload-using-workbench.png              | Bin 49690 -> 23979 bytes
 doc/images/workbench-dashboard.png                 | Bin 94257 -> 57930 bytes
 doc/images/workbench-move-selected.png             | Bin 47833 -> 15450 bytes
 doc/install/install-keep-web.html.textile.liquid   |   2 +-
 doc/install/install-keepproxy.html.textile.liquid  |   2 +-
 doc/install/install-ws.html.textile.liquid         | 205 ++++++++++++++++++
 doc/user/cwl/cwl-runner.html.textile.liquid        |   8 +-
 .../getting_started/workbench.html.textile.liquid  |   2 +-
 doc/user/topics/arv-run.html.textile.liquid        |   4 +
 ...nning-pipeline-command-line.html.textile.liquid |   4 +
 ...tutorial-pipeline-workbench.html.textile.liquid |   6 +-
 sdk/cli/bin/crunch-job                             |   6 +-
 sdk/cwl/arvados_cwl/__init__.py                    | 143 ++++++++----
 sdk/cwl/arvados_cwl/arvcontainer.py                |  58 ++---
 sdk/cwl/arvados_cwl/arvdocker.py                   |   5 +-
 sdk/cwl/arvados_cwl/arvjob.py                      |  46 ++--
 sdk/cwl/arvados_cwl/arvworkflow.py                 |  14 +-
 sdk/cwl/arvados_cwl/done.py                        |   3 +
 sdk/cwl/arvados_cwl/runner.py                      |  23 +-
 sdk/cwl/setup.py                                   |   2 +-
 sdk/cwl/tests/test_container.py                    |  71 ++----
 sdk/cwl/tests/test_submit.py                       | 148 ++++++++++---
 sdk/dev-jobs.dockerfile                            |  38 ++++
 sdk/go/httpserver/id_generator.go                  |  31 +++
 sdk/go/httpserver/request_limiter.go               |  26 ++-
 sdk/go/stats/duration.go                           |  35 +++
 sdk/go/stats/duration_test.go                      |  23 ++
 sdk/python/arvados/arvfile.py                      |   2 +-
 sdk/python/arvados/keep.py                         | 134 +++++++-----
 sdk/python/tests/test_events.py                    |  57 +++--
 sdk/python/tests/test_keep_client.py               |  76 ++++---
 services/api/Gemfile                               |  19 +-
 services/api/Gemfile.lock                          | 239 +++++++++++----------
 .../api/app/controllers/application_controller.rb  |  18 +-
 .../arvados/v1/collections_controller.rb           |   4 +-
 .../arvados/v1/container_requests_controller.rb    |   1 +
 .../arvados/v1/containers_controller.rb            |   1 +
 .../controllers/arvados/v1/groups_controller.rb    |   9 +-
 .../controllers/arvados/v1/schema_controller.rb    |  22 +-
 .../arvados/v1/user_agreements_controller.rb       |   3 +-
 .../app/controllers/arvados/v1/users_controller.rb |   2 +-
 .../arvados/v1/virtual_machines_controller.rb      |   2 +-
 .../api/app/controllers/database_controller.rb     |   2 +-
 services/api/app/middlewares/arvados_api_token.rb  |   2 +-
 services/api/app/models/arvados_model.rb           |   9 +-
 services/api/app/models/blob.rb                    |   4 +-
 services/api/app/models/collection.rb              |  15 +-
 services/api/app/models/commit_ancestor.rb         |   6 +-
 services/api/app/models/container_request.rb       |  19 +-
 services/api/app/models/job.rb                     |  42 ++--
 services/api/app/models/link.rb                    |   1 -
 services/api/app/models/log.rb                     |   1 -
 services/api/app/models/node.rb                    |   6 +-
 services/api/app/models/repository.rb              |   2 +-
 services/api/app/models/user.rb                    |   4 +-
 services/api/config/boot.rb                        |   2 +-
 services/api/config/initializers/inflections.rb    |   8 +-
 services/api/config/initializers/load_config.rb    |   4 +-
 .../api/config/initializers/preload_all_models.rb  |   2 +-
 ...add_output_and_log_uuid_to_container_request.rb |  22 ++
 ..._log_uuids_to_container_request_search_index.rb |  21 ++
 services/api/db/structure.sql                      |  12 +-
 services/api/lib/crunch_dispatch.rb                |  10 +-
 services/api/lib/current_api_client.rb             |  10 +-
 services/api/lib/eventbus.rb                       |  35 +--
 services/api/lib/load_param.rb                     |   4 +-
 services/api/lib/salvage_collection.rb             |   2 +-
 services/api/script/arvados-git-sync.rb            |   4 +-
 .../api/script/migrate-gitolite-to-uuid-storage.rb |   4 +-
 services/api/test/factories/user.rb                |   2 +-
 services/api/test/fixtures/container_requests.yml  |   2 +
 .../api_client_authorizations_controller_test.rb   |   2 +-
 .../arvados/v1/collections_controller_test.rb      |   8 +-
 .../v1/container_requests_controller_test.rb       |  22 ++
 .../api/test/functional/arvados/v1/filters_test.rb |   6 +-
 .../arvados/v1/groups_controller_test.rb           |  19 +-
 .../functional/arvados/v1/jobs_controller_test.rb  |   2 +-
 .../functional/arvados/v1/links_controller_test.rb |   4 +-
 .../arvados/v1/repositories_controller_test.rb     |  13 +-
 .../arvados/v1/schema_controller_test.rb           |   2 +-
 .../functional/arvados/v1/users_controller_test.rb |   4 +-
 .../arvados/v1/virtual_machines_controller_test.rb |   1 -
 services/api/test/helpers/users_test_helper.rb     |   2 +-
 .../api/test/integration/collections_api_test.rb   |  14 +-
 .../integration/collections_performance_test.rb    |   6 +-
 services/api/test/integration/cross_origin_test.rb |   2 +-
 .../api/test/integration/database_reset_test.rb    |   9 +-
 .../api/test/integration/user_sessions_test.rb     |   2 +-
 services/api/test/integration/websocket_test.rb    | 136 ++++++++----
 services/api/test/test_helper.rb                   |  32 ++-
 services/api/test/unit/app_version_test.rb         |   7 +-
 services/api/test/unit/authorized_key_test.rb      |   2 +-
 .../api/test/unit/collection_performance_test.rb   |   5 +-
 services/api/test/unit/collection_test.rb          |  16 +-
 services/api/test/unit/commit_test.rb              |  18 +-
 services/api/test/unit/container_request_test.rb   |  12 +-
 services/api/test/unit/container_test.rb           |  55 +++--
 services/api/test/unit/fail_jobs_test.rb           |   4 +-
 services/api/test/unit/job_test.rb                 |   6 +-
 services/api/test/unit/log_test.rb                 |   2 +-
 services/api/test/unit/node_test.rb                |   2 +-
 services/api/test/unit/owner_test.rb               |   3 +-
 services/api/test/unit/permission_test.rb          |  14 +-
 services/api/test/unit/pipeline_instance_test.rb   |   1 -
 services/api/test/unit/salvage_collection_test.rb  |  12 +-
 services/api/test/unit/user_test.rb                |   1 -
 services/api/test/websocket_runner.rb              |  53 -----
 services/keepstore/azure_blob_volume.go            |   2 +-
 services/keepstore/azure_blob_volume_test.go       |   2 +-
 services/keepstore/bufferpool.go                   |   3 +-
 services/keepstore/config.go                       |  22 +-
 services/keepstore/config_test.go                  |   2 +-
 services/keepstore/count.go                        |  44 ++++
 services/keepstore/handler_test.go                 |   2 +-
 services/keepstore/handlers.go                     |  91 +++++---
 services/keepstore/keepstore.go                    |  13 +-
 services/keepstore/logging_router.go               |  46 +++-
 services/keepstore/pull_worker.go                  |   3 +-
 services/keepstore/s3_volume.go                    | 127 ++++++++++-
 services/keepstore/s3_volume_test.go               |  32 ++-
 services/keepstore/trash_worker.go                 |   2 +-
 services/keepstore/usage.go                        |   4 +
 services/keepstore/volume.go                       |  45 +++-
 services/keepstore/volume_unix.go                  |  10 +-
 .../keepproxy.service => ws/arvados-ws.service}    |   6 +-
 services/ws/config.go                              |   6 +-
 services/ws/doc.go                                 |  49 +++++
 services/ws/event.go                               |   1 +
 services/ws/{pg.go => event_source.go}             |  18 +-
 services/ws/handler.go                             |  47 ++--
 services/ws/log.go                                 |  29 +++
 services/ws/main.go                                |  32 +--
 services/ws/permission.go                          |   2 +-
 services/ws/router.go                              |  58 +++--
 services/ws/session_v0.go                          |   6 +-
 services/ws/session_v1.go                          |   2 +-
 tools/arvbash/arvbash.sh                           | 124 +++++++++++
 tools/arvbox/lib/arvbox/docker/Dockerfile.base     |   2 +
 tools/arvbox/lib/arvbox/docker/common.sh           |   4 +
 .../lib/arvbox/docker/service/api/run-service      |   4 +-
 169 files changed, 2409 insertions(+), 976 deletions(-)
 create mode 100644 apps/workbench/app/views/projects/_show_processes.html.erb
 create mode 100755 build/build-dev-docker-jobs-image.sh
 create mode 100644 doc/_includes/_container_scheduling_parameters.liquid
 create mode 100644 doc/_includes/_crunch1only_begin.liquid
 create mode 100644 doc/_includes/_crunch1only_end.liquid
 create mode 100644 doc/_includes/_notebox_begin_warning.liquid
 create mode 100644 doc/install/install-ws.html.textile.liquid
 create mode 100644 sdk/dev-jobs.dockerfile
 create mode 100644 sdk/go/httpserver/id_generator.go
 create mode 100644 sdk/go/stats/duration.go
 create mode 100644 sdk/go/stats/duration_test.go
 create mode 100644 services/api/db/migrate/20161115171221_add_output_and_log_uuid_to_container_request.rb
 create mode 100644 services/api/db/migrate/20161115174218_add_output_and_log_uuids_to_container_request_search_index.rb
 create mode 100644 services/api/test/functional/arvados/v1/container_requests_controller_test.rb
 delete mode 100644 services/api/test/websocket_runner.rb
 create mode 100644 services/keepstore/count.go
 copy services/{keepproxy/keepproxy.service => ws/arvados-ws.service} (55%)
 create mode 100644 services/ws/doc.go
 rename services/ws/{pg.go => event_source.go} (89%)
 create mode 100755 tools/arvbash/arvbash.sh

       via  2a62b82e96a6f56357e913117e04c36b1ccf7585 (commit)
       via  2ec950c960fa8e95de9593fdf09f40833861c36e (commit)
       via  324ffadbac85457cba901c7c195b8a20a32a8c4d (commit)
       via  83ec78f608d67622ccc87b67cc16276a48a0430a (commit)
       via  7ef471d9d0513b0b06cf0007e575f16eda93ec29 (commit)
       via  45dd8543ac140a3326e6a8063bd43bd6c4ad05c8 (commit)
       via  d63601c63f651ab9fe4fefb5a7e8d76bf0495da3 (commit)
       via  8595030d0314e6f88a245e66f90cce0a306b6867 (commit)
       via  dd2320c6877939365caf79767b5244d86a288437 (commit)
       via  fd717a88835be62ae1b5c4b1ecd74c21cab0e744 (commit)
       via  b9381df7b730762ee7e32a9ab58c6cbf3dcaed89 (commit)
       via  21c1d7b918eb4019a8341f433979333da876218d (commit)
       via  beeab71df7bda32524c6afe5bf19f435ee3e0ade (commit)
       via  252047244ba1c285a689e13ea0d59effa9d837ee (commit)
       via  85f965a266837c9a2637c1180d8789c7d6dd0944 (commit)
       via  cb6baf8dc12be24ae664335dce29529252ac35df (commit)
       via  49d18afe093301bbb48892e05b2c9c732e038abe (commit)
       via  9f1ce358ac564abf62965cccf2bf3afa3bcfef95 (commit)
       via  d25dedaec8ea386c18b7f61c08a3097ba3c4f26c (commit)
       via  bd6b334c80cea328a51a8612d40ef16bdd6ab2e2 (commit)
       via  84861876fd077f1892c6419fecfd348d7588747c (commit)
       via  03f812007d8a28d7023dddb6f1fafb72e65ec525 (commit)
       via  9e9190af52e4fc448f623aa2f4dd1fee803d99cd (commit)
       via  2993bc9aea20c2295d1b2b3cbecf3b1f7e06bb17 (commit)
       via  9be8a468534acaf324e9c18b831677f0ae067e60 (commit)
       via  b0cbc39917df391c0f0e7f31d9d46c5cc9070520 (commit)
       via  0250713cc4e6d7fdf41fd7c0a99c6307e2eac72e (commit)
       via  0d2bed2452f0840612be0b1bc792ffeff576d065 (commit)
       via  4d164d025cd1e8c5c719b1e0f670e6cb10a6fe5e (commit)
       via  66622dea0bb8725d0cbec0976175d11162b17815 (commit)
       via  87143451d4c6535576fe7232729ae9ce93a26d30 (commit)
       via  e1b133b13f9ac50a87051d07c36a3904d6f01028 (commit)
       via  0f1fca27f3b4b40ff8f6be729e12f2feeba05f8f (commit)
       via  cc0cd658d3db6820fba9daf380da4fa177b38f5c (commit)
       via  44702d2cdc6f6f76c16d9d5da9ae3225ab07de2f (commit)
       via  15aca78766bda903480e327340d67d3d882ee69f (commit)
       via  5f38e62616773fcc97d795d1dd707c7fce801f2a (commit)
       via  1a8d456e6e8b7026dfe1c1b1176e8d46f9f374e7 (commit)
       via  e64e03d71d1ad9c4ed9354fce55be7af17b4a56e (commit)
       via  2b3af1807efa6bc4b3aaa298c1e24998d8ddd1c5 (commit)
       via  79e63a733c48b04d7c9b6bcc6120af72b3f14641 (commit)
       via  2eaa77dc327c024f2faa3fbd322e7054454b6442 (commit)
       via  277ad30ef4824ab6363ba24f10f62d9fb6544ad7 (commit)
       via  515e3f5c3dac3076217bd59545d01604008c01e6 (commit)
       via  5646a899f667ee14efc32e9db84c72c0938ac6dc (commit)
       via  b9e5c8b32858338850da3e12ce27570b828898b3 (commit)
       via  bde488e7eff0e9a94cbf9709d28de13603857a3d (commit)
       via  dfd9492ee371ce86f7b4543106eb5060bc98928c (commit)
       via  dcc80dc5b02e46170400d42ca72672cceff03ba2 (commit)
       via  b5d3273b5aa503b22fdbeab7f8979e720e30c119 (commit)
       via  7a76d3b2fc40c41e2d028daa0ee150b47421f0c5 (commit)
       via  8735f2aa77b9e3c51449738e429399e4501ec94e (commit)
       via  03406083ed63e5a0c118216e3e33fc8823c808a7 (commit)
       via  7bf6588ce6589a194df2c7f45b9a443025c2ff67 (commit)
       via  d71a3ed8a91b7491df1f675389e1038d1a781e8e (commit)
       via  a12d68dd2e2646beb4ae68e4dd0825272e279508 (commit)
       via  b738c7e7357a143dade94dc5a1bad2b69bac2b27 (commit)
       via  8f39c027e4895a8d872093c3dd16aa51b26a3731 (commit)
       via  e4664336c420836bf26f423faf2af9316302da93 (commit)
       via  16942303133cb4e6be76d4d33a211d1e0ce2ea38 (commit)
       via  52f9f7b3cf35efb7e8d1189b80f9b0f7afe0e111 (commit)
       via  1bb30a981efa038bf07f139f996eb9d77749fced (commit)
       via  1d45a45d1a1ebb0c02b4a9dd7ece73a55a6b24db (commit)
       via  92fbda579013213b6f0d101f4fee71d149f3cb02 (commit)
       via  f17da3a5f54ccaad3ec4f38dedac8b6c50a5cb0a (commit)
       via  86f04235021d84afa0d28d105111422e0dd15738 (commit)
       via  7c2b38632ae2af8a2f0f5c8fa2e55523220a3335 (commit)
       via  8c014223e42683d308798475c021bad7a794e998 (commit)
       via  13341931986ffc0cef740f0422fe482e14437b48 (commit)
       via  4f678d4c1a5aa2aec73e0145e81c2629dee6689f (commit)
       via  239115f5ff0bc2c82fe7d7be78cab1752cfba372 (commit)
       via  d13a369065b4e72047c8c267d6ceaa5c84d50c3d (commit)
       via  064d34b61f3c3102ec7bc66d7d40867acfc6464c (commit)
       via  a983bbeec3db12156cc96741034ffe8d4053866b (commit)
       via  adda93976055fde37e92fa3fde7c08f529687132 (commit)
       via  65ab86d8af08548d0f264dfc7f462fba0880ea7e (commit)
       via  15635ff9fa2c964fb45467c9846ed92f7f6388b9 (commit)
       via  e2598332f9ecd40ab0dfd025bd1e5eba02ad8673 (commit)
       via  0e1522f8e0cbe5d1626a7f66dc4c28b7b1bf0efc (commit)
       via  2f6b00cc2a8765e9f8e07f98239f8f1f06887e2e (commit)
       via  e4a51416586f73593ac68bf0d2a74c53a4875f7e (commit)
       via  81ccd9264d0742ba1bf0b9dfbc31ff4f15929f3e (commit)
       via  6f461f4d0a996da85140982846a5d5c10ccfaae4 (commit)
       via  9c3cc0f61751720cfdea62717934746d1aa32b72 (commit)
       via  4ed4b6554535849341673efb7f80392dd5fba946 (commit)
       via  09a2e88c51e5432e607f2a38466e55b4ba15e887 (commit)
       via  7588bb6abd3886af9e1f3078db573a691a974771 (commit)
       via  55e693dc29608f9b9975bb40c2fe4cf9c51df3f1 (commit)
       via  726becee14c0f4aaa1b0f72fe33a16ee1de7ccd4 (commit)
       via  e41abf0cefe61a69a5dac27647066116eba5af96 (commit)
       via  5debe2b77df18d381d7d59976e76f4ea6c9d5ce0 (commit)
       via  f3c31e7c71f076f5feafbb3f14e210f4d0de9012 (commit)
       via  372a67d4364d4776aaa8a5ae9a4dd0ac16a0c524 (commit)
       via  6e38822705235d01fbb7d51626b073174a65e46e (commit)
       via  08d5c8073523b4c5d72413d8e2095f3b68b58190 (commit)
       via  7692a12d05d54f0e209dcc2ff9ae9152560c7d52 (commit)
       via  3afec6824121eea6aed8c2d25567a5e8974d3100 (commit)
       via  185bc33b57a164463a7b67fca84b7596f6f79ad3 (commit)
       via  e5fd39ad76ac8bcb5e7bfe0c6938fdeb7c8ab4e3 (commit)
       via  29356d0c02566aa33b3dbb9513dc701bad8fd16b (commit)
       via  b1e2f45d0a926617c991410feda842a5056ff5f0 (commit)
       via  bf08ad601e8c69e812dcfd5fd88cd711d35647ae (commit)
       via  8c716ec575b5f7679a2ab95ebea944a46ff756c1 (commit)
       via  d542de20d8617f5823ab8f675c114f78aaf4a924 (commit)
       via  515a58c0ef8634fca2397a8609f868524a42132c (commit)
       via  4568673894b4a752503ad403bd391767ac1805e5 (commit)
       via  b45d7c92a23390c8be246219a1c84b8736854581 (commit)
       via  78889e115e6fffd5eb82e54a541bd4858f804f91 (commit)
       via  161a519439dcad4d77ca400ec48bb58cb685b54a (commit)
       via  c1ebef70f3b66080b51ef700383f44d70736f495 (commit)
       via  5977b70a38e7102a6a369074897af990944c8934 (commit)
       via  1071e1163f894c2a73df76cd400d102748e5281d (commit)
       via  322d784513c37abc8fa1d1c05dd5f41019868735 (commit)
       via  6f2ac70b70d4b64b728b815d5be429d5a165e2f0 (commit)
       via  c654baedb04251a741c840860041768ec661d3e7 (commit)
       via  329b27f1407d900f8de7872077e6c91ebb32107c (commit)
       via  9f53e085c98249ba79d85ba59e6f1ca624fede10 (commit)
       via  c40265a873d73f03a9ca077f18fe305d883fb4a5 (commit)
       via  a07c8bfac95524f3074d11cda0d6689b5e7bf9cc (commit)
       via  d53262e8e07785a9d2566966ee41ab8949ef1962 (commit)
       via  ce924ad9cdbab7ede0df7409d43b2660c1329979 (commit)
       via  2d8150ed20d97491930d99a52d923872bca73939 (commit)
       via  47f987da576d7dac80e5a03ad6613b5eb1f58660 (commit)
       via  ad825d1c73655d3f62fd485a1bc32fc3e76531f6 (commit)
       via  a4e81f561cce966074a72ff6219826a89152451a (commit)
       via  a4762b52738e12c39d93c14501a2f62463b05d07 (commit)
       via  5fe0d5e7496fad7bd1c4bab0e5ca5f348c0eec63 (commit)
       via  4bccbaed84c6b398f4cb4dbc7a9bc345e79d6550 (commit)
       via  b6bc9bbe45a68a07b1ea8139e1f4d698873739a7 (commit)
       via  a9a677e1655c461e742e46cc3c239f8605f4fc6b (commit)
       via  8668135855b400e7f6047ecacd9cfa27fed723f7 (commit)
       via  20d3523229805ee800ec11bf4ab6e41c4e18eea6 (commit)
       via  ddce4f6de6f0a77b45e9f3358eb6f0c1f1870fba (commit)
       via  e7a865275a832420b9d63c0ab3ebf87eaca57d26 (commit)
       via  d8db04f0165e57ab8021f2c1e4ff7061e494ceba (commit)
       via  a078c965c1ed1e54f678ad93305b91b6c7dcb1a9 (commit)
       via  53a9aa7651a9fed6d88f4ece0f8d4cd10a77a63f (commit)
       via  5a553b7b1fc75a84f8784eeb812361616911accd (commit)
       via  6361d996c7a2c7d7648abfcf1699aa989e552f22 (commit)
       via  15488bac7de5fa73c2695589c6436a6848615e84 (commit)
       via  b21e623902cb32b6d5a2fcf2c6ac9d92d472cb58 (commit)
       via  5cd2757a79e7be7ed00156a69191893c3bb7e1c6 (commit)
       via  42f433ac4486c18fa6408d5f942dc394e5ff149e (commit)
       via  9cc813f25cb1d0c912c90ad8ed58166cbaaeff1c (commit)
       via  72912c6b25fcb10c0acc540daf4f8a25d802784b (commit)
       via  8886788e82b7e45c9211f6bf3a23c601ed0b88c1 (commit)
       via  279db7d614a25c5e81c6efda5c8950988682974a (commit)
       via  0b228d13b944b4b94165bf22a8f796070bd711bf (commit)
       via  014f72fcdac2b26f9f2bbf257707eb59c676eebb (commit)
       via  1671eb078bc4d254ebdfe5ac8dd40c83b371e8ac (commit)
       via  554350245276ee082417cdcfcb8fd3f4f593e00f (commit)
       via  c2f6922a1997ca73d46866f03474a23c76c7afa5 (commit)
       via  450cf2524ff41b2a0fcea09e2e6ccbf3bdbf78ed (commit)
       via  c632b87902785c4bbc5242b095e53bcf657f6aae (commit)
       via  61925eaac13ef601879178d6343de9f9714f3da1 (commit)
       via  c6b3743bae761b8802cf7e016e54833d833ba9f7 (commit)
       via  722670d1113e1af551d8470841a38d8d9218b4e8 (commit)
       via  2bb3fb59369abd48211a872ca1a8e49cb70be5f3 (commit)
       via  e34688e4a78595aaabe7307f0184df84ff703c6f (commit)
       via  b8a539e3a0803083b1fcbb23755fc12cffff6f31 (commit)
       via  2e9514e54e31bb37c438cf5d611d49a7c15235b4 (commit)
       via  fd308c814745d9c4eed5c3f79b44b9f9142df28a (commit)
      from  b0ba939812720869fca0a75b07d42518d4953345 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 2a62b82e96a6f56357e913117e04c36b1ccf7585
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Dec 6 13:29:26 2016 -0500

    8460: Add install doc page.

diff --git a/doc/install/install-ws.html.textile.liquid b/doc/install/install-ws.html.textile.liquid
new file mode 100644
index 0000000..ffbaf5e
--- /dev/null
+++ b/doc/install/install-ws.html.textile.liquid
@@ -0,0 +1,205 @@
+---
+layout: default
+navsection: installguide
+title: Install the websocket server
+...
+
+{% include 'notebox_begin_warning' %}
+
+This websocket server is an alternative to the puma server that comes with the API server. It is available as an *experimental pre-release* and is not recommended for production sites.
+
+{% include 'notebox_end' %}
+
+The arvados-ws server provides event notifications to websocket clients. It can be installed anywhere with access to Postgres database and the Arvados API server, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for additional information.
+
+By convention, we use the following hostname for the websocket service.
+
+<notextile>
+<pre><code>ws.<span class="userinput">uuid_prefix.your.domain</span></code></pre>
+</notextile>
+
+The above hostname should resolve from anywhere on the internet.
+
+h2. Install arvados-ws
+
+Typically arvados-ws runs on the same host as the API server.
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-ws</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-ws</span>
+</code></pre>
+</notextile>
+
+Verify that @arvados-ws@ is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-ws -h</span>
+Usage of arvados-ws:
+  -config path
+        path to config file (default "/etc/arvados/ws/ws.yml")
+  -dump-config
+        show current configuration and exit
+</code></pre>
+</notextile>
+
+h3. Create a configuration file
+
+Create @/etc/arvados/ws/ws.yml@ using the following template. Replace @xxxxxxxx@ with the "password you generated during database setup":install-postgresql.html#api.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-ws -h</span>
+Client:
+  APIHost: <span class="userinput">uuid_prefix.your.domain</span>:443
+Listen: ":<span class="userinput">9003</span>"
+Postgres:
+  dbname: arvados_production
+  host: localhost
+  password: <span class="userinput">xxxxxxxx</span>
+  user: arvados
+</code></pre>
+</notextile>
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the arvados-ws service should already be set up. Start it and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-ws</span>
+~$ <span class="userinput">sudo systemctl status arvados-ws</span>
+● arvados-ws.service - Arvados websocket server
+   Loaded: loaded (/lib/systemd/system/arvados-ws.service; enabled)
+   Active: active (running) since Tue 2016-12-06 11:20:48 EST; 10s ago
+     Docs: https://doc.arvados.org/
+ Main PID: 9421 (arvados-ws)
+   CGroup: /system.slice/arvados-ws.service
+           └─9421 /usr/bin/arvados-ws
+
+Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"level":"info","msg":"started","time":"2016-12-06T11:20:48.207617188-05:00"}
+Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:20:48.244956506-05:00"}
+Dec 06 11:20:48 zzzzz systemd[1]: Started Arvados websocket server.
+</code></pre>
+</notextile>
+
+If it is not running, use @journalctl@ to check logs for errors:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo journalctl -n10 -u arvados-ws</span>
+...
+Dec 06 11:12:48 zzzzz systemd[1]: Starting Arvados websocket server...
+Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"level":"info","msg":"started","time":"2016-12-06T11:12:48.030496636-05:00"}
+Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"error":"pq: password authentication failed for user \"arvados\"","level":"fatal","msg":"db.Ping failed","time":"2016-12-06T11:12:48.058206400-05:00"}
+</code></pre>
+</notextile>
+
+Skip ahead to "confirm the service is working":#confirm.
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the arvados-ws daemon.  {% include 'install_runit' %}
+
+Create a supervised service.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-ws</span>
+~$ <span class="userinput">cd /etc/service/arvados-ws</span>
+~$ <span class="userinput">sudo mkdir log log/main</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec arvados-ws 2>&1\n' | sudo tee run</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
+~$ <span class="userinput">sudo chmod +x run log/run</span>
+~$ <span class="userinput">sudo sv exit .</span>
+~$ <span class="userinput">cd -</span>
+</code></pre>
+</notextile>
+
+Use @sv stat@ and check the log file to verify the service is running.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-ws</span>
+run: /etc/service/arvados-ws: (pid 12520) 2s; run: log: (pid 12519) 2s
+~$ <span class="userinput">tail /etc/service/arvados-ws/log/main/current</span>
+{"level":"info","msg":"started","time":"2016-12-06T11:56:20.669171449-05:00"}
+{"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:56:20.708847627-05:00"}
+</code></pre>
+</notextile>
+
+h3(#confirm). Confirm the service is working
+
+Confirm the service is listening on its assigned port and responding to requests.
+
+<notextile>
+<pre><code>~$ <span class="userinput">curl http://0.0.0.0:<b>9003</b>/status.json</span>
+{"Clients":1}
+</code></pre>
+</notextile>
+
+h3. Set up a reverse proxy with SSL support
+
+The arvados-ws service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
+
+This is best achieved by putting a reverse proxy with SSL support in front of arvados-ws, running on port 443 and passing requests to arvados-ws on port 9003 (or whatever port you chose in your configuration file).
+
+For example, using Nginx:
+
+<notextile><pre>
+upstream arvados-ws {
+  server                127.0.0.1:<span class="userinput">9003</span>;
+}
+
+server {
+  listen                <span class="userinput">[your public IP address]</span>:443 ssl;
+  server_name           ws.<span class="userinput">uuid_prefix.your.domain</span>;
+
+  proxy_connect_timeout 90s;
+  proxy_read_timeout    300s;
+
+  ssl                   on;
+  ssl_certificate       <span class="userinput"/>YOUR/PATH/TO/cert.pem</span>;
+  ssl_certificate_key   <span class="userinput"/>YOUR/PATH/TO/cert.key</span>;
+
+  location / {
+    proxy_pass          http://arvados-ws;
+    proxy_set_header    Upgrade         $http_upgrade;
+    proxy_set_header    Connection      "upgrade";
+    proxy_set_header    Host            $host;
+    proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
+  }
+}
+</pre></notextile>
+
+If Nginx is already configured to proxy @ws@ requests to puma, move that configuration out of the way or change its @server_name@ so it doesn't conflict.
+
+h3. Update API server configuration
+
+Ensure the websocket server address is correct in the API server configuration file @/etc/arvados/api/application.yml at .
+
+<notextile>
+<pre><code>websocket_address: wss://ws.<span class="userinput">uuid_prefix.your.domain</span>/websocket
+</code></pre>
+</notextile>
+
+Restart the Nginx to reload the API server configuration.
+
+<notextile>
+<pre><code>$ sudo nginx -s reload</span>
+</code></pre>
+</notextile>
+
+h3. Verify DNS and proxy setup
+
+Use a host elsewhere on the Internet to confirm that your DNS, proxy, and SSL are configured correctly.
+
+<notextile>
+<pre><code>$ <span class="userinput">curl https://ws.<b>uuid_prefix.your.domain</b>/status.json</span>
+{"Clients":1}
+</code></pre>
+</notextile>
diff --git a/services/ws/doc.go b/services/ws/doc.go
index 1f01b68..c2d3187 100644
--- a/services/ws/doc.go
+++ b/services/ws/doc.go
@@ -2,7 +2,7 @@
 // cache-invalidation event feed at "ws://.../websocket") to
 // websocket clients.
 //
-// See https://doc.arvados.org/install/install-arvados-ws.html.
+// See https://doc.arvados.org/install/install-ws.html.
 //
 // Usage
 //

commit 2ec950c960fa8e95de9593fdf09f40833861c36e
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Dec 6 11:07:00 2016 -0500

    8460: Add json mime type.

diff --git a/services/ws/router.go b/services/ws/router.go
index 13c5ac3..974459f 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -130,6 +130,7 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 func jsonHandler(fn func() interface{}) http.HandlerFunc {
 	return func(resp http.ResponseWriter, req *http.Request) {
 		logger := logger(req.Context())
+		resp.Header().Set("Content-Type", "application/json")
 		enc := json.NewEncoder(resp)
 		err := enc.Encode(fn())
 		if err != nil {

commit 324ffadbac85457cba901c7c195b8a20a32a8c4d
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Dec 6 11:06:44 2016 -0500

    8460: Fix binary path.

diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
index 6de6848..ebccf0c 100644
--- a/services/ws/arvados-ws.service
+++ b/services/ws/arvados-ws.service
@@ -6,7 +6,7 @@ AssertPathExists=/etc/arvados/ws/ws.yml
 
 [Service]
 Type=notify
-ExecStart=/usr/bin/ws
+ExecStart=/usr/bin/arvados-ws
 Restart=always
 
 [Install]

commit 83ec78f608d67622ccc87b67cc16276a48a0430a
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 5 15:40:25 2016 -0500

    8460: golint fixes.

diff --git a/services/ws/config.go b/services/ws/config.go
index e2d69d0..0faa863 100644
--- a/services/ws/config.go
+++ b/services/ws/config.go
@@ -6,7 +6,7 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 )
 
-type Config struct {
+type wsConfig struct {
 	Client    arvados.Client
 	Postgres  pgConfig
 	Listen    string
@@ -18,8 +18,8 @@ type Config struct {
 	ServerEventQueue int
 }
 
-func DefaultConfig() Config {
-	return Config{
+func defaultConfig() wsConfig {
+	return wsConfig{
 		Client: arvados.Client{
 			APIHost: "localhost:443",
 		},
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index bb32374..ea90ec7 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -44,7 +44,7 @@ type pgEventSource struct {
 	eventsOut  uint64
 }
 
-var _ DebugStatuser = (*pgEventSource)(nil)
+var _ debugStatuser = (*pgEventSource)(nil)
 
 func (ps *pgEventSource) setup() {
 	ps.shutdown = make(chan error, 1)
diff --git a/services/ws/log.go b/services/ws/log.go
index ddfbb9e..1fbfbad 100644
--- a/services/ws/log.go
+++ b/services/ws/log.go
@@ -32,7 +32,7 @@ func logger(ctx context.Context) *logrus.Entry {
 }
 
 // loggerConfig sets up logging to behave as configured.
-func loggerConfig(cfg Config) {
+func loggerConfig(cfg wsConfig) {
 	lvl, err := logrus.ParseLevel(cfg.LogLevel)
 	if err != nil {
 		logrus.Fatal(err)
diff --git a/services/ws/main.go b/services/ws/main.go
index 650a249..77ebf9e 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -15,7 +15,7 @@ func main() {
 
 	configPath := flag.String("config", "/etc/arvados/ws/ws.yml", "`path` to config file")
 	dumpConfig := flag.Bool("dump-config", false, "show current configuration and exit")
-	cfg := DefaultConfig()
+	cfg := defaultConfig()
 	flag.Parse()
 
 	err := config.LoadFile(&cfg, *configPath)
@@ -47,7 +47,7 @@ func main() {
 		Handler: &router{
 			Config:         &cfg,
 			eventSource:    eventSource,
-			newPermChecker: func() permChecker { return NewPermChecker(cfg.Client) },
+			newPermChecker: func() permChecker { return newPermChecker(cfg.Client) },
 		},
 	}
 	// Bootstrap the eventSource by attaching a dummy subscriber
diff --git a/services/ws/permission.go b/services/ws/permission.go
index 42f7b37..e467e06 100644
--- a/services/ws/permission.go
+++ b/services/ws/permission.go
@@ -18,7 +18,7 @@ type permChecker interface {
 	Check(uuid string) (bool, error)
 }
 
-func NewPermChecker(ac arvados.Client) permChecker {
+func newPermChecker(ac arvados.Client) permChecker {
 	ac.AuthToken = ""
 	return &cachingPermChecker{
 		Client:     &ac,
diff --git a/services/ws/router.go b/services/ws/router.go
index 3f9800f..13c5ac3 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -22,7 +22,7 @@ type wsConn interface {
 }
 
 type router struct {
-	Config         *Config
+	Config         *wsConfig
 	eventSource    eventSource
 	newPermChecker func() permChecker
 
@@ -41,7 +41,7 @@ type routerDebugStatus struct {
 	ReqsActive   int64
 }
 
-type DebugStatuser interface {
+type debugStatuser interface {
 	DebugStatus() interface{}
 }
 
@@ -53,8 +53,8 @@ func (rtr *router) setup() {
 		QueueSize:   rtr.Config.ClientEventQueue,
 	}
 	rtr.mux = http.NewServeMux()
-	rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
-	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
+	rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0))
+	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
 	rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
 	rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
 }
@@ -98,7 +98,7 @@ func (rtr *router) DebugStatus() interface{} {
 		"HTTP":     rtr.status,
 		"Outgoing": rtr.handler.DebugStatus(),
 	}
-	if es, ok := rtr.eventSource.(DebugStatuser); ok {
+	if es, ok := rtr.eventSource.(debugStatuser); ok {
 		s["EventSource"] = es.DebugStatus()
 	}
 	return s
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index 9a9707b..1cb3021 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -33,7 +33,7 @@ type v0session struct {
 	setupOnce     sync.Once
 }
 
-func NewSessionV0(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
+func newSessionV0(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
 	sess := &v0session{
 		sendq:       sendq,
 		ws:          ws,
diff --git a/services/ws/session_v1.go b/services/ws/session_v1.go
index 763fe59..701dce0 100644
--- a/services/ws/session_v1.go
+++ b/services/ws/session_v1.go
@@ -5,6 +5,6 @@ import (
 	"errors"
 )
 
-func NewSessionV1(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
+func newSessionV1(ws wsConn, sendq chan<- interface{}, db *sql.DB, pc permChecker) (session, error) {
 	return nil, errors.New("Not implemented")
 }

commit 7ef471d9d0513b0b06cf0007e575f16eda93ec29
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 5 15:20:16 2016 -0500

    8460: Log entry at startup.

diff --git a/services/ws/main.go b/services/ws/main.go
index c4e1078..650a249 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -34,6 +34,7 @@ func main() {
 		return
 	}
 
+	log.Info("started")
 	eventSource := &pgEventSource{
 		DataSource: cfg.Postgres.ConnectionString(),
 		QueueSize:  cfg.ServerEventQueue,

commit 45dd8543ac140a3326e6a8063bd43bd6c4ad05c8
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 5 15:20:03 2016 -0500

    8460: Cancel context before returning from handler.

diff --git a/services/ws/handler.go b/services/ws/handler.go
index ca9231c..b07b78c 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -31,6 +31,7 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
 	h.setupOnce.Do(h.setup)
 
 	ctx, cancel := context.WithCancel(ws.Request().Context())
+	defer cancel()
 	log := logger(ctx)
 
 	incoming := eventSource.NewSink()

commit d63601c63f651ab9fe4fefb5a7e8d76bf0495da3
Merge: 8595030 b9381df
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 5 14:36:15 2016 -0500

    8460: Merge branch 'master' into 8460-websocket-go


commit 8595030d0314e6f88a245e66f90cce0a306b6867
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 5 14:35:25 2016 -0500

    8460: Add simple /status.json

diff --git a/services/ws/router.go b/services/ws/router.go
index 78cedce..3f9800f 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -55,7 +55,8 @@ func (rtr *router) setup() {
 	rtr.mux = http.NewServeMux()
 	rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
 	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
-	rtr.mux.HandleFunc("/debug.json", rtr.serveDebugStatus)
+	rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
+	rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
 }
 
 func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -103,14 +104,9 @@ func (rtr *router) DebugStatus() interface{} {
 	return s
 }
 
-func (rtr *router) serveDebugStatus(resp http.ResponseWriter, req *http.Request) {
-	rtr.setupOnce.Do(rtr.setup)
-	logger := logger(req.Context())
-	logger.Debug("status")
-	enc := json.NewEncoder(resp)
-	err := enc.Encode(rtr.DebugStatus())
-	if err != nil {
-		logger.WithError(err).Error("status encode failed")
+func (rtr *router) Status() interface{} {
+	return map[string]interface{}{
+		"Clients": atomic.LoadInt64(&rtr.status.ReqsActive),
 	}
 }
 
@@ -130,3 +126,16 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	}).Info("accept request")
 	rtr.mux.ServeHTTP(resp, req)
 }
+
+func jsonHandler(fn func() interface{}) http.HandlerFunc {
+	return func(resp http.ResponseWriter, req *http.Request) {
+		logger := logger(req.Context())
+		enc := json.NewEncoder(resp)
+		err := enc.Encode(fn())
+		if err != nil {
+			msg := "encode failed"
+			logger.WithError(err).Error(msg)
+			http.Error(resp, msg, http.StatusInternalServerError)
+		}
+	}
+}

commit dd2320c6877939365caf79767b5244d86a288437
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Dec 3 03:52:43 2016 -0500

    8460: status.json -> debug.json

diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 1653879..bb32374 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -44,6 +44,8 @@ type pgEventSource struct {
 	eventsOut  uint64
 }
 
+var _ DebugStatuser = (*pgEventSource)(nil)
+
 func (ps *pgEventSource) setup() {
 	ps.shutdown = make(chan error, 1)
 	ps.sinks = make(map[*pgEventSink]bool)
@@ -172,7 +174,7 @@ func (ps *pgEventSource) DB() *sql.DB {
 	return ps.db
 }
 
-func (ps *pgEventSource) Status() interface{} {
+func (ps *pgEventSource) DebugStatus() interface{} {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
 	blocked := 0
diff --git a/services/ws/handler.go b/services/ws/handler.go
index 8d8edca..ca9231c 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -192,7 +192,7 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
 	return
 }
 
-func (h *handler) Status() interface{} {
+func (h *handler) DebugStatus() interface{} {
 	h.mtx.Lock()
 	defer h.mtx.Unlock()
 
diff --git a/services/ws/router.go b/services/ws/router.go
index 073a398..78cedce 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -33,16 +33,16 @@ type router struct {
 	lastReqID  int64
 	lastReqMtx sync.Mutex
 
-	status routerStatus
+	status routerDebugStatus
 }
 
-type routerStatus struct {
+type routerDebugStatus struct {
 	ReqsReceived int64
 	ReqsActive   int64
 }
 
-type Statuser interface {
-	Status() interface{}
+type DebugStatuser interface {
+	DebugStatus() interface{}
 }
 
 type sessionFactory func(wsConn, chan<- interface{}, *sql.DB, permChecker) (session, error)
@@ -55,7 +55,7 @@ func (rtr *router) setup() {
 	rtr.mux = http.NewServeMux()
 	rtr.mux.Handle("/websocket", rtr.makeServer(NewSessionV0))
 	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(NewSessionV1))
-	rtr.mux.HandleFunc("/status.json", rtr.serveStatus)
+	rtr.mux.HandleFunc("/debug.json", rtr.serveDebugStatus)
 }
 
 func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -92,23 +92,23 @@ func (rtr *router) newReqID() string {
 	return strconv.FormatInt(id, 36)
 }
 
-func (rtr *router) Status() interface{} {
+func (rtr *router) DebugStatus() interface{} {
 	s := map[string]interface{}{
 		"HTTP":     rtr.status,
-		"Outgoing": rtr.handler.Status(),
+		"Outgoing": rtr.handler.DebugStatus(),
 	}
-	if es, ok := rtr.eventSource.(Statuser); ok {
-		s["EventSource"] = es.Status()
+	if es, ok := rtr.eventSource.(DebugStatuser); ok {
+		s["EventSource"] = es.DebugStatus()
 	}
 	return s
 }
 
-func (rtr *router) serveStatus(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) serveDebugStatus(resp http.ResponseWriter, req *http.Request) {
 	rtr.setupOnce.Do(rtr.setup)
 	logger := logger(req.Context())
 	logger.Debug("status")
 	enc := json.NewEncoder(resp)
-	err := enc.Encode(rtr.Status())
+	err := enc.Encode(rtr.DebugStatus())
 	if err != nil {
 		logger.WithError(err).Error("status encode failed")
 	}

commit fd717a88835be62ae1b5c4b1ecd74c21cab0e744
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Dec 3 03:44:45 2016 -0500

    8460: Rename pg -> event_source.go

diff --git a/services/ws/pg.go b/services/ws/event_source.go
similarity index 100%
rename from services/ws/pg.go
rename to services/ws/event_source.go

commit 0f1fca27f3b4b40ff8f6be729e12f2feeba05f8f
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Dec 1 10:35:50 2016 -0500

    8460: fixup 15aca78 stats.Duration

diff --git a/services/ws/doc.go b/services/ws/doc.go
index d280568..1f01b68 100644
--- a/services/ws/doc.go
+++ b/services/ws/doc.go
@@ -1,4 +1,6 @@
-// Arvados-ws is an Arvados event feed for Websocket clients.
+// Arvados-ws exposes Arvados APIs (currently just one, the
+// cache-invalidation event feed at "ws://.../websocket") to
+// websocket clients.
 //
 // See https://doc.arvados.org/install/install-arvados-ws.html.
 //
@@ -34,6 +36,10 @@
 //
 // A log is printed each time a client connects or disconnects.
 //
+// Enable additional logs by configuring:
+//
+//     LogLevel: debug
+//
 // Runtime status
 //
 // GET /debug.json responds with debug stats.
diff --git a/services/ws/handler.go b/services/ws/handler.go
index dace39b..8d8edca 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -2,12 +2,12 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"sync"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/stats"
 )
 
 type handler struct {
@@ -16,7 +16,7 @@ type handler struct {
 	QueueSize   int
 
 	mtx       sync.Mutex
-	lastDelay map[chan interface{}]time.Duration
+	lastDelay map[chan interface{}]stats.Duration
 	setupOnce sync.Once
 }
 
@@ -27,7 +27,7 @@ type handlerStats struct {
 	EventCount   uint64
 }
 
-func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsConn, chan<- interface{}) (session, error)) (stats handlerStats) {
+func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsConn, chan<- interface{}) (session, error)) (hStats handlerStats) {
 	h.setupOnce.Do(h.setup)
 
 	ctx, cancel := context.WithCancel(ws.Request().Context())
@@ -132,14 +132,14 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
 			log.Debug("sent")
 
 			if e != nil {
-				stats.QueueDelayNs += t0.Sub(e.Ready)
+				hStats.QueueDelayNs += t0.Sub(e.Ready)
 				h.mtx.Lock()
-				h.lastDelay[queue] = time.Since(e.Ready)
+				h.lastDelay[queue] = stats.Duration(time.Since(e.Ready))
 				h.mtx.Unlock()
 			}
-			stats.WriteDelayNs += time.Since(t0)
-			stats.EventBytes += uint64(len(buf))
-			stats.EventCount++
+			hStats.WriteDelayNs += time.Since(t0)
+			hStats.EventBytes += uint64(len(buf))
+			hStats.EventCount++
 		}
 	}()
 
@@ -201,10 +201,8 @@ func (h *handler) Status() interface{} {
 		QueueMin      int
 		QueueMax      int
 		QueueTotal    uint64
-		queueDelayMin time.Duration
-		QueueDelayMin string
-		queueDelayMax time.Duration
-		QueueDelayMax string
+		QueueDelayMin stats.Duration
+		QueueDelayMax stats.Duration
 	}
 	for q, lastDelay := range h.lastDelay {
 		s.QueueCount++
@@ -216,18 +214,16 @@ func (h *handler) Status() interface{} {
 		if s.QueueMin > n || s.QueueCount == 1 {
 			s.QueueMin = n
 		}
-		if (s.queueDelayMin > lastDelay || s.queueDelayMin == 0) && lastDelay > 0 {
-			s.queueDelayMin = lastDelay
+		if (s.QueueDelayMin > lastDelay || s.QueueDelayMin == 0) && lastDelay > 0 {
+			s.QueueDelayMin = lastDelay
 		}
-		if s.queueDelayMax < lastDelay {
-			s.queueDelayMax = lastDelay
+		if s.QueueDelayMax < lastDelay {
+			s.QueueDelayMax = lastDelay
 		}
 	}
-	s.QueueDelayMin = fmt.Sprintf("%.06f", s.queueDelayMin.Seconds())
-	s.QueueDelayMax = fmt.Sprintf("%.06f", s.queueDelayMax.Seconds())
 	return &s
 }
 
 func (h *handler) setup() {
-	h.lastDelay = make(map[chan interface{}]time.Duration)
+	h.lastDelay = make(map[chan interface{}]stats.Duration)
 }
diff --git a/services/ws/pg.go b/services/ws/pg.go
index dc899c5..1653879 100644
--- a/services/ws/pg.go
+++ b/services/ws/pg.go
@@ -2,13 +2,13 @@ package main
 
 import (
 	"database/sql"
-	"fmt"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/stats"
 	"github.com/lib/pq"
 )
 
@@ -184,7 +184,7 @@ func (ps *pgEventSource) Status() interface{} {
 		"EventsOut":    atomic.LoadUint64(&ps.eventsOut),
 		"Queue":        len(ps.queue),
 		"QueueLimit":   cap(ps.queue),
-		"QueueDelay":   fmt.Sprintf("%.06f", ps.lastQDelay.Seconds()),
+		"QueueDelay":   stats.Duration(ps.lastQDelay),
 		"Sinks":        len(ps.sinks),
 		"SinksBlocked": blocked,
 	}

commit cc0cd658d3db6820fba9daf380da4fa177b38f5c
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Dec 1 02:29:11 2016 -0500

    8460: Add godoc page.

diff --git a/services/ws/doc.go b/services/ws/doc.go
new file mode 100644
index 0000000..d280568
--- /dev/null
+++ b/services/ws/doc.go
@@ -0,0 +1,43 @@
+// Arvados-ws is an Arvados event feed for Websocket clients.
+//
+// See https://doc.arvados.org/install/install-arvados-ws.html.
+//
+// Usage
+//
+//     arvados-ws [-config /etc/arvados/ws/ws.yml] [-dump-config]
+//
+// Minimal configuration
+//
+//     Client:
+//       APIHost: localhost:443
+//     Listen: ":1234"
+//     Postgres:
+//       dbname: arvados_production
+//       host: localhost
+//       password: xyzzy
+//       user: arvados
+//
+// Options
+//
+// -config path
+//
+// Load configuration from the given file instead of the default
+// /etc/arvados/ws/ws.yml
+//
+// -dump-config
+//
+// Print the loaded configuration to stdout and exit.
+//
+// Logs
+//
+// Logs are printed to stderr, formatted as JSON.
+//
+// A log is printed each time a client connects or disconnects.
+//
+// Runtime status
+//
+// GET /debug.json responds with debug stats.
+//
+// GET /status.json responds with health check results and
+// activity/usage metrics.
+package main

commit 44702d2cdc6f6f76c16d9d5da9ae3225ab07de2f
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Dec 1 02:28:38 2016 -0500

    8460: Add systemd unit file.

diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
new file mode 100644
index 0000000..6de6848
--- /dev/null
+++ b/services/ws/arvados-ws.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Arvados websocket server
+Documentation=https://doc.arvados.org/
+After=network.target
+AssertPathExists=/etc/arvados/ws/ws.yml
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/ws
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/ws/main.go b/services/ws/main.go
index 0031dc0..c4e1078 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/config"
+	"github.com/coreos/go-systemd/daemon"
 )
 
 func main() {
@@ -48,8 +49,14 @@ func main() {
 			newPermChecker: func() permChecker { return NewPermChecker(cfg.Client) },
 		},
 	}
+	// Bootstrap the eventSource by attaching a dummy subscriber
+	// and hanging up.
 	eventSource.NewSink().Stop()
 
+	if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
+		log.WithError(err).Warn("error notifying init daemon")
+	}
+
 	log.WithField("Listen", srv.Addr).Info("listening")
 	log.Fatal(srv.ListenAndServe())
 }

commit 15aca78766bda903480e327340d67d3d882ee69f
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Dec 1 01:33:55 2016 -0500

    8460: Move loggedDuration from keepstore to sdk pkg as stats.Duration.

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 2c28817..379c616 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -91,6 +91,7 @@ sdk/go/httpserver
 sdk/go/manifest
 sdk/go/blockdigest
 sdk/go/streamer
+sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
 tools/crunchstat-summary
@@ -757,6 +758,7 @@ gostuff=(
     sdk/go/manifest
     sdk/go/streamer
     sdk/go/crunchrunner
+    sdk/go/stats
     lib/crunchstat
     services/arv-git-httpd
     services/crunchstat
diff --git a/sdk/go/stats/duration.go b/sdk/go/stats/duration.go
new file mode 100644
index 0000000..103dea0
--- /dev/null
+++ b/sdk/go/stats/duration.go
@@ -0,0 +1,35 @@
+package stats
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Duration is a duration that is displayed as a number of seconds in
+// fixed-point notation.
+type Duration time.Duration
+
+// MarshalJSON implements json.Marshaler.
+func (d Duration) MarshalJSON() ([]byte, error) {
+	return []byte(d.String()), nil
+}
+
+// String implements fmt.Stringer.
+func (d Duration) String() string {
+	return fmt.Sprintf("%.6f", time.Duration(d).Seconds())
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (d *Duration) UnmarshalJSON(data []byte) error {
+	return d.Set(string(data))
+}
+
+// Value implements flag.Value
+func (d *Duration) Set(s string) error {
+	sec, err := strconv.ParseFloat(s, 64)
+	if err == nil {
+		*d = Duration(sec * float64(time.Second))
+	}
+	return err
+}
diff --git a/sdk/go/stats/duration_test.go b/sdk/go/stats/duration_test.go
new file mode 100644
index 0000000..730e646
--- /dev/null
+++ b/sdk/go/stats/duration_test.go
@@ -0,0 +1,23 @@
+package stats
+
+import (
+	"testing"
+	"time"
+)
+
+func TestString(t *testing.T) {
+	d := Duration(123123123123 * time.Nanosecond)
+	if s, expect := d.String(), "123.123123"; s != expect {
+		t.Errorf("got %s, expect %s", s, expect)
+	}
+}
+
+func TestSet(t *testing.T) {
+	var d Duration
+	if err := d.Set("123.456"); err != nil {
+		t.Fatal(err)
+	}
+	if got, expect := time.Duration(d).Nanoseconds(), int64(123456000000); got != expect {
+		t.Errorf("got %d, expect %d", got, expect)
+	}
+}
diff --git a/services/keepstore/logging_router.go b/services/keepstore/logging_router.go
index bfd006e..e34f858 100644
--- a/services/keepstore/logging_router.go
+++ b/services/keepstore/logging_router.go
@@ -5,12 +5,12 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strings"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+	"git.curoverse.com/arvados.git/sdk/go/stats"
 	log "github.com/Sirupsen/logrus"
 )
 
@@ -97,25 +97,11 @@ func (loggingRouter *LoggingRESTRouter) ServeHTTP(wrappedResp http.ResponseWrite
 	}
 
 	lgr.WithFields(log.Fields{
-		"timeTotal":      loggedDuration(tDone.Sub(tStart)),
-		"timeToStatus":   loggedDuration(resp.sentHdr.Sub(tStart)),
-		"timeWriteBody":  loggedDuration(tDone.Sub(resp.sentHdr)),
+		"timeTotal":      stats.Duration(tDone.Sub(tStart)),
+		"timeToStatus":   stats.Duration(resp.sentHdr.Sub(tStart)),
+		"timeWriteBody":  stats.Duration(tDone.Sub(resp.sentHdr)),
 		"respStatusCode": resp.Status,
 		"respStatus":     statusText,
 		"respBytes":      resp.Length,
 	}).Info("response")
 }
-
-type loggedDuration time.Duration
-
-// MarshalJSON formats a duration as a number of seconds, using
-// fixed-point notation with no more than 6 decimal places.
-func (d loggedDuration) MarshalJSON() ([]byte, error) {
-	return []byte(d.String()), nil
-}
-
-// String formats a duration as a number of seconds, using
-// fixed-point notation with no more than 6 decimal places.
-func (d loggedDuration) String() string {
-	return fmt.Sprintf("%.6f", time.Duration(d).Seconds())
-}

commit 2eaa77dc327c024f2faa3fbd322e7054454b6442
Merge: 8c01422 bde488e
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Nov 30 15:09:13 2016 -0500

    8460: Merge branch 'master' into 8460-websocket-go


commit 8c014223e42683d308798475c021bad7a794e998
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Nov 24 14:18:59 2016 -0500

    8460: More statistics in status.json: events in/out, reqs received, lowest client queue delay.

diff --git a/services/ws/event.go b/services/ws/event.go
index 280035b..fa2a5df 100644
--- a/services/ws/event.go
+++ b/services/ws/event.go
@@ -22,6 +22,7 @@ type eventSource interface {
 type event struct {
 	LogID    uint64
 	Received time.Time
+	Ready    time.Time
 	Serial   uint64
 
 	db     *sql.DB
diff --git a/services/ws/handler.go b/services/ws/handler.go
index d2c119a..dace39b 100644
--- a/services/ws/handler.go
+++ b/services/ws/handler.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"sync"
 	"time"
@@ -15,7 +16,7 @@ type handler struct {
 	QueueSize   int
 
 	mtx       sync.Mutex
-	queues    map[chan interface{}]struct{}
+	lastDelay map[chan interface{}]time.Duration
 	setupOnce sync.Once
 }
 
@@ -37,11 +38,11 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
 
 	queue := make(chan interface{}, h.QueueSize)
 	h.mtx.Lock()
-	h.queues[queue] = struct{}{}
+	h.lastDelay[queue] = 0
 	h.mtx.Unlock()
 	defer func() {
 		h.mtx.Lock()
-		delete(h.queues, queue)
+		delete(h.lastDelay, queue)
 		h.mtx.Unlock()
 	}()
 
@@ -131,7 +132,10 @@ func (h *handler) Handle(ws wsConn, eventSource eventSource, newSession func(wsC
 			log.Debug("sent")
 
 			if e != nil {
-				stats.QueueDelayNs += t0.Sub(e.Received)
+				stats.QueueDelayNs += t0.Sub(e.Ready)
+				h.mtx.Lock()
+				h.lastDelay[queue] = time.Since(e.Ready)
+				h.mtx.Unlock()
 			}
 			stats.WriteDelayNs += time.Since(t0)
 			stats.EventBytes += uint64(len(buf))
@@ -193,21 +197,37 @@ func (h *handler) Status() interface{} {
 	defer h.mtx.Unlock()
 
 	var s struct {
-		QueueCount int
-		QueueMax   int
-		QueueTotal uint64
+		QueueCount    int
+		QueueMin      int
+		QueueMax      int
+		QueueTotal    uint64
+		queueDelayMin time.Duration
+		QueueDelayMin string
+		queueDelayMax time.Duration
+		QueueDelayMax string
 	}
-	for q := range h.queues {
+	for q, lastDelay := range h.lastDelay {
+		s.QueueCount++
 		n := len(q)
 		s.QueueTotal += uint64(n)
 		if s.QueueMax < n {
 			s.QueueMax = n
 		}
+		if s.QueueMin > n || s.QueueCount == 1 {
+			s.QueueMin = n
+		}
+		if (s.queueDelayMin > lastDelay || s.queueDelayMin == 0) && lastDelay > 0 {
+			s.queueDelayMin = lastDelay
+		}
+		if s.queueDelayMax < lastDelay {
+			s.queueDelayMax = lastDelay
+		}
 	}
-	s.QueueCount = len(h.queues)
+	s.QueueDelayMin = fmt.Sprintf("%.06f", s.queueDelayMin.Seconds())
+	s.QueueDelayMax = fmt.Sprintf("%.06f", s.queueDelayMax.Seconds())
 	return &s
 }
 
 func (h *handler) setup() {
-	h.queues = make(map[chan interface{}]struct{})
+	h.lastDelay = make(map[chan interface{}]time.Duration)
 }
diff --git a/services/ws/pg.go b/services/ws/pg.go
index b6b064e..dc899c5 100644
--- a/services/ws/pg.go
+++ b/services/ws/pg.go
@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/lib/pq"
@@ -39,6 +40,8 @@ type pgEventSource struct {
 	shutdown   chan error
 
 	lastQDelay time.Duration
+	eventsIn   uint64
+	eventsOut  uint64
 }
 
 func (ps *pgEventSource) setup() {
@@ -89,9 +92,11 @@ func (ps *pgEventSource) run() {
 				WithField("serial", e.Serial).
 				WithField("detail", e.Detail()).
 				Debug("event ready")
-			ps.lastQDelay = time.Now().Sub(e.Received)
+			e.Ready = time.Now()
+			ps.lastQDelay = e.Ready.Sub(e.Received)
 
 			ps.mtx.Lock()
+			atomic.AddUint64(&ps.eventsOut, uint64(len(ps.sinks)))
 			for sink := range ps.sinks {
 				sink.channel <- e
 			}
@@ -136,6 +141,7 @@ func (ps *pgEventSource) run() {
 				db:       ps.db,
 			}
 			logger(nil).WithField("event", e).Debug("incoming")
+			atomic.AddUint64(&ps.eventsIn, 1)
 			ps.queue <- e
 			go e.Detail()
 		}
@@ -174,6 +180,8 @@ func (ps *pgEventSource) Status() interface{} {
 		blocked += len(sink.channel)
 	}
 	return map[string]interface{}{
+		"EventsIn":     atomic.LoadUint64(&ps.eventsIn),
+		"EventsOut":    atomic.LoadUint64(&ps.eventsOut),
 		"Queue":        len(ps.queue),
 		"QueueLimit":   cap(ps.queue),
 		"QueueDelay":   fmt.Sprintf("%.06f", ps.lastQDelay.Seconds()),
diff --git a/services/ws/router.go b/services/ws/router.go
index 18eaf73..073a398 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -37,7 +37,8 @@ type router struct {
 }
 
 type routerStatus struct {
-	Connections int64
+	ReqsReceived int64
+	ReqsActive   int64
 }
 
 type Statuser interface {
@@ -97,7 +98,7 @@ func (rtr *router) Status() interface{} {
 		"Outgoing": rtr.handler.Status(),
 	}
 	if es, ok := rtr.eventSource.(Statuser); ok {
-		s["Incoming"] = es.Status()
+		s["EventSource"] = es.Status()
 	}
 	return s
 }
@@ -115,8 +116,9 @@ func (rtr *router) serveStatus(resp http.ResponseWriter, req *http.Request) {
 
 func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	rtr.setupOnce.Do(rtr.setup)
-	atomic.AddInt64(&rtr.status.Connections, 1)
-	defer atomic.AddInt64(&rtr.status.Connections, -1)
+	atomic.AddInt64(&rtr.status.ReqsReceived, 1)
+	atomic.AddInt64(&rtr.status.ReqsActive, 1)
+	defer atomic.AddInt64(&rtr.status.ReqsActive, -1)
 
 	logger := logger(req.Context()).
 		WithField("RequestID", rtr.newReqID())
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index a60a4a3..9a9707b 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -164,9 +164,11 @@ func (sub *v0subscribe) sendOldEvents(sess *v0session) {
 			// same thing all over again.
 			time.Sleep(100 * time.Millisecond)
 		}
+		now := time.Now()
 		e := &event{
 			LogID:    id,
-			Received: time.Now(),
+			Received: now,
+			Ready:    now,
 			db:       sess.db,
 		}
 		if sub.match(sess, e) {

commit 161a519439dcad4d77ca400ec48bb58cb685b54a
Author: Tom Clegg <tom at curoverse.com>
Date:   Sun Nov 20 00:48:46 2016 -0500

    8460: Move logging setup to log.go, use fixed-width timestamps.

diff --git a/services/ws/log.go b/services/ws/log.go
index fbf21b9..ddfbb9e 100644
--- a/services/ws/log.go
+++ b/services/ws/log.go
@@ -11,10 +11,17 @@ var (
 	rootLogger   = logrus.New()
 )
 
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
+// contextWithLogger returns a new child context such that
+// logger(child) returns the given logger.
 func contextWithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
 	return context.WithValue(ctx, loggerCtxKey, logger)
 }
 
+// logger returns the logger suitable for the given context -- the one
+// attached by contextWithLogger() if applicable, otherwise the
+// top-level logger with no fields/values.
 func logger(ctx context.Context) *logrus.Entry {
 	if ctx != nil {
 		if logger, ok := ctx.Value(loggerCtxKey).(*logrus.Entry); ok {
@@ -23,3 +30,25 @@ func logger(ctx context.Context) *logrus.Entry {
 	}
 	return rootLogger.WithFields(nil)
 }
+
+// loggerConfig sets up logging to behave as configured.
+func loggerConfig(cfg Config) {
+	lvl, err := logrus.ParseLevel(cfg.LogLevel)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	rootLogger.Level = lvl
+	switch cfg.LogFormat {
+	case "text":
+		rootLogger.Formatter = &logrus.TextFormatter{
+			FullTimestamp:   true,
+			TimestampFormat: rfc3339NanoFixed,
+		}
+	case "json":
+		rootLogger.Formatter = &logrus.JSONFormatter{
+			TimestampFormat: rfc3339NanoFixed,
+		}
+	default:
+		logrus.WithField("LogFormat", cfg.LogFormat).Fatal("unknown log format")
+	}
+}
diff --git a/services/ws/main.go b/services/ws/main.go
index 33728dc..0031dc0 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/config"
-	"github.com/Sirupsen/logrus"
 )
 
 func main() {
@@ -23,24 +22,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	lvl, err := logrus.ParseLevel(cfg.LogLevel)
-	if err != nil {
-		log.Fatal(err)
-	}
-	rootLogger.Level = lvl
-	switch cfg.LogFormat {
-	case "text":
-		rootLogger.Formatter = &logrus.TextFormatter{
-			FullTimestamp:   true,
-			TimestampFormat: time.RFC3339Nano,
-		}
-	case "json":
-		rootLogger.Formatter = &logrus.JSONFormatter{
-			TimestampFormat: time.RFC3339Nano,
-		}
-	default:
-		log.WithField("LogFormat", cfg.LogFormat).Fatal("unknown log format")
-	}
+	loggerConfig(cfg)
 
 	if *dumpConfig {
 		txt, err := config.Dump(&cfg)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list