[ARVADOS] updated: 0e7cfa403cb80fc68d6c10c4c912e4d618fa085a
git at public.curoverse.com
git at public.curoverse.com
Thu Nov 20 09:49:08 EST 2014
Summary of changes:
apps/workbench/Gemfile | 2 +-
apps/workbench/Gemfile.lock | 12 +-
apps/workbench/app/assets/javascripts/event_log.js | 250 ++++++++-------
.../workbench/app/assets/javascripts/filterable.js | 41 ++-
.../app/assets/javascripts/infinite_scroll.js | 7 +-
.../app/assets/javascripts/pipeline_instances.js | 10 +-
.../app/assets/javascripts/select_modal.js | 31 +-
.../app/assets/javascripts/selection.js.erb | 3 +-
.../workbench/app/assets/stylesheets/jobs.css.scss | 5 +
.../app/controllers/application_controller.rb | 14 +-
.../app/controllers/collections_controller.rb | 39 +--
apps/workbench/app/controllers/jobs_controller.rb | 6 +
.../controllers/pipeline_instances_controller.rb | 26 +-
apps/workbench/app/controllers/users_controller.rb | 6 +-
apps/workbench/app/helpers/application_helper.rb | 11 +-
apps/workbench/app/helpers/jobs_helper.rb | 3 +-
apps/workbench/app/models/arvados_api_client.rb | 7 +-
apps/workbench/app/models/arvados_base.rb | 4 +-
apps/workbench/app/models/arvados_resource_list.rb | 168 ++++++----
.../app/views/application/_choose.html.erb | 9 +-
.../app/views/application/_paging.html.erb | 22 +-
.../app/views/collections/_choose_rows.html.erb | 22 +-
.../app/views/collections/_index_tbody.html.erb | 2 +-
.../collections/_show_chooser_preview.html.erb | 2 +-
.../app/views/collections/_show_files.html.erb | 23 +-
.../app/views/collections/_show_recent.html.erb | 4 +-
apps/workbench/app/views/jobs/_show_log.html.erb | 5 +-
apps/workbench/app/views/jobs/push_logs.js.erb | 3 +
.../app/views/layouts/application.html.erb | 2 +-
apps/workbench/app/views/layouts/body.html.erb | 13 -
.../views/pipeline_instances/_show_inputs.html.erb | 5 +-
.../views/pipeline_instances/_show_recent.html.erb | 44 +--
..._recent.html.erb => _show_recent_rows.html.erb} | 45 +--
.../app/views/pipeline_instances/index.html.erb | 14 +-
.../app/views/projects/_show_sharing.html.erb | 3 +-
.../workbench/app/views/users/_show_admin.html.erb | 2 +-
apps/workbench/config/routes.rb | 1 +
.../pipeline_instances_controller_test.rb | 2 +-
.../integration/filterable_infinite_scroll_test.rb | 29 ++
.../test/integration/pipeline_instances_test.rb | 144 +++++++--
.../test/unit/arvados_resource_list_test.rb | 108 +++++--
crunch_scripts/run-command | 5 +-
doc/_includes/_ssh_addkey.liquid | 2 +-
doc/api/methods/jobs.html.textile.liquid | 6 +-
doc/api/schema/Job.html.textile.liquid | 13 +-
doc/images/workbench-move-selected.png | Bin 11267 -> 47833 bytes
doc/sdk/cli/index.html.textile.liquid | 1 -
doc/sdk/cli/subcommands.html.textile.liquid | 8 +-
doc/user/topics/arv-run.html.textile.liquid | 2 +-
doc/user/topics/run-command.html.textile.liquid | 3 +-
.../tutorial-firstscript.html.textile.liquid | 2 +-
.../tutorials/tutorial-keep.html.textile.liquid | 14 +-
...tutorial-pipeline-workbench.html.textile.liquid | 2 +-
sdk/cli/bin/arv | 8 +-
sdk/cli/bin/arv-tag | 15 +-
sdk/cli/bin/crunch-job | 91 +++---
sdk/go/keepclient/keepclient.go | 22 +-
sdk/go/keepclient/keepclient_test.go | 124 ++++----
sdk/go/keepclient/root_sorter.go | 57 ++++
sdk/go/keepclient/root_sorter_test.go | 58 ++++
sdk/go/keepclient/support.go | 65 +---
sdk/python/arvados/arvfile.py | 29 ++
sdk/python/arvados/collection.py | 113 +++++--
sdk/python/arvados/keep.py | 120 +++----
sdk/python/arvados/stream.py | 119 ++++---
sdk/python/arvados/util.py | 4 +-
sdk/python/setup.py | 6 +-
sdk/python/tests/arvados_testutil.py | 16 +
sdk/python/tests/run_test_server.py | 14 +-
sdk/python/tests/test_collections.py | 275 +++++++---------
sdk/python/tests/test_keep_client.py | 77 ++++-
sdk/python/tests/test_stream.py | 160 ++++++++++
sdk/ruby/arvados.gemspec | 2 +-
.../api/app/controllers/application_controller.rb | 78 ++---
.../arvados/v1/collections_controller.rb | 23 +-
.../app/controllers/arvados/v1/jobs_controller.rb | 84 +++--
services/api/app/models/collection.rb | 106 ++++++-
services/api/app/models/job.rb | 59 ++--
services/api/app/models/node.rb | 60 +++-
services/api/config/application.default.yml | 22 +-
services/api/config/unbound.template | 4 +
...141111133038_add_arvados_sdk_version_to_jobs.rb | 13 +
services/api/db/structure.sql | 7 +-
services/api/lib/load_param.rb | 7 +-
services/api/script/restart-dns-server | 34 ++
services/api/script/restart-dnsmasq | 21 --
services/api/test/fixtures/jobs.yml | 17 +
services/api/test/fixtures/keep_services.yml | 6 +-
services/api/test/fixtures/links.yml | 14 +
services/api/test/fixtures/pipeline_templates.yml | 39 ++-
services/api/test/fixtures/repositories.yml | 5 +
.../arvados/v1/collections_controller_test.rb | 84 +++--
.../arvados/v1/commits_controller_test.rb | 3 +-
.../arvados/v1/groups_controller_test.rb | 67 ++--
.../arvados/v1/job_reuse_controller_test.rb | 81 +++--
.../arvados/v1/repositories_controller_test.rb | 15 +-
services/api/test/helpers/git_test_helper.rb | 4 +-
services/api/test/integration/keep_proxy_test.rb | 17 +-
services/api/test/integration/websocket_test.rb | 2 +-
services/api/test/test.git.tar | Bin 153600 -> 194560 bytes
services/api/test/unit/collection_test.rb | 40 ++-
services/api/test/unit/job_test.rb | 93 +++++-
services/fuse/arvados_fuse/__init__.py | 28 +-
services/fuse/setup.py | 3 +-
services/fuse/tests/test_mount.py | 72 ++++-
services/keepproxy/keepproxy_test.go | 14 +-
services/keepstore/handler_test.go | 80 ++---
services/keepstore/handlers.go | 26 --
services/keepstore/keepstore.go | 8 +-
services/keepstore/logging_router.go | 41 +++
.../nodemanager/arvnodeman/computenode/__init__.py | 353 ---------------------
.../computenode/{ => dispatch}/__init__.py | 188 +++--------
.../arvnodeman/computenode/dispatch/slurm.py | 49 +++
.../arvnodeman/computenode/driver/__init__.py | 64 ++++
.../arvnodeman/computenode/{ => driver}/dummy.py | 3 +-
.../arvnodeman/computenode/{ => driver}/ec2.py | 3 +-
services/nodemanager/arvnodeman/config.py | 13 +-
services/nodemanager/arvnodeman/daemon.py | 56 ++--
services/nodemanager/arvnodeman/launcher.py | 11 +-
services/nodemanager/doc/ec2.example.cfg | 5 +
services/nodemanager/tests/test_computenode.py | 279 ----------------
...computenode.py => test_computenode_dispatch.py} | 136 ++++----
.../tests/test_computenode_dispatch_slurm.py | 58 ++++
...enode_ec2.py => test_computenode_driver_ec2.py} | 2 +-
services/nodemanager/tests/test_config.py | 18 ++
services/nodemanager/tests/test_daemon.py | 55 +++-
services/nodemanager/tests/testutil.py | 6 +
127 files changed, 2904 insertions(+), 2254 deletions(-)
create mode 100644 apps/workbench/app/views/jobs/push_logs.js.erb
copy apps/workbench/app/views/pipeline_instances/{_show_recent.html.erb => _show_recent_rows.html.erb} (58%)
create mode 100644 apps/workbench/test/integration/filterable_infinite_scroll_test.rb
create mode 100644 sdk/go/keepclient/root_sorter.go
create mode 100644 sdk/go/keepclient/root_sorter_test.go
create mode 100644 sdk/python/arvados/arvfile.py
create mode 100644 services/api/config/unbound.template
create mode 100644 services/api/db/migrate/20141111133038_add_arvados_sdk_version_to_jobs.rb
create mode 100755 services/api/script/restart-dns-server
delete mode 100755 services/api/script/restart-dnsmasq
create mode 100644 services/keepstore/logging_router.go
copy services/nodemanager/arvnodeman/computenode/{ => dispatch}/__init__.py (64%)
create mode 100644 services/nodemanager/arvnodeman/computenode/dispatch/slurm.py
create mode 100644 services/nodemanager/arvnodeman/computenode/driver/__init__.py
rename services/nodemanager/arvnodeman/computenode/{ => driver}/dummy.py (96%)
rename services/nodemanager/arvnodeman/computenode/{ => driver}/ec2.py (98%)
copy services/nodemanager/tests/{test_computenode.py => test_computenode_dispatch.py} (77%)
create mode 100644 services/nodemanager/tests/test_computenode_dispatch_slurm.py
rename services/nodemanager/tests/{test_computenode_ec2.py => test_computenode_driver_ec2.py} (98%)
via 0e7cfa403cb80fc68d6c10c4c912e4d618fa085a (commit)
via 84ff06d37f920ef6cf400189ea19edcf6158993c (commit)
via a65acb3945d8646c616b4f8a2654d1a346fdaf2d (commit)
via e9a19734c359fa52ec07a3ea4bb81b6382864596 (commit)
via 7b9112dbc270ea338fee756f583bb76870f2e391 (commit)
via b77fb69914fda0aedecddf2e62076cb404265647 (commit)
via 2834445f90c62644fde2cd46f9a4328600caba59 (commit)
via a325b04603deacebc07d322e691d7cc4e38b2f24 (commit)
via d270d652cb21c601513fb89e77a64270fc80bf21 (commit)
via 9d7efc36748d8b83af3d5dbdcd5326834fb317ff (commit)
via 6f31d0c9d4b6a0d3070e933a84cfd843722e81b1 (commit)
via 6c68141eb50255128cf38b5717b15b16f2a8cdff (commit)
via ffc7c1b20530dda13c1adf3aa48122ac881bbc29 (commit)
via 6874da0dd56d6c0320880dfb3bd4da34a3c0a7d3 (commit)
via 4af4a2cf12d3b87a4fdbd09115bc258961b706bb (commit)
via 22381d218295075e63c06f0b60bcb24cee7b30b6 (commit)
via 8d324f6d853ce38ab816ce12e8557b0e47ce9c2f (commit)
via 89285c343d05bc7ceef55a46c4ebb38d9b047baf (commit)
via 1a2a05a4714a4bc538800d08d353b0306d5e49ad (commit)
via b1f65fcd4fa5fca5400f63fa25823fef6229a158 (commit)
via 40b15b9523c1a89769c8df5bafa250075af0c567 (commit)
via b1f3c944c3217b6e42a377f4b0a04985c9cea908 (commit)
via 0e5cd501a34ebcf6e138b4693f4fa4094bff24b4 (commit)
via 22c135139279edcf5be0218eb90c375f2433a4b9 (commit)
via 3735d52b65928e3626a8e223acadc318a3d31097 (commit)
via f9ba8a730dbc1a457485886aa4591991e9dbbf0d (commit)
via 97a2555a220d424b029d39b6b6e62f28bc3bc148 (commit)
via 3ce8039e7ca278e4816ca90cd973bdbe08404b87 (commit)
via 9f52191b13196b1926f7b8b0c7c35aac56e02198 (commit)
via 69fae522bf6fe6dbaa64ce7d75ef4ce5616ced72 (commit)
via 22ae84c0d72cdeab882017a8caf2ade4465ddd59 (commit)
via c621e2e901f5b5de4f72b1c6a9d8816aa5745292 (commit)
via aea8c62eaec4d170842b01e0fd1ddb5e256152b8 (commit)
via 14b762d2dd37a0a533532839aaf0488c11205e42 (commit)
via 1a520a700456325a8a7427d74fa11ed15b62a4b2 (commit)
via e82958936210b6bbc8cc04b237229fdf6fac295c (commit)
via 8deda2012ea7e25610f2cbd0a271e131d9364503 (commit)
via 732f7f0740dc7e5d5cff54f9cc30d4cd4597cb27 (commit)
via 763af5413e6195fe004846abf86a7ae134168990 (commit)
via f72ca2ac98200452700dc386fa439a45b8bb15e3 (commit)
via b75e2a12f618e2b69f3af1672abfb4773e202bed (commit)
via 84d5db363e4c10881fdb5317b01f46bdc33c002a (commit)
via dea90c956d96d06e39df0f6567c3bb6582a5b3dd (commit)
via 75a07d8175c9ee9a1c212cd1abe1c0d64a91cfc7 (commit)
via 07bf4bb6d787c276f338846eb46c2b527a323896 (commit)
via 18103a8f198e2038c978e1d38cef3fc3bf971d1b (commit)
via ff0716748c5e94b03dcfd47e9875be3bc7bcb9e6 (commit)
via 84a1d6a622193dae18c715e0421353da2aecf1f7 (commit)
via 4805867bc1004ae9d755a6375d592748a7b29585 (commit)
via 645a998c13b7dcd21af72187d953b8069288d6d2 (commit)
via 28c6bbeeed6b228d51666914bcd845d05500107e (commit)
via aa6f07c9687670ee98e15d9afa2bd47416839dc6 (commit)
via 7bd04ed26901e34e3e5788093bee12be4406dd7b (commit)
via ff7b22c70cd77073d9bdbebac0bf03d43745ed0c (commit)
via 372cff3e04b4243eb68d9ae3e296aebcee459aea (commit)
via d3229c7f727f40436cad66fba5f3345e0b3eede5 (commit)
via 95a61743fd1df435313042c17e581d4b7b457a2a (commit)
via 78447c70f491db25dc99eca60ebcc1b7a5362ec3 (commit)
via 664919d58c3689cd9e0a25547ec1e02d9adda38c (commit)
via 97ddc11818d059e42989f302fa15db4c5e19d5b3 (commit)
via 9a46fb0033514db6dd5e8ab4ccf793f78458a734 (commit)
via bfd5100c4e0bed1910875099937169f1b0c8ce7b (commit)
via d9f45bd01d2f5691a9aaca4f1fc00465497ef00c (commit)
via 0adcabd351108e83b630e27c48ce85f2b3cb3e14 (commit)
via e0f4747a2b24af68b474bb2da12e8799440c9acc (commit)
via a128baa147df78cadb5c1657dfa740fad80570ea (commit)
via 43e019a1e37effa0ccde680b52a61fbba1f858d4 (commit)
via 0e5dfbbcefcb462617ad89ef116a5b18f9e3ebc8 (commit)
via de35a630edcbe2ddcadf002ab3efe027d77fc046 (commit)
via 7395af3b19b8fc422fe2456c6bead1e7f4c60989 (commit)
via a75ef0985d2a385165168cbb488cf97382308549 (commit)
via 11374252ee3c26240420bb3aa0d0433fad71731f (commit)
via 5d87bcae66870127445e5eb018a69d606e93e244 (commit)
via 83e73eda7123a6de741198d759841fd186fc2a11 (commit)
via 06afd90ff09a83982451733f1e39974299d9a2d9 (commit)
via 4b3222657e160eaf61459aac843e8377100a53e4 (commit)
via abe8fc4c5818165ffb696448c81b3bf1bb5dc1dd (commit)
via 4c110b5f4e8e92d613f70427f27bdfae2ea376c7 (commit)
via 2ac2528dc0e5b700a2640e28dc66e474db9ad329 (commit)
via 9cd99d3070cd69664a31f8e25fad68fe8b7c43cf (commit)
via d78db2b1b518abdac0893e65123504fb85319010 (commit)
via 11236731eda455b5df6448733e4c0090a5f727c4 (commit)
via 8c682e4ffcd8fdf9bcaf8f062f1d0a6517ea9e5d (commit)
via f798c910fa1966b0471451de4f24cc4f55e07290 (commit)
via 6de0a4ab0323e69a5499e59af03c9c5e6125fed7 (commit)
via f51138289b7625bde6cc0bb9840ccf681944941a (commit)
via 97cd8d6023f918babc5e6cca7f7a1228f4a32d59 (commit)
via 84b6c8f2d915ad6c1e2843f62bc47e510f2d6f54 (commit)
via 2070784e52d89bbcf636062532a8ce37629737db (commit)
via 59f750a109a706d29862530dc3d282db75736d38 (commit)
via 8b7ed36ff057c4a483f4ee8a1c66929425b366c6 (commit)
via d4224bfee9d5ece132bc50db96de48da3620d5e5 (commit)
via 35c53e2525e0d29cb8ffae486718b4fde88b273f (commit)
via f5a1a9210e7638babe33d16938246e93ff4dd334 (commit)
via 3a31350c6265cb1135d3d4d40af436aae91a9894 (commit)
via c805714d8bbc51adb32a3a3ae4b82ea41eaa0de0 (commit)
via 9325dabcc2b6c32a589ba86cc052946b4875eb1d (commit)
via 95323ba327cadff2d1cb8c3f38c5dbb65dbd3694 (commit)
via eb185aab22bd916981ea874fd82e6a1fecce7cd2 (commit)
via 069b22b2d1196d0437253e60e616840ea141e1a9 (commit)
via 4d01f18d6a1129fb3f2f0efc4bda31795519fda2 (commit)
via e363a7973a911a6484b24d4e7f92f363d4d3e521 (commit)
via 06e402b11ad4d503feb5fa45845cb27c93478cfc (commit)
via afdab87268e4ee2f19f92e02b219696081959378 (commit)
via 809b122ee02d566b3bddb26df928e57efa39ed87 (commit)
via f5c574f45a6352c7d675e8c111eef5083a435213 (commit)
via f81483a374fbdfe6d5f36e5f5e55f92455afa155 (commit)
via 1ac0e3da5fbd477bd2acbf5bd8baa3c86f08194b (commit)
via dff161c1eaaecd6b557e88709fdf5f0dcb52821b (commit)
via cf1db3981ad0c6416d2572ceeb512743a9df24c7 (commit)
via 3894b788440fce678a28b4c7a8780f2b7767449d (commit)
via 5be66e86ac84d7d46c8503c3c136cb144222e4bf (commit)
via fd207a0e0fc3aa02eba81b567b0b30d144ac3fbf (commit)
via 9ef386f6e405db49df44a24e4cdfe5a0c0742f62 (commit)
via d833a28612bb22e1489536b3b0dfe8eb00196fda (commit)
via 754ff84aab18b623b37f21377c639694e6ea0ed7 (commit)
via 53192c327dcce6159d21f6cc27f7d5c0bfc9e7b0 (commit)
via 1a001045ccce247a13266a5ef619ffbc07308226 (commit)
via 9966893484226206474d7d9936d884cc56582f15 (commit)
via ac21f0d45a76294aaca0c0c0fdf06eb72d03368d (commit)
via a0e70cf1da033f7d94e728ab919bd8cfcabf3743 (commit)
via c28b9b91028c90497455bfde188915982377b825 (commit)
via e158f485053be1e840073b321033d60d686a55a8 (commit)
via 8f9f0dece977ccf5a778f3b3bd2379375e723c18 (commit)
via b626a85eb86fd4909712852040cd305c71c37ee5 (commit)
via 5141c3ee23e89696773e227a93236ef2a51543c2 (commit)
via 45bfc1104dd30fb97a586de5ff96d6b739f7bb2b (commit)
via 6bcb827b1405ce8ca5d767108bc4356ecc2972f4 (commit)
via 093cebc3a4baed645fd5d4a5331c2ff837de8c77 (commit)
via 76c9a858643680d90f9694d49c212a54fe5b759f (commit)
via 53381c90fd5ef26d3a0cc08c8d35318b0b92557a (commit)
via bc4992f9e11cd0639cf6461736963d66830c5af8 (commit)
via 39e39addee0a0de050c2b4d1f88fb7d56da7436b (commit)
via 333fbec95590cfcb30cfff7dc61e6b8cc41edf18 (commit)
via 91f0b180a8474321738577a12250b5a1b8f5bcbc (commit)
via a844af57de33b0094ce23c873d044f15de1ed138 (commit)
via d1cdd7eddf071e41643bc958b77d0056410d01ea (commit)
via 26aa2f1a1c8c9883beac1538c318279190f91c8a (commit)
via 7bf8f6c701e28e574c137b0c942522e8f8ee4d8c (commit)
via 7924077e3db2898de26c86599d9f311e02d6db46 (commit)
via c1d9150de2977f7eb85e7cb058d5c41ae6f06173 (commit)
via b2610e066d569b323fe55ffee615e61ca4e461d1 (commit)
via 6c7bfabc03237419decad9674c90ea7af906b615 (commit)
via c7b7e4d44e9349c9104fbfcd33c5e4b97fe17a71 (commit)
via e61de9add861db4c043341f3926acb95ded37862 (commit)
via 1237511f9da5ee20588d44be4db5f9e37cfc6400 (commit)
via fa3e996ee452bd9be853dd9e93aaec15623708f5 (commit)
via 1304e044aa87a65145bf8b6d4bc141586556c0ed (commit)
via 47b0cb35b5ee933757c4342bb75fd286c1dac8cb (commit)
via e02ef893c4f6cf881e449c248782b2ac21b49b0f (commit)
via 9405b5f224b205e681e378bd2d90d18f3638139b (commit)
via eee66a4fd87be84379929d17864f244b0a68fd74 (commit)
via a4b55a80251d666ca39d0f8d201c70e493b4c661 (commit)
via 0198b4937df2be79e41ea76ea6c31841441f4f2c (commit)
via cac035d7a171b90f0dc714d17b500b4062c0c58b (commit)
via 5cfa1aab53ba24d800d2dfa9ac1d88477063ab7f (commit)
via 61f8ceeb1d9b0786507d44ddceffce43fb9d13ae (commit)
via 8e209b3cc2065c4a52d7bdcb2c4c651b3f450d0d (commit)
via e993707e7a20ec37f4023bcf709fc77adce3857d (commit)
via 2d10f584cafa6348a933a5714ec8a7744957005d (commit)
via 16527c125c469451d810951ded0860294950a1f0 (commit)
via cba838830cb4b47420573ba0187e0eebbc5b6f1f (commit)
via fbd2326841005c1a288c65502dc038f26923ed65 (commit)
via 0f9bca457448372de1d15dcd9ed4548c324ff14f (commit)
via 3bbb988777079718338e3e6cb9c6c9b5399be800 (commit)
via 7b4fc9f10ad0df6e532299dbfc57ad4ba97ee015 (commit)
via 9547602dd2a0daf680635cec3249da759d3fd274 (commit)
via 911c97b06e4578e7c74db814ae1587f23ef18604 (commit)
via 7c76d9218ef91b14ede8a06fe9a96a5016361e17 (commit)
via f628e38317c20513cb73f81633d6a7c0663b8fd5 (commit)
via 7d197c6f7a676ac0e73f855fb4e6a3cd103dc55c (commit)
via 008cec917d19fadc011fdb1d61d7b54ffe1d4dda (commit)
via b917e8a2ebf8c51729ecde1d3cc3396d2fcf0034 (commit)
via 83369774c123b427f519bd2a2229415fac383a6a (commit)
via d3cb9460f0a30f9602508bed7507c906b8a6d91d (commit)
via 1491cdd65b8a3ef75f66516004bc0680651da201 (commit)
via 29e012f772b1c1d4c286e143b50a9de9b8a6918c (commit)
via a5b4a7e8da51d84fe200d88dc39bd5431bb60a11 (commit)
via 723afcb87896a7dcee99bb41b37ef368464f4f40 (commit)
via cffe9566efaa838ada850353dd6e51ec4cc3ebff (commit)
via 88d7932586705439e7a02b403e65202164920660 (commit)
via 1c0e350af2a30f4627ad879545c9c6fc6554d92a (commit)
via 81bfa2fd57ee5e4b0042d418115f0e50170ee5eb (commit)
via 2361a1b4d619a0f405f98c507cff6219b237bb2e (commit)
via 47530892a8a6b174786316c3881e22dc0864c859 (commit)
via 6ac59725fedc0a958d60d85740b47deb25addb5b (commit)
via 974ad63fc24161cb228003f4ef4cc32d5da578ff (commit)
via 87567c9440a65a42fdde62e513b40dc5a8585fdf (commit)
via c1f7fd8aa615d4a91f919ce18a6085ae766a5517 (commit)
from 854af8067c6486193eb00917e2515f81ba776770 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0e7cfa403cb80fc68d6c10c4c912e4d618fa085a
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Thu Nov 20 15:49:03 2014 +0100
4233: refactor to separate addDatum function
- this anticipates being able to add multiple datapoints from one log line, as for memory log lines
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 8c248ac..d2e9d05 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -65,7 +65,7 @@ $(document).on('ajax:complete ready', function() {
*/
function processLogLineForChart( logLine ) {
try {
- var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+ var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*)/);
if( !match ) {
match = logLine.match(/((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
if( match ) {
@@ -73,117 +73,48 @@ function processLogLineForChart( logLine ) {
}
}
if( match ) {
- // the timestamp comes first
- var timestamp = match[1].replace('_','T');
- // for the series use the task number (4th term) and then the first word after 'crunchstat:'
- var series = 'T' + match[4] + '-' + match[5];
- if( $.inArray( series, jobGraphSeries) < 0 ) {
- var newIndex = jobGraphSeries.push(series) - 1;
- jobGraphSortedSeries.push(newIndex);
- jobGraphSortedSeries.sort( function(a,b) {
- var matchA = jobGraphSeries[a].match(/^T(\d+)-(.*)/);
- var matchB = jobGraphSeries[b].match(/^T(\d+)-(.*)/);
- var termA = ('000000' + matchA[1]).slice(-6) + matchA[2];
- var termB = ('000000' + matchB[1]).slice(-6) + matchB[2];
- return termA > termB;
- });
- jobGraphMaxima[series] = null;
- window.recreate = true;
- }
- var intervalData = match[7].trim().split(' ');
- var dt = parseFloat(intervalData[0]);
- var dsum = 0.0;
- for(var i=2; i < intervalData.length; i += 2 ) {
- dsum += parseFloat(intervalData[i]);
- }
- var datum = dsum/dt;
- if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
- if( isJobSeriesRescalable(series) ) {
- // use old maximum to get a scale conversion
- var scaleConversion = jobGraphMaxima[series]/datum;
- // set new maximum and rescale the series
- jobGraphMaxima[series] = datum;
- rescaleJobGraphSeries( series, scaleConversion );
- }
- // and special calculation for cpus
- if( /-cpu$/.test(series) ) {
- // divide the stat by the number of cpus
- var cpuCountMatch = match[6].match(/(\d+) cpus/);
- if( cpuCountMatch ) {
- datum = datum / cpuCountMatch[1];
- }
+ var rawDetailData = '';
+ var datum = null;
+
+ // we are interested in "-- interval" recordings
+ var intervalMatch = match[6].match(/(.*) -- interval (.*)/);
+ if( intervalMatch ) {
+ var intervalData = intervalMatch[2].trim().split(' ');
+ var dt = parseFloat(intervalData[0]);
+ var dsum = 0.0;
+ for(var i=2; i < intervalData.length; i += 2 ) {
+ dsum += parseFloat(intervalData[i]);
}
- }
- // scale
- var scaledDatum = null;
- if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
- scaledDatum = datum/jobGraphMaxima[series]
+ datum = dsum/dt;
+ rawDetailData = intervalMatch[2];
} else {
- scaledDatum = datum;
- }
- // identify x axis point, searching from the end of the array (most recent)
- var found = false;
- for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
- if( jobGraphData[i]['t'] === timestamp ) {
- found = true;
- jobGraphData[i][series] = scaledDatum;
- jobGraphData[i]['raw-'+series] = match[7];
- break;
- } else if( jobGraphData[i]['t'] < timestamp ) {
- // we've gone far enough back in time and this data is supposed to be sorted
- break;
+ // we are also interested in memory ("mem") recordings
+ var memoryMatch = match[6].match(/(\d+) cache (\d+) swap (\d+) pgmajfault (\d+) rss/);
+ if( memoryMatch ) {
+ datum = parseInt(memoryMatch[4]);
+ rawDetailData = match[6];
+ } else {
+ // not interested
+ return;
}
}
- // index counter from previous loop will have gone one too far, so add one
- var insertAt = i+1;
- if(!found) {
- // create a new x point for this previously unrecorded timestamp
- var entry = { 't': timestamp };
- entry[series] = scaledDatum;
- entry['raw-'+series] = match[7];
- jobGraphData.splice( insertAt, 0, entry );
- var shifted = [];
- // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
- while( jobGraphData.length > 0
- && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
- shifted.push(jobGraphData.shift());
- }
- if( shifted.length > 0 ) {
- // from those that we dropped, are any of them maxima? if so we need to rescale
- jobGraphSeries.forEach( function(series) {
- // test that every shifted entry in this series was either not a number (in which case we don't care)
- // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
- // because otherwise we just scrolled off something that was a maximum point
- // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
- if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
- && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
- // check the remaining displayed points and find the new (scaled) maximum
- var seriesMax = null;
- jobGraphData.forEach( function(entry) {
- if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
- seriesMax = entry[series];
- }
- });
- if( seriesMax !== null && seriesMax !== 0 ) {
- // set new actual maximum using the new maximum as the conversion conversion and rescale the series
- jobGraphMaxima[series] *= seriesMax;
- var scaleConversion = 1/seriesMax;
- rescaleJobGraphSeries( series, scaleConversion );
- }
- else {
- // we no longer have any data points displaying for this series
- jobGraphMaxima[series] = null;
- }
- }
- });
- }
- // add a 10 minute old null data point to keep the chart honest if the oldest point is less than 9.5 minutes old
- if( jobGraphData.length > 0
- && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 9.5*60000 > Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
- var tenMinutesBefore = (new Date(Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf() - 600*1000)).toISOString().replace('Z','');
- jobGraphData.unshift( { 't': tenMinutesBefore } );
+
+ // the timestamp comes first
+ var timestamp = match[1].replace('_','T');
+ // for the series use the task number (4th term) and then the first word after 'crunchstat:'
+ var series = 'T' + match[4] + '-' + match[5];
+
+ // special calculation for cpus
+ if( /-cpu$/.test(series) ) {
+ // divide the stat by the number of cpus
+ var cpuCountMatch = intervalMatch[1].match(/(\d+) cpus/);
+ if( cpuCountMatch ) {
+ datum = datum / cpuCountMatch[1];
}
}
+
+ addJobGraphDatum( timestamp, datum, series, rawDetailData );
+
window.redraw = true;
}
} catch( err ) {
@@ -191,6 +122,104 @@ function processLogLineForChart( logLine ) {
}
}
+function addJobGraphDatum(timestamp, datum, series, rawDetailData) {
+ if( $.inArray( series, jobGraphSeries ) < 0 ) {
+ var newIndex = jobGraphSeries.push(series) - 1;
+ jobGraphSortedSeries.push(newIndex);
+ jobGraphSortedSeries.sort( function(a,b) {
+ var matchA = jobGraphSeries[a].match(/^T(\d+)-(.*)/);
+ var matchB = jobGraphSeries[b].match(/^T(\d+)-(.*)/);
+ var termA = ('000000' + matchA[1]).slice(-6) + matchA[2];
+ var termB = ('000000' + matchB[1]).slice(-6) + matchB[2];
+ return termA > termB;
+ });
+ jobGraphMaxima[series] = null;
+ window.recreate = true;
+ }
+
+ if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
+ if( isJobSeriesRescalable(series) ) {
+ // use old maximum to get a scale conversion
+ var scaleConversion = jobGraphMaxima[series]/datum;
+ // set new maximum and rescale the series
+ jobGraphMaxima[series] = datum;
+ rescaleJobGraphSeries( series, scaleConversion );
+ }
+ }
+
+ // scale
+ var scaledDatum = null;
+ if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
+ scaledDatum = datum/jobGraphMaxima[series]
+ } else {
+ scaledDatum = datum;
+ }
+ // identify x axis point, searching from the end of the array (most recent)
+ var found = false;
+ for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
+ if( jobGraphData[i]['t'] === timestamp ) {
+ found = true;
+ jobGraphData[i][series] = scaledDatum;
+ jobGraphData[i]['raw-'+series] = rawDetailData;
+ break;
+ } else if( jobGraphData[i]['t'] < timestamp ) {
+ // we've gone far enough back in time and this data is supposed to be sorted
+ break;
+ }
+ }
+ // index counter from previous loop will have gone one too far, so add one
+ var insertAt = i+1;
+ if(!found) {
+ // create a new x point for this previously unrecorded timestamp
+ var entry = { 't': timestamp };
+ entry[series] = scaledDatum;
+ entry['raw-'+series] = rawDetailData;
+ jobGraphData.splice( insertAt, 0, entry );
+ var shifted = [];
+ // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
+ while( jobGraphData.length > 0
+ && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
+ shifted.push(jobGraphData.shift());
+ }
+ if( shifted.length > 0 ) {
+ // from those that we dropped, are any of them maxima? if so we need to rescale
+ jobGraphSeries.forEach( function(series) {
+ // test that every shifted entry in this series was either not a number (in which case we don't care)
+ // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
+ // because otherwise we just scrolled off something that was a maximum point
+ // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
+ if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
+ && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
+ // check the remaining displayed points and find the new (scaled) maximum
+ var seriesMax = null;
+ jobGraphData.forEach( function(entry) {
+ if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
+ seriesMax = entry[series];
+ }
+ });
+ if( seriesMax !== null && seriesMax !== 0 ) {
+ // set new actual maximum using the new maximum as the conversion conversion and rescale the series
+ jobGraphMaxima[series] *= seriesMax;
+ var scaleConversion = 1/seriesMax;
+ rescaleJobGraphSeries( series, scaleConversion );
+ }
+ else {
+ // we no longer have any data points displaying for this series
+ jobGraphMaxima[series] = null;
+ }
+ }
+ });
+ }
+ // add a 10 minute old null data point to keep the chart honest if the oldest point is less than 9.9 minutes old
+ if( jobGraphData.length > 0
+ && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 9.9*60000 > Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
+ var tenMinutesBefore = (new Date(Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf() - 600*1000)).toISOString().replace('Z','');
+ jobGraphData.unshift( { 't': tenMinutesBefore } );
+ }
+ }
+
+}
+
function createJobGraph(elementName) {
delete jobGraph;
var emptyGraph = false;
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 1ae8e50..31bbfa9 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -57,6 +57,9 @@ class JobsController < ApplicationController
end
def push_logs
+ @push_logs = stderr_log_records([@object.uuid]).select do |log_record|
+ log_record.properties[:text] =~ /crunchstat: (mem.*swap.*rss|.*-- interval)/
+ end
end
def index_pane_list
diff --git a/apps/workbench/app/views/jobs/push_logs.js.erb b/apps/workbench/app/views/jobs/push_logs.js.erb
index 2501781..40ce037 100644
--- a/apps/workbench/app/views/jobs/push_logs.js.erb
+++ b/apps/workbench/app/views/jobs/push_logs.js.erb
@@ -1,3 +1,3 @@
-<% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']]).each do |log_record| %>
+<% @push_logs.each do |log_record| %>
processLogLineForChart( '<%=j log_record.properties[:text] %>' );
<% end %>
\ No newline at end of file
commit 84ff06d37f920ef6cf400189ea19edcf6158993c
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Thu Nov 20 14:21:08 2014 +0100
4233: try to match older log date format
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 542bb56..8c248ac 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -66,6 +66,12 @@ $(document).on('ajax:complete ready', function() {
function processLogLineForChart( logLine ) {
try {
var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+ if( !match ) {
+ match = logLine.match(/((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+ if( match ) {
+ match[1] = (new Date(match[1] + ' UTC')).toISOString().replace('Z','');
+ }
+ }
if( match ) {
// the timestamp comes first
var timestamp = match[1].replace('_','T');
commit a65acb3945d8646c616b4f8a2654d1a346fdaf2d
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Thu Nov 20 14:18:07 2014 +0100
4233: better hover presentation (left aligned, clearer task numbers)
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 74f15f7..542bb56 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -218,7 +218,8 @@ function createJobGraph(elementName) {
s += "<div class='morris-hover-point' style='color: ";
s += options.lineColors[sortedIndex];
s += "'>";
- s += options.labels[sortedIndex];
+ var labelMatch = options.labels[sortedIndex].match(/^T(\d+)-(.*)/);
+ s += 'Task ' + labelMatch[1] + ' ' + labelMatch[2];
s += ": ";
if ( !(typeof datum === 'undefined') ) {
if( isJobSeriesRescalable( series ) ) {
diff --git a/apps/workbench/app/assets/stylesheets/jobs.css.scss b/apps/workbench/app/assets/stylesheets/jobs.css.scss
index f76c70b..565c354 100644
--- a/apps/workbench/app/assets/stylesheets/jobs.css.scss
+++ b/apps/workbench/app/assets/stylesheets/jobs.css.scss
@@ -8,3 +8,8 @@
font-size: .8em;
border: 2px solid black;
}
+
+.morris-hover-point {
+ text-align: left;
+ width: 100%;
+}
\ No newline at end of file
commit e9a19734c359fa52ec07a3ea4bb81b6382864596
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Thu Nov 20 13:39:11 2014 +0100
4233: initial chart points added via AJAX
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 5365182..1ae8e50 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -56,6 +56,9 @@ class JobsController < ApplicationController
super
end
+ def push_logs
+ end
+
def index_pane_list
if params[:uuid]
%w(Recent Provenance)
diff --git a/apps/workbench/app/helpers/jobs_helper.rb b/apps/workbench/app/helpers/jobs_helper.rb
index aca0999..46f2601 100644
--- a/apps/workbench/app/helpers/jobs_helper.rb
+++ b/apps/workbench/app/helpers/jobs_helper.rb
@@ -24,9 +24,8 @@ module JobsHelper
filters += extra_filters if extra_filters
last_entry = Log.order('id DESC').limit(1).filter(filters).results.first
if last_entry
- filters += [["event_at", ">=", last_entry.event_at - 5.minutes]]
+ filters += [["event_at", ">=", last_entry.event_at - 10.minutes]]
Log.order('id DESC')
- .limit(250)
.filter(filters)
.results
else
diff --git a/apps/workbench/app/views/jobs/_show_log.html.erb b/apps/workbench/app/views/jobs/_show_log.html.erb
index 6775cb8..64cd96a 100644
--- a/apps/workbench/app/views/jobs/_show_log.html.erb
+++ b/apps/workbench/app/views/jobs/_show_log.html.erb
@@ -22,10 +22,7 @@
window.jobGraphSeries = [];
window.jobGraphSortedSeries = [];
window.jobGraphMaxima = {};
- <% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']])
- .each do |log_record| %>
- processLogLineForChart( '<%=j log_record.properties[:text] %>' );
- <% end %>
+ $.get('<%= push_logs_job_path(@object) %>');
createJobGraph('log_graph_div');
<% end %>
diff --git a/apps/workbench/app/views/jobs/push_logs.js.erb b/apps/workbench/app/views/jobs/push_logs.js.erb
new file mode 100644
index 0000000..2501781
--- /dev/null
+++ b/apps/workbench/app/views/jobs/push_logs.js.erb
@@ -0,0 +1,3 @@
+<% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']]).each do |log_record| %>
+ processLogLineForChart( '<%=j log_record.properties[:text] %>' );
+<% end %>
\ No newline at end of file
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 9f9696f..820303d 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -22,6 +22,7 @@ ArvadosWorkbench::Application.routes.draw do
resources :job_tasks
resources :jobs do
post 'cancel', :on => :member
+ get 'push_logs', :on => :member
end
match '/logout' => 'sessions#destroy', via: [:get, :post]
get '/logged_out' => 'sessions#index'
commit 7b9112dbc270ea338fee756f583bb76870f2e391
Merge: 854af80 b77fb69
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Thu Nov 20 13:19:03 2014 +0100
Merge branch 'master' into 4233-graph-job-stats
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list