[ARVADOS] updated: 0e7cfa403cb80fc68d6c10c4c912e4d618fa085a

git at public.curoverse.com git at public.curoverse.com
Thu Nov 20 09:49:08 EST 2014


Summary of changes:
 apps/workbench/Gemfile                             |   2 +-
 apps/workbench/Gemfile.lock                        |  12 +-
 apps/workbench/app/assets/javascripts/event_log.js | 250 ++++++++-------
 .../workbench/app/assets/javascripts/filterable.js |  41 ++-
 .../app/assets/javascripts/infinite_scroll.js      |   7 +-
 .../app/assets/javascripts/pipeline_instances.js   |  10 +-
 .../app/assets/javascripts/select_modal.js         |  31 +-
 .../app/assets/javascripts/selection.js.erb        |   3 +-
 .../workbench/app/assets/stylesheets/jobs.css.scss |   5 +
 .../app/controllers/application_controller.rb      |  14 +-
 .../app/controllers/collections_controller.rb      |  39 +--
 apps/workbench/app/controllers/jobs_controller.rb  |   6 +
 .../controllers/pipeline_instances_controller.rb   |  26 +-
 apps/workbench/app/controllers/users_controller.rb |   6 +-
 apps/workbench/app/helpers/application_helper.rb   |  11 +-
 apps/workbench/app/helpers/jobs_helper.rb          |   3 +-
 apps/workbench/app/models/arvados_api_client.rb    |   7 +-
 apps/workbench/app/models/arvados_base.rb          |   4 +-
 apps/workbench/app/models/arvados_resource_list.rb | 168 ++++++----
 .../app/views/application/_choose.html.erb         |   9 +-
 .../app/views/application/_paging.html.erb         |  22 +-
 .../app/views/collections/_choose_rows.html.erb    |  22 +-
 .../app/views/collections/_index_tbody.html.erb    |   2 +-
 .../collections/_show_chooser_preview.html.erb     |   2 +-
 .../app/views/collections/_show_files.html.erb     |  23 +-
 .../app/views/collections/_show_recent.html.erb    |   4 +-
 apps/workbench/app/views/jobs/_show_log.html.erb   |   5 +-
 apps/workbench/app/views/jobs/push_logs.js.erb     |   3 +
 .../app/views/layouts/application.html.erb         |   2 +-
 apps/workbench/app/views/layouts/body.html.erb     |  13 -
 .../views/pipeline_instances/_show_inputs.html.erb |   5 +-
 .../views/pipeline_instances/_show_recent.html.erb |  44 +--
 ..._recent.html.erb => _show_recent_rows.html.erb} |  45 +--
 .../app/views/pipeline_instances/index.html.erb    |  14 +-
 .../app/views/projects/_show_sharing.html.erb      |   3 +-
 .../workbench/app/views/users/_show_admin.html.erb |   2 +-
 apps/workbench/config/routes.rb                    |   1 +
 .../pipeline_instances_controller_test.rb          |   2 +-
 .../integration/filterable_infinite_scroll_test.rb |  29 ++
 .../test/integration/pipeline_instances_test.rb    | 144 +++++++--
 .../test/unit/arvados_resource_list_test.rb        | 108 +++++--
 crunch_scripts/run-command                         |   5 +-
 doc/_includes/_ssh_addkey.liquid                   |   2 +-
 doc/api/methods/jobs.html.textile.liquid           |   6 +-
 doc/api/schema/Job.html.textile.liquid             |  13 +-
 doc/images/workbench-move-selected.png             | Bin 11267 -> 47833 bytes
 doc/sdk/cli/index.html.textile.liquid              |   1 -
 doc/sdk/cli/subcommands.html.textile.liquid        |   8 +-
 doc/user/topics/arv-run.html.textile.liquid        |   2 +-
 doc/user/topics/run-command.html.textile.liquid    |   3 +-
 .../tutorial-firstscript.html.textile.liquid       |   2 +-
 .../tutorials/tutorial-keep.html.textile.liquid    |  14 +-
 ...tutorial-pipeline-workbench.html.textile.liquid |   2 +-
 sdk/cli/bin/arv                                    |   8 +-
 sdk/cli/bin/arv-tag                                |  15 +-
 sdk/cli/bin/crunch-job                             |  91 +++---
 sdk/go/keepclient/keepclient.go                    |  22 +-
 sdk/go/keepclient/keepclient_test.go               | 124 ++++----
 sdk/go/keepclient/root_sorter.go                   |  57 ++++
 sdk/go/keepclient/root_sorter_test.go              |  58 ++++
 sdk/go/keepclient/support.go                       |  65 +---
 sdk/python/arvados/arvfile.py                      |  29 ++
 sdk/python/arvados/collection.py                   | 113 +++++--
 sdk/python/arvados/keep.py                         | 120 +++----
 sdk/python/arvados/stream.py                       | 119 ++++---
 sdk/python/arvados/util.py                         |   4 +-
 sdk/python/setup.py                                |   6 +-
 sdk/python/tests/arvados_testutil.py               |  16 +
 sdk/python/tests/run_test_server.py                |  14 +-
 sdk/python/tests/test_collections.py               | 275 +++++++---------
 sdk/python/tests/test_keep_client.py               |  77 ++++-
 sdk/python/tests/test_stream.py                    | 160 ++++++++++
 sdk/ruby/arvados.gemspec                           |   2 +-
 .../api/app/controllers/application_controller.rb  |  78 ++---
 .../arvados/v1/collections_controller.rb           |  23 +-
 .../app/controllers/arvados/v1/jobs_controller.rb  |  84 +++--
 services/api/app/models/collection.rb              | 106 ++++++-
 services/api/app/models/job.rb                     |  59 ++--
 services/api/app/models/node.rb                    |  60 +++-
 services/api/config/application.default.yml        |  22 +-
 services/api/config/unbound.template               |   4 +
 ...141111133038_add_arvados_sdk_version_to_jobs.rb |  13 +
 services/api/db/structure.sql                      |   7 +-
 services/api/lib/load_param.rb                     |   7 +-
 services/api/script/restart-dns-server             |  34 ++
 services/api/script/restart-dnsmasq                |  21 --
 services/api/test/fixtures/jobs.yml                |  17 +
 services/api/test/fixtures/keep_services.yml       |   6 +-
 services/api/test/fixtures/links.yml               |  14 +
 services/api/test/fixtures/pipeline_templates.yml  |  39 ++-
 services/api/test/fixtures/repositories.yml        |   5 +
 .../arvados/v1/collections_controller_test.rb      |  84 +++--
 .../arvados/v1/commits_controller_test.rb          |   3 +-
 .../arvados/v1/groups_controller_test.rb           |  67 ++--
 .../arvados/v1/job_reuse_controller_test.rb        |  81 +++--
 .../arvados/v1/repositories_controller_test.rb     |  15 +-
 services/api/test/helpers/git_test_helper.rb       |   4 +-
 services/api/test/integration/keep_proxy_test.rb   |  17 +-
 services/api/test/integration/websocket_test.rb    |   2 +-
 services/api/test/test.git.tar                     | Bin 153600 -> 194560 bytes
 services/api/test/unit/collection_test.rb          |  40 ++-
 services/api/test/unit/job_test.rb                 |  93 +++++-
 services/fuse/arvados_fuse/__init__.py             |  28 +-
 services/fuse/setup.py                             |   3 +-
 services/fuse/tests/test_mount.py                  |  72 ++++-
 services/keepproxy/keepproxy_test.go               |  14 +-
 services/keepstore/handler_test.go                 |  80 ++---
 services/keepstore/handlers.go                     |  26 --
 services/keepstore/keepstore.go                    |   8 +-
 services/keepstore/logging_router.go               |  41 +++
 .../nodemanager/arvnodeman/computenode/__init__.py | 353 ---------------------
 .../computenode/{ => dispatch}/__init__.py         | 188 +++--------
 .../arvnodeman/computenode/dispatch/slurm.py       |  49 +++
 .../arvnodeman/computenode/driver/__init__.py      |  64 ++++
 .../arvnodeman/computenode/{ => driver}/dummy.py   |   3 +-
 .../arvnodeman/computenode/{ => driver}/ec2.py     |   3 +-
 services/nodemanager/arvnodeman/config.py          |  13 +-
 services/nodemanager/arvnodeman/daemon.py          |  56 ++--
 services/nodemanager/arvnodeman/launcher.py        |  11 +-
 services/nodemanager/doc/ec2.example.cfg           |   5 +
 services/nodemanager/tests/test_computenode.py     | 279 ----------------
 ...computenode.py => test_computenode_dispatch.py} | 136 ++++----
 .../tests/test_computenode_dispatch_slurm.py       |  58 ++++
 ...enode_ec2.py => test_computenode_driver_ec2.py} |   2 +-
 services/nodemanager/tests/test_config.py          |  18 ++
 services/nodemanager/tests/test_daemon.py          |  55 +++-
 services/nodemanager/tests/testutil.py             |   6 +
 127 files changed, 2904 insertions(+), 2254 deletions(-)
 create mode 100644 apps/workbench/app/views/jobs/push_logs.js.erb
 copy apps/workbench/app/views/pipeline_instances/{_show_recent.html.erb => _show_recent_rows.html.erb} (58%)
 create mode 100644 apps/workbench/test/integration/filterable_infinite_scroll_test.rb
 create mode 100644 sdk/go/keepclient/root_sorter.go
 create mode 100644 sdk/go/keepclient/root_sorter_test.go
 create mode 100644 sdk/python/arvados/arvfile.py
 create mode 100644 services/api/config/unbound.template
 create mode 100644 services/api/db/migrate/20141111133038_add_arvados_sdk_version_to_jobs.rb
 create mode 100755 services/api/script/restart-dns-server
 delete mode 100755 services/api/script/restart-dnsmasq
 create mode 100644 services/keepstore/logging_router.go
 copy services/nodemanager/arvnodeman/computenode/{ => dispatch}/__init__.py (64%)
 create mode 100644 services/nodemanager/arvnodeman/computenode/dispatch/slurm.py
 create mode 100644 services/nodemanager/arvnodeman/computenode/driver/__init__.py
 rename services/nodemanager/arvnodeman/computenode/{ => driver}/dummy.py (96%)
 rename services/nodemanager/arvnodeman/computenode/{ => driver}/ec2.py (98%)
 copy services/nodemanager/tests/{test_computenode.py => test_computenode_dispatch.py} (77%)
 create mode 100644 services/nodemanager/tests/test_computenode_dispatch_slurm.py
 rename services/nodemanager/tests/{test_computenode_ec2.py => test_computenode_driver_ec2.py} (98%)

       via  0e7cfa403cb80fc68d6c10c4c912e4d618fa085a (commit)
       via  84ff06d37f920ef6cf400189ea19edcf6158993c (commit)
       via  a65acb3945d8646c616b4f8a2654d1a346fdaf2d (commit)
       via  e9a19734c359fa52ec07a3ea4bb81b6382864596 (commit)
       via  7b9112dbc270ea338fee756f583bb76870f2e391 (commit)
       via  b77fb69914fda0aedecddf2e62076cb404265647 (commit)
       via  2834445f90c62644fde2cd46f9a4328600caba59 (commit)
       via  a325b04603deacebc07d322e691d7cc4e38b2f24 (commit)
       via  d270d652cb21c601513fb89e77a64270fc80bf21 (commit)
       via  9d7efc36748d8b83af3d5dbdcd5326834fb317ff (commit)
       via  6f31d0c9d4b6a0d3070e933a84cfd843722e81b1 (commit)
       via  6c68141eb50255128cf38b5717b15b16f2a8cdff (commit)
       via  ffc7c1b20530dda13c1adf3aa48122ac881bbc29 (commit)
       via  6874da0dd56d6c0320880dfb3bd4da34a3c0a7d3 (commit)
       via  4af4a2cf12d3b87a4fdbd09115bc258961b706bb (commit)
       via  22381d218295075e63c06f0b60bcb24cee7b30b6 (commit)
       via  8d324f6d853ce38ab816ce12e8557b0e47ce9c2f (commit)
       via  89285c343d05bc7ceef55a46c4ebb38d9b047baf (commit)
       via  1a2a05a4714a4bc538800d08d353b0306d5e49ad (commit)
       via  b1f65fcd4fa5fca5400f63fa25823fef6229a158 (commit)
       via  40b15b9523c1a89769c8df5bafa250075af0c567 (commit)
       via  b1f3c944c3217b6e42a377f4b0a04985c9cea908 (commit)
       via  0e5cd501a34ebcf6e138b4693f4fa4094bff24b4 (commit)
       via  22c135139279edcf5be0218eb90c375f2433a4b9 (commit)
       via  3735d52b65928e3626a8e223acadc318a3d31097 (commit)
       via  f9ba8a730dbc1a457485886aa4591991e9dbbf0d (commit)
       via  97a2555a220d424b029d39b6b6e62f28bc3bc148 (commit)
       via  3ce8039e7ca278e4816ca90cd973bdbe08404b87 (commit)
       via  9f52191b13196b1926f7b8b0c7c35aac56e02198 (commit)
       via  69fae522bf6fe6dbaa64ce7d75ef4ce5616ced72 (commit)
       via  22ae84c0d72cdeab882017a8caf2ade4465ddd59 (commit)
       via  c621e2e901f5b5de4f72b1c6a9d8816aa5745292 (commit)
       via  aea8c62eaec4d170842b01e0fd1ddb5e256152b8 (commit)
       via  14b762d2dd37a0a533532839aaf0488c11205e42 (commit)
       via  1a520a700456325a8a7427d74fa11ed15b62a4b2 (commit)
       via  e82958936210b6bbc8cc04b237229fdf6fac295c (commit)
       via  8deda2012ea7e25610f2cbd0a271e131d9364503 (commit)
       via  732f7f0740dc7e5d5cff54f9cc30d4cd4597cb27 (commit)
       via  763af5413e6195fe004846abf86a7ae134168990 (commit)
       via  f72ca2ac98200452700dc386fa439a45b8bb15e3 (commit)
       via  b75e2a12f618e2b69f3af1672abfb4773e202bed (commit)
       via  84d5db363e4c10881fdb5317b01f46bdc33c002a (commit)
       via  dea90c956d96d06e39df0f6567c3bb6582a5b3dd (commit)
       via  75a07d8175c9ee9a1c212cd1abe1c0d64a91cfc7 (commit)
       via  07bf4bb6d787c276f338846eb46c2b527a323896 (commit)
       via  18103a8f198e2038c978e1d38cef3fc3bf971d1b (commit)
       via  ff0716748c5e94b03dcfd47e9875be3bc7bcb9e6 (commit)
       via  84a1d6a622193dae18c715e0421353da2aecf1f7 (commit)
       via  4805867bc1004ae9d755a6375d592748a7b29585 (commit)
       via  645a998c13b7dcd21af72187d953b8069288d6d2 (commit)
       via  28c6bbeeed6b228d51666914bcd845d05500107e (commit)
       via  aa6f07c9687670ee98e15d9afa2bd47416839dc6 (commit)
       via  7bd04ed26901e34e3e5788093bee12be4406dd7b (commit)
       via  ff7b22c70cd77073d9bdbebac0bf03d43745ed0c (commit)
       via  372cff3e04b4243eb68d9ae3e296aebcee459aea (commit)
       via  d3229c7f727f40436cad66fba5f3345e0b3eede5 (commit)
       via  95a61743fd1df435313042c17e581d4b7b457a2a (commit)
       via  78447c70f491db25dc99eca60ebcc1b7a5362ec3 (commit)
       via  664919d58c3689cd9e0a25547ec1e02d9adda38c (commit)
       via  97ddc11818d059e42989f302fa15db4c5e19d5b3 (commit)
       via  9a46fb0033514db6dd5e8ab4ccf793f78458a734 (commit)
       via  bfd5100c4e0bed1910875099937169f1b0c8ce7b (commit)
       via  d9f45bd01d2f5691a9aaca4f1fc00465497ef00c (commit)
       via  0adcabd351108e83b630e27c48ce85f2b3cb3e14 (commit)
       via  e0f4747a2b24af68b474bb2da12e8799440c9acc (commit)
       via  a128baa147df78cadb5c1657dfa740fad80570ea (commit)
       via  43e019a1e37effa0ccde680b52a61fbba1f858d4 (commit)
       via  0e5dfbbcefcb462617ad89ef116a5b18f9e3ebc8 (commit)
       via  de35a630edcbe2ddcadf002ab3efe027d77fc046 (commit)
       via  7395af3b19b8fc422fe2456c6bead1e7f4c60989 (commit)
       via  a75ef0985d2a385165168cbb488cf97382308549 (commit)
       via  11374252ee3c26240420bb3aa0d0433fad71731f (commit)
       via  5d87bcae66870127445e5eb018a69d606e93e244 (commit)
       via  83e73eda7123a6de741198d759841fd186fc2a11 (commit)
       via  06afd90ff09a83982451733f1e39974299d9a2d9 (commit)
       via  4b3222657e160eaf61459aac843e8377100a53e4 (commit)
       via  abe8fc4c5818165ffb696448c81b3bf1bb5dc1dd (commit)
       via  4c110b5f4e8e92d613f70427f27bdfae2ea376c7 (commit)
       via  2ac2528dc0e5b700a2640e28dc66e474db9ad329 (commit)
       via  9cd99d3070cd69664a31f8e25fad68fe8b7c43cf (commit)
       via  d78db2b1b518abdac0893e65123504fb85319010 (commit)
       via  11236731eda455b5df6448733e4c0090a5f727c4 (commit)
       via  8c682e4ffcd8fdf9bcaf8f062f1d0a6517ea9e5d (commit)
       via  f798c910fa1966b0471451de4f24cc4f55e07290 (commit)
       via  6de0a4ab0323e69a5499e59af03c9c5e6125fed7 (commit)
       via  f51138289b7625bde6cc0bb9840ccf681944941a (commit)
       via  97cd8d6023f918babc5e6cca7f7a1228f4a32d59 (commit)
       via  84b6c8f2d915ad6c1e2843f62bc47e510f2d6f54 (commit)
       via  2070784e52d89bbcf636062532a8ce37629737db (commit)
       via  59f750a109a706d29862530dc3d282db75736d38 (commit)
       via  8b7ed36ff057c4a483f4ee8a1c66929425b366c6 (commit)
       via  d4224bfee9d5ece132bc50db96de48da3620d5e5 (commit)
       via  35c53e2525e0d29cb8ffae486718b4fde88b273f (commit)
       via  f5a1a9210e7638babe33d16938246e93ff4dd334 (commit)
       via  3a31350c6265cb1135d3d4d40af436aae91a9894 (commit)
       via  c805714d8bbc51adb32a3a3ae4b82ea41eaa0de0 (commit)
       via  9325dabcc2b6c32a589ba86cc052946b4875eb1d (commit)
       via  95323ba327cadff2d1cb8c3f38c5dbb65dbd3694 (commit)
       via  eb185aab22bd916981ea874fd82e6a1fecce7cd2 (commit)
       via  069b22b2d1196d0437253e60e616840ea141e1a9 (commit)
       via  4d01f18d6a1129fb3f2f0efc4bda31795519fda2 (commit)
       via  e363a7973a911a6484b24d4e7f92f363d4d3e521 (commit)
       via  06e402b11ad4d503feb5fa45845cb27c93478cfc (commit)
       via  afdab87268e4ee2f19f92e02b219696081959378 (commit)
       via  809b122ee02d566b3bddb26df928e57efa39ed87 (commit)
       via  f5c574f45a6352c7d675e8c111eef5083a435213 (commit)
       via  f81483a374fbdfe6d5f36e5f5e55f92455afa155 (commit)
       via  1ac0e3da5fbd477bd2acbf5bd8baa3c86f08194b (commit)
       via  dff161c1eaaecd6b557e88709fdf5f0dcb52821b (commit)
       via  cf1db3981ad0c6416d2572ceeb512743a9df24c7 (commit)
       via  3894b788440fce678a28b4c7a8780f2b7767449d (commit)
       via  5be66e86ac84d7d46c8503c3c136cb144222e4bf (commit)
       via  fd207a0e0fc3aa02eba81b567b0b30d144ac3fbf (commit)
       via  9ef386f6e405db49df44a24e4cdfe5a0c0742f62 (commit)
       via  d833a28612bb22e1489536b3b0dfe8eb00196fda (commit)
       via  754ff84aab18b623b37f21377c639694e6ea0ed7 (commit)
       via  53192c327dcce6159d21f6cc27f7d5c0bfc9e7b0 (commit)
       via  1a001045ccce247a13266a5ef619ffbc07308226 (commit)
       via  9966893484226206474d7d9936d884cc56582f15 (commit)
       via  ac21f0d45a76294aaca0c0c0fdf06eb72d03368d (commit)
       via  a0e70cf1da033f7d94e728ab919bd8cfcabf3743 (commit)
       via  c28b9b91028c90497455bfde188915982377b825 (commit)
       via  e158f485053be1e840073b321033d60d686a55a8 (commit)
       via  8f9f0dece977ccf5a778f3b3bd2379375e723c18 (commit)
       via  b626a85eb86fd4909712852040cd305c71c37ee5 (commit)
       via  5141c3ee23e89696773e227a93236ef2a51543c2 (commit)
       via  45bfc1104dd30fb97a586de5ff96d6b739f7bb2b (commit)
       via  6bcb827b1405ce8ca5d767108bc4356ecc2972f4 (commit)
       via  093cebc3a4baed645fd5d4a5331c2ff837de8c77 (commit)
       via  76c9a858643680d90f9694d49c212a54fe5b759f (commit)
       via  53381c90fd5ef26d3a0cc08c8d35318b0b92557a (commit)
       via  bc4992f9e11cd0639cf6461736963d66830c5af8 (commit)
       via  39e39addee0a0de050c2b4d1f88fb7d56da7436b (commit)
       via  333fbec95590cfcb30cfff7dc61e6b8cc41edf18 (commit)
       via  91f0b180a8474321738577a12250b5a1b8f5bcbc (commit)
       via  a844af57de33b0094ce23c873d044f15de1ed138 (commit)
       via  d1cdd7eddf071e41643bc958b77d0056410d01ea (commit)
       via  26aa2f1a1c8c9883beac1538c318279190f91c8a (commit)
       via  7bf8f6c701e28e574c137b0c942522e8f8ee4d8c (commit)
       via  7924077e3db2898de26c86599d9f311e02d6db46 (commit)
       via  c1d9150de2977f7eb85e7cb058d5c41ae6f06173 (commit)
       via  b2610e066d569b323fe55ffee615e61ca4e461d1 (commit)
       via  6c7bfabc03237419decad9674c90ea7af906b615 (commit)
       via  c7b7e4d44e9349c9104fbfcd33c5e4b97fe17a71 (commit)
       via  e61de9add861db4c043341f3926acb95ded37862 (commit)
       via  1237511f9da5ee20588d44be4db5f9e37cfc6400 (commit)
       via  fa3e996ee452bd9be853dd9e93aaec15623708f5 (commit)
       via  1304e044aa87a65145bf8b6d4bc141586556c0ed (commit)
       via  47b0cb35b5ee933757c4342bb75fd286c1dac8cb (commit)
       via  e02ef893c4f6cf881e449c248782b2ac21b49b0f (commit)
       via  9405b5f224b205e681e378bd2d90d18f3638139b (commit)
       via  eee66a4fd87be84379929d17864f244b0a68fd74 (commit)
       via  a4b55a80251d666ca39d0f8d201c70e493b4c661 (commit)
       via  0198b4937df2be79e41ea76ea6c31841441f4f2c (commit)
       via  cac035d7a171b90f0dc714d17b500b4062c0c58b (commit)
       via  5cfa1aab53ba24d800d2dfa9ac1d88477063ab7f (commit)
       via  61f8ceeb1d9b0786507d44ddceffce43fb9d13ae (commit)
       via  8e209b3cc2065c4a52d7bdcb2c4c651b3f450d0d (commit)
       via  e993707e7a20ec37f4023bcf709fc77adce3857d (commit)
       via  2d10f584cafa6348a933a5714ec8a7744957005d (commit)
       via  16527c125c469451d810951ded0860294950a1f0 (commit)
       via  cba838830cb4b47420573ba0187e0eebbc5b6f1f (commit)
       via  fbd2326841005c1a288c65502dc038f26923ed65 (commit)
       via  0f9bca457448372de1d15dcd9ed4548c324ff14f (commit)
       via  3bbb988777079718338e3e6cb9c6c9b5399be800 (commit)
       via  7b4fc9f10ad0df6e532299dbfc57ad4ba97ee015 (commit)
       via  9547602dd2a0daf680635cec3249da759d3fd274 (commit)
       via  911c97b06e4578e7c74db814ae1587f23ef18604 (commit)
       via  7c76d9218ef91b14ede8a06fe9a96a5016361e17 (commit)
       via  f628e38317c20513cb73f81633d6a7c0663b8fd5 (commit)
       via  7d197c6f7a676ac0e73f855fb4e6a3cd103dc55c (commit)
       via  008cec917d19fadc011fdb1d61d7b54ffe1d4dda (commit)
       via  b917e8a2ebf8c51729ecde1d3cc3396d2fcf0034 (commit)
       via  83369774c123b427f519bd2a2229415fac383a6a (commit)
       via  d3cb9460f0a30f9602508bed7507c906b8a6d91d (commit)
       via  1491cdd65b8a3ef75f66516004bc0680651da201 (commit)
       via  29e012f772b1c1d4c286e143b50a9de9b8a6918c (commit)
       via  a5b4a7e8da51d84fe200d88dc39bd5431bb60a11 (commit)
       via  723afcb87896a7dcee99bb41b37ef368464f4f40 (commit)
       via  cffe9566efaa838ada850353dd6e51ec4cc3ebff (commit)
       via  88d7932586705439e7a02b403e65202164920660 (commit)
       via  1c0e350af2a30f4627ad879545c9c6fc6554d92a (commit)
       via  81bfa2fd57ee5e4b0042d418115f0e50170ee5eb (commit)
       via  2361a1b4d619a0f405f98c507cff6219b237bb2e (commit)
       via  47530892a8a6b174786316c3881e22dc0864c859 (commit)
       via  6ac59725fedc0a958d60d85740b47deb25addb5b (commit)
       via  974ad63fc24161cb228003f4ef4cc32d5da578ff (commit)
       via  87567c9440a65a42fdde62e513b40dc5a8585fdf (commit)
       via  c1f7fd8aa615d4a91f919ce18a6085ae766a5517 (commit)
      from  854af8067c6486193eb00917e2515f81ba776770 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 0e7cfa403cb80fc68d6c10c4c912e4d618fa085a
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date:   Thu Nov 20 15:49:03 2014 +0100

    4233: refactor to separate addDatum function
      - this anticipates being able to add multiple datapoints from one log line, as for memory log lines

diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 8c248ac..d2e9d05 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -65,7 +65,7 @@ $(document).on('ajax:complete ready', function() {
  */
 function processLogLineForChart( logLine ) {
     try {
-        var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+        var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*)/);
         if( !match ) {
             match = logLine.match(/((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
             if( match ) {
@@ -73,117 +73,48 @@ function processLogLineForChart( logLine ) {
             }
         }
         if( match ) {
-            // the timestamp comes first
-            var timestamp = match[1].replace('_','T');
-            // for the series use the task number (4th term) and then the first word after 'crunchstat:'
-            var series = 'T' + match[4] + '-' + match[5];
-            if( $.inArray( series, jobGraphSeries) < 0 ) {
-                var newIndex = jobGraphSeries.push(series) - 1;
-                jobGraphSortedSeries.push(newIndex);
-                jobGraphSortedSeries.sort( function(a,b) {
-                    var matchA = jobGraphSeries[a].match(/^T(\d+)-(.*)/);
-                    var matchB = jobGraphSeries[b].match(/^T(\d+)-(.*)/);
-                    var termA = ('000000' + matchA[1]).slice(-6) + matchA[2];
-                    var termB = ('000000' + matchB[1]).slice(-6) + matchB[2];
-                    return termA > termB;
-                });
-                jobGraphMaxima[series] = null;
-                window.recreate = true;
-            }
-            var intervalData = match[7].trim().split(' ');
-            var dt = parseFloat(intervalData[0]);
-            var dsum = 0.0;
-            for(var i=2; i < intervalData.length; i += 2 ) {
-                dsum += parseFloat(intervalData[i]);
-            }
-            var datum = dsum/dt;
-            if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
-                if( isJobSeriesRescalable(series) ) {
-                    // use old maximum to get a scale conversion
-                    var scaleConversion = jobGraphMaxima[series]/datum;
-                    // set new maximum and rescale the series
-                    jobGraphMaxima[series] = datum;
-                    rescaleJobGraphSeries( series, scaleConversion );
-                }
-                // and special calculation for cpus
-                if( /-cpu$/.test(series) ) {
-                    // divide the stat by the number of cpus
-                    var cpuCountMatch = match[6].match(/(\d+) cpus/);
-                    if( cpuCountMatch ) {
-                        datum = datum / cpuCountMatch[1];
-                    }
+            var rawDetailData = '';
+            var datum = null;
+
+            // we are interested in "-- interval" recordings
+            var intervalMatch = match[6].match(/(.*) -- interval (.*)/);
+            if( intervalMatch ) {
+                var intervalData = intervalMatch[2].trim().split(' ');
+                var dt = parseFloat(intervalData[0]);
+                var dsum = 0.0;
+                for(var i=2; i < intervalData.length; i += 2 ) {
+                    dsum += parseFloat(intervalData[i]);
                 }
-            }
-            // scale
-            var scaledDatum = null;
-            if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
-                scaledDatum = datum/jobGraphMaxima[series]
+                datum = dsum/dt;
+                rawDetailData = intervalMatch[2];
             } else {
-                scaledDatum = datum;
-            }
-            // identify x axis point, searching from the end of the array (most recent)
-            var found = false;
-            for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
-                if( jobGraphData[i]['t'] === timestamp ) {
-                    found = true;
-                    jobGraphData[i][series] = scaledDatum;
-                    jobGraphData[i]['raw-'+series] = match[7];
-                    break;
-                } else if( jobGraphData[i]['t'] < timestamp  ) {
-                    // we've gone far enough back in time and this data is supposed to be sorted
-                    break;
+                // we are also interested in memory ("mem") recordings
+                var memoryMatch = match[6].match(/(\d+) cache (\d+) swap (\d+) pgmajfault (\d+) rss/);
+                if( memoryMatch ) {
+                    datum = parseInt(memoryMatch[4]);
+                    rawDetailData = match[6];
+                } else {
+                    // not interested
+                    return;
                 }
             }
-            // index counter from previous loop will have gone one too far, so add one
-            var insertAt = i+1;
-            if(!found) {
-                // create a new x point for this previously unrecorded timestamp
-                var entry = { 't': timestamp };
-                entry[series] = scaledDatum;
-                entry['raw-'+series] = match[7];
-                jobGraphData.splice( insertAt, 0, entry );
-                var shifted = [];
-                // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
-                while( jobGraphData.length > 0
-                         && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
-                    shifted.push(jobGraphData.shift());
-                }
-                if( shifted.length > 0 ) {
-                    // from those that we dropped, are any of them maxima? if so we need to rescale
-                    jobGraphSeries.forEach( function(series) {
-                        // test that every shifted entry in this series was either not a number (in which case we don't care)
-                        // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
-                        // because otherwise we just scrolled off something that was a maximum point
-                        // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
-                        if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
-                              && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
-                            // check the remaining displayed points and find the new (scaled) maximum
-                            var seriesMax = null;
-                            jobGraphData.forEach( function(entry) {
-                                if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
-                                    seriesMax = entry[series];
-                                }
-                            });
-                            if( seriesMax !== null && seriesMax !== 0 ) {
-                                // set new actual maximum using the new maximum as the conversion conversion and rescale the series
-                                jobGraphMaxima[series] *= seriesMax;
-                                var scaleConversion = 1/seriesMax;
-                                rescaleJobGraphSeries( series, scaleConversion );
-                            }
-                            else {
-                                // we no longer have any data points displaying for this series
-                                jobGraphMaxima[series] = null;
-                            }
-                        }
-                    });
-                }
-                // add a 10 minute old null data point to keep the chart honest if the oldest point is less than 9.5 minutes old
-                if( jobGraphData.length > 0
-                      && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 9.5*60000 > Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
-                    var tenMinutesBefore = (new Date(Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf() - 600*1000)).toISOString().replace('Z','');
-                    jobGraphData.unshift( { 't': tenMinutesBefore } );
+
+            // the timestamp comes first
+            var timestamp = match[1].replace('_','T');
+            // for the series use the task number (4th term) and then the first word after 'crunchstat:'
+            var series = 'T' + match[4] + '-' + match[5];
+
+            // special calculation for cpus
+            if( /-cpu$/.test(series) ) {
+                // divide the stat by the number of cpus
+                var cpuCountMatch = intervalMatch[1].match(/(\d+) cpus/);
+                if( cpuCountMatch ) {
+                    datum = datum / cpuCountMatch[1];
                 }
             }
+
+            addJobGraphDatum( timestamp, datum, series, rawDetailData );
+
             window.redraw = true;
         }
     } catch( err ) {
@@ -191,6 +122,104 @@ function processLogLineForChart( logLine ) {
     }
 }
 
+function addJobGraphDatum(timestamp, datum, series, rawDetailData) {
+    if( $.inArray( series, jobGraphSeries ) < 0 ) {
+        var newIndex = jobGraphSeries.push(series) - 1;
+        jobGraphSortedSeries.push(newIndex);
+        jobGraphSortedSeries.sort( function(a,b) {
+            var matchA = jobGraphSeries[a].match(/^T(\d+)-(.*)/);
+            var matchB = jobGraphSeries[b].match(/^T(\d+)-(.*)/);
+            var termA = ('000000' + matchA[1]).slice(-6) + matchA[2];
+            var termB = ('000000' + matchB[1]).slice(-6) + matchB[2];
+            return termA > termB;
+        });
+        jobGraphMaxima[series] = null;
+        window.recreate = true;
+    }
+
+    if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
+        if( isJobSeriesRescalable(series) ) {
+            // use old maximum to get a scale conversion
+            var scaleConversion = jobGraphMaxima[series]/datum;
+            // set new maximum and rescale the series
+            jobGraphMaxima[series] = datum;
+            rescaleJobGraphSeries( series, scaleConversion );
+        }
+    }
+
+    // scale
+    var scaledDatum = null;
+    if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
+        scaledDatum = datum/jobGraphMaxima[series]
+    } else {
+        scaledDatum = datum;
+    }
+    // identify x axis point, searching from the end of the array (most recent)
+    var found = false;
+    for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
+        if( jobGraphData[i]['t'] === timestamp ) {
+            found = true;
+            jobGraphData[i][series] = scaledDatum;
+            jobGraphData[i]['raw-'+series] = rawDetailData;
+            break;
+        } else if( jobGraphData[i]['t'] < timestamp  ) {
+            // we've gone far enough back in time and this data is supposed to be sorted
+            break;
+        }
+    }
+    // index counter from previous loop will have gone one too far, so add one
+    var insertAt = i+1;
+    if(!found) {
+        // create a new x point for this previously unrecorded timestamp
+        var entry = { 't': timestamp };
+        entry[series] = scaledDatum;
+        entry['raw-'+series] = rawDetailData;
+        jobGraphData.splice( insertAt, 0, entry );
+        var shifted = [];
+        // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
+        while( jobGraphData.length > 0
+                 && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
+            shifted.push(jobGraphData.shift());
+        }
+        if( shifted.length > 0 ) {
+            // from those that we dropped, are any of them maxima? if so we need to rescale
+            jobGraphSeries.forEach( function(series) {
+                // test that every shifted entry in this series was either not a number (in which case we don't care)
+                // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
+                // because otherwise we just scrolled off something that was a maximum point
+                // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
+                if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
+                      && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
+                    // check the remaining displayed points and find the new (scaled) maximum
+                    var seriesMax = null;
+                    jobGraphData.forEach( function(entry) {
+                        if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
+                            seriesMax = entry[series];
+                        }
+                    });
+                    if( seriesMax !== null && seriesMax !== 0 ) {
+                        // set new actual maximum using the new maximum as the conversion conversion and rescale the series
+                        jobGraphMaxima[series] *= seriesMax;
+                        var scaleConversion = 1/seriesMax;
+                        rescaleJobGraphSeries( series, scaleConversion );
+                    }
+                    else {
+                        // we no longer have any data points displaying for this series
+                        jobGraphMaxima[series] = null;
+                    }
+                }
+            });
+        }
+        // add a 10 minute old null data point to keep the chart honest if the oldest point is less than 9.9 minutes old
+        if( jobGraphData.length > 0
+              && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 9.9*60000 > Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
+            var tenMinutesBefore = (new Date(Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf() - 600*1000)).toISOString().replace('Z','');
+            jobGraphData.unshift( { 't': tenMinutesBefore } );
+        }
+    }
+
+}
+
 function createJobGraph(elementName) {
     delete jobGraph;
     var emptyGraph = false;
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 1ae8e50..31bbfa9 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -57,6 +57,9 @@ class JobsController < ApplicationController
   end
 
   def push_logs
+    @push_logs = stderr_log_records([@object.uuid]).select do |log_record|
+      log_record.properties[:text] =~ /crunchstat: (mem.*swap.*rss|.*-- interval)/
+    end
   end
 
   def index_pane_list
diff --git a/apps/workbench/app/views/jobs/push_logs.js.erb b/apps/workbench/app/views/jobs/push_logs.js.erb
index 2501781..40ce037 100644
--- a/apps/workbench/app/views/jobs/push_logs.js.erb
+++ b/apps/workbench/app/views/jobs/push_logs.js.erb
@@ -1,3 +1,3 @@
-<% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']]).each do |log_record| %>
+<% @push_logs.each do |log_record| %>
 	processLogLineForChart( '<%=j log_record.properties[:text] %>' );
 <% end %>
\ No newline at end of file

commit 84ff06d37f920ef6cf400189ea19edcf6158993c
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date:   Thu Nov 20 14:21:08 2014 +0100

    4233: try to match older log date format

diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 542bb56..8c248ac 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -66,6 +66,12 @@ $(document).on('ajax:complete ready', function() {
 function processLogLineForChart( logLine ) {
     try {
         var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+        if( !match ) {
+            match = logLine.match(/((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+            if( match ) {
+                match[1] = (new Date(match[1] + ' UTC')).toISOString().replace('Z','');
+            }
+        }
         if( match ) {
             // the timestamp comes first
             var timestamp = match[1].replace('_','T');

commit a65acb3945d8646c616b4f8a2654d1a346fdaf2d
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date:   Thu Nov 20 14:18:07 2014 +0100

    4233: better hover presentation (left aligned, clearer task numbers)

diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index 74f15f7..542bb56 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -218,7 +218,8 @@ function createJobGraph(elementName) {
                 s += "<div class='morris-hover-point' style='color: ";
                 s += options.lineColors[sortedIndex];
                 s += "'>";
-                s += options.labels[sortedIndex];
+                var labelMatch = options.labels[sortedIndex].match(/^T(\d+)-(.*)/);
+                s += 'Task ' + labelMatch[1] + ' ' + labelMatch[2];
                 s += ": ";
                 if ( !(typeof datum === 'undefined') ) {
                     if( isJobSeriesRescalable( series ) ) {
diff --git a/apps/workbench/app/assets/stylesheets/jobs.css.scss b/apps/workbench/app/assets/stylesheets/jobs.css.scss
index f76c70b..565c354 100644
--- a/apps/workbench/app/assets/stylesheets/jobs.css.scss
+++ b/apps/workbench/app/assets/stylesheets/jobs.css.scss
@@ -8,3 +8,8 @@
     font-size: .8em;
     border: 2px solid black;
 }
+
+.morris-hover-point {
+	text-align: left;
+	width: 100%;
+}
\ No newline at end of file

commit e9a19734c359fa52ec07a3ea4bb81b6382864596
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date:   Thu Nov 20 13:39:11 2014 +0100

    4233: initial chart points added via AJAX

diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 5365182..1ae8e50 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -56,6 +56,9 @@ class JobsController < ApplicationController
     super
   end
 
+  def push_logs
+  end
+
   def index_pane_list
     if params[:uuid]
       %w(Recent Provenance)
diff --git a/apps/workbench/app/helpers/jobs_helper.rb b/apps/workbench/app/helpers/jobs_helper.rb
index aca0999..46f2601 100644
--- a/apps/workbench/app/helpers/jobs_helper.rb
+++ b/apps/workbench/app/helpers/jobs_helper.rb
@@ -24,9 +24,8 @@ module JobsHelper
     filters += extra_filters if extra_filters
     last_entry = Log.order('id DESC').limit(1).filter(filters).results.first
     if last_entry
-      filters += [["event_at", ">=", last_entry.event_at - 5.minutes]]
+      filters += [["event_at", ">=", last_entry.event_at - 10.minutes]]
       Log.order('id DESC')
-         .limit(250)
          .filter(filters)
          .results
     else
diff --git a/apps/workbench/app/views/jobs/_show_log.html.erb b/apps/workbench/app/views/jobs/_show_log.html.erb
index 6775cb8..64cd96a 100644
--- a/apps/workbench/app/views/jobs/_show_log.html.erb
+++ b/apps/workbench/app/views/jobs/_show_log.html.erb
@@ -22,10 +22,7 @@
   window.jobGraphSeries = [];
   window.jobGraphSortedSeries = [];
   window.jobGraphMaxima = {};
-  <% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']])
-        .each do |log_record| %>
-    processLogLineForChart( '<%=j log_record.properties[:text] %>' );
-  <% end %>
+  $.get('<%= push_logs_job_path(@object) %>');
   createJobGraph('log_graph_div');
 <% end %>
 
diff --git a/apps/workbench/app/views/jobs/push_logs.js.erb b/apps/workbench/app/views/jobs/push_logs.js.erb
new file mode 100644
index 0000000..2501781
--- /dev/null
+++ b/apps/workbench/app/views/jobs/push_logs.js.erb
@@ -0,0 +1,3 @@
+<% stderr_log_records([@object.uuid],[['properties','~','crunchstat:.*-- interval']]).each do |log_record| %>
+	processLogLineForChart( '<%=j log_record.properties[:text] %>' );
+<% end %>
\ No newline at end of file
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 9f9696f..820303d 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -22,6 +22,7 @@ ArvadosWorkbench::Application.routes.draw do
   resources :job_tasks
   resources :jobs do
     post 'cancel', :on => :member
+    get 'push_logs', :on => :member
   end
   match '/logout' => 'sessions#destroy', via: [:get, :post]
   get '/logged_out' => 'sessions#index'

commit 7b9112dbc270ea338fee756f583bb76870f2e391
Merge: 854af80 b77fb69
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date:   Thu Nov 20 13:19:03 2014 +0100

    Merge branch 'master' into 4233-graph-job-stats


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list