[ARVADOS] updated: 1.1.3-255-gd275d76
Git user
git at public.curoverse.com
Wed Mar 28 14:52:23 EDT 2018
Summary of changes:
.../pipeline_instances/_running_component.html.erb | 4 +-
.../views/work_units/_component_detail.html.erb | 2 +-
.../test/integration/anonymous_access_test.rb | 6 +-
build/run-build-packages-one-target.sh | 17 ++-
doc/sdk/python/cookbook.html.textile.liquid | 42 +++++++
doc/user/cwl/cwl-extensions.html.textile.liquid | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 13 ++-
sdk/cwl/setup.py | 2 +-
sdk/go/arvados/keep_service.go | 34 +++++-
sdk/python/arvados/collection.py | 6 +
services/keep-balance/balance.go | 108 +++++++++++++----
services/keep-balance/balance_run_test.go | 128 ++++++++++++++++++---
services/keep-balance/balance_test.go | 48 +++++++-
services/keep-balance/block_state.go | 12 +-
services/keep-balance/keep_service.go | 26 +++++
services/keepstore/azure_blob_volume.go | 2 +-
services/keepstore/handlers.go | 4 +-
services/keepstore/mounts_test.go | 10 +-
services/keepstore/pull_worker_test.go | 2 +-
services/keepstore/volume.go | 24 ++--
.../arvnodeman/computenode/dispatch/__init__.py | 6 +
services/nodemanager/arvnodeman/daemon.py | 5 +-
services/nodemanager/arvnodeman/jobqueue.py | 13 ++-
services/nodemanager/arvnodeman/status.py | 22 +++-
services/nodemanager/tests/integration_test.py | 84 ++++++++++----
.../nodemanager/tests/test_computenode_dispatch.py | 10 ++
services/nodemanager/tests/test_daemon.py | 23 +++-
services/nodemanager/tests/test_jobqueue.py | 12 +-
services/nodemanager/tests/test_status.py | 26 +++++
29 files changed, 572 insertions(+), 121 deletions(-)
via d275d768486e8ad353aea08bd60c61f921d799c9 (commit)
via 714fb0cc841b9d619af48eceee922d8b6b594e92 (commit)
via f1ee0cb2f85c4ea988700da2e3957565b6861ad5 (commit)
via b17321254d9bdfb0939d3705b4b110d7df75afe2 (commit)
via 66aaa7d715eedd38265ee0987ef2a5cf7b623fd9 (commit)
via 9e92ae4c7fb2ca196b1dd9bc96e2944f1c8f8f8e (commit)
via f728be219a1a3cd2cc65f52c3160d76d17a96656 (commit)
via a15ab44d2b2dad42a32dc0179ccdd686586893f3 (commit)
via f94fdceb54ce870be790b7830756d8c04ba2e26e (commit)
via d85b7e29a4b7304e746ae2511b64473d88efc6e5 (commit)
via 9059fc9d13db73bde93b589955a941af44b4447d (commit)
via d63d7dc79ed74beaaceda15ca88344de12258da3 (commit)
via 50542359e38f9963ff6864c919e94edb286d69d8 (commit)
via 1d5934b254afc9e6082a154b5c365df968b3a65f (commit)
via b60d064e281243088d3dd63fcb99478344b2a8de (commit)
via 9b53c7be9d77f2175944e80f0daf701f32fc3e1f (commit)
via 25b62d2b7ba5273928ee98085fa2f4674caa67e6 (commit)
via cfc90c7c90afe0e953aaf1ec0a4ebf539898d8f5 (commit)
via 44426069373d8657706d796f37618b9d7f6eca59 (commit)
via 41e612b59ad85ee7f22ebf3239ec8ff1cbb463c5 (commit)
via 8e96f8cacd728b1a1b4316ecb2fb7cd233a97144 (commit)
via 741acb186f89237011fd7bf371218246d7f85403 (commit)
via 61f4a861fb993f68240d884c3e9cc1e750d0753b (commit)
via 388c54e21fafcfbf01e0fe6775a6b62134605990 (commit)
via 93d9c63e1fd4d4c268aa9050406b9a212f3d4a22 (commit)
via 9746d8b3d8c259836799656ff6e5f401d9d4d492 (commit)
via 2dd214985e9727bf0b5fbf11b0e39c3e7d3cc5c4 (commit)
from c087f6a8ff1220d4abf8201e04ecd60c53815ff6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d275d768486e8ad353aea08bd60c61f921d799c9
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Mar 28 15:51:14 2018 -0300
12085: Idle node times tracking, with tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
index 340668e..9106ea6 100644
--- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
@@ -413,6 +413,12 @@ class ComputeNodeMonitorActor(config.actor_class):
#if state == 'idle' and self.arvados_node['job_uuid']:
# state = 'busy'
+ # Update idle node times tracker
+ if state == 'idle':
+ status.tracker.idle_in(self.arvados_node['hostname'])
+ else:
+ status.tracker.idle_out(self.arvados_node['hostname'])
+
return state
def in_state(self, *states):
diff --git a/services/nodemanager/arvnodeman/status.py b/services/nodemanager/arvnodeman/status.py
index b2eb7c8..20e30e1 100644
--- a/services/nodemanager/arvnodeman/status.py
+++ b/services/nodemanager/arvnodeman/status.py
@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function
from future import standard_library
import http.server
+import time
import json
import logging
import socketserver
@@ -82,10 +83,16 @@ class Tracker(object):
'actor_exceptions': 0
}
self._version = {'Version' : __version__}
+ self._idle_nodes = {}
def get_json(self):
with self._mtx:
- return json.dumps(dict(self._latest, **self._version))
+ times = {'idle_times' : {}}
+ now = time.time()
+ for node, ts in self._idle_nodes.items():
+ times['idle_times'][node] = int(now - ts)
+ return json.dumps(
+ dict(dict(self._latest, **self._version), **times))
def keys(self):
with self._mtx:
@@ -104,4 +111,17 @@ class Tracker(object):
self._latest.setdefault(counter, 0)
self._latest[counter] += value
+ def idle_in(self, nodename):
+ with self._mtx:
+ if self._idle_nodes.get(nodename):
+ return
+ self._idle_nodes[nodename] = time.time()
+
+ def idle_out(self, nodename):
+ with self._mtx:
+ try:
+ del self._idle_nodes[nodename]
+ except KeyError:
+ pass
+
tracker = Tracker()
diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index d93c940..5775aa6 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -376,16 +376,26 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
self.assertTrue(self.node_state('down'))
def test_in_idle_state(self):
+ idle_nodes_before = status.tracker._idle_nodes.keys()
self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
self.assertTrue(self.node_state('idle'))
self.assertFalse(self.node_state('busy'))
self.assertTrue(self.node_state('idle', 'busy'))
+ idle_nodes_after = status.tracker._idle_nodes.keys()
+ new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before]
+ # There should be 1 additional idle node
+ self.assertEqual(1, len(new_idle_nodes))
def test_in_busy_state(self):
+ idle_nodes_before = status.tracker._idle_nodes.keys()
self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
self.assertFalse(self.node_state('idle'))
self.assertTrue(self.node_state('busy'))
self.assertTrue(self.node_state('idle', 'busy'))
+ idle_nodes_after = status.tracker._idle_nodes.keys()
+ new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before]
+ # There shouldn't be any additional idle node
+ self.assertEqual(0, len(new_idle_nodes))
def test_init_shutdown_scheduling(self):
self.make_actor()
diff --git a/services/nodemanager/tests/test_status.py b/services/nodemanager/tests/test_status.py
index a3f0d17..910ad8a 100644
--- a/services/nodemanager/tests/test_status.py
+++ b/services/nodemanager/tests/test_status.py
@@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function
from future import standard_library
import json
+import mock
import requests
import unittest
@@ -72,6 +73,31 @@ class StatusServerUpdates(unittest.TestCase):
resp = srv.get_status()
self.assertEqual(count, resp['a_counter'])
+ @mock.patch('time.time')
+ def test_idle_times(self, time_mock):
+ with TestServer() as srv:
+ resp = srv.get_status()
+ # Test idle_times initial value
+ self.assertIn('idle_times', resp)
+ self.assertEqual({}, resp['idle_times'])
+ # Test add an idle node
+ time_mock.return_value = 10
+ status.tracker.idle_in('compute42')
+ time_mock.return_value += 10
+ resp = srv.get_status()
+ self.assertEqual(10, resp['idle_times']['compute42'])
+ # Test adding the same idle node a 2nd time
+ time_mock.return_value += 10
+ status.tracker.idle_in('compute42')
+ time_mock.return_value += 10
+ resp = srv.get_status()
+ # Idle timestamp doesn't get reset if already exists
+ self.assertEqual(30, resp['idle_times']['compute42'])
+ # Test remove idle node
+ status.tracker.idle_out('compute42')
+ resp = srv.get_status()
+ self.assertNotIn('compute42', resp['idle_times'])
+
class StatusServerDisabled(unittest.TestCase):
def test_config_disabled(self):
commit 714fb0cc841b9d619af48eceee922d8b6b594e92
Merge: c087f6a f1ee0cb
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Mar 27 18:31:04 2018 -0300
Merge branch 'master' into 12085-anm-metrics
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list