[ARVADOS] updated: 314f3c34442e0daffbea4ed62adea9e722673810
Git user
git at public.curoverse.com
Wed Feb 15 15:14:04 EST 2017
Summary of changes:
.../nodemanager/arvnodeman/computenode/dispatch/slurm.py | 9 +++++----
services/nodemanager/arvnodeman/daemon.py | 6 +++---
services/nodemanager/arvnodeman/launcher.py | 2 +-
services/nodemanager/arvnodeman/nodelist.py | 14 ++++++++++++--
services/nodemanager/tests/test_nodelist.py | 9 ++++++---
5 files changed, 27 insertions(+), 13 deletions(-)
via 314f3c34442e0daffbea4ed62adea9e722673810 (commit)
via 15c1329834e02814fb5b510bf13f1ea2a97ee028 (commit)
via 0e6c1a287933d8e55508c2457b7ce31a2bb5a965 (commit)
from 908959eafe8a9925cb2c204d6511095f702c1667 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 314f3c34442e0daffbea4ed62adea9e722673810
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Feb 15 15:08:39 2017 -0500
6520: Check that hostname is set before setting weight.
diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py b/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py
index 6c5cfc8..cc344e5 100644
--- a/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/slurm.py
@@ -79,8 +79,9 @@ class ComputeNodeShutdownActor(SlurmMixin, ShutdownActorBase):
class ComputeNodeUpdateActor(UpdateActorBase):
def sync_node(self, cloud_node, arvados_node):
- try:
- subprocess.check_output(['scontrol', 'update', 'NodeName=' + arvados_node["hostname"], 'Weight=%i' % int(cloud_node.size.price * 1000)])
- except OSError:
- self._logger.warn("Unable to set slurm node weight.", exc_info=True)
+ if arvados_node.get("hostname"):
+ try:
+ subprocess.check_output(['scontrol', 'update', 'NodeName=' + arvados_node["hostname"], 'Weight=%i' % int(cloud_node.size.price * 1000)])
+ except:
+ self._logger.error("Unable to set slurm node weight.", exc_info=True)
return super(ComputeNodeUpdateActor, self).sync_node(cloud_node, arvados_node)
commit 15c1329834e02814fb5b510bf13f1ea2a97ee028
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Feb 15 14:56:16 2017 -0500
6520: Apply node size fixup in CloudNodeListMonitorActor
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 6be87b1..dcbc87c 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -155,7 +155,6 @@ class NodeManagerDaemonActor(actor_class):
start_time = self._cloud_driver.node_start_time(cloud_node)
shutdown_timer = cnode.ShutdownTimer(start_time,
self.shutdown_windows)
- cloud_node.size = self.server_calculator.find_size(cloud_node.size.id)
actor = self._node_actor.start(
cloud_node=cloud_node,
cloud_node_start_time=start_time,
@@ -273,9 +272,9 @@ class NodeManagerDaemonActor(actor_class):
def _total_price(self):
cost = 0
- cost += sum(self.server_calculator.find_size(self.sizes_booting[c].id).price
- for c in self.booting.iterkeys())
- cost += sum(self.server_calculator.find_size(c.cloud_node.size.id).price
+ cost += sum(self.sizes_booting[c].price
+ for c in self.booting.iterkeys())
+ cost += sum(c.cloud_node.size.price
for c in self.cloud_nodes.nodes.itervalues())
return cost
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 4fe5f8a..9c45318 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -79,7 +79,7 @@ def launch_pollers(config, server_calculator):
timer = TimedCallBackActor.start(poll_time / 10.0).tell_proxy()
cloud_node_poller = CloudNodeListMonitorActor.start(
- config.new_cloud_client(), timer, poll_time, max_poll_time).tell_proxy()
+ config.new_cloud_client(), timer, server_calculator, poll_time, max_poll_time).tell_proxy()
arvados_node_poller = ArvadosNodeListMonitorActor.start(
config.new_arvados_client(), timer, poll_time, max_poll_time).tell_proxy()
job_queue_poller = JobQueueMonitorActor.start(
diff --git a/services/nodemanager/arvnodeman/nodelist.py b/services/nodemanager/arvnodeman/nodelist.py
index 5b02fde..37142fe 100644
--- a/services/nodemanager/arvnodeman/nodelist.py
+++ b/services/nodemanager/arvnodeman/nodelist.py
@@ -52,6 +52,11 @@ class CloudNodeListMonitorActor(clientactor.RemotePollLoopActor):
nodes, and sends it to subscribers.
"""
+ def __init__(self, client, timer_actor, server_calc, *args, **kwargs):
+ super(CloudNodeListMonitorActor, self).__init__(
+ client, timer_actor, *args, **kwargs)
+ self._calculator = server_calc
+
def is_common_error(self, exception):
return self._client.is_cloud_exception(exception)
@@ -59,5 +64,10 @@ class CloudNodeListMonitorActor(clientactor.RemotePollLoopActor):
return node.id
def _send_request(self):
- n = self._client.list_nodes()
- return n
+ nodes = self._client.list_nodes()
+ for n in nodes:
+ # Replace with libcloud NodeSize object with compatible
+ # CloudSizeWrapper object which merges the size info reported from
+ # the cloud with size information from the configuration file.
+ n.size = self._calculator.find_size(n.size.id)
+ return nodes
diff --git a/services/nodemanager/tests/test_nodelist.py b/services/nodemanager/tests/test_nodelist.py
index 8ceaca6..3096035 100644
--- a/services/nodemanager/tests/test_nodelist.py
+++ b/services/nodemanager/tests/test_nodelist.py
@@ -6,6 +6,7 @@ import unittest
import mock
import arvnodeman.nodelist as nodelist
+from libcloud.compute.base import NodeSize
from . import testutil
class ArvadosNodeListMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
@@ -66,7 +67,7 @@ class CloudNodeListMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
self.name = 'test{}.example.com'.format(count)
self.private_ips = ['10.0.0.{}'.format(count)]
self.public_ips = []
- self.size = None
+ self.size = testutil.MockSize(1)
self.state = 0
@@ -77,11 +78,13 @@ class CloudNodeListMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
def test_id_is_subscription_key(self):
node = self.MockNode(1)
- self.build_monitor([[node]])
+ mock_calc = mock.MagicMock()
+ mock_calc.find_size.return_value = testutil.MockSize(2)
+ self.build_monitor([[node]], mock_calc)
self.monitor.subscribe_to('1', self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with(node)
-
+ self.assertEqual(testutil.MockSize(2), node.size)
if __name__ == '__main__':
unittest.main()
commit 0e6c1a287933d8e55508c2457b7ce31a2bb5a965
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Feb 15 12:28:36 2017 -0500
6520: Update cloud_node.size to match server calculator configuration.
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 63c9d19..6be87b1 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -155,6 +155,7 @@ class NodeManagerDaemonActor(actor_class):
start_time = self._cloud_driver.node_start_time(cloud_node)
shutdown_timer = cnode.ShutdownTimer(start_time,
self.shutdown_windows)
+ cloud_node.size = self.server_calculator.find_size(cloud_node.size.id)
actor = self._node_actor.start(
cloud_node=cloud_node,
cloud_node_start_time=start_time,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list