[ARVADOS] updated: eace27b5fe2e945d68ef2d617fbae8f23f2b808a
Git user
git at public.curoverse.com
Mon Jun 5 16:40:35 EDT 2017
Summary of changes:
.../arvnodeman/computenode/driver/__init__.py | 7 +-----
services/nodemanager/arvnodeman/config.py | 4 ++--
services/nodemanager/tests/integration_test.py | 3 ++-
.../nodemanager/tests/test_computenode_dispatch.py | 18 +++++++--------
services/nodemanager/tests/test_daemon.py | 26 +++++++++++-----------
services/nodemanager/tests/test_jobqueue.py | 8 ++++---
6 files changed, 32 insertions(+), 34 deletions(-)
via eace27b5fe2e945d68ef2d617fbae8f23f2b808a (commit)
from 81aa936c50bda0257d62dd59e45d5afba73a4bac (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit eace27b5fe2e945d68ef2d617fbae8f23f2b808a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Jun 5 16:40:30 2017 -0400
10312: Fix unit tests.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/__init__.py b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
index 29b0484..4420341 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
@@ -25,7 +25,7 @@ class BaseComputeNodeDriver(RetryMixin):
Subclasses must implement arvados_create_kwargs, sync_node,
node_fqdn, and node_start_time.
"""
- CLOUD_ERRORS = NETWORK_ERRORS + (cloud_types.LibcloudError,)
+ CLOUD_ERRORS = NETWORK_ERRORS + (cloud_types.LibcloudError, BaseHTTPError)
@RetryMixin._retry()
def _create_driver(self, driver_class, **auth_kwargs):
@@ -211,11 +211,6 @@ class BaseComputeNodeDriver(RetryMixin):
# libcloud compute drivers typically raise bare Exceptions to
# represent API errors. Return True for any exception that is
# exactly an Exception, or a better-known higher-level exception.
- if (type(exception) is BaseHTTPError and
- exception.message and
- (exception.message.startswith("InvalidInstanceID.NotFound") or
- exception.message.startswith("InstanceLimitExceeded"))):
- return True
return (isinstance(exception, cls.CLOUD_ERRORS) or
type(exception) is Exception)
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index f4ffff5..f884295 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -53,7 +53,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
'Manage': {'address': '127.0.0.1',
'port': '-1'},
'Logging': {'file': '/dev/stderr',
- 'level': 'WARNING'},
+ 'level': 'WARNING'}
}.iteritems():
if not self.has_section(sec_name):
self.add_section(sec_name)
@@ -107,7 +107,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
module = importlib.import_module('arvnodeman.computenode.driver.' +
self.get('Cloud', 'provider'))
driver_class = module.ComputeNodeDriver.DEFAULT_DRIVER
- if self.get('Cloud', 'driver_class'):
+ if self.has_option('Cloud', 'driver_class'):
d = self.get('Cloud', 'driver_class').split('.')
mod = '.'.join(d[:-1])
cls = d[-1]
diff --git a/services/nodemanager/tests/integration_test.py b/services/nodemanager/tests/integration_test.py
index 53cbc10..4813564 100755
--- a/services/nodemanager/tests/integration_test.py
+++ b/services/nodemanager/tests/integration_test.py
@@ -300,4 +300,5 @@ def main():
exit(code)
-main()
+if __name__ == '__main__':
+ main()
diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index c718dad..b950cc1 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -90,27 +90,27 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
- def test_unknown_basehttperror_not_retried(self):
+ def test_basehttperror_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
- BaseHTTPError(400, "Unknown"),
+ BaseHTTPError(500, "Try again"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- finished = threading.Event()
- self.setup_actor.subscribe(lambda _: finished.set())
- assert(finished.wait(self.TIMEOUT))
- self.assertEqual(0, self.cloud_client.post_create_node.call_count)
+ self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ self.assertEqual(1, self.cloud_client.post_create_node.call_count)
- def test_known_basehttperror_retried(self):
+ def test_instance_exceeded_not_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "InstanceLimitExceeded"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- self.wait_for_assignment(self.setup_actor, 'cloud_node')
- self.assertEqual(1, self.cloud_client.post_create_node.call_count)
+ done = self.FUTURE_CLASS()
+ self.setup_actor.subscribe(done.set)
+ done.get(self.TIMEOUT)
+ self.assertEqual(0, self.cloud_client.post_create_node.call_count)
def test_failed_post_create_retried(self):
self.make_mocks()
diff --git a/services/nodemanager/tests/test_daemon.py b/services/nodemanager/tests/test_daemon.py
index 04ff9b6..c8af72a 100644
--- a/services/nodemanager/tests/test_daemon.py
+++ b/services/nodemanager/tests/test_daemon.py
@@ -305,7 +305,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
cloud_node = testutil.cloud_node_mock(4)
arv_node = testutil.arvados_node_mock(4, crunch_worker_state='down')
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_arvados_nodes([arv_node])
self.daemon.update_cloud_nodes([cloud_node])
@@ -322,13 +322,13 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
cloud_node = testutil.cloud_node_mock(1)
setup = self.start_node_boot(cloud_node)
self.daemon.update_cloud_nodes([cloud_node])
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
def test_no_duplication_when_booted_node_listed(self):
cloud_node = testutil.cloud_node_mock(2)
setup = self.start_node_boot(cloud_node, id_num=2)
- self.daemon.node_up(setup)
+ self.daemon.node_setup_finished(setup)
self.daemon.update_cloud_nodes([cloud_node]).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
@@ -337,14 +337,14 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
# even it doesn't appear in the listing (e.g., because of delays
# propagating tags).
setup = self.start_node_boot()
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
def test_booted_unlisted_node_counted(self):
setup = self.start_node_boot(id_num=1)
- self.daemon.node_up(setup)
+ self.daemon.node_setup_finished(setup)
self.daemon.update_server_wishlist(
[testutil.MockSize(1)]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
@@ -352,7 +352,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
def test_booted_node_can_shutdown(self):
setup = self.start_node_boot()
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
monitor = self.monitor_list()[0].proxy()
self.daemon.update_server_wishlist([])
@@ -370,7 +370,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
def test_booted_node_lifecycle(self):
cloud_node = testutil.cloud_node_mock(6)
setup = self.start_node_boot(cloud_node, id_num=6)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
monitor = self.monitor_list()[0].proxy()
self.daemon.update_server_wishlist([])
@@ -393,7 +393,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
def test_booted_node_shut_down_when_never_listed(self):
setup = self.start_node_boot()
self.cloud_factory().node_start_time.return_value = time.time() - 3601
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.assertFalse(self.node_shutdown.start.called)
now = time.time()
@@ -406,7 +406,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
cloud_node = testutil.cloud_node_mock(2)
setup = self.start_node_boot(cloud_node)
self.cloud_factory().node_start_time.return_value = time.time() - 3601
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.monitor_list()[0].tell_proxy().consider_shutdown()
@@ -419,7 +419,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
arv_node = testutil.arvados_node_mock(4, crunch_worker_state='down')
setup = self.start_node_boot(cloud_node, arv_node)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.monitor_list()[0].proxy().cloud_node_start_time = time.time()-3601
self.daemon.update_cloud_nodes([cloud_node])
@@ -431,7 +431,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
cloud_node = testutil.cloud_node_mock(3)
arv_node = testutil.arvados_node_mock(3)
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
@@ -443,7 +443,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
cloud_node = testutil.cloud_node_mock(5)
arv_node = testutil.arvados_node_mock(5, job_uuid=True)
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
@@ -581,7 +581,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
new_node.stop_if_no_cloud_node.reset_mock()
self.daemon.shutdown().get(self.TIMEOUT)
self.assertTrue(new_node.stop_if_no_cloud_node.called)
- self.daemon.node_up(new_node).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(new_node).get(self.TIMEOUT)
self.assertTrue(new_node.stop.called)
self.timer.deliver()
self.assertTrue(
diff --git a/services/nodemanager/tests/test_jobqueue.py b/services/nodemanager/tests/test_jobqueue.py
index 554ff1e..9f7730b 100644
--- a/services/nodemanager/tests/test_jobqueue.py
+++ b/services/nodemanager/tests/test_jobqueue.py
@@ -142,7 +142,7 @@ class JobQueueMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
def test_subscribers_get_server_lists(self, mock_squeue):
mock_squeue.return_value = ""
- self.build_monitor([{'items': [1, 2]}], self.MockCalculator())
+ self.build_monitor([{'items': [1, 2]}], self.MockCalculator(), True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),
@@ -155,7 +155,8 @@ class JobQueueMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
- [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]))
+ [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]),
+ True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),
@@ -168,7 +169,8 @@ class JobQueueMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
- [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]))
+ [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]),
+ True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list