[ARVADOS] updated: 85c71a36173550f14fb1d5f4092f2050ec8dc033
git at public.curoverse.com
git at public.curoverse.com
Tue Jan 26 09:47:20 EST 2016
Summary of changes:
.../nodemanager/arvnodeman/computenode/__init__.py | 59 +++++++++++++---------
.../arvnodeman/computenode/driver/__init__.py | 5 ++
services/nodemanager/arvnodeman/launcher.py | 2 +-
services/nodemanager/tests/testutil.py | 7 +++
4 files changed, 49 insertions(+), 24 deletions(-)
via 85c71a36173550f14fb1d5f4092f2050ec8dc033 (commit)
from 9aa6066eef7a67809f33871447d4312af6f22749 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 85c71a36173550f14fb1d5f4092f2050ec8dc033
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Jan 25 17:02:40 2016 -0500
8206: Add test to support retry on create_driver.
diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index 1bea4eb..bc8ada5 100644
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -45,35 +45,48 @@ def arvados_node_missing(arvados_node, fresh_time):
return not timestamp_fresh(arvados_timestamp(arvados_node["last_ping_at"]), fresh_time)
def _retry(errors=()):
- """Retry decorator for an actor method that makes remote requests.
+ """Retry decorator for an method that makes remote requests.
+
+ Use this function to decorate method, and pass in a tuple of exceptions to
+ catch. If the original method raises a known cloud driver error, or any of
+ the given exception types, this decorator will either go into a
+ sleep-and-retry loop with exponential backoff either by sleeping (if
+ self._timer is None) or by scheduling retries of the method (if self._timer
+ is a timer actor.)
- Use this function to decorator an actor method, and pass in a
- tuple of exceptions to catch. This decorator will schedule
- retries of that method with exponential backoff if the
- original method raises a known cloud driver error, or any of the
- given exception types.
"""
+
def decorator(orig_func):
@functools.wraps(orig_func)
def retry_wrapper(self, *args, **kwargs):
start_time = time.time()
- try:
- return orig_func(self, *args, **kwargs)
- except Exception as error:
- if not (isinstance(error, errors) or
- self._cloud.is_cloud_exception(error)):
- raise
- self._logger.warning(
- "Client error: %s - waiting %s seconds",
- error, self.retry_wait)
- self._timer.schedule(start_time + self.retry_wait,
- getattr(self._later,
- orig_func.__name__),
- *args, **kwargs)
- self.retry_wait = min(self.retry_wait * 2,
- self.max_retry_wait)
- else:
- self.retry_wait = self.min_retry_wait
+ while True:
+ try:
+ ret = orig_func(self, *args, **kwargs)
+ except Exception as error:
+ if not (isinstance(error, errors) or
+ self._cloud.is_cloud_exception(error)):
+ raise
+ self._logger.warning(
+ "Client error: %s - waiting %s seconds",
+ error, self.retry_wait)
+ if self._timer:
+ # reschedule to be called again
+ self._timer.schedule(start_time + self.retry_wait,
+ getattr(self._later,
+ orig_func.__name__),
+ *args, **kwargs)
+ else:
+ # sleep on it.
+ time.sleep(self.retry_wait)
+ self.retry_wait = min(self.retry_wait * 2,
+ self.max_retry_wait)
+ if self._timer:
+ # expect to be called again by timer so don't loop
+ return
+ else:
+ self.retry_wait = self.min_retry_wait
+ return ret
return retry_wrapper
return decorator
diff --git a/services/nodemanager/arvnodeman/computenode/driver/__init__.py b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
index f0d5314..779209b 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
@@ -2,6 +2,7 @@
from __future__ import absolute_import, print_function
+import logging
from operator import attrgetter
import libcloud.common.types as cloud_types
@@ -45,6 +46,10 @@ class BaseComputeNodeDriver(object):
"""
self.min_retry_wait = retry_wait
self.max_retry_wait = max_retry_wait
+ self.retry_wait = retry_wait
+ self._cloud = type(self)
+ self._logger = logging.getLogger(str(self._cloud))
+ self._timer = None
self.real = self._create_driver(driver_class, **auth_kwargs)
self.list_kwargs = list_kwargs
self.create_kwargs = create_kwargs
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index e8c2fe6..1ee7928 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -71,7 +71,7 @@ def launch_pollers(config, server_calculator):
timer = TimedCallBackActor.start(poll_time / 10.0).proxy()
cloud_node_poller = CloudNodeListMonitorActor.start(
- config.new_cloud_client(), timer, poll_time, max_poll_time).proxy()
+ config.new_cloud_client(timer), timer, poll_time, max_poll_time).proxy()
arvados_node_poller = ArvadosNodeListMonitorActor.start(
config.new_arvados_client(), timer, poll_time, max_poll_time).proxy()
job_queue_poller = JobQueueMonitorActor.start(
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index e543c28..b266ed6 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -130,6 +130,13 @@ class DriverTestMixin(object):
def driver_method_args(self, method_name):
return getattr(self.driver_mock(), method_name).call_args
+ def test_driver_create_retry(self):
+ driver_mock2 = mock.MagicMock(name='driver_mock2')
+ self.driver_mock.side_effect = (Exception("oops"), driver_mock2)
+ kwargs = {'user_id': 'foo'}
+ driver = self.new_driver(auth_kwargs=kwargs)
+ self.assertTrue(self.driver_mock.called)
+ self.assertIs(driver.real, driver_mock2)
class RemotePollLoopActorTestMixin(ActorTestMixin):
def build_monitor(self, *args, **kwargs):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list