[ARVADOS] created: 6dd8a072ec2e305df687f72dd294d760ae6c8e23

[Git user]

        at  6dd8a072ec2e305df687f72dd294d760ae6c8e23 (commit)


commit 6dd8a072ec2e305df687f72dd294d760ae6c8e23
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 8 16:56:10 2017 -0400

    11345: Clamp retry-after to (0, max_retry_wait).  Deindent retry_wrapper a bit for readability.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index b11b2de..d2c3d0c 100644
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -73,56 +73,66 @@ class RetryMixin(object):
             @functools.wraps(orig_func)
             def retry_wrapper(self, *args, **kwargs):
                 while True:
+                    should_retry = False
                     try:
                         ret = orig_func(self, *args, **kwargs)
-                    except Exception as error:
-                        should_retry = False
-
-                        if isinstance(error, BaseHTTPError):
-                            if error.headers and error.headers.get("retry-after"):
-                                try:
-                                    self.retry_wait = int(error.headers["retry-after"])
-                                    should_retry = True
-                                except ValueError:
-                                    pass
-                            if error.code == 429 or error.code >= 500:
+                    except BaseHTTPError as error:
+                        if error.headers and error.headers.get("retry-after"):
+                            try:
+                                self.retry_wait = int(error.headers["retry-after"])
+                                if self.retry_wait < 0 or self.retry_wait > self.max_retry_wait:
+                                    self.retry_wait = self.max_retry_wait
                                 should_retry = True
-                        elif isinstance(error, CLOUD_ERRORS) or isinstance(error, errors) or type(error) is Exception:
+                            except ValueError:
+                                pass
+                        if error.code == 429 or error.code >= 500:
                             should_retry = True
-
-                        if not should_retry:
-                            self.retry_wait = self.min_retry_wait
-                            self._logger.warning(
-                                "Re-raising error (no retry): %s",
-                                error, exc_info=error)
-                            raise
-
-                        self._logger.warning(
-                            "Client error: %s - %s %s seconds",
-                            error,
-                            "scheduling retry in" if self._timer else "sleeping",
-                            self.retry_wait,
-                            exc_info=error)
-
-                        if self._timer:
-                            start_time = time.time()
-                            # reschedule to be called again
-                            self._timer.schedule(start_time + self.retry_wait,
-                                                 getattr(self._later,
-                                                         orig_func.__name__),
-                                                 *args, **kwargs)
-                        else:
-                            # sleep on it.
-                            time.sleep(self.retry_wait)
-
-                        self.retry_wait = min(self.retry_wait * 2,
-                                              self.max_retry_wait)
-                        if self._timer:
-                            # expect to be called again by timer so don't loop
-                            return
+                    except CLOUD_ERRORS as error:
+                        should_retry = True
+                    except errors as error:
+                        should_retry = True
+                    except Exception as error:
+                        # As a libcloud workaround for drivers that don't use
+                        # typed exceptions, consider bare Exception() objects
+                        # retryable.
+                        should_retry = type(error) is Exception
                     else:
+                        # No exception,
                         self.retry_wait = self.min_retry_wait
                         return ret
+
+                    # Only got here if an exception was caught.  Now determine what to do about it.
+                    if not should_retry:
+                        self.retry_wait = self.min_retry_wait
+                        self._logger.warning(
+                            "Re-raising error (no retry): %s",
+                            error, exc_info=error)
+                        raise
+
+                    self._logger.warning(
+                        "Client error: %s - %s %s seconds",
+                        error,
+                        "scheduling retry in" if self._timer else "sleeping",
+                        self.retry_wait,
+                        exc_info=error)
+
+                    if self._timer:
+                        start_time = time.time()
+                        # reschedule to be called again
+                        self._timer.schedule(start_time + self.retry_wait,
+                                             getattr(self._later,
+                                                     orig_func.__name__),
+                                             *args, **kwargs)
+                    else:
+                        # sleep on it.
+                        time.sleep(self.retry_wait)
+
+                    self.retry_wait = min(self.retry_wait * 2,
+                                          self.max_retry_wait)
+                    if self._timer:
+                        # expect to be called again by timer so don't loop
+                        return
+
             return retry_wrapper
         return decorator
 
diff --git a/services/nodemanager/tests/integration_test.py b/services/nodemanager/tests/integration_test.py
index ad0e683..f024b0c 100755
--- a/services/nodemanager/tests/integration_test.py
+++ b/services/nodemanager/tests/integration_test.py
@@ -332,7 +332,7 @@ def main():
              "34t0i-dz642-h42bg3hq4bdfpf3": "ReqNodeNotAvail",
              "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"
          }),
-        "test6": (
+        "test_retry_create": (
             [
                 (r".*Daemon started", set_squeue),
                 (r".*Rate limit exceeded - scheduling retry in 12 seconds", noop),

-----------------------------------------------------------------------


hooks/post-receive
--