[ARVADOS] updated: b65b2b9df921793fd1d4fca20b1e0738db446489

[Git user]

Summary of changes:
 .../nodemanager/arvnodeman/computenode/__init__.py | 32 ++++++++++++++--------
 1 file changed, 21 insertions(+), 11 deletions(-)

       via  b65b2b9df921793fd1d4fca20b1e0738db446489 (commit)
      from  2e185453aeda50d3c7f3443b5b17455cebf425de (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b65b2b9df921793fd1d4fca20b1e0738db446489
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Sep 25 10:44:26 2017 -0300

    12195: Log warning message when a retry-after header is not an integer.
    Make sure min_retry_wait & max_retry_wait have sensible values.
    Check that the effective retry wait is within bounds and doesn't get reset
    when retry-after is zero, so it continues the exponential backoff plan.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index ce114e1..00b023f 100644
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -61,10 +61,9 @@ class RetryMixin(object):
     is a timer actor.)
 
     """
-    def __init__(self, retry_wait, max_retry_wait,
-                 logger, cloud, timer=None):
-        self.min_retry_wait = retry_wait
-        self.max_retry_wait = max_retry_wait
+    def __init__(self, retry_wait, max_retry_wait, logger, cloud, timer=None):
+        self.min_retry_wait = max(1, retry_wait)
+        self.max_retry_wait = max(self.min_retry_wait, max_retry_wait)
         self.retry_wait = retry_wait
         self._logger = logger
         self._cloud = cloud
@@ -80,16 +79,24 @@ class RetryMixin(object):
                     try:
                         ret = orig_func(self, *args, **kwargs)
                     except RateLimitReachedError as error:
-                        self.retry_wait = error.retry_after
+                        # If retry-after is zero, continue with exponential
+                        # backoff.
+                        if error.retry_after != 0:
+                            self.retry_wait = error.retry_after
                         should_retry = True
                     except BaseHTTPError as error:
                         if error.headers and error.headers.get("retry-after"):
                             try:
-                                self.retry_wait = \
-                                    int(error.headers["retry-after"])
+                                retry_after = int(error.headers["retry-after"])
+                                # If retry-after is zero, continue with
+                                # exponential backoff.
+                                if retry_after != 0:
+                                    self.retry_wait = retry_after
                                 should_retry = True
                             except ValueError:
-                                pass
+                                self._logger.warning(
+                                    "Unrecognizable Retry-After header: %s",
+                                    error, exc_info=error)
                         if error.code == 429 or error.code >= 500:
                             should_retry = True
                     except CLOUD_ERRORS as error:
@@ -106,9 +113,6 @@ class RetryMixin(object):
                         self.retry_wait = self.min_retry_wait
                         return ret
 
-                    if self.retry_wait < 0 or self.retry_wait > self.max_retry_wait:
-                        self.retry_wait = self.max_retry_wait
-
                     # Only got here if an exception was caught.  Now determine what to do about it.
                     if not should_retry:
                         self.retry_wait = self.min_retry_wait
@@ -117,6 +121,12 @@ class RetryMixin(object):
                             error, exc_info=error)
                         raise
 
+                    # Retry wait out of bounds?
+                    if self.retry_wait < self.min_retry_wait:
+                        self.retry_wait = self.min_retry_wait
+                    elif self.retry_wait > self.max_retry_wait:
+                        self.retry_wait = self.max_retry_wait
+
                     self._logger.warning(
                         "Client error: %s - %s %s seconds",
                         error,

-----------------------------------------------------------------------


hooks/post-receive
--