[ARVADOS] created: 2e185453aeda50d3c7f3443b5b17455cebf425de

[Git user]

        at  2e185453aeda50d3c7f3443b5b17455cebf425de (commit)


commit 2e185453aeda50d3c7f3443b5b17455cebf425de
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Wed Sep 20 17:01:08 2017 -0300

    12195: Catch the special case when a RateLimitReachedError exception is raised,
    as it doesn't have a header attribute like its parent class.
    Updated integration test to add a retry with this kind of error.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index 8a4e5f3..ce114e1 100644
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -12,7 +12,7 @@ import re
 import time
 
 from ..config import CLOUD_ERRORS
-from libcloud.common.exceptions import BaseHTTPError
+from libcloud.common.exceptions import BaseHTTPError, RateLimitReachedError
 
 ARVADOS_TIMEFMT = '%Y-%m-%dT%H:%M:%SZ'
 ARVADOS_TIMESUBSEC_RE = re.compile(r'(\.\d+)Z$')
@@ -79,12 +79,14 @@ class RetryMixin(object):
                     should_retry = False
                     try:
                         ret = orig_func(self, *args, **kwargs)
+                    except RateLimitReachedError as error:
+                        self.retry_wait = error.retry_after
+                        should_retry = True
                     except BaseHTTPError as error:
                         if error.headers and error.headers.get("retry-after"):
                             try:
-                                self.retry_wait = int(error.headers["retry-after"])
-                                if self.retry_wait < 0 or self.retry_wait > self.max_retry_wait:
-                                    self.retry_wait = self.max_retry_wait
+                                self.retry_wait = \
+                                    int(error.headers["retry-after"])
                                 should_retry = True
                             except ValueError:
                                 pass
@@ -104,6 +106,9 @@ class RetryMixin(object):
                         self.retry_wait = self.min_retry_wait
                         return ret
 
+                    if self.retry_wait < 0 or self.retry_wait > self.max_retry_wait:
+                        self.retry_wait = self.max_retry_wait
+
                     # Only got here if an exception was caught.  Now determine what to do about it.
                     if not should_retry:
                         self.retry_wait = self.min_retry_wait
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 9cb582c..c8558a4 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -11,7 +11,7 @@ from arvnodeman.computenode import ARVADOS_TIMEFMT
 
 from libcloud.compute.base import NodeSize, Node, NodeDriver, NodeState, NodeImage
 from libcloud.compute.drivers.gce import GCEDiskType
-from libcloud.common.exceptions import BaseHTTPError
+from libcloud.common.exceptions import BaseHTTPError, RateLimitReachedError
 
 all_nodes = []
 create_calls = 0
@@ -129,8 +129,11 @@ class RetryDriver(FakeDriver):
         global create_calls
         create_calls += 1
         if create_calls < 2:
+            raise RateLimitReachedError(429, "Rate limit exceeded",
+                                        retry_after=12)
+        elif create_calls < 3:
             raise BaseHTTPError(429, "Rate limit exceeded",
-                                {'retry-after': '12'})
+                                {'retry-after': '2'})
         else:
             return super(RetryDriver, self).create_node(name=name,
                     size=size,
diff --git a/services/nodemanager/tests/integration_test.py b/services/nodemanager/tests/integration_test.py
index bdd3ffd..d5b5554 100755
--- a/services/nodemanager/tests/integration_test.py
+++ b/services/nodemanager/tests/integration_test.py
@@ -388,6 +388,7 @@ def main():
             [
                 (r".*Daemon started", set_squeue),
                 (r".*Rate limit exceeded - scheduling retry in 12 seconds", noop),
+                (r".*Rate limit exceeded - scheduling retry in 2 seconds", noop),
                 (r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", noop),
             ],
             {},

-----------------------------------------------------------------------


hooks/post-receive
--