[ARVADOS] created: 7024063c50c7d2b5b629a2127bea226ad8a4629f

git at public.curoverse.com git at public.curoverse.com
Fri Oct 23 16:54:50 EDT 2015


        at  7024063c50c7d2b5b629a2127bea226ad8a4629f (commit)


commit 7024063c50c7d2b5b629a2127bea226ad8a4629f
Author: Bryan Cosca <bcosc at curoverse.com>
Date:   Wed Sep 2 16:59:01 2015 -0400

    6600: Added Retryloop to task_set_output(), current_task(), and current_job() to python SDK

diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 1df6470..b74f828 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -23,6 +23,7 @@ from collection import CollectionReader, CollectionWriter, ResumableCollectionWr
 from keep import *
 from stream import *
 from arvfile import StreamFileReader
+from retry import RetryLoop
 import errors
 import util
 
@@ -37,36 +38,60 @@ logger.addHandler(log_handler)
 logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG')
                 else logging.WARNING)
 
-def task_set_output(self,s):
-    api('v1').job_tasks().update(uuid=self['uuid'],
-                                 body={
-            'output':s,
-            'success':True,
-            'progress':1.0
-            }).execute()
+def task_set_output(self, s, num_retries=5):
+    for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0):
+        try:
+            return api('v1').job_tasks().update(
+                uuid=self['uuid'],
+                body={
+                    'output':s,
+                    'success':True,
+                    'progress':1.0
+                }).execute()
+        except errors.ApiError as error:
+            if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
+                logger.debug("task_set_output: job_tasks().update() raised {}, retrying with {} tries left".format(repr(error),tries_left))
+            else:
+                raise
 
 _current_task = None
-def current_task():
+def current_task(num_retries=5):
     global _current_task
     if _current_task:
         return _current_task
-    t = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
-    t = UserDict.UserDict(t)
-    t.set_output = types.MethodType(task_set_output, t)
-    t.tmpdir = os.environ['TASK_WORK']
-    _current_task = t
-    return t
+
+    for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
+        try:
+            task = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
+            task = UserDict.UserDict(task)
+            task.set_output = types.MethodType(task_set_output, task)
+            task.tmpdir = os.environ['TASK_WORK']
+            _current_task = task
+            return task
+        except errors.ApiError as error:
+            if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
+                logger.debug("current_task: job_tasks().get() raised {}, retrying with {} tries left".format(repr(error),tries_left))
+            else:
+                raise
 
 _current_job = None
-def current_job():
+def current_job(num_retries=5):
     global _current_job
     if _current_job:
         return _current_job
-    t = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
-    t = UserDict.UserDict(t)
-    t.tmpdir = os.environ['JOB_WORK']
-    _current_job = t
-    return t
+
+    for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
+        try:
+            job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
+            job = UserDict.UserDict(job)
+            job.tmpdir = os.environ['JOB_WORK']
+            _current_job = job
+            return job
+        except errors.ApiError as error:
+            if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
+                logger.debug("current_job: jobs().get() raised {}, retrying with {} tries left".format(repr(error),tries_left))
+            else:
+                raise
 
 def getjobparam(*args):
     return current_job()['script_parameters'].get(*args)
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 6e2a078..ea318c6 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -43,6 +43,10 @@ def mock_responses(body, *codes, **headers):
     return mock.patch('httplib2.Http.request', side_effect=queue_with((
         (fake_httplib2_response(code, **headers), body) for code in codes)))
 
+def mock_api_responses(api_client, body, codes, headers={}):
+    return mock.patch.object(api_client._http, 'request', side_effect=queue_with((
+        (fake_httplib2_response(code, **headers), body) for code in codes)))
+
 
 class FakeCurl:
     @classmethod
diff --git a/sdk/python/tests/test_retry_job_helpers.py b/sdk/python/tests/test_retry_job_helpers.py
new file mode 100644
index 0000000..6e562a0
--- /dev/null
+++ b/sdk/python/tests/test_retry_job_helpers.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+import run_test_server
+import json
+import arvados
+import arvados_testutil as tutil
+from apiclient import http as apiclient_http
+
+
+ at tutil.skip_sleep
+class ApiClientRetryTestMixin(object):
+
+    TEST_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+    TEST_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
+
+    @classmethod
+    def setUpClass(cls):
+        run_test_server.run()
+
+    def setUp(self):
+        # Patch arvados.api() to return our mock API, so we can mock
+        # its http requests.
+        self.api_client = arvados.api('v1', cache=False)
+        self.api_patch = mock.patch('arvados.api', return_value=self.api_client)
+        self.api_patch.start()
+
+    def tearDown(self):
+        self.api_patch.stop()
+
+    def run_method(self):
+        raise NotImplementedError("test subclasses must define run_method")
+
+    def test_immediate_success(self):
+        with tutil.mock_api_responses(self.api_client, '{}', [200]):
+            self.run_method()
+
+    def test_immediate_failure(self):
+        with tutil.mock_api_responses(self.api_client, '{}', [400]), self.assertRaises(self.DEFAULT_EXCEPTION):
+            self.run_method()
+
+    def test_retry_then_success(self):
+        with tutil.mock_api_responses(self.api_client, '{}', [500, 200]):
+            self.run_method()
+
+    def test_error_after_default_retries_exhausted(self):
+        with tutil.mock_api_responses(self.api_client, '{}', [500, 500, 500, 500, 500, 500, 200]), self.assertRaises(self.DEFAULT_EXCEPTION):
+            self.run_method()
+
+    def test_no_retry_after_immediate_success(self):
+        with tutil.mock_api_responses(self.api_client, '{}', [200, 400]):
+            self.run_method()
+
+
+class CurrentJobTestCase(ApiClientRetryTestMixin, unittest.TestCase):
+
+    DEFAULT_EXCEPTION = arvados.errors.ApiError
+
+    def setUp(self):
+        super(CurrentJobTestCase, self).setUp()
+        os.environ['JOB_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+        os.environ['JOB_WORK'] = '.'
+
+    def tearDown(self):
+        del os.environ['JOB_UUID']
+        del os.environ['JOB_WORK']
+        arvados._current_job = None
+        super(CurrentJobTestCase, self).tearDown()
+
+    def run_method(self):
+        arvados.current_job()
+
+
+class CurrentTaskTestCase(ApiClientRetryTestMixin, unittest.TestCase):
+
+    DEFAULT_EXCEPTION = arvados.errors.ApiError
+
+    def setUp(self):
+        super(CurrentTaskTestCase, self).setUp()
+        os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+        os.environ['TASK_WORK'] = '.'
+
+    def tearDown(self):
+        del os.environ['TASK_UUID']
+        del os.environ['TASK_WORK']
+        arvados._current_task = None
+        super(CurrentTaskTestCase, self).tearDown()
+
+    def run_method(self):
+        arvados.current_task()
+
+
+class TaskSetOutputTestCase(CurrentTaskTestCase, unittest.TestCase):
+
+    DEFAULT_EXCEPTION = arvados.errors.ApiError
+
+    def tearDown(self):
+        super(TaskSetOutputTestCase, self).tearDown()
+        run_test_server.reset()
+
+    def run_method(self, locator=ApiClientRetryTestMixin.TEST_LOCATOR):
+        arvados.task_set_output({'uuid':self.TEST_UUID},s=locator)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list