[ARVADOS] updated: 111db6f3f99d7209c850d623139142529dc90c9f
git at public.curoverse.com
git at public.curoverse.com
Fri Sep 11 16:36:51 EDT 2015
Summary of changes:
sdk/python/arvados/__init__.py | 12 +--
sdk/python/tests/arvados_testutil.py | 20 +++-
sdk/python/tests/test_task_output_retry.py | 107 ++++++++-------------
...tput_retry.py => test_task_output_retry_old.py} | 0
4 files changed, 60 insertions(+), 79 deletions(-)
copy sdk/python/tests/{test_task_output_retry.py => test_task_output_retry_old.py} (100%)
via 111db6f3f99d7209c850d623139142529dc90c9f (commit)
from a8688fdd6fbacc7cbcf95785a929e82a2c30bec6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 111db6f3f99d7209c850d623139142529dc90c9f
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Fri Sep 11 16:36:44 2015 -0400
6600: Beginning to model task_set_output unit tests after test_keep_client
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 54ae12f..77e4138 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -43,11 +43,10 @@ def task_set_output(self,s,api_client=None,num_retries=5):
if not api_client:
api_client = api('v1')
- loop = RetryLoop(num_retries=num_retries, backoff_start=0)
- for tries_left in loop:
+ for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0): # change this to 2 after tests are finished
try:
- result = api_client.job_tasks().update(uuid=self['uuid'],
- body={
+ return api_client.job_tasks().update(uuid=self['uuid'],
+ body={
'output':s,
'success':True,
'progress':1.0
@@ -57,11 +56,6 @@ def task_set_output(self,s,api_client=None,num_retries=5):
logger.debug("task_set_output: job_tasks().update() raised {}, retrying with {} tries left".format(repr(error),tries_left))
else:
raise
- else:
- loop.save_result(result)
- if loop.success():
- logger.debug("task_set_output is successful")
- return result
_current_task = None
def current_task(api_client=None, num_retries=5):
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 6e2a078..53e5ba5 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -43,7 +43,6 @@ def mock_responses(body, *codes, **headers):
return mock.patch('httplib2.Http.request', side_effect=queue_with((
(fake_httplib2_response(code, **headers), body) for code in codes)))
-
class FakeCurl:
@classmethod
def make(cls, code, body='', headers={}):
@@ -117,6 +116,25 @@ def mock_keep_responses(body, *codes, **headers):
cm.responses = responses
return mock.patch('pycurl.Curl', cm)
+def mock_api_responses(body, *codes, **headers):
+ cm = mock.MagicMock()
+ if isinstance(body, tuple):
+ codes = list(codes)
+ codes.insert(0, body)
+ responses = [
+ FakeCurl.make(code=code, body=b, headers=headers)
+ for b, code in codes
+ ]
+ else:
+ responses = [
+ FakeCurl.make(code=code, body=body, headers=headers)
+ for code in codes
+ ]
+ cm.side_effect = queue_with(responses)
+ cm.responses = responses
+ return cm
+
+
class MockStreamReader(object):
def __init__(self, name='.', *data):
diff --git a/sdk/python/tests/test_task_output_retry.py b/sdk/python/tests/test_task_output_retry.py
index 8fd9f9a..50001aa 100644
--- a/sdk/python/tests/test_task_output_retry.py
+++ b/sdk/python/tests/test_task_output_retry.py
@@ -8,84 +8,53 @@ import hashlib
import arvados
import arvados_testutil as tutil
- at tutil.skip_sleep
-class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
+class ApiClientRetrytestMixin(object):
- def test_mock_retry_until_break(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
+ TEST_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ TEST_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
- def test_mock_success(self):
+ def setUp(self):
api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
- def test_mock_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
+ def run_method(self, *args, **kwargs):
+ raise NotImplementedError("test subclasses must define run_method")
- def test_mock_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+ def check_success(self, expected=None, *args, **kwargs):
+ if expected is None:
+ expected = self.DEFAULT_EXPECT
+ self.assertEqual(expected, self.run_method(*args, **kwargs))
- def test_mock_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(404, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+ def check_exception(self, error_class=None, *args, **kwargs):
+ if error_class is None:
+ error_class = self.DEFAULT_EXCEPTION
+ self.assertRaises(error_class, self.run_method, *args, **kwargs)
- def test_mock_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+ def test_immediate_success(self):
+ with self.TEST_API_PATCHER(self.DEFAULT_EXPECT, 200):
+ self.check_success()
- def test_mock_retry_success_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+ def test_retry_then_success(self):
+ with self.TEST_API_PATCHER(self.DEFAULT_EXPECT, 500, 200):
+ self.check_success(num_retries=3)
- def test_mock_retry_fail_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+ def test_success_after_default_retries_exhausted(self):
+ with self.TEST_API_PATCHER(self.DEFAULT_EXPECT, 500, 200):
+ self.check_success(num_retries=5)
- def test_mock_success_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+ def test_error_after_default_retries_exhausted(self):
+ with self.TEST_API_PATCHER(self.DEFAULT_EXPECT, 500, 500, 500, 500, 500, 200):
+ self.check_exception()
- def test_mock_retry_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+ def test_no_retry_after_immediate_success(self):
+ with self.TEST_API_PATCHER(self.DEFAULT_EXPECT, 200, 400):
+ self.check_success()
- def test_mock_retry_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+class TaskSetOutputTestCase(ApiClientRetrytestMixin, unittest.TestCase):
+
+ DEFAULT_EXCEPT = ApiClientRetrytestMixin.TEST_UUID
+ DEFAULT_RETURN = True
+ DEFAULT_EXCEPTION = arvados.errors.HttpError
+ TEST_API_PATCHER = staticmethod(tutil.mock_api_responses)
+
+ def run_method(self, locator=ApiClientRetrytestMixin.TEST_LOCATOR, *args, **kwargs):
+ return arvados.job_tasks().update(locator, *args, **kwargs)
diff --git a/sdk/python/tests/test_task_output_retry.py b/sdk/python/tests/test_task_output_retry_old.py
similarity index 100%
copy from sdk/python/tests/test_task_output_retry.py
copy to sdk/python/tests/test_task_output_retry_old.py
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list