[ARVADOS] updated: 4f5836306e2c9cb475f0513394249f5fce8d3d8c
git at public.curoverse.com
git at public.curoverse.com
Thu Aug 27 16:55:06 EDT 2015
Summary of changes:
sdk/python/arvados/__init__.py | 34 +++-
sdk/python/tests/test_crunch_job_retry.py | 99 +++++++++++
sdk/python/tests/test_crunch_task_retry.py | 99 +++++++++++
sdk/python/tests/test_init.py | 192 ----------------------
sdk/python/tests/test_one_task_per_input_retry.py | 28 ++++
sdk/python/tests/test_task_output_retry.py | 84 ++++++++++
6 files changed, 336 insertions(+), 200 deletions(-)
create mode 100644 sdk/python/tests/test_crunch_job_retry.py
create mode 100644 sdk/python/tests/test_crunch_task_retry.py
delete mode 100644 sdk/python/tests/test_init.py
create mode 100644 sdk/python/tests/test_one_task_per_input_retry.py
create mode 100644 sdk/python/tests/test_task_output_retry.py
via 4f5836306e2c9cb475f0513394249f5fce8d3d8c (commit)
via 08e7453a6c2af6d86486310d11afd7e9b331e04a (commit)
from 58f25f3a8ff57c5db8124b553a81eefa62482a81 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4f5836306e2c9cb475f0513394249f5fce8d3d8c
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 27 16:54:46 2015 -0400
6600: Added retry tests for crunch job/task
diff --git a/sdk/python/tests/test_crunch_task_retry.py b/sdk/python/tests/test_crunch_task_retry.py
new file mode 100644
index 0000000..b203ea7
--- /dev/null
+++ b/sdk/python/tests/test_crunch_task_retry.py
@@ -0,0 +1,99 @@
+
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class CurrentTaskTestCase(unittest.TestCase, tutil.ApiClientMock):
+
+ os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ os.environ['TASK_WORK'] = '.'
+
+ def tearDown(self):
+ super(CurrentTaskTestCase,self).tearDown()
+ try:
+ arvados._current_task = None
+ except:
+ raise
+
+ def test_mock_retry_until_break(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_success_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
commit 08e7453a6c2af6d86486310d11afd7e9b331e04a
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 27 16:54:27 2015 -0400
6600: Added retry tests for crunch job/task
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index fc701c0..e622102 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -120,7 +120,7 @@ def current_task(api_client=None, num_retries=None):
if retry.check_http_response_success(error.resp.status) == False:
print "Non-retryable ApiError raised, logging error: {}".format(error)
raise
- output_retry_loop.save_result(error)
+ current_task_retry_loop.save_result(error)
else:
if tries_left > 0:
print "Retrying with {} tries left".format(tries_left)
@@ -132,7 +132,7 @@ def current_task(api_client=None, num_retries=None):
if not retry.check_http_response_success(error.status_code):
if retry.check_http_response_success(error.status_code) == False:
print "False retry code"
- output_retry_loop.save_result(error)
+ current_task_retry_loop.save_result(error)
else:
if tries_left > 0:
print "Retrying with {} tries left".format(tries_left+1)
@@ -170,18 +170,35 @@ def current_job(api_client=None, num_retries=None):
try:
result = api_client.jobs().get(uuid=os.environ['JOB_UUID']).execute()
except errors.ApiError as error:
+ print error
if not retry.check_http_response_success(error.resp.status):
if retry.check_http_response_success(error.resp.status) == False:
print "Non-retryable ApiError raised, logging error: {}".format(error)
- output_retry_loop.save_result(error)
raise
+ current_job_retry_loop.save_result(error)
else:
if tries_left > 0:
print "Retrying with {} tries left".format(tries_left)
- else:
- current_job_retry_loop.save_result(result)
+ else:
+ print "No more retries left (api)"
+ raise
+ except errors.HttpError as error:
+ print "HttpError raised"
+ if not retry.check_http_response_success(error.status_code):
+ if retry.check_http_response_success(error.status_code) == False:
+ print "False retry code"
+ current_job_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left+1)
+ else:
+ print "No more retries left (http)"
+ raise
+ else:
+ print "Task_set_output saved?"
+ current_job_retry_loop.save_result(result)
if current_job_retry_loop.success():
- result = UserDict.UserDict(result)
+ result = UserDict.UserDict(current_job_retry_loop.last_result())
result.tmpdir = os.environ['JOB_WORK']
_current_job = result
return result
@@ -216,8 +233,9 @@ class job_setup:
if not api_client:
api_client = api('v1')
- if not num_retries:
- num_retries = 2
+# if not num_retries:
+# num_retries = 2
+# print num_retries
job_input = current_job()['script_parameters']['input']
cr = CollectionReader(job_input, api_client=api_client)
diff --git a/sdk/python/tests/test_crunch_job_retry.py b/sdk/python/tests/test_crunch_job_retry.py
new file mode 100644
index 0000000..ccd26c7
--- /dev/null
+++ b/sdk/python/tests/test_crunch_job_retry.py
@@ -0,0 +1,99 @@
+
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class CurrentJobTestCase(unittest.TestCase, tutil.ApiClientMock):
+
+ os.environ['JOB_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ os.environ['JOB_WORK'] = '.'
+
+ def tearDown(self):
+ super(CurrentJobTestCase,self).tearDown()
+ try:
+ arvados._current_job = None
+ except:
+ raise
+
+ def test_mock_retry_until_break(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_success_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
diff --git a/sdk/python/tests/test_init.py b/sdk/python/tests/test_init.py
deleted file mode 100644
index 93b4d15..0000000
--- a/sdk/python/tests/test_init.py
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-import Queue
-
-import arvados
-import arvados.collection
-import arvados_testutil as tutil
-import run_test_server
-from arvados_testutil import FakeCurl
-from arvados_testutil import queue_with
-from arvados_testutil import mock_responses
-
-def mock_responses(*codes, **headers):
- body = ''
- mock_api = mock.MagicMock()
- mock_api.job_tasks().get().update().execute.side_effect=tutil.queue_with((tutil.fake_httplib2_response(code, **headers), body) for code in codes)
- return mock_api
-
- at tutil.skip_sleep
-class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
- def mock_call_update(self, api_mock, code, body):
- self._mock_api_call(api_mock, code, body)
-
- def api_client_mock(self, status=200):
- api_client = super(TaskSetOutputTestCase, self).api_client_mock()
- self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
- return api_client
-
- api_client = mock.MagicMock()
-
- def test_mock_retry_until_break(self,code=500):
- api_client = self.api_client_mock(code)
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_success(self,code=200):
- api_client = self.api_client_mock(code)
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_fail(self,code=400):
- api_client = self.api_client_mock(code)
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_success(self,api_client=api_client):
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_fail(self,api_client=api_client):
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_retry(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_success_retry(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail_retry(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_success_retry_retry(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_success(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_fail(self,api_client=api_client):
-# api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
-
- at tutil.skip_sleep
-class CurrentTaskTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- os.environ['TASK_WORK'] = '.'
-
- def test_mock_retry_until_break(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=0)
-
- def test_mock_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(200), "{}")]
- arvados.current_task(api_client=api_client,num_retries=0)
-
- def test_mock_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=0)
-
- def test_mock_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=0)
-
- def test_mock_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
-# api_client = mock_responses(500,500,500)
-# print api_client.job_tasks().get().execute.side_effect
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_success_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_success_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
- arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
- with self.assertRaises(arvados.errors.ApiError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
diff --git a/sdk/python/tests/test_one_task_per_input_retry.py b/sdk/python/tests/test_one_task_per_input_retry.py
new file mode 100644
index 0000000..ddb723e
--- /dev/null
+++ b/sdk/python/tests/test_one_task_per_input_retry.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+from arvados import job_setup
+
+ at tutil.skip_sleep
+class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def mock_call_update(self, api_mock, code, body):
+ self._mock_api_call(api_mock, code, body)
+
+ def api_client_mock(self, status=200):
+ api_client = super(TaskSetOutputTestCase, self).api_client_mock()
+ self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+ return api_client
+
+ api_client = mock.MagicMock()
+
+ def test_mock_retry_until_break(self,code=500):
+ j = job_setup()
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ j.one_task_per_input_file(api_client=api_client,num_retries=1)
diff --git a/sdk/python/tests/test_task_output_retry.py b/sdk/python/tests/test_task_output_retry.py
new file mode 100644
index 0000000..ae02403
--- /dev/null
+++ b/sdk/python/tests/test_task_output_retry.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def mock_call_update(self, api_mock, code, body):
+ self._mock_api_call(api_mock, code, body)
+
+ def api_client_mock(self, status=200):
+ api_client = super(TaskSetOutputTestCase, self).api_client_mock()
+ self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+ return api_client
+
+ api_client = mock.MagicMock()
+
+ def test_mock_retry_until_break(self,code=500):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_success(self,code=200):
+ api_client = self.api_client_mock(code)
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_fail(self,code=400):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_success(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_fail(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_success_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list