[ARVADOS] created: 27928441e62c9c58115a3e1cf26aba8393a65f18
git at public.curoverse.com
git at public.curoverse.com
Wed Sep 2 16:59:32 EDT 2015
at 27928441e62c9c58115a3e1cf26aba8393a65f18 (commit)
commit 27928441e62c9c58115a3e1cf26aba8393a65f18
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Wed Sep 2 16:59:01 2015 -0400
6600: Added retries to PySDK functions task_set_output, current_task, current_jo, including unit tests.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 1df6470..bb73453 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -37,36 +37,156 @@ logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG')
else logging.WARNING)
-def task_set_output(self,s):
- api('v1').job_tasks().update(uuid=self['uuid'],
- body={
- 'output':s,
- 'success':True,
- 'progress':1.0
- }).execute()
+def task_set_output(self,s,api_client=None,num_retries=None):
+
+ if not api_client:
+ api_client = api('v1')
+
+ if not num_retries:
+ num_retries = 5
+
+ output_retry_loop = RetryLoop(num_retries=num_retries, backoff_start=0)
+ for tries_left in output_retry_loop:
+ try:
+ result = api_client.job_tasks().update(uuid=self['uuid'],
+ body={
+ 'output':s,
+ 'success':True,
+ 'progress':1.0
+ }).execute()
+ except errors.ApiError as error:
+ if not retry.check_http_response_success(error.resp.status):
+ if retry.check_http_response_success(error.resp.status) == False:
+ logger.debug("Non-retryable ApiError raised, logging error: {}".format(error))
+ output_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ logger.debug("Retrying with {} tries left".format(tries_left))
+ except errors.HttpError as error:
+ if not retry.check_http_response_success(error.status_code):
+ if retry.check_http_response_success(error.status_code) == False:
+ logger.debug("HttpError returned {}, saving error and not retrying.".format(error.status_code))
+ output_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ logger.debug("HttpError returned {}, retrying with {} tries left".format(error.status_code,tries_left+1))
+ else:
+ output_retry_loop.save_result(result)
+ if output_retry_loop.success():
+ logger.debug("task_set_output is successful")
+ return output_retry_loop.last_result()
+ else:
+ logger.debug("task_set_output has failed.")
+ raise
_current_task = None
-def current_task():
+def current_task(api_client=None, num_retries=None):
global _current_task
if _current_task:
return _current_task
- t = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
- t = UserDict.UserDict(t)
- t.set_output = types.MethodType(task_set_output, t)
- t.tmpdir = os.environ['TASK_WORK']
- _current_task = t
- return t
+ if not api_client:
+ api_client = api('v1')
+ if not num_retries:
+ num_retries = 5
+
+ current_task_retry_loop = RetryLoop(num_retries=num_retries, backoff_start=0)
+ for tries_left in current_task_retry_loop:
+ try:
+ print "Trying api call"
+ result = api_client.job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
+ print result
+ except errors.ApiError as error:
+ print error
+ if not retry.check_http_response_success(error.resp.status):
+ if retry.check_http_response_success(error.resp.status) == False:
+ print "Non-retryable ApiError raised, logging error: {}".format(error)
+ raise
+ current_task_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left)
+ else:
+ print "No more retries left (api)"
+ raise
+ except errors.HttpError as error:
+ print "HttpError raised"
+ if not retry.check_http_response_success(error.status_code):
+ if retry.check_http_response_success(error.status_code) == False:
+ print "False retry code"
+ current_task_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left+1)
+ else:
+ print "No more retries left (http)"
+ raise
+ else:
+ print "Task_set_output saved?"
+ current_task_retry_loop.save_result(result)
+ if current_task_retry_loop.success():
+ print "Current_task_retry_loop succeeded"
+ result = UserDict.UserDict(current_task_retry_loop.last_result())
+ result.set_output = types.MethodType(task_set_output, result)
+ result.tmpdir = os.environ['TASK_WORK']
+ _current_task = result
+ return result
+ else:
+ print "current_task failed."
+ raise
_current_job = None
-def current_job():
+def current_job(api_client=None, num_retries=None):
global _current_job
if _current_job:
return _current_job
- t = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
- t = UserDict.UserDict(t)
- t.tmpdir = os.environ['JOB_WORK']
- _current_job = t
- return t
+
+ if not api_client:
+ api_client = api('v1')
+
+ if not num_retries:
+ num_retries = 0
+
+ current_job_retry_loop = RetryLoop(num_retries=num_retries, backoff_start=0)
+ for tries_left in current_job_retry_loop:
+ try:
+ result = api_client.jobs().get(uuid=os.environ['JOB_UUID']).execute()
+ except errors.ApiError as error:
+ print error
+ if not retry.check_http_response_success(error.resp.status):
+ if retry.check_http_response_success(error.resp.status) == False:
+ print "Non-retryable ApiError raised, logging error: {}".format(error)
+ raise
+ current_job_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left)
+ else:
+ print "No more retries left (api)"
+ raise
+ except errors.HttpError as error:
+ print "HttpError raised"
+ if not retry.check_http_response_success(error.status_code):
+ if retry.check_http_response_success(error.status_code) == False:
+ print "False retry code"
+ current_job_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left+1)
+ else:
+ print "No more retries left (http)"
+ raise
+ else:
+ print "Task_set_output saved?"
+ current_job_retry_loop.save_result(result)
+ if current_job_retry_loop.success():
+ result = UserDict.UserDict(current_job_retry_loop.last_result())
+ result.tmpdir = os.environ['JOB_WORK']
+ _current_job = result
+ return result
+ else:
+ print "current_job failed."
+ raise
+
def getjobparam(*args):
return current_job()['script_parameters'].get(*args)
diff --git a/sdk/python/tests/test_crunch_job_retry.py b/sdk/python/tests/test_crunch_job_retry.py
new file mode 100644
index 0000000..ccd26c7
--- /dev/null
+++ b/sdk/python/tests/test_crunch_job_retry.py
@@ -0,0 +1,99 @@
+
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class CurrentJobTestCase(unittest.TestCase, tutil.ApiClientMock):
+
+ os.environ['JOB_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ os.environ['JOB_WORK'] = '.'
+
+ def tearDown(self):
+ super(CurrentJobTestCase,self).tearDown()
+ try:
+ arvados._current_job = None
+ except:
+ raise
+
+ def test_mock_retry_until_break(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=0)
+
+ def test_mock_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_success_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_job(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.jobs().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_job(api_client=api_client,num_retries=2)
+
diff --git a/sdk/python/tests/test_crunch_task_retry.py b/sdk/python/tests/test_crunch_task_retry.py
new file mode 100644
index 0000000..b203ea7
--- /dev/null
+++ b/sdk/python/tests/test_crunch_task_retry.py
@@ -0,0 +1,99 @@
+
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class CurrentTaskTestCase(unittest.TestCase, tutil.ApiClientMock):
+
+ os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ os.environ['TASK_WORK'] = '.'
+
+ def tearDown(self):
+ super(CurrentTaskTestCase,self).tearDown()
+ try:
+ arvados._current_task = None
+ except:
+ raise
+
+ def test_mock_retry_until_break(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=0)
+
+ def test_mock_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_success_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.current_task(api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self):
+ api_client = mock.MagicMock()
+ api_client.job_tasks().get().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.current_task(api_client=api_client,num_retries=2)
+
diff --git a/sdk/python/tests/test_task_output_retry.py b/sdk/python/tests/test_task_output_retry.py
new file mode 100644
index 0000000..ae02403
--- /dev/null
+++ b/sdk/python/tests/test_task_output_retry.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados_testutil as tutil
+
+ at tutil.skip_sleep
+class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def mock_call_update(self, api_mock, code, body):
+ self._mock_api_call(api_mock, code, body)
+
+ def api_client_mock(self, status=200):
+ api_client = super(TaskSetOutputTestCase, self).api_client_mock()
+ self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+ return api_client
+
+ api_client = mock.MagicMock()
+
+ def test_mock_retry_until_break(self,code=500):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_success(self,code=200):
+ api_client = self.api_client_mock(code)
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_fail(self,code=400):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_success(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_fail(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(404), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
+
+ def test_mock_retry_success_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_fail_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_success_retry_retry(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}")]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_success(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ tutil.fake_httplib2_response(200)]
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_retry_retry_fail(self,api_client=api_client):
+ api_client.job_tasks().update().execute.side_effect = [arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(500), "{}"),
+ arvados.errors.ApiError(tutil.fake_httplib2_response(400), "{}")]
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list