[ARVADOS] updated: 10e69ac63c2588bd4c4da8fceaf8979b0ab637ab
git at public.curoverse.com
git at public.curoverse.com
Tue Aug 25 16:55:25 EDT 2015
Summary of changes:
sdk/python/arvados/__init__.py | 24 ++++--
sdk/python/tests/test_init.py | 149 ++++++++---------------------------
sdk/python/tests/test_init_psuedo.py | 39 +++++++++
sdk/python/tests/test_init_temp.py | 23 ------
4 files changed, 86 insertions(+), 149 deletions(-)
create mode 100644 sdk/python/tests/test_init_psuedo.py
delete mode 100644 sdk/python/tests/test_init_temp.py
via 10e69ac63c2588bd4c4da8fceaf8979b0ab637ab (commit)
from 28bdfc61b7bb2d6bed3b62af8a4b328f3e7d767d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 10e69ac63c2588bd4c4da8fceaf8979b0ab637ab
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Tue Aug 25 16:54:41 2015 -0400
6600: Unit tests for single ApiErrors work
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 1591539..8e3af9f 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -48,6 +48,7 @@ def task_set_output(self,s,api_client=None,num_retries=None):
if not num_retries:
num_retries = 0
+ print api_client
output_retry_loop = RetryLoop(num_retries=num_retries, backoff_start=0)
for tries_left in output_retry_loop:
try:
@@ -59,19 +60,26 @@ def task_set_output(self,s,api_client=None,num_retries=None):
'progress':1.0
}).execute()
except errors.ApiError as error:
- print "ApiError Raised"
- #logger.debug("ApiError {}, retrying with {} tries_left".format(error,tries_left+1))
- # How do we check if its a retryable ApiError or not? ApiError inherits HttpError but how do we check for status code?
+ if not retry.check_http_response_success(error.resp.status):
+ if retry.check_http_response_success(error.resp.status) == False:
+ print "Non-retryable ApiError raised, logging error: {}".format(error)
+ raise
+ output_retry_loop.save_result(error)
+ else:
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left)
+ else:
+ print "Retrying with {} tries left".format(tries_left)
except errors.HttpError as error:
print "HttpError raised"
if not retry.check_http_response_success(error.status_code):
if retry.check_http_response_success(error.status_code) == False:
print "False retry code"
#logger.debug("HttpError returned {}, saving error and not retrying.".format(error.status_code))
- raise
- #output_retry_loop.save_result(error)
+ output_retry_loop.save_result(error)
else:
- print "Retrying with tries left"
+ if tries_left > 0:
+ print "Retrying with {} tries left".format(tries_left+1)
#logger.debug("HttpError returned {}, retrying with {} tries left".format(error.status_code,tries_left+1))
else:
print "Output is good"
@@ -80,8 +88,8 @@ def task_set_output(self,s,api_client=None,num_retries=None):
print "Output Retry loop is successful"
return output_retry_loop.last_result()
else:
- print "Output Retry loop is not successful"
- self.tries_left = num_retries + 1
+ print "Output Retry loop is not successful, task_set_output_failed."
+ raise
_current_task = None
def current_task(api_client=None):
diff --git a/sdk/python/tests/test_init.py b/sdk/python/tests/test_init.py
index 533548e..081e653 100644
--- a/sdk/python/tests/test_init.py
+++ b/sdk/python/tests/test_init.py
@@ -3,6 +3,7 @@
import mock
import os
import unittest
+import hashlib
import arvados
import arvados.collection
@@ -14,121 +15,33 @@ from arvados_testutil import mock_responses
@tutil.skip_sleep
class SDKTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- def test_mock_keep_responses_works(self):
- api_client = mock.MagicMock()
- req_mock = tutil.mock_keep_responses("retry error reporting test", 500, 500, 403, 403)
- with req_mock, self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client)
- self.assertEqual(err_check,False) # Need a spy assert to see why ApiError/HttpError isn't getting called
-
-########## Psuedocode work start ##########
-
-# Add mock_api_responses to tutil
-
- at tutil.skip_sleep
-class TaskSetOutputTests:
-
- api_client=mock.MagicMock()
-
- def check_for_errors(self, errors*,num_retries,api_client):
- with req_mock, self.assertRaises(all_errors) as err_check: # look for all errors using calls
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client,num_retries=2)
- self.assertEqual(err_check,errors*,True) # Spy to check to see if you asserted the right errors
- self.assertTrue(num_retries_needed=num_retries) # Spy to check to see if you asserted the correct num_retries
-
- def test_task_set_output_zero_retries_success(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 200) # Mock a 200 call
- self.assertTrue(check_for_errors('Success', 0, api_client)
-
- def test_task_set_output_zero_retries_tempfail(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 500) # Mock a 500 call
- self.assertTrue(check_for_errors('Success', 0, api_client)
-
- def test_task_set_output_zero_retries_permfail(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 404) # Mock a 404 call
- self.assertTrue(check_for_errors('PermFail', 0, api_client))
-
- def test_task_set_output_zero_retries_permfail(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 200, 408) # Mock a 200, 408 call
- self.assertTrue(check_for_errors('Success', 0, api_client))
-
- def test_task_set_output_one_retry_success(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 500, 200) # Mock a 500, 200 call
- self.assertTrue(check_for_errors('TempFail', 'Success', 1, api_client))
-
- def test_task_set_output_one_retry_tempfail(self,api_client=api_client):
- req_mock = tutil.mock_api_responses('', 500, 501) # Mock a 500, 501 call
- self.assertTrue(check_for_errors('TempFail', 'TempFail', 1, api_client)
-
- def test_task_set_output_one_retry_permfail(self):
- req_mock = tutil.mock_api_responses('', 500, 404) # Mock a 500, 404 call
- self.assertTrue(check_for_errors('TempFail', 'PermFail', 1, api_client))
-
- def test_task_set_output_two_retries_success(self):
- req_mock = tutil.mock_api_responses('', 500, 408, 200) # Mock a 500, 408, 200 call
- self.assertTrue(check_for_errors('TempFail', 'TempFail', 'Success', 2, api_client))
-
- def test_task_set_output_two_retries_permfail(self):
- req_mock = tutil.mock_api_responses('', 500, 408, 404) # Mock a 500, 408, 404 call
- self.assertTrue(check_for_errors('TempFail', 'TempFail', 'PermFail', 2, api_client))
-
- def test_task_set_output_two_retries_success_after_one(self):
- req_mock = tutil.mock_api_responses('', 500, 200, 500) # Mock a 500, 200, 500 call
- self.assertTrue(check_for_errors('TempFail', 'Success', 1, api_client))
-
- def test_task_set_output_two_retries_forever_tempfail(self):
- req_mock = tutil.mock_api_responses('', 500, 508, 501) # Mock a 500, 200, 500 call
- self.assertTrue(check_for_errors('TempFail', 'TempFail', 'TempFail', 1, api_client))
-
-
-
-
-########## Psuedocode work end ##########
-
-
-
- #with tutil.mock_keep_responses('foo', 500, 500, 403, 403):
- # arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client)
-
-# def check_errors_from_last_retry(self, verb, exc_class):
-# api_client = self.mock_keep_services(count=2)
-# req_mock = tutil.mock_keep_responses(
-# "retry error reporting test")
-# with req_mock, tutil.skip_sleep, \
-# self.assertRaises(exc_class) as err_check:
-# keep_client = arvados.KeepClient(api_client=api_client)
-# getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0',
-# num_retries=3)
-# self.assertEqual([403, 403], [
-# getattr(error, 'status_code', None)
-# for error in err_check.exception.request_errors().itervalues()])
-
-# def test_get_error_reflects_last_retry(self):
-# self.check_errors_from_last_retry('get', arvados.errors.KeepReadError)
-
-#### tests for functions, not methods ####
-
-#class TestSDK(unittest.TestCase, tutil.ApiClientMock):
-
-# FAKE_HASH = '7idummyhashb0b1f1f03e4e6f69c4a4e+28243'
-
- #mock_api = mock.MagicMock()
- #mock_instance = arvados_testutil.ApiClientMock
- #mock_api = mock_instance.api_client_mock()
-
-# def test_task_set_output_success(self, mock_input=FAKE_HASH, api_client=mock_api):
-# api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=200,reason="Mock 200")
-# arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-
- #arvados_testutil.ApiClientMock.api_client_mock()
-
-# def test_task_set_output_failure(self, mock_input=FAKE_HASH, api_client=mock_api):
-# api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=404,reason="Mock 404")
-# arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-
-# def test_task_set_output_retryable(self, mock_input=FAKE_HASH, api_client=mock_api):
-# api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=408,reason="Mock 500")
-# arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-
- # need a spy to make mock_api return down if retry_num = 1 and mock_api return up if retry_num = 2
+ def mock_call_update(self, api_mock, code, body):
+ self._mock_api_call(api_mock, code, body)
+
+ def api_client_mock(self, status=200):
+ api_client = super(SDKTestCase, self).api_client_mock()
+ self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+ return api_client
+
+ def mock_multi(self, body, *codes, **headers):
+ headers.setdefault('x-keep-replicas-stored', 2)
+ return tutil.mock_keep_responses(body, *codes, **headers)
+
+ def test_mock_500(self,code=500):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_200(self,code=200):
+ api_client = self.api_client_mock(code)
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_400(self,code=400):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_200_500(self,code=400):
+ api_client = self.api_client_mock(code)
+ with tutil.mock_responses('', 500, 404, 1) as patch:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client.patch,num_retries=1)
diff --git a/sdk/python/tests/test_init_psuedo.py b/sdk/python/tests/test_init_psuedo.py
new file mode 100644
index 0000000..8c989cf
--- /dev/null
+++ b/sdk/python/tests/test_init_psuedo.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados.collection
+import arvados_testutil as tutil
+import run_test_server
+from arvados_testutil import FakeCurl
+from arvados_testutil import queue_with
+from arvados_testutil import mock_responses
+
+ at tutil.skip_sleep
+class SDKTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def mock_call_update(self, api_mock, code, body):
+ self._mock_api_call(api_mock, code, body)
+
+ def api_client_mock(self, status=200):
+ api_client = super(SDKTestCase, self).api_client_mock()
+ self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+ return api_client
+
+ def test_mock_500(self,code=500):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_200(self,code=200):
+ api_client = self.api_client_mock(code)
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+ def test_mock_400(self,code=400):
+ api_client = self.api_client_mock(code)
+ with self.assertRaises(arvados.errors.ApiError) as err_check:
+ arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
diff --git a/sdk/python/tests/test_init_temp.py b/sdk/python/tests/test_init_temp.py
deleted file mode 100644
index cf6672b..0000000
--- a/sdk/python/tests/test_init_temp.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-
-import arvados
-import arvados.collection
-import arvados_testutil as tutil
-import run_test_server
-from arvados_testutil import FakeCurl
-from arvados_testutil import queue_with
-from arvados_testutil import mock_responses
-
- at tutil.skip_sleep
-class SDKTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- def test_mock_keep_responses_works(self):
- api_client = self.api_client_mock()
- req_mock = self._mock_api_call(api_client.job_tasks().update().execute, 500,'Error')
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list