[ARVADOS] updated: 03f9677371f7c502b65c950bb48aafc37c72c39d

git at public.curoverse.com git at public.curoverse.com
Wed Aug 26 10:51:22 EDT 2015


Summary of changes:

       via  03f9677371f7c502b65c950bb48aafc37c72c39d (commit)
       via  3bd885bf51ab982a157ea8edfd6d61adc5d84a5e (commit)
       via  8297c260f59d0abd93ac3430be6be71b0cc0125a (commit)
      from  10e69ac63c2588bd4c4da8fceaf8979b0ab637ab (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 03f9677371f7c502b65c950bb48aafc37c72c39d
Merge: 3bd885b 10e69ac
Author: Bryan Cosca <bcosc at curoverse.com>
Date:   Wed Aug 26 10:51:07 2015 -0400

    Merge branch '6600-pysdk-api-retries' of git.curoverse.com:arvados into 6600-pysdk-api-retries


commit 3bd885bf51ab982a157ea8edfd6d61adc5d84a5e
Author: Bryan Cosca <bcosc at curoverse.com>
Date:   Wed Aug 26 10:51:02 2015 -0400

    flailing

diff --git a/sdk/python/tests/test_init_psuedo.py b/sdk/python/tests/test_init_psuedo.py
new file mode 100644
index 0000000..8c989cf
--- /dev/null
+++ b/sdk/python/tests/test_init_psuedo.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python   
+
+import mock
+import os
+import unittest
+import hashlib
+
+import arvados
+import arvados.collection
+import arvados_testutil as tutil
+import run_test_server
+from arvados_testutil import FakeCurl
+from arvados_testutil import queue_with
+from arvados_testutil import mock_responses
+
+ at tutil.skip_sleep
+class SDKTestCase(unittest.TestCase, tutil.ApiClientMock):
+    def mock_call_update(self, api_mock, code, body):
+        self._mock_api_call(api_mock, code, body)
+
+    def api_client_mock(self, status=200):
+        api_client = super(SDKTestCase, self).api_client_mock()
+        self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+        return api_client
+
+    def test_mock_500(self,code=500):
+        api_client = self.api_client_mock(code)
+        with self.assertRaises(arvados.errors.ApiError) as err_check:
+            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+    def test_mock_200(self,code=200):
+        api_client = self.api_client_mock(code)
+        arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+    def test_mock_400(self,code=400):
+        api_client = self.api_client_mock(code)
+        with self.assertRaises(arvados.errors.ApiError) as err_check:
+            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+

commit 8297c260f59d0abd93ac3430be6be71b0cc0125a
Author: Bryan Cosca <bcosc at curoverse.com>
Date:   Wed Aug 26 10:50:36 2015 -0400

    fixing mess

diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index c506dd2..8e3af9f 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -48,9 +48,11 @@ def task_set_output(self,s,api_client=None,num_retries=None):
     if not num_retries:
         num_retries = 0
 
+    print api_client
     output_retry_loop = RetryLoop(num_retries=num_retries, backoff_start=0)
     for tries_left in output_retry_loop:
         try:
+            print "Trying api call"
             result = api_client.job_tasks().update(uuid=self['uuid'],
                                                 body={
                                                       'output':s,
@@ -58,19 +60,36 @@ def task_set_output(self,s,api_client=None,num_retries=None):
                                                       'progress':1.0
                                                      }).execute()            
         except errors.ApiError as error:
-            logger.debug("ApiError {}, retrying with {} tries_left".format(error,tries_left+1))
-            # How do we check if its a retryable ApiError or not? ApiError inherits HttpError but how do we check for status code?
+            if not retry.check_http_response_success(error.resp.status):
+                if retry.check_http_response_success(error.resp.status) == False:
+                    print "Non-retryable ApiError raised, logging error: {}".format(error)
+                    raise
+                    output_retry_loop.save_result(error)
+                else:
+                    if tries_left > 0:
+                        print "Retrying with {} tries left".format(tries_left)
+            else:
+                print "Retrying with {} tries left".format(tries_left)
         except errors.HttpError as error:
+            print "HttpError raised"
             if not retry.check_http_response_success(error.status_code):
                 if retry.check_http_response_success(error.status_code) == False:
-                    logger.debug("HttpError returned {}, saving error and not retrying.".format(error.status_code))
+                    print "False retry code"
+                    #logger.debug("HttpError returned {}, saving error and not retrying.".format(error.status_code))
                     output_retry_loop.save_result(error)
                 else:
-                    logger.debug("HttpError returned {}, retrying with {} tries left".format(error.status_code,tries_left+1))
+                    if tries_left > 0:
+                        print "Retrying with {} tries left".format(tries_left+1)
+                    #logger.debug("HttpError returned {}, retrying with {} tries left".format(error.status_code,tries_left+1))
         else:
+            print "Output is good"
             output_retry_loop.save_result(result)
     if output_retry_loop.success():
-        return output_retry_loop.last_result()	    
+        print "Output Retry loop is successful"
+        return output_retry_loop.last_result()
+    else:
+        print "Output Retry loop is not successful, task_set_output_failed."
+        raise
 
 _current_task = None
 def current_task(api_client=None):
diff --git a/sdk/python/tests/test_init.py b/sdk/python/tests/test_init.py
index 98bc43e..081e653 100644
--- a/sdk/python/tests/test_init.py
+++ b/sdk/python/tests/test_init.py
@@ -3,6 +3,7 @@
 import mock
 import os
 import unittest
+import hashlib
 
 import arvados
 import arvados.collection
@@ -14,121 +15,33 @@ from arvados_testutil import mock_responses
 
 @tutil.skip_sleep
 class SDKTestCase(unittest.TestCase, tutil.ApiClientMock):
-
-    def test_mock_keep_responses_works(self):
-        api_client = mock.MagicMock()
-        req_mock = tutil.mock_keep_responses("retry error reporting test", 500, 500, 403, 403)
-        with req_mock, self.assertRaises(arvados.errors.ApiError) as err_check:
-            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client)
-            self.assertEqual(err_check,False) # Need a spy assert to see why ApiError/HttpError isn't getting called
-
-########## Psuedocode work start ##########
-
-# Add mock_api_responses to tutil
-
- at tutil.skip_sleep
-class TaskSetOutputTests:
-
-    api_client=mock.MagicMock()
-
-    def check_for_errors(self, errors*,num_retries,api_client):
-         with req_mock, self.assertRaises(all_errors) as err_check: # look for all errors using calls
-            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client,num_retries=2)
-            self.assertEqual(err_check,errors*,True) # Spy to check to see if you asserted the right errors
-            self.assertTrue(num_retries_needed=num_retries) # Spy to check to see if you asserted the correct num_retries
-
-    def test_task_set_output_zero_retries_success(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 200) # Mock a 200 call
-        self.assertTrue(check_for_errors('Success', 0, api_client)
-
-    def test_task_set_output_zero_retries_tempfail(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 500) # Mock a 500 call
-        self.assertTrue(check_for_errors('Success', 0, api_client)
-
-    def test_task_set_output_zero_retries_permfail(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 404) # Mock a 404 call
-        self.assertTrue(check_for_errors('PermFail', 0, api_client))
-
-    def test_task_set_output_zero_retries_permfail(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 200, 408) # Mock a 200, 408 call
-        self.assertTrue(check_for_errors('Success', 0, api_client))
-
-    def test_task_set_output_one_retry_success(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 500, 200) # Mock a 500, 200 call
-        self.assertTrue(check_for_errors('TempFail', 'Success', 1, api_client))
-    
-    def test_task_set_output_one_retry_tempfail(self,api_client=api_client):
-        req_mock = tutil.mock_api_responses('', 500, 501) # Mock a 500, 501 call
-        self.assertTrue(check_for_errors('TempFail', 'TempFail', 1, api_client)    
-
-    def test_task_set_output_one_retry_permfail(self):
-        req_mock = tutil.mock_api_responses('', 500, 404) # Mock a 500, 404 call
-        self.assertTrue(check_for_errors('TempFail', 'PermFail', 1, api_client))
-
-    def test_task_set_output_two_retries_success(self):
-        req_mock = tutil.mock_api_responses('', 500, 408, 200) # Mock a 500, 408, 200 call
-        self.assertTrue(check_for_errors('TempFail', 'TempFail', 'Success', 2, api_client))
-
-    def test_task_set_output_two_retries_permfail(self):
-        req_mock = tutil.mock_api_responses('', 500, 408, 404) # Mock a 500, 408, 404 call
-        self.assertTrue(check_for_errors('TempFail', 'TempFail', 'PermFail', 2, api_client))
-
-    def test_task_set_output_two_retries_success_after_one(self):
-        req_mock = tutil.mock_api_responses('', 500, 200, 500) # Mock a 500, 200, 500 call
-        self.assertTrue(check_for_errors('TempFail', 'Success', 1, api_client))
-
-    def test_task_set_output_two_retries_forever_tempfail(self):
-        req_mock = tutil.mock_api_responses('', 500, 508, 501) # Mock a 500, 200, 500 call
-        self.assertTrue(check_for_errors('TempFail', 'TempFail', 'TempFail', 1, api_client))
-
-
-
-
-########## Psuedocode work end ##########
-
-
-            
-        #with tutil.mock_keep_responses('foo', 500, 500, 403, 403):
-        #    arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='7idummyhashb0b1f1f03e4e6f69c4a4e+28243',api_client=api_client)
-    
-#    def check_errors_from_last_retry(self, verb, exc_class):
-#        api_client = self.mock_keep_services(count=2)
-#        req_mock = tutil.mock_keep_responses(
-#            "retry error reporting test")
-#        with req_mock, tutil.skip_sleep, \
-#                self.assertRaises(exc_class) as err_check:
-#            keep_client = arvados.KeepClient(api_client=api_client)
-#            getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0',
-#                                       num_retries=3)
-#        self.assertEqual([403, 403], [
-#                getattr(error, 'status_code', None)
-#                for error in err_check.exception.request_errors().itervalues()])
-
-#    def test_get_error_reflects_last_retry(self):
-#        self.check_errors_from_last_retry('get', arvados.errors.KeepReadError)
-
-#### tests for functions, not methods ####
-
-#class TestSDK(unittest.TestCase, tutil.ApiClientMock):
-
-#    FAKE_HASH = '7idummyhashb0b1f1f03e4e6f69c4a4e+28243'
-
-    #mock_api = mock.MagicMock()
-    #mock_instance = arvados_testutil.ApiClientMock
-    #mock_api = mock_instance.api_client_mock()
-
-#    def test_task_set_output_success(self, mock_input=FAKE_HASH, api_client=mock_api):
-#        api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=200,reason="Mock 200")
-#        arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-        
-        #arvados_testutil.ApiClientMock.api_client_mock()    
-
-#    def test_task_set_output_failure(self, mock_input=FAKE_HASH, api_client=mock_api):
-#        api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=404,reason="Mock 404")
-#        arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-
-#    def test_task_set_output_retryable(self, mock_input=FAKE_HASH, api_client=mock_api):
-#        api_client.job_tasks().update().execute.side_effect = arvados.errors.HttpError(status_code=408,reason="Mock 500")
-#        arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s=mock_input,api_client=api_client)
-
-        # need a spy to make mock_api return down if retry_num = 1 and mock_api return up if retry_num = 2
+    def mock_call_update(self, api_mock, code, body):
+        self._mock_api_call(api_mock, code, body)
+
+    def api_client_mock(self, status=200):
+        api_client = super(SDKTestCase, self).api_client_mock()
+        self.mock_call_update(api_client.job_tasks().update, status, 'foo_file')
+        return api_client
+
+    def mock_multi(self, body, *codes, **headers):
+        headers.setdefault('x-keep-replicas-stored', 2)        
+        return tutil.mock_keep_responses(body, *codes, **headers)    
+
+    def test_mock_500(self,code=500):
+        api_client = self.api_client_mock(code)
+        with self.assertRaises(arvados.errors.ApiError) as err_check:
+            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+    def test_mock_200(self,code=200):
+        api_client = self.api_client_mock(code)
+        arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+    def test_mock_400(self,code=400):
+        api_client = self.api_client_mock(code)
+        with self.assertRaises(arvados.errors.ApiError) as err_check:
+            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
+
+    def test_mock_200_500(self,code=400):
+        api_client = self.api_client_mock(code)
+        with tutil.mock_responses('', 500, 404, 1) as patch:
+            arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client.patch,num_retries=1)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list