[ARVADOS] updated: 6fb3726cbd0ee75d85249eca568e75159fff66e5
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 8 16:43:25 EDT 2015
Summary of changes:
sdk/python/arvados/__init__.py | 6 +-
sdk/python/tests/test_crunch_job_retry.py | 104 -------------------------
sdk/python/tests/test_crunch_task_retry.py | 103 ------------------------
sdk/python/tests/test_task_output_retry.py | 97 -----------------------
sdk/python/tests/test_task_output_retry_old.py | 91 ----------------------
5 files changed, 3 insertions(+), 398 deletions(-)
delete mode 100644 sdk/python/tests/test_crunch_job_retry.py
delete mode 100644 sdk/python/tests/test_crunch_task_retry.py
delete mode 100644 sdk/python/tests/test_task_output_retry.py
delete mode 100644 sdk/python/tests/test_task_output_retry_old.py
via 6fb3726cbd0ee75d85249eca568e75159fff66e5 (commit)
from 72070e2405d6c3635d9e937d4faa475290c227df (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6fb3726cbd0ee75d85249eca568e75159fff66e5
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Oct 8 16:43:19 2015 -0400
6600: Simplifying code
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 708dbf4..bf1533a 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -43,7 +43,7 @@ def task_set_output(self,s,api_client=None,num_retries=5):
if not api_client:
api_client = api('v1')
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0): # change this to 2 after tests are finished
+ for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2): # change this to 2 after tests are finished
try:
api_client.job_tasks().update(uuid=self['uuid'],
body={
@@ -67,7 +67,7 @@ def current_task(api_client=None, num_retries=5):
if not api_client:
api_client = api('v1')
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0):
+ for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
result = api_client.job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
result = UserDict.UserDict(result)
@@ -90,7 +90,7 @@ def current_job(api_client=None, num_retries=5):
if not api_client:
api_client = api('v1')
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0):
+ for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
result = api_client.jobs().get(uuid=os.environ['JOB_UUID']).execute()
result = UserDict.UserDict(result)
diff --git a/sdk/python/tests/test_crunch_job_retry.py b/sdk/python/tests/test_crunch_job_retry.py
deleted file mode 100644
index caf9365..0000000
--- a/sdk/python/tests/test_crunch_job_retry.py
+++ /dev/null
@@ -1,104 +0,0 @@
-
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-
-import arvados
-import arvados_testutil as tutil
-
- at tutil.skip_sleep
-class CurrentJobTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- os.environ['JOB_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- os.environ['JOB_WORK'] = '.'
-
- def tearDown(self):
- super(CurrentJobTestCase,self).tearDown()
- try:
- arvados._current_job = None
- except:
- raise
-
- def test_mock_retry_until_break(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client)
-
- def test_mock_success(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
- arvados.current_job(api_client=api_client)
-
- def test_mock_fail(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client)
-
- def test_mock_retry_success(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(404, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_success_retry(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}")]
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail_retry(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_success_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_success(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_job(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.jobs().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_job(api_client=api_client,num_retries=2)
-
diff --git a/sdk/python/tests/test_crunch_task_retry.py b/sdk/python/tests/test_crunch_task_retry.py
deleted file mode 100644
index 2d07065..0000000
--- a/sdk/python/tests/test_crunch_task_retry.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-
-import arvados
-import arvados_testutil as tutil
-
- at tutil.skip_sleep
-class CurrentTaskTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- os.environ['TASK_WORK'] = '.'
-
- def tearDown(self):
- super(CurrentTaskTestCase,self).tearDown()
- try:
- arvados._current_task = None
- except:
- raise
-
- def test_mock_retry_until_break(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client)
-
- def test_mock_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200)]
- arvados.current_task(api_client=api_client)
-
- def test_mock_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client)
-
- def test_mock_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(404, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_success_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}")]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_success_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.current_task(api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().get().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.current_task(api_client=api_client,num_retries=2)
-
diff --git a/sdk/python/tests/test_task_output_retry.py b/sdk/python/tests/test_task_output_retry.py
deleted file mode 100644
index b3045d5..0000000
--- a/sdk/python/tests/test_task_output_retry.py
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-import run_test_server
-import mimetypes
-import json
-import arvados
-import arvados_testutil as tutil
-from apiclient import http as apiclient_http
-
-if not mimetypes.inited:
- mimetypes.init()
-
-class ApiClientRetrytestMixin(object):
-
- TEST_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- TEST_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
-
- @classmethod
- def setUpClass(cls):
- run_test_server.run()
- mock_responses = {
- 'arvados.job_tasks().update().execute': (tutil.fake_httplib2_response(500, **cls.ERROR_HEADERS), "")
- }
- req_builder = apiclient_http.RequestMockBuilder(mock_responses)
- cls.api = arvados.api('v1', requestBuilder=req_builder)
-
- def tearDown(cls):
- run_test_server.reset()
-
- def new_client(self, **kwargs):
- return self.api_client(**kwargs)
-
- def run_method(self, *args, **kwargs):
- raise NotImplementedError("test subclasses must define run_method")
-
- def check_success(self, expected=None, *args, **kwargs):
- try:
- self.run_method
- except: # This seems really iffy, because random exceptions can occur.
- self.assertTrue(False)
-
- def check_exception(self, error_class=None, *args, **kwargs):
- if error_class is None:
- error_class = self.DEFAULT_EXCEPTION
- self.assertRaises(error_class, self.run_method)
-
- def test_immediate_success(self):
- with tutil.mock_responses('', 200):
- self.check_success()
-
- def test_immediate_failure(self):
- with tutil.mock_responses('', 400):
- self.check_exception()
-
- def test_retry_then_success(self):
- with tutil.mock_responses('', 500, 200):
- self.check_success()
-
- def test_error_after_default_retries_exhausted(self):
- with tutil.mock_responses('', 500, 500, 500, 500, 500, 500):
- self.check_exception()
-
- def test_no_retry_after_immediate_success(self):
- with tutil.mock_responses('', 200, 400):
- self.check_success()
-
-class TaskSetOutputTestCase(ApiClientRetrytestMixin, unittest.TestCase):
-
- ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
- DEFAULT_EXCEPTION = arvados.errors.ApiError
-
- def run_method(self, locator=ApiClientRetrytestMixin.TEST_LOCATOR, api_client=None, *args, **kwargs):
- arvados.task_set_output({'uuid':self.TEST_UUID},s=locator,api_client=api_client)
-
-class CurrentJobTestCase(ApiClientRetrytestMixin, unittest.TestCase):
-
- ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
- DEFAULT_EXCEPTION = arvados.errors.ApiError
- os.environ['JOB_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- os.environ['JOB_WORK'] = '.'
-
- def run_method(self, api_client=None, *args, **kwargs):
- arvados.current_job(api_client=api_client)
-
-class CurrentTaskTestCase(ApiClientRetrytestMixin, unittest.TestCase):
-
- ERROR_HEADERS = {'Content-Type': mimetypes.types_map['.json']}
- DEFAULT_EXCEPTION = arvados.errors.ApiError
- os.environ['TASK_UUID'] = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
- os.environ['TASK_WORK'] = '.'
-
- def run_method(self, api_client=None, *args, **kwargs):
- arvados.current_task(api_client=api_client)
diff --git a/sdk/python/tests/test_task_output_retry_old.py b/sdk/python/tests/test_task_output_retry_old.py
deleted file mode 100644
index 8fd9f9a..0000000
--- a/sdk/python/tests/test_task_output_retry_old.py
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/env python
-
-import mock
-import os
-import unittest
-import hashlib
-
-import arvados
-import arvados_testutil as tutil
-
- at tutil.skip_sleep
-class TaskSetOutputTestCase(unittest.TestCase, tutil.ApiClientMock):
-
- def test_mock_retry_until_break(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
-
- def test_mock_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
-
- def test_mock_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client)
-
- def test_mock_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(404, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=1)
-
- def test_mock_retry_success_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_fail_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}"),
- arvados.errors.HttpError(500, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_success_retry_retry(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [tutil.fake_httplib2_response(200),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}")]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_success(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- tutil.fake_httplib2_response(200)]
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-
- def test_mock_retry_retry_fail(self):
- api_client = mock.MagicMock()
- api_client.job_tasks().update().execute.side_effect = [arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(500, "{}"),
- arvados.errors.HttpError(400, "{}")]
- with self.assertRaises(arvados.errors.HttpError) as err_check:
- arvados.task_set_output({'uuid':'zzzzz-zzzzz-zzzzzzzzzzzzzzz'},s='d41d8cd98f00b204e9800998ecf8427e+0',api_client=api_client,num_retries=2)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list