[ARVADOS] updated: 86ab369fd599da54a51bf80668a26b969a3f94db
Git user
git at public.curoverse.com
Mon Apr 3 21:54:56 EDT 2017
Summary of changes:
sdk/python/arvados/commands/put.py | 4 +-
sdk/python/arvados/keep.py | 9 ++++-
sdk/python/tests/keepstub.py | 72 ++++++++++++++++++++++--------------
sdk/python/tests/test_arv_put.py | 1 +
sdk/python/tests/test_collections.py | 6 +--
sdk/python/tests/test_errors.py | 14 ++++---
sdk/python/tests/test_keep_client.py | 2 +-
7 files changed, 67 insertions(+), 41 deletions(-)
discards fd8940c4b389e7549a16cf56b43100cdf7cd998d (commit)
discards 2685e2eadb2bec951c956c2558c39391ca8d8a7c (commit)
discards 63eac50bdba525faf1c40d5d4b49afdc6ca5c494 (commit)
discards d39d59e9d415254ed5f5c2901de5aa4ef26ec060 (commit)
discards 5901b74965a0ad5c924d2894fc1f3450567d4633 (commit)
via 86ab369fd599da54a51bf80668a26b969a3f94db (commit)
via 5f238a36c6c3745bef12468c0e4c216ed65f566a (commit)
via 3fcd28c65a035f96b082e1a8bb337fb0cb115fa6 (commit)
via 44a87969d906535ceef83847f2b4211f1199cc3e (commit)
via d0b76e970ab97e6ddf0441e7bff95977ae0c40e8 (commit)
via 3622c512ac41da978017d5bd3e104d6e96006114 (commit)
via fd548e8929999ac199c99056f35e9c309e880cf7 (commit)
via a5957935a4fbe27c9b04dffa6ce06f6b44ef7e1c (commit)
via 5f17b00166dca53ccd55ed1f347dc0ab6e25051c (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (fd8940c4b389e7549a16cf56b43100cdf7cd998d)
\
N -- N -- N (86ab369fd599da54a51bf80668a26b969a3f94db)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 86ab369fd599da54a51bf80668a26b969a3f94db
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 3 21:50:54 2017 -0400
11308: Update assertion to accommodate different exception in py3.
diff --git a/sdk/python/tests/test_errors.py b/sdk/python/tests/test_errors.py
index 23b3f08..dd7b87e 100644
--- a/sdk/python/tests/test_errors.py
+++ b/sdk/python/tests/test_errors.py
@@ -57,10 +57,14 @@ class KeepRequestErrorTestCase(unittest.TestCase):
test_exc = arv_error.KeepRequestError(message, self.REQUEST_ERRORS[:])
exc_report = self.traceback_str(test_exc)
self.assertRegex(exc_report, r"^(arvados\.errors\.)?KeepRequestError: ")
- for expect_substr in [message, "raised IOError", "raised MemoryError",
- "test MemoryError", "second test IOError",
- "responded with 500 Internal Server Error"]:
- self.assertIn(expect_substr, exc_report)
+ self.assertIn(message, exc_report)
+ for expect_re in [
+ r"raised (IOError|OSError)", # IOError in Python2, OSError in Python3
+ r"raised MemoryError",
+ r"test MemoryError",
+ r"second test IOError",
+ r"responded with 500 Internal Server Error"]:
+ self.assertRegex(exc_report, expect_re)
# Assert the report maintains order of listed services.
last_index = -1
for service_key, _ in self.REQUEST_ERRORS:
commit 5f238a36c6c3745bef12468c0e4c216ed65f566a
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 3 21:50:30 2017 -0400
11308: Add stderr assertion for easier debugging.
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 755b4b5..e0ff7a7 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -787,6 +787,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=self.ENVIRON)
stdout, stderr = pipe.communicate(text.encode())
+ self.assertRegex(stderr.decode(), r'INFO: Collection (updated:|saved as)')
search_key = ('portable_data_hash'
if '--portable-data-hash' in extra_args else 'uuid')
collection_list = arvados.api('v1').collections().list(
commit 3fcd28c65a035f96b082e1a8bb337fb0cb115fa6
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 3 17:45:30 2017 -0400
11308: Fix keepstub bugs and Python 3 compatibility.
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index 870e776..4992196 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -10,6 +10,9 @@ import socketserver
import sys
import time
+_debug = os.environ.get('ARVADOS_DEBUG', None)
+
+
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
allow_reuse_address = 1
@@ -60,6 +63,9 @@ class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
class Handler(http.server.BaseHTTPRequestHandler, object):
+
+ protocol_version = 'HTTP/1.1'
+
def wfile_bandwidth_write(self, data_to_write):
if self.server.bandwidth is None and self.server.delays['mid_write'] == 0:
self.wfile.write(data_to_write)
@@ -72,7 +78,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
while num_sent_bytes < num_bytes:
if num_sent_bytes > self.server.bandwidth and not outage_happened:
self.server._do_delay('mid_write')
- target_time += self.delays['mid_write']
+ target_time += self.server.delays['mid_write']
outage_happened = True
num_write_bytes = min(BYTES_PER_WRITE,
num_bytes - num_sent_bytes)
@@ -96,7 +102,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
while bytes_to_read > bytes_read:
if bytes_read > self.server.bandwidth and not outage_happened:
self.server._do_delay('mid_read')
- target_time += self.delays['mid_read']
+ target_time += self.server.delays['mid_read']
outage_happened = True
next_bytes_to_read = min(BYTES_PER_READ,
bytes_to_read - bytes_read)
@@ -107,9 +113,29 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
self.server._sleep_at_least(target_time - time.time())
return data
+ def finish(self, *args, **kwargs):
+ try:
+ return super(Handler, self).finish(*args, **kwargs)
+ except Exception as err:
+ if _debug:
+ raise
+
def handle(self, *args, **kwargs):
+ try:
+ return super(Handler, self).handle(*args, **kwargs)
+ except:
+ if _debug:
+ raise
+
+ def handle_one_request(self, *args, **kwargs):
+ self._sent_continue = False
self.server._do_delay('request')
- return super(Handler, self).handle(*args, **kwargs)
+ return super(Handler, self).handle_one_request(*args, **kwargs)
+
+ def handle_expect_100(self):
+ self.server._do_delay('request_body')
+ self._sent_continue = True
+ return super(Handler, self).handle_expect_100()
def do_GET(self):
self.server._do_delay('response')
@@ -120,6 +146,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
if datahash not in self.server.store:
return self.send_response(404)
self.send_response(200)
+ self.send_header('Connection', 'close')
self.send_header('Content-type', 'application/octet-stream')
self.end_headers()
self.server._do_delay('response_body')
@@ -135,16 +162,15 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
if datahash not in self.server.store:
return self.send_response(404)
self.send_response(200)
+ self.send_header('Connection', 'close')
self.send_header('Content-type', 'application/octet-stream')
self.send_header('Content-length', str(len(self.server.store[datahash])))
self.end_headers()
self.server._do_delay('response_close')
-
- def handle_expect_100(self):
- self.server._do_delay('request_body')
+ self.close_connection = True
def do_PUT(self):
- if sys.version_info < (3, 0):
+ if not self._sent_continue and self.headers.get('expect') == '100-continue':
# The comments at https://bugs.python.org/issue1491
# implies that Python 2.7 BaseHTTPRequestHandler was
# patched to support 100 Continue, but reading the actual
@@ -152,33 +178,23 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
# to send the response on the socket directly.
self.server._do_delay('request_body')
self.wfile.write("{} {} {}\r\n\r\n".format(
- self.protocol_version, 100, "Continue"))
+ self.protocol_version, 100, "Continue").encode())
data = self.rfile_bandwidth_read(
int(self.headers.get('content-length')))
datahash = hashlib.md5(data).hexdigest()
self.server.store[datahash] = data
+ resp = '{}+{}\n'.format(datahash, len(data)).encode()
self.server._do_delay('response')
self.send_response(200)
+ self.send_header('Connection', 'close')
self.send_header('Content-type', 'text/plain')
+ self.send_header('Content-length', len(resp))
self.end_headers()
self.server._do_delay('response_body')
- self.wfile_bandwidth_write(datahash + '+' + str(len(data)))
+ self.wfile_bandwidth_write(resp)
self.server._do_delay('response_close')
+ self.close_connection = True
def log_request(self, *args, **kwargs):
- if os.environ.get('ARVADOS_DEBUG', None):
+ if _debug:
super(Handler, self).log_request(*args, **kwargs)
-
- def finish(self, *args, **kwargs):
- """Ignore exceptions, notably "Broken pipe" when client times out."""
- try:
- return super(Handler, self).finish(*args, **kwargs)
- except:
- pass
-
- def handle_one_request(self, *args, **kwargs):
- """Ignore exceptions, notably "Broken pipe" when client times out."""
- try:
- return super(Handler, self).handle_one_request(*args, **kwargs)
- except:
- pass
commit 44a87969d906535ceef83847f2b4211f1199cc3e
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 3 17:36:54 2017 -0400
11308: pep8
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index 0bb074c..870e776 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -38,7 +38,7 @@ class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
def setdelays(self, **kwargs):
"""In future requests, induce delays at the given checkpoints."""
for (k, v) in kwargs.items():
- self.delays.get(k) # NameError if unknown key
+ self.delays.get(k) # NameError if unknown key
self.delays[k] = v
def setbandwidth(self, bandwidth):
@@ -61,7 +61,7 @@ class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
class Handler(http.server.BaseHTTPRequestHandler, object):
def wfile_bandwidth_write(self, data_to_write):
- if self.server.bandwidth == None and self.server.delays['mid_write'] == 0:
+ if self.server.bandwidth is None and self.server.delays['mid_write'] == 0:
self.wfile.write(data_to_write)
else:
BYTES_PER_WRITE = int(self.server.bandwidth/4) or 32768
@@ -85,7 +85,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
return None
def rfile_bandwidth_read(self, bytes_to_read):
- if self.server.bandwidth == None and self.server.delays['mid_read'] == 0:
+ if self.server.bandwidth is None and self.server.delays['mid_read'] == 0:
return self.rfile.read(bytes_to_read)
else:
BYTES_PER_READ = int(self.server.bandwidth/4) or 32768
commit d0b76e970ab97e6ddf0441e7bff95977ae0c40e8
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 3 01:06:08 2017 -0400
11308: Close socket after pycurl.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 7a9b5bf..ce34e18 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -290,6 +290,7 @@ class KeepClient(object):
self._result = {'error': None}
self._usable = True
self._session = None
+ self._socket = None
self.get_headers = {'Accept': 'application/octet-stream'}
self.get_headers.update(headers)
self.put_headers = headers
@@ -320,8 +321,7 @@ class KeepClient(object):
except:
ua.close()
- @staticmethod
- def _socket_open(family, socktype, protocol, address=None):
+ def _socket_open(self, family, socktype, protocol, address=None):
"""Because pycurl doesn't have CURLOPT_TCP_KEEPALIVE"""
s = socket.socket(family, socktype, protocol)
s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
@@ -329,6 +329,7 @@ class KeepClient(object):
if hasattr(socket, 'TCP_KEEPIDLE'):
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 75)
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 75)
+ self._socket = s
return s
def get(self, locator, method="GET", timeout=None):
@@ -342,7 +343,8 @@ class KeepClient(object):
self._headers = {}
response_body = BytesIO()
curl.setopt(pycurl.NOSIGNAL, 1)
- curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
+ curl.setopt(pycurl.OPENSOCKETFUNCTION,
+ lambda *args, **kwargs: self._socket_open(*args, **kwargs))
curl.setopt(pycurl.URL, url.encode('utf-8'))
curl.setopt(pycurl.HTTPHEADER, [
'{}: {}'.format(k,v) for k,v in self.get_headers.items()])
@@ -356,6 +358,10 @@ class KeepClient(object):
curl.perform()
except Exception as e:
raise arvados.errors.HttpError(0, str(e))
+ finally:
+ if self._socket:
+ self._socket.close()
+ self._socket = None
self._result = {
'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
'body': response_body.getvalue(),
@@ -420,7 +426,8 @@ class KeepClient(object):
body_reader = BytesIO(body)
response_body = BytesIO()
curl.setopt(pycurl.NOSIGNAL, 1)
- curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
+ curl.setopt(pycurl.OPENSOCKETFUNCTION,
+ lambda *args, **kwargs: self._socket_open(*args, **kwargs))
curl.setopt(pycurl.URL, url.encode('utf-8'))
# Using UPLOAD tells cURL to wait for a "go ahead" from the
# Keep server (in the form of a HTTP/1.1 "100 Continue"
@@ -440,6 +447,10 @@ class KeepClient(object):
curl.perform()
except Exception as e:
raise arvados.errors.HttpError(0, str(e))
+ finally:
+ if self._socket:
+ self._socket.close()
+ self._socket = None
self._result = {
'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
'body': response_body.getvalue().decode('utf-8'),
commit 3622c512ac41da978017d5bd3e104d6e96006114
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Apr 2 19:40:26 2017 -0400
11308: Expect full class name in traceback.
diff --git a/sdk/python/tests/test_errors.py b/sdk/python/tests/test_errors.py
index 5b32da7..23b3f08 100644
--- a/sdk/python/tests/test_errors.py
+++ b/sdk/python/tests/test_errors.py
@@ -49,14 +49,14 @@ class KeepRequestErrorTestCase(unittest.TestCase):
message = "test plain traceback string"
test_exc = arv_error.KeepRequestError(message)
exc_report = self.traceback_str(test_exc)
- self.assertRegex(exc_report, r"^KeepRequestError: ")
+ self.assertRegex(exc_report, r"^(arvados\.errors\.)?KeepRequestError: ")
self.assertIn(message, exc_report)
def test_traceback_str_with_request_errors(self):
message = "test traceback shows Keep services"
test_exc = arv_error.KeepRequestError(message, self.REQUEST_ERRORS[:])
exc_report = self.traceback_str(test_exc)
- self.assertRegex(exc_report, r"^KeepRequestError: ")
+ self.assertRegex(exc_report, r"^(arvados\.errors\.)?KeepRequestError: ")
for expect_substr in [message, "raised IOError", "raised MemoryError",
"test MemoryError", "second test IOError",
"responded with 500 Internal Server Error"]:
commit fd548e8929999ac199c99056f35e9c309e880cf7
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Apr 2 19:11:06 2017 -0400
11308: assertRegexpMatches -> assertRegex
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 23f00b6..c70ecd4 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -90,7 +90,7 @@ class VersionChecker(object):
# Python 2 writes version info on stderr.
self.assertEqual(out.getvalue(), '')
v = err.getvalue()
- self.assertRegexpMatches(v, "[0-9]+\.[0-9]+\.[0-9]+$\n")
+ self.assertRegex(v, r"[0-9]+\.[0-9]+\.[0-9]+$\n")
class FakeCurl(object):
@@ -261,3 +261,13 @@ class ArvadosBaseTestCase(unittest.TestCase):
testfile.write(text)
testfile.flush()
return testfile
+
+if sys.version_info < (3, 0):
+ # There is no assert[Not]Regex that works in both Python 2 and 3,
+ # so we backport Python 3 style to Python 2.
+ def assertRegex(self, *args, **kwargs):
+ return self.assertRegexpMatches(*args, **kwargs)
+ def assertNotRegex(self, *args, **kwargs):
+ return self.assertNotRegexpMatches(*args, **kwargs)
+ unittest.TestCase.assertRegex = assertRegex
+ unittest.TestCase.assertNotRegex = assertNotRegex
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index 1634803..ff3fbb4 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -83,15 +83,15 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
self.assertEqual(out.getvalue(), '')
if expect_ok:
- self.assertNotRegexpMatches(
+ self.assertNotRegex(
err.getvalue(), "refusing to store",
msg=repr((supported, img_id)))
else:
- self.assertRegexpMatches(
+ self.assertRegex(
err.getvalue(), "refusing to store",
msg=repr((supported, img_id)))
if not supported:
- self.assertRegexpMatches(
+ self.assertRegex(
err.getvalue(),
"server does not specify supported image formats",
msg=repr((supported, img_id)))
@@ -112,4 +112,4 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
api()._rootDesc = fakeDD
self.run_arv_keepdocker(
['--force', '--force-image-format', 'testimage'], err)
- self.assertRegexpMatches(err.getvalue(), "forcing incompatible image")
+ self.assertRegex(err.getvalue(), "forcing incompatible image")
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index a738403..755b4b5 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -773,7 +773,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
# The manifest text stored in the API server under the same
# manifest UUID must use signed locators.
c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
- self.assertRegexpMatches(
+ self.assertRegex(
c['manifest_text'],
r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
@@ -808,7 +808,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
self.assertEqual(col['uuid'], updated_col['uuid'])
# Get the manifest and check that the new file is being included
c = arv_put.api_client.collections().get(uuid=updated_col['uuid']).execute()
- self.assertRegexpMatches(c['manifest_text'], r'^\. .*:44:file2\n')
+ self.assertRegex(c['manifest_text'], r'^\. .*:44:file2\n')
def test_put_collection_with_high_redundancy(self):
# Write empty data: we're not testing CollectionWriter, just
@@ -826,7 +826,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
"Test unnamed collection",
['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
username = pwd.getpwuid(os.getuid()).pw_name
- self.assertRegexpMatches(
+ self.assertRegex(
link['name'],
r'^Saved at .* by {}@'.format(re.escape(username)))
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 3a1afce..a128560 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -1048,7 +1048,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
# c1 changed, so c2 mod will go to a conflict file
c1.apply(d)
- self.assertRegexpMatches(c1.portable_manifest_text(), r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
+ self.assertRegex(
+ c1.portable_manifest_text(),
+ r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
def test_conflict_add(self):
c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
@@ -1061,7 +1063,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
# c1 added count1.txt, so c2 add will go to a conflict file
c1.apply(d)
- self.assertRegexpMatches(c1.portable_manifest_text(), r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
+ self.assertRegex(
+ c1.portable_manifest_text(),
+ r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
def test_conflict_del(self):
c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
@@ -1072,7 +1076,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
# c1 deleted, so c2 mod will go to a conflict file
c1.apply(d)
- self.assertRegexpMatches(c1.portable_manifest_text(), r"\. 5348b82a029fd9e971a811ce1f71360b\+43 0:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
+ self.assertRegex(
+ c1.portable_manifest_text(),
+ r"\. 5348b82a029fd9e971a811ce1f71360b\+43 0:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
def test_notify(self):
c1 = Collection()
@@ -1154,12 +1160,16 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
def test_create_and_save(self):
c = self.create_count_txt()
c.save()
- self.assertRegexpMatches(c.manifest_text(), r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
+ self.assertRegex(
+ c.manifest_text(),
+ r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
def test_create_and_save_new(self):
c = self.create_count_txt()
c.save_new()
- self.assertRegexpMatches(c.manifest_text(), r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
+ self.assertRegex(
+ c.manifest_text(),
+ r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
def test_create_diff_apply(self):
c1 = self.create_count_txt()
@@ -1218,7 +1228,9 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c2.save()
c1.update()
- self.assertRegexpMatches(c1.manifest_text(), r"\. e65075d550f9b5bf9992fa1d71a131be\+3\S* 7ac66c0f148de9519b8bd264312c4d64\+7\S* 0:3:count\.txt 3:7:count\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
+ self.assertRegex(
+ c1.manifest_text(),
+ r"\. e65075d550f9b5bf9992fa1d71a131be\+3\S* 7ac66c0f148de9519b8bd264312c4d64\+7\S* 0:3:count\.txt 3:7:count\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
if __name__ == '__main__':
diff --git a/sdk/python/tests/test_errors.py b/sdk/python/tests/test_errors.py
index 2ccd594..5b32da7 100644
--- a/sdk/python/tests/test_errors.py
+++ b/sdk/python/tests/test_errors.py
@@ -49,14 +49,14 @@ class KeepRequestErrorTestCase(unittest.TestCase):
message = "test plain traceback string"
test_exc = arv_error.KeepRequestError(message)
exc_report = self.traceback_str(test_exc)
- self.assertTrue(exc_report.startswith("KeepRequestError: "))
+ self.assertRegex(exc_report, r"^KeepRequestError: ")
self.assertIn(message, exc_report)
def test_traceback_str_with_request_errors(self):
message = "test traceback shows Keep services"
test_exc = arv_error.KeepRequestError(message, self.REQUEST_ERRORS[:])
exc_report = self.traceback_str(test_exc)
- self.assertTrue(exc_report.startswith("KeepRequestError: "))
+ self.assertRegex(exc_report, r"^KeepRequestError: ")
for expect_substr in [message, "raised IOError", "raised MemoryError",
"test MemoryError", "second test IOError",
"responded with 500 Internal Server Error"]:
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 5ea2cc1..35070f3 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -39,7 +39,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_KeepBasicRWTest(self):
self.assertEqual(0, self.keep_client.upload_counter.get())
foo_locator = self.keep_client.put('foo')
- self.assertRegexpMatches(
+ self.assertRegex(
foo_locator,
'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
@@ -56,7 +56,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_KeepBinaryRWTest(self):
blob_str = b'\xff\xfe\xf7\x00\x01\x02'
blob_locator = self.keep_client.put(blob_str)
- self.assertRegexpMatches(
+ self.assertRegex(
blob_locator,
'^7fc7c53b45e53926ba52821140fef396\+6',
('wrong locator from Keep.put(<binarydata>):' + blob_locator))
@@ -69,7 +69,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
for i in range(0,23):
blob_data = blob_data + blob_data
blob_locator = self.keep_client.put(blob_data)
- self.assertRegexpMatches(
+ self.assertRegex(
blob_locator,
'^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
@@ -81,7 +81,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_KeepSingleCopyRWTest(self):
blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03'
blob_locator = self.keep_client.put(blob_data, copies=1)
- self.assertRegexpMatches(
+ self.assertRegex(
blob_locator,
'^c902006bc98a3eb4a3663b65ab4a6fab\+8',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
@@ -91,7 +91,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_KeepEmptyCollectionTest(self):
blob_locator = self.keep_client.put('', copies=1)
- self.assertRegexpMatches(
+ self.assertRegex(
blob_locator,
'^d41d8cd98f00b204e9800998ecf8427e\+0',
('wrong locator from Keep.put(""): ' + blob_locator))
@@ -99,7 +99,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_unicode_must_be_ascii(self):
# If unicode type, must only consist of valid ASCII
foo_locator = self.keep_client.put(u'foo')
- self.assertRegexpMatches(
+ self.assertRegex(
foo_locator,
'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
@@ -115,7 +115,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
def test_KeepHeadTest(self):
locator = self.keep_client.put('test_head')
- self.assertRegexpMatches(
+ self.assertRegex(
locator,
'^b9a772c7049325feb7130fff1f8333e9\+9',
'wrong md5 hash from Keep.put for "test_head": ' + locator)
@@ -133,7 +133,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
run_test_server.authorize_with('active')
keep_client = arvados.KeepClient()
foo_locator = keep_client.put('foo')
- self.assertRegexpMatches(
+ self.assertRegex(
foo_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
@@ -144,7 +144,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
# GET with an unsigned locator => NotFound
bar_locator = keep_client.put('bar')
unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
- self.assertRegexpMatches(
+ self.assertRegex(
bar_locator,
r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("bar"): ' + bar_locator)
@@ -196,7 +196,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
def _put_foo_and_check(self):
signed_locator = self.keep_client.put('foo')
- self.assertRegexpMatches(
+ self.assertRegex(
signed_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
@@ -254,7 +254,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='')
baz_locator = keep_client.put('baz')
- self.assertRegexpMatches(
+ self.assertRegex(
baz_locator,
'^73feffa4b7f6bb68e44cf984c85f6e88\+3',
'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
@@ -270,7 +270,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
keep_client = arvados.KeepClient(api_client=self.api_client,
proxy='', local_store='')
baz_locator = keep_client.put('baz2')
- self.assertRegexpMatches(
+ self.assertRegex(
baz_locator,
'^91f372a266fe2bf2823cb8ec7fda31ce\+4',
'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
@@ -874,7 +874,7 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
mocks[i].getopt(pycurl.URL).decode())
# Disk services are tried next.
for i in range(gateways, gateways+disks):
- self.assertRegexpMatches(
+ self.assertRegex(
mocks[i].getopt(pycurl.URL).decode(),
r'keep0x')
@@ -898,7 +898,7 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
mocks[i].getopt(pycurl.URL).decode())
# Disk services are tried next.
for i in range(gateways, gateways+disks):
- self.assertRegexpMatches(
+ self.assertRegex(
mocks[i].getopt(pycurl.URL).decode(),
r'keep0x')
commit a5957935a4fbe27c9b04dffa6ce06f6b44ef7e1c
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Apr 2 16:28:46 2017 -0400
11308: Fix traceback.format_exc() usage.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 96b6090..2dee1dc 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -460,7 +460,8 @@ class ArvPutUploadJob(object):
# we have a custom signal handler in place that raises SystemExit with
# the catched signal's code.
if not isinstance(e, SystemExit) or e.code != -2:
- self.logger.warning("Abnormal termination:\n{}".format(traceback.format_exc(e)))
+ self.logger.warning("Abnormal termination:\n{}".format(
+ traceback.format_exc()))
raise
finally:
if not self.dry_run:
commit 5f17b00166dca53ccd55ed1f347dc0ab6e25051c
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Apr 2 01:26:36 2017 -0400
11308: Fix string/bytes confusion for Python 3
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 59a73b4..a06170d 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -117,6 +117,7 @@ _cast_orig = apiclient_discovery._cast
def _cast_objects_too(value, schema_type):
global _cast_orig
if (type(value) != type('') and
+ type(value) != type(b'') and
(schema_type == 'object' or schema_type == 'array')):
return json.dumps(value)
else:
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index c6cb1c9..9f95ee0 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -112,7 +112,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
def readall(self, size=2**20, num_retries=None):
while True:
data = self.read(size, num_retries=num_retries)
- if data == '':
+ if len(data) == 0:
break
yield data
@@ -124,23 +124,23 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
data = [cache_data]
self._filepos += len(cache_data)
else:
- data = ['']
+ data = [b'']
data_size = len(data[-1])
- while (data_size < size) and ('\n' not in data[-1]):
+ while (data_size < size) and (b'\n' not in data[-1]):
next_read = self.read(2 ** 20, num_retries=num_retries)
if not next_read:
break
data.append(next_read)
data_size += len(next_read)
- data = ''.join(data)
+ data = b''.join(data)
try:
- nextline_index = data.index('\n') + 1
+ nextline_index = data.index(b'\n') + 1
except ValueError:
nextline_index = len(data)
nextline_index = min(nextline_index, size)
self._filepos -= len(data) - nextline_index
self._readline_cache = (self.tell(), data[nextline_index:])
- return data[:nextline_index]
+ return data[:nextline_index].decode()
@_FileLikeObjectBase._before_close
@retry_method
@@ -175,7 +175,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
data_size += len(s)
if data_size >= sizehint:
break
- return ''.join(data).splitlines(True)
+ return b''.join(data).decode().splitlines(True)
def size(self):
raise NotImplementedError()
@@ -212,9 +212,9 @@ class StreamFileReader(ArvadosFileReaderBase):
def read(self, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at the current file position"""
if size == 0:
- return ''
+ return b''
- data = ''
+ data = b''
available_chunks = locators_and_ranges(self.segments, self._filepos, size)
if available_chunks:
lr = available_chunks[0]
@@ -230,13 +230,13 @@ class StreamFileReader(ArvadosFileReaderBase):
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
if size == 0:
- return ''
+ return b''
data = []
for lr in locators_and_ranges(self.segments, start, size):
data.append(self._stream.readfrom(lr.locator+lr.segment_offset, lr.segment_size,
num_retries=num_retries))
- return ''.join(data)
+ return b''.join(data)
def as_manifest(self):
segs = []
@@ -316,6 +316,8 @@ class _BufferBlock(object):
"""
if self._state == _BufferBlock.WRITABLE:
+ if not isinstance(data, bytes) and not isinstance(data, memoryview):
+ data = data.encode()
while (self.write_pointer+len(data)) > len(self.buffer_block):
new_buffer_block = bytearray(len(self.buffer_block) * 2)
new_buffer_block[0:self.write_pointer] = self.buffer_block[0:self.write_pointer]
@@ -944,7 +946,7 @@ class ArvadosFile(object):
with self.lock:
if size == 0 or offset >= self.size():
- return ''
+ return b''
readsegs = locators_and_ranges(self._segments, offset, size)
prefetch = locators_and_ranges(self._segments, offset + size, config.KEEP_BLOCK_SIZE, limit=32)
@@ -964,7 +966,7 @@ class ArvadosFile(object):
self.parent._my_block_manager().block_prefetch(lr.locator)
locs.add(lr.locator)
- return ''.join(data)
+ return b''.join(data)
def _repack_writes(self, num_retries):
"""Test if the buffer block has more data than actual segments.
@@ -1001,6 +1003,8 @@ class ArvadosFile(object):
necessary.
"""
+ if not isinstance(data, bytes) and not isinstance(data, memoryview):
+ data = data.encode()
if len(data) == 0:
return
@@ -1157,7 +1161,7 @@ class ArvadosFileReader(ArvadosFileReaderBase):
data.append(rd)
self._filepos += len(rd)
rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
- return ''.join(data)
+ return b''.join(data)
else:
data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
self._filepos += len(data)
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index 868b478..f59d92f 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -33,7 +33,7 @@ class SafeHTTPCache(object):
return self._dir
def _filename(self, url):
- return os.path.join(self._dir, hashlib.md5(url).hexdigest()+'.tmp')
+ return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
def get(self, url):
filename = self._filename(url)
@@ -50,7 +50,7 @@ class SafeHTTPCache(object):
return None
try:
try:
- f = os.fdopen(fd, 'w')
+ f = os.fdopen(fd, 'wb')
except:
os.close(fd)
raise
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 0d88084..1c68c8e 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -220,7 +220,11 @@ class CollectionWriter(CollectionBase):
self.do_queued_work()
def write(self, newdata):
- if hasattr(newdata, '__iter__'):
+ if isinstance(newdata, bytes):
+ pass
+ elif isinstance(newdata, str):
+ newdata = newdata.encode()
+ elif hasattr(newdata, '__iter__'):
for s in newdata:
self.write(s)
return
@@ -260,7 +264,7 @@ class CollectionWriter(CollectionBase):
return self._last_open
def flush_data(self):
- data_buffer = ''.join(self._data_buffer)
+ data_buffer = b''.join(self._data_buffer)
if data_buffer:
self._current_stream_locators.append(
self._my_keep().put(
@@ -350,10 +354,11 @@ class CollectionWriter(CollectionBase):
sending manifest_text() to the API server's "create
collection" endpoint.
"""
- return self._my_keep().put(self.manifest_text(), copies=self.replication)
+ return self._my_keep().put(self.manifest_text().encode(),
+ copies=self.replication)
def portable_data_hash(self):
- stripped = self.stripped_manifest()
+ stripped = self.stripped_manifest().encode()
return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped))
def manifest_text(self):
@@ -1078,7 +1083,7 @@ class RichCollectionBase(CollectionBase):
# then return API server's PDH response.
return self._portable_data_hash
else:
- stripped = self.portable_manifest_text()
+ stripped = self.portable_manifest_text().encode()
return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped))
@synchronized
@@ -1336,7 +1341,7 @@ class Collection(RichCollectionBase):
# mode. Return an exception, or None if successful.
try:
self._manifest_text = self._my_keep().get(
- self._manifest_locator, num_retries=self.num_retries)
+ self._manifest_locator, num_retries=self.num_retries).decode()
except Exception as e:
return e
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index a7ef0be..354ae50 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -99,7 +99,7 @@ def docker_image_format(image_hash):
cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash],
stdout=subprocess.PIPE)
try:
- image_id = next(cmd.stdout).strip()
+ image_id = next(cmd.stdout).decode().strip()
if image_id.startswith('sha256:'):
return 'v2'
elif ':' not in image_id:
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index ef86fef..96b6090 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -281,13 +281,13 @@ class ResumeCache(object):
@classmethod
def make_path(cls, args):
md5 = hashlib.md5()
- md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost'))
+ md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost').encode())
realpaths = sorted(os.path.realpath(path) for path in args.paths)
- md5.update('\0'.join(realpaths))
+ md5.update(b'\0'.join([p.encode() for p in realpaths]))
if any(os.path.isdir(path) for path in realpaths):
- md5.update("-1")
+ md5.update(b'-1')
elif args.filename:
- md5.update(args.filename)
+ md5.update(args.filename.encode())
return os.path.join(
arv_cmd.make_home_conf_dir(cls.CACHE_DIR, 0o700, 'raise'),
md5.hexdigest())
@@ -667,11 +667,11 @@ class ArvPutUploadJob(object):
if self.use_cache:
# Set up cache file name from input paths.
md5 = hashlib.md5()
- md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost'))
+ md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost').encode())
realpaths = sorted(os.path.realpath(path) for path in self.paths)
- md5.update('\0'.join(realpaths))
+ md5.update(b'\0'.join([p.encode() for p in realpaths]))
if self.filename:
- md5.update(self.filename)
+ md5.update(self.filename.encode())
cache_filename = md5.hexdigest()
cache_filepath = os.path.join(
arv_cmd.make_home_conf_dir(self.CACHE_DIR, 0o700, 'raise'),
@@ -754,8 +754,8 @@ class ArvPutUploadJob(object):
def portable_data_hash(self):
pdh = self._my_collection().portable_data_hash()
- m = self._my_collection().stripped_manifest()
- local_pdh = hashlib.md5(m).hexdigest() + '+' + str(len(m))
+ m = self._my_collection().stripped_manifest().encode()
+ local_pdh = '{}+{}'.format(hashlib.md5(m).hexdigest(), len(m))
if pdh != local_pdh:
logger.warning("\n".join([
"arv-put: API server provided PDH differs from local manifest.",
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index ee91491..7a9b5bf 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -90,7 +90,7 @@ class KeepLocator(object):
return getattr(self, data_name)
def setter(self, hex_str):
if not arvados.util.is_hex(hex_str, length):
- raise ValueError("{} is not a {}-digit hex string: {}".
+ raise ValueError("{} is not a {}-digit hex string: {!r}".
format(name, length, hex_str))
setattr(self, data_name, hex_str)
return property(getter, setter)
@@ -442,7 +442,7 @@ class KeepClient(object):
raise arvados.errors.HttpError(0, str(e))
self._result = {
'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
- 'body': response_body.getvalue(),
+ 'body': response_body.getvalue().decode('utf-8'),
'headers': self._headers,
'error': False,
}
@@ -851,7 +851,7 @@ class KeepClient(object):
The weight is md5(h + u) where u is the last 15 characters of
the service endpoint's UUID.
"""
- return hashlib.md5(data_hash + service_uuid[-15:]).hexdigest()
+ return hashlib.md5((data_hash + service_uuid[-15:]).encode()).hexdigest()
def weighted_service_roots(self, locator, force_rebuild=False, need_writable=False):
"""Return an array of Keep service endpoints, in the order in
@@ -1141,7 +1141,7 @@ class KeepClient(object):
"""
md5 = hashlib.md5(data).hexdigest()
locator = '%s+%d' % (md5, len(data))
- with open(os.path.join(self.local_store, md5 + '.tmp'), 'w') as f:
+ with open(os.path.join(self.local_store, md5 + '.tmp'), 'wb') as f:
f.write(data)
os.rename(os.path.join(self.local_store, md5 + '.tmp'),
os.path.join(self.local_store, md5))
@@ -1155,8 +1155,8 @@ class KeepClient(object):
raise arvados.errors.NotFoundError(
"Invalid data locator: '%s'" % loc_s)
if locator.md5sum == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
- return ''
- with open(os.path.join(self.local_store, locator.md5sum), 'r') as f:
+ return b''
+ with open(os.path.join(self.local_store, locator.md5sum), 'rb') as f:
return f.read()
def is_cached(self, locator):
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 5955816..1fe5d35 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -80,13 +80,13 @@ class StreamReader(object):
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
if size == 0:
- return ''
+ return b''
if self._keep is None:
self._keep = KeepClient(num_retries=self.num_retries)
data = []
for lr in locators_and_ranges(self._data_locators, start, size):
data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size])
- return ''.join(data)
+ return b''.join(data)
def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 5992371..23f00b6 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -21,6 +21,11 @@ import sys
import tempfile
import unittest
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
# Use this hostname when you want to make sure the traffic will be
# instantly refused. 100::/64 is a dedicated black hole.
TEST_HOST = '100::'
@@ -47,33 +52,55 @@ def fake_httplib2_response(code, **headers):
return httplib2.Response(headers)
def mock_responses(body, *codes, **headers):
+ if not isinstance(body, bytes) and hasattr(body, 'encode'):
+ body = body.encode()
return mock.patch('httplib2.Http.request', side_effect=queue_with((
(fake_httplib2_response(code, **headers), body) for code in codes)))
def mock_api_responses(api_client, body, codes, headers={}):
+ if not isinstance(body, bytes) and hasattr(body, 'encode'):
+ body = body.encode()
return mock.patch.object(api_client._http, 'request', side_effect=queue_with((
(fake_httplib2_response(code, **headers), body) for code in codes)))
def str_keep_locator(s):
- return '{}+{}'.format(hashlib.md5(s).hexdigest(), len(s))
+ return '{}+{}'.format(hashlib.md5(s if isinstance(s, bytes) else s.encode()).hexdigest(), len(s))
@contextlib.contextmanager
def redirected_streams(stdout=None, stderr=None):
+ if stdout == StringIO:
+ stdout = StringIO()
+ if stderr == StringIO:
+ stderr = StringIO()
orig_stdout, sys.stdout = sys.stdout, stdout or sys.stdout
orig_stderr, sys.stderr = sys.stderr, stderr or sys.stderr
try:
- yield
+ yield (stdout, stderr)
finally:
sys.stdout = orig_stdout
sys.stderr = orig_stderr
+class VersionChecker(object):
+ def assertVersionOutput(self, out, err):
+ if sys.version_info >= (3, 0):
+ self.assertEqual(err.getvalue(), '')
+ v = out.getvalue()
+ else:
+ # Python 2 writes version info on stderr.
+ self.assertEqual(out.getvalue(), '')
+ v = err.getvalue()
+ self.assertRegexpMatches(v, "[0-9]+\.[0-9]+\.[0-9]+$\n")
+
+
class FakeCurl(object):
@classmethod
- def make(cls, code, body='', headers={}):
+ def make(cls, code, body=b'', headers={}):
+ if not isinstance(body, bytes) and hasattr(body, 'encode'):
+ body = body.encode()
return mock.Mock(spec=cls, wraps=cls(code, body, headers))
- def __init__(self, code=200, body='', headers={}):
+ def __init__(self, code=200, body=b'', headers={}):
self._opt = {}
self._got_url = None
self._writer = None
@@ -146,7 +173,9 @@ def mock_keep_responses(body, *codes, **headers):
class MockStreamReader(object):
def __init__(self, name='.', *data):
self._name = name
- self._data = ''.join(data)
+ self._data = b''.join([
+ b if isinstance(b, bytes) else b.encode()
+ for b in data])
self._data_locators = [str_keep_locator(d) for d in data]
self.num_retries = 0
@@ -190,7 +219,7 @@ class ApiClientMock(object):
mock_method.return_value = body
else:
mock_method.side_effect = arvados.errors.ApiError(
- fake_httplib2_response(code), "{}")
+ fake_httplib2_response(code), b"{}")
class ArvadosBaseTestCase(unittest.TestCase):
@@ -227,7 +256,7 @@ class ArvadosBaseTestCase(unittest.TestCase):
tmpfile.write(leaf)
return tree_root
- def make_test_file(self, text="test"):
+ def make_test_file(self, text=b"test"):
testfile = tempfile.NamedTemporaryFile()
testfile.write(text)
testfile.flush()
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index 28bd483..0bb074c 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -7,6 +7,7 @@ import hashlib
import os
import re
import socketserver
+import sys
import time
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
@@ -88,7 +89,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
return self.rfile.read(bytes_to_read)
else:
BYTES_PER_READ = int(self.server.bandwidth/4) or 32768
- data = ''
+ data = b''
outage_happened = False
bytes_read = 0
target_time = time.time()
@@ -139,15 +140,21 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
self.end_headers()
self.server._do_delay('response_close')
- def do_PUT(self):
+ def handle_expect_100(self):
self.server._do_delay('request_body')
- # The comments at https://bugs.python.org/issue1491 implies that Python
- # 2.7 BaseHTTPRequestHandler was patched to support 100 Continue, but
- # reading the actual code that ships in Debian it clearly is not, so we
- # need to send the response on the socket directly.
- self.wfile_bandwidth_write("%s %d %s\r\n\r\n" %
- (self.protocol_version, 100, "Continue"))
- data = self.rfile_bandwidth_read(int(self.headers.getheader('content-length')))
+
+ def do_PUT(self):
+ if sys.version_info < (3, 0):
+ # The comments at https://bugs.python.org/issue1491
+ # implies that Python 2.7 BaseHTTPRequestHandler was
+ # patched to support 100 Continue, but reading the actual
+ # code that ships in Debian it clearly is not, so we need
+ # to send the response on the socket directly.
+ self.server._do_delay('request_body')
+ self.wfile.write("{} {} {}\r\n\r\n".format(
+ self.protocol_version, 100, "Continue"))
+ data = self.rfile_bandwidth_read(
+ int(self.headers.get('content-length')))
datahash = hashlib.md5(data).hexdigest()
self.server.store[datahash] = data
self.server._do_delay('response')
diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py
index a2dcaa0..7eefd62 100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@ -32,7 +32,7 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers):
def api_error_response(self, code, *errors):
return (fake_httplib2_response(code, **self.ERROR_HEADERS),
json.dumps({'errors': errors,
- 'error_token': '1234567890+12345678'}))
+ 'error_token': '1234567890+12345678'}).encode())
def test_new_api_objects_with_cache(self):
clients = [arvados.api('v1', cache=True) for index in [0, 1]]
@@ -84,7 +84,7 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers):
mock_responses = {
'arvados.humans.delete': (
fake_httplib2_response(500, **self.ERROR_HEADERS),
- "")
+ b"")
}
req_builder = apiclient_http.RequestMockBuilder(mock_responses)
api = arvados.api('v1', requestBuilder=req_builder)
@@ -101,9 +101,13 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers):
def test_ordered_json_model(self):
mock_responses = {
- 'arvados.humans.get': (None, json.dumps(collections.OrderedDict(
- (c, int(c, 16)) for c in string.hexdigits))),
- }
+ 'arvados.humans.get': (
+ None,
+ json.dumps(collections.OrderedDict(
+ (c, int(c, 16)) for c in string.hexdigits
+ )).encode(),
+ ),
+ }
req_builder = apiclient_http.RequestMockBuilder(mock_responses)
api = arvados.api('v1',
requestBuilder=req_builder, model=OrderedJsonModel())
diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py
index c8eb5d1..cc207c2 100644
--- a/sdk/python/tests/test_arv_copy.py
+++ b/sdk/python/tests/test_arv_copy.py
@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
-import io
import os
import sys
import tempfile
@@ -11,7 +10,7 @@ import unittest
import arvados.commands.arv_copy as arv_copy
from . import arvados_testutil as tutil
-class ArvCopyTestCase(unittest.TestCase):
+class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker):
def run_copy(self, args):
sys.argv = ['arv-copy'] + args
return arv_copy.main()
@@ -21,10 +20,8 @@ class ArvCopyTestCase(unittest.TestCase):
self.run_copy(['-x=unknown'])
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with tutil.redirected_streams(stdout=out, stderr=err):
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_copy(['--version'])
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index 2257227..1634803 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -4,7 +4,6 @@
from __future__ import absolute_import
import arvados
import hashlib
-import io
import mock
import os
import subprocess
@@ -22,7 +21,7 @@ class StopTest(Exception):
pass
-class ArvKeepdockerTestCase(unittest.TestCase):
+class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
def run_arv_keepdocker(self, args, err):
sys.argv = ['arv-keepdocker'] + args
log_handler = logging.StreamHandler(err)
@@ -37,21 +36,19 @@ class ArvKeepdockerTestCase(unittest.TestCase):
self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with tutil.redirected_streams(stdout=out, stderr=err):
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_arv_keepdocker(['--version'], sys.stderr)
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
@mock.patch('arvados.commands.keepdocker.find_image_hashes',
return_value=['abc123'])
@mock.patch('arvados.commands.keepdocker.find_one_image_hash',
return_value='abc123')
def test_image_format_compatibility(self, _1, _2):
- old_id = hashlib.sha256('old').hexdigest()
- new_id = 'sha256:'+hashlib.sha256('new').hexdigest()
+ old_id = hashlib.sha256(b'old').hexdigest()
+ new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
for supported, img_id, expect_ok in [
(['v1'], old_id, True),
(['v1'], new_id, False),
@@ -68,8 +65,8 @@ class ArvKeepdockerTestCase(unittest.TestCase):
else:
fakeDD['dockerImageFormats'] = supported
- err = io.BytesIO()
- out = io.BytesIO()
+ err = tutil.StringIO()
+ out = tutil.StringIO()
with tutil.redirected_streams(stdout=out), \
mock.patch('arvados.api') as api, \
@@ -101,8 +98,8 @@ class ArvKeepdockerTestCase(unittest.TestCase):
fakeDD = arvados.api('v1')._rootDesc
fakeDD['dockerImageFormats'] = ['v1']
- err = io.BytesIO()
- out = io.BytesIO()
+ err = tutil.StringIO()
+ out = tutil.StringIO()
with tutil.redirected_streams(stdout=out), \
mock.patch('arvados.api') as api, \
mock.patch('arvados.commands.keepdocker.popen_docker',
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index ae26ae7..6bb089b 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -4,7 +4,6 @@
from __future__ import absolute_import
from builtins import str
from builtins import range
-import io
import os
import random
import sys
@@ -15,9 +14,10 @@ import arvados.errors as arv_error
import arvados.commands.ls as arv_ls
from . import run_test_server
-from .arvados_testutil import str_keep_locator, redirected_streams
+from . import arvados_testutil as tutil
+from .arvados_testutil import str_keep_locator, redirected_streams, StringIO
-class ArvLsTestCase(run_test_server.TestCaseWithServers):
+class ArvLsTestCase(run_test_server.TestCaseWithServers, tutil.VersionChecker):
FAKE_UUID = 'zzzzz-4zz18-12345abcde12345'
def newline_join(self, seq):
@@ -39,8 +39,8 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
return coll_info, api_client
def run_ls(self, args, api_client):
- self.stdout = io.BytesIO()
- self.stderr = io.BytesIO()
+ self.stdout = StringIO()
+ self.stderr = StringIO()
return arv_ls.main(args, self.stdout, self.stderr, api_client)
def test_plain_listing(self):
@@ -85,10 +85,7 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
self.assertNotEqual('', self.stderr.getvalue())
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with redirected_streams(stdout=out, stderr=err):
+ with redirected_streams(stdout=StringIO, stderr=StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_ls(['--version'], None)
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
diff --git a/sdk/python/tests/test_arv_normalize.py b/sdk/python/tests/test_arv_normalize.py
index 426d41a..b0272f4 100644
--- a/sdk/python/tests/test_arv_normalize.py
+++ b/sdk/python/tests/test_arv_normalize.py
@@ -6,22 +6,28 @@ import sys
import tempfile
import unittest
+from . import arvados_testutil as tutil
-class ArvNormalizeTestCase(unittest.TestCase):
+
+class ArvNormalizeTestCase(unittest.TestCase, tutil.VersionChecker):
def run_arv_normalize(self, args=[]):
p = subprocess.Popen([sys.executable, 'bin/arv-normalize'] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
- (stdout, stderr) = p.communicate()
- return p.returncode, stdout, stderr
+ out, err = p.communicate()
+ sys.stdout.write(out.decode())
+ sys.stderr.write(err.decode())
+ return p.returncode
def test_unsupported_arg(self):
- returncode, out, err = self.run_arv_normalize(['-x=unknown'])
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
+ returncode = self.run_arv_normalize(['-x=unknown'])
self.assertNotEqual(0, returncode)
def test_version_argument(self):
- returncode, out, err = self.run_arv_normalize(['--version'])
- self.assertEqual(b'', out)
- self.assertNotEqual(b'', err)
- self.assertRegexpMatches(err.decode(), "^bin/arv-normalize [0-9]+\.[0-9]+\.[0-9]+$")
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
+ returncode = self.run_arv_normalize(['--version'])
+ self.assertVersionOutput(out, err)
self.assertEqual(0, returncode)
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 05e5e06..a738403 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -8,7 +8,6 @@ standard_library.install_aliases()
from builtins import str
from builtins import range
import apiclient
-import io
import mock
import os
import pwd
@@ -24,11 +23,6 @@ import threading
import hashlib
import random
-if sys.version_info >= (3, 0):
- from io import StringIO
-else:
- from cStringIO import StringIO
-
import arvados
import arvados.commands.put as arv_put
from . import arvados_testutil as tutil
@@ -291,7 +285,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
def test_writer_works_with_cache(self):
with tempfile.NamedTemporaryFile() as f:
- f.write('foo')
+ f.write(b'foo')
f.flush()
cwriter = arv_put.ArvPutUploadJob([f.name])
cwriter.start(save_collection=False)
@@ -310,7 +304,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
def test_progress_reporting(self):
with tempfile.NamedTemporaryFile() as f:
- f.write('foo')
+ f.write(b'foo')
f.flush()
for expect_count in (None, 8):
progression, reporter = self.make_progress_tester()
@@ -544,13 +538,15 @@ class ArvadosPutReportTest(ArvadosBaseTestCase):
arv_put.human_progress(count, None)))
-class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
+class ArvadosPutTest(run_test_server.TestCaseWithServers,
+ ArvadosBaseTestCase,
+ tutil.VersionChecker):
MAIN_SERVER = {}
Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
def call_main_with_args(self, args):
- self.main_stdout = StringIO()
- self.main_stderr = StringIO()
+ self.main_stdout = tutil.StringIO()
+ self.main_stderr = tutil.StringIO()
return arv_put.main(args, self.main_stdout, self.main_stderr)
def call_main_on_test_file(self, args=[]):
@@ -575,13 +571,11 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
super(ArvadosPutTest, self).tearDown()
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with tutil.redirected_streams(stdout=out, stderr=err):
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.call_main_with_args(['--version'])
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
def test_simple_file_put(self):
self.call_main_on_test_file()
@@ -651,7 +645,7 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
def test_api_error_handling(self):
coll_save_mock = mock.Mock(name='arv.collection.Collection().save_new()')
coll_save_mock.side_effect = arvados.errors.ApiError(
- fake_httplib2_response(403), '{}')
+ fake_httplib2_response(403), b'{}')
with mock.patch('arvados.collection.Collection.save_new',
new=coll_save_mock):
with self.assertRaises(SystemExit) as exc_test:
@@ -719,7 +713,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
0)
except ValueError as error:
- self.assertIn(BAD_UUID, error.message)
+ self.assertIn(BAD_UUID, str(error))
else:
self.assertFalse(result, "incorrectly found nonexistent project")
@@ -739,7 +733,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
[sys.executable, arv_put.__file__, '--stream'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=self.ENVIRON)
- pipe.stdin.write('stdin test\n')
+ pipe.stdin.write(b'stdin test\n')
pipe.stdin.close()
deadline = time.time() + 5
while (pipe.poll() is None) and (time.time() < deadline):
@@ -751,7 +745,8 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
elif returncode != 0:
sys.stdout.write(pipe.stdout.read())
self.fail("arv-put returned exit code {}".format(returncode))
- self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
+ self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11',
+ pipe.stdout.read().decode())
def test_ArvPutSignedManifest(self):
# ArvPutSignedManifest runs "arv-put foo" and then attempts to get
@@ -791,11 +786,12 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
[sys.executable, arv_put.__file__] + extra_args,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=self.ENVIRON)
- stdout, stderr = pipe.communicate(text)
+ stdout, stderr = pipe.communicate(text.encode())
search_key = ('portable_data_hash'
if '--portable-data-hash' in extra_args else 'uuid')
collection_list = arvados.api('v1').collections().list(
- filters=[[search_key, '=', stdout.strip()]]).execute().get('items', [])
+ filters=[[search_key, '=', stdout.decode().strip()]]
+ ).execute().get('items', [])
self.assertEqual(1, len(collection_list))
return collection_list[0]
diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py
index 0f33dfc..8823837 100644
--- a/sdk/python/tests/test_arv_run.py
+++ b/sdk/python/tests/test_arv_run.py
@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
-import io
import os
import sys
import tempfile
@@ -11,7 +10,7 @@ import unittest
import arvados.commands.run as arv_run
from . import arvados_testutil as tutil
-class ArvRunTestCase(unittest.TestCase):
+class ArvRunTestCase(unittest.TestCase, tutil.VersionChecker):
def run_arv_run(self, args):
sys.argv = ['arv-run'] + args
return arv_run.main()
@@ -21,10 +20,8 @@ class ArvRunTestCase(unittest.TestCase):
self.run_arv_run(['-x=unknown'])
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with tutil.redirected_streams(stdout=out, stderr=err):
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_arv_run(['--version'])
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
diff --git a/sdk/python/tests/test_arv_ws.py b/sdk/python/tests/test_arv_ws.py
index 4a0b823..86a21cc 100644
--- a/sdk/python/tests/test_arv_ws.py
+++ b/sdk/python/tests/test_arv_ws.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python
from __future__ import absolute_import
-import io
import os
import sys
import tempfile
@@ -11,7 +10,7 @@ import arvados.errors as arv_error
import arvados.commands.ws as arv_ws
from . import arvados_testutil as tutil
-class ArvWsTestCase(unittest.TestCase):
+class ArvWsTestCase(unittest.TestCase, tutil.VersionChecker):
def run_ws(self, args):
return arv_ws.main(args)
@@ -20,10 +19,8 @@ class ArvWsTestCase(unittest.TestCase):
self.run_ws(['-x=unknown'])
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
- with tutil.redirected_streams(stdout=out, stderr=err):
+ with tutil.redirected_streams(
+ stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_ws(['--version'])
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertVersionOutput(out, err)
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 0b2b741..e3ba090 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -68,27 +68,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_truncate(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_truncate",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.read(12))
+ self.assertEqual(b"0123456789", writer.read(12))
writer.truncate(8)
# Make sure reading off the end doesn't break
- self.assertEqual("", writer.read(12))
+ self.assertEqual(b"", writer.read(12))
self.assertEqual(writer.size(), 8)
writer.seek(0, os.SEEK_SET)
- self.assertEqual("01234567", writer.read(12))
+ self.assertEqual(b"01234567", writer.read(12))
self.assertIsNone(c.manifest_locator())
self.assertTrue(c.modified())
@@ -97,27 +102,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertFalse(c.modified())
def test_write_to_end(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_append",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
writer.seek(5, os.SEEK_SET)
- self.assertEqual("56789", writer.read(8))
+ self.assertEqual(b"56789", writer.read(8))
writer.seek(10, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 13)
writer.seek(5, os.SEEK_SET)
- self.assertEqual("56789foo", writer.read(8))
+ self.assertEqual(b"56789foo", writer.read(8))
self.assertIsNone(c.manifest_locator())
self.assertTrue(c.modified())
@@ -126,35 +136,39 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_append")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
+ self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
def test_append(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "a+")
- self.assertEqual(writer.read(20), "0123456789")
+ self.assertEqual(writer.read(20), b"0123456789")
writer.seek(0, os.SEEK_SET)
writer.write("hello")
- self.assertEqual(writer.read(20), "0123456789hello")
+ self.assertEqual(writer.read(20), b"0123456789hello")
writer.seek(0, os.SEEK_SET)
writer.write("world")
- self.assertEqual(writer.read(20), "0123456789helloworld")
+ self.assertEqual(writer.read(20), b"0123456789helloworld")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
def test_write_at_beginning(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(0, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("foo3456789", writer.readfrom(0, 13))
+ self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
def test_write_empty(self):
@@ -168,7 +182,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection(keep_client=keep) as c:
writer = c.open("count.txt", "w")
- writer.write("0123456789")
+ writer.write(b"0123456789")
self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
@@ -187,51 +201,51 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_in_middle(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(3, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("012foo6789", writer.readfrom(0, 13))
+ self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
def test_write_at_end(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(7, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456foo", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
def test_write_across_segment_boundary(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("012345678901234", writer.readfrom(0, 15))
+ self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
writer.seek(7, os.SEEK_SET)
writer.write("foobar")
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
+ self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
def test_write_across_several_segments(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("012301230123", writer.readfrom(0, 15))
+ self.assertEqual(b"012301230123", writer.readfrom(0, 15))
writer.seek(2, os.SEEK_SET)
writer.write("abcdefg")
self.assertEqual(writer.size(), 12)
- self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
+ self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
def test_write_large(self):
@@ -283,15 +297,17 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("0123456789")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 20))
self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
writer.flush()
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
def test_rewrite_append_existing_file(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
@@ -300,16 +316,18 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("abcdefghij")
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
writer.arvadosfile.flush()
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
def test_rewrite_over_existing_file(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
@@ -318,13 +336,13 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("abcdefghij")
self.assertEqual(writer.size(), 15)
- self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
writer.arvadosfile.flush()
self.assertEqual(writer.size(), 15)
- self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
def test_write_large_rewrite(self):
@@ -338,7 +356,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- text = ''.join(["0123456789" for a in range(0, 100)])
+ text = b''.join([b"0123456789" for a in range(0, 100)])
for b in range(0, 100000):
writer.write(text)
writer.seek(0, os.SEEK_SET)
@@ -353,12 +371,15 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_create(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
- "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
- "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name":"test_create",
+ "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
+ "replication_desired":None,
+ }, {
+ "uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
+ "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
+ })
with Collection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
@@ -371,7 +392,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_create")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
+ self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
def test_create_subdir(self):
@@ -444,7 +465,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_create_multiple")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
+ self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
@@ -478,7 +499,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
stream = []
n = 0
blocks = {}
- for d in ['01234', '34567', '67890']:
+ for d in [b'01234', b'34567', b'67890']:
loc = tutil.str_keep_locator(d)
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
@@ -490,27 +511,30 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
# read() needs to return all the data requested if possible, even if it
# crosses uncached blocks: https://arvados.org/issues/5856
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('12345678', sfile.read(8))
+ self.assertEqual(b'12345678', sfile.read(8))
def test_successive_reads(self):
# Override StreamFileReaderTestCase.test_successive_reads
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('1234', sfile.read(4))
- self.assertEqual('5678', sfile.read(4))
- self.assertEqual('9', sfile.read(4))
- self.assertEqual('', sfile.read(4))
+ self.assertEqual(b'1234', sfile.read(4))
+ self.assertEqual(b'5678', sfile.read(4))
+ self.assertEqual(b'9', sfile.read(4))
+ self.assertEqual(b'', sfile.read(4))
def test_tell_after_block_read(self):
# Override StreamFileReaderTestCase.test_tell_after_block_read
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('12345678', sfile.read(8))
+ self.assertEqual(b'12345678', sfile.read(8))
self.assertEqual(8, sfile.tell())
def test_prefetch(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
+ "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
+ })
with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
- self.assertEqual("0123", r.read(4))
+ self.assertEqual(b"0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
@@ -572,17 +596,17 @@ class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall(**kwargs))
+ return b''.join(reader.readall(**kwargs))
class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall_decompressed(**kwargs))
+ return b''.join(reader.readall_decompressed(**kwargs))
class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readlines(**kwargs))
+ return ''.join(reader.readlines(**kwargs)).encode()
class ArvadosFileTestCase(unittest.TestCase):
@@ -612,13 +636,13 @@ class BlockManagerTest(unittest.TestCase):
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
bufferblock.append("bar")
self.assertEqual(bufferblock.size(), 6)
- self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
+ self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
@@ -632,7 +656,7 @@ class BlockManagerTest(unittest.TestCase):
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
@@ -642,21 +666,23 @@ class BlockManagerTest(unittest.TestCase):
bufferblock2.append("bar")
self.assertEqual(bufferblock2.size(), 6)
- self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
+ self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
def test_bufferblock_get(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with arvados.arvfile._BlockManager(keep) as blockmanager:
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
- self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
- self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
+ self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
+ self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
def test_bufferblock_commit(self):
mockkeep = mock.MagicMock()
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index 1c00551..6adab7b 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -34,15 +34,17 @@ class CacheTestThread(threading.Thread):
for x in range(16):
try:
data_in = _random(128)
- data_in = hashlib.md5(data_in).hexdigest() + bytes("\n") + bytes(data_in)
+ data_in = bytes(hashlib.md5(data_in).hexdigest()) + bytes("\n") + bytes(data_in)
c.set(url, data_in)
data_out = c.get(url)
digest, _, content = data_out.partition("\n")
- if digest != hashlib.md5(content).hexdigest():
+ if digest != bytes(hashlib.md5(content).hexdigest()):
self.ok = False
- except Exception as err:
- self.ok = False
- print("cache failed: {}".format(err), file=sys.stderr)
+ finally:
+ pass
+ #except Exception as err:
+ # self.ok = False
+ # print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
class CacheTest(unittest.TestCase):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 259c5aa..3a1afce 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -42,13 +42,13 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
self.assertEqual(cw.current_stream_name(), '.',
'current_stream_name() should be "." now')
cw.set_current_file_name('foo.txt')
- cw.write('foo')
+ cw.write(b'foo')
self.assertEqual(cw.current_file_name(), 'foo.txt',
'current_file_name() should be foo.txt now')
cw.start_new_file('bar.txt')
- cw.write('bar')
+ cw.write(b'bar')
cw.start_new_stream('baz')
- cw.write('baz')
+ cw.write(b'baz')
cw.set_current_file_name('baz.txt')
self.assertEqual(cw.manifest_text(),
". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
@@ -58,8 +58,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
return cw.portable_data_hash()
def test_keep_local_store(self):
- self.assertEqual(self.keep_client.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
- self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
+ self.assertEqual(self.keep_client.put(b'foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
+ self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), b'foo', 'wrong data from Keep.get')
def test_local_collection_writer(self):
self.assertEqual(self.write_foo_bar_baz(),
@@ -74,20 +74,20 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
for s in cr.all_streams():
for f in s.all_files():
got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]]
- expected = [[3, '.', 'foo.txt', 'foo'],
- [3, '.', 'bar.txt', 'bar'],
- [3, './baz', 'baz.txt', 'baz']]
+ expected = [[3, '.', 'foo.txt', b'foo'],
+ [3, '.', 'bar.txt', b'bar'],
+ [3, './baz', 'baz.txt', b'baz']]
self.assertEqual(got,
expected)
stream0 = cr.all_streams()[0]
self.assertEqual(stream0.readfrom(0, 0),
- '',
+ b'',
'reading zero bytes should have returned empty string')
self.assertEqual(stream0.readfrom(0, 2**26),
- 'foobar',
+ b'foobar',
'reading entire stream failed')
self.assertEqual(stream0.readfrom(2**26, 0),
- '',
+ b'',
'reading zero bytes should have returned empty string')
def _test_subset(self, collection, expected):
@@ -104,50 +104,50 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
def test_collection_manifest_subset(self):
foobarbaz = self.write_foo_bar_baz()
self._test_subset(foobarbaz,
- [[3, '.', 'bar.txt', 'bar'],
- [3, '.', 'foo.txt', 'foo'],
- [3, './baz', 'baz.txt', 'baz']])
+ [[3, '.', 'bar.txt', b'bar'],
+ [3, '.', 'foo.txt', b'foo'],
+ [3, './baz', 'baz.txt', b'baz']])
self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
- (self.keep_client.put("foo"),
- self.keep_client.put("bar"))),
- [[3, '.', 'bar.txt', 'bar'],
- [3, '.', 'foo.txt', 'foo']])
+ (self.keep_client.put(b"foo"),
+ self.keep_client.put(b"bar"))),
+ [[3, '.', 'bar.txt', b'bar'],
+ [3, '.', 'foo.txt', b'foo']])
self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
- (self.keep_client.put("foo"),
- self.keep_client.put("bar"))),
- [[2, '.', 'fo.txt', 'fo'],
- [4, '.', 'obar.txt', 'obar']])
+ (self.keep_client.put(b"foo"),
+ self.keep_client.put(b"bar"))),
+ [[2, '.', 'fo.txt', b'fo'],
+ [4, '.', 'obar.txt', b'obar']])
self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
- (self.keep_client.put("foo"),
- self.keep_client.put("bar"))),
- [[2, '.', 'ar.txt', 'ar'],
- [2, '.', 'fo.txt', 'fo'],
- [2, '.', 'ob.txt', 'ob'],
- [0, '.', 'zero.txt', '']])
+ (self.keep_client.put(b"foo"),
+ self.keep_client.put(b"bar"))),
+ [[2, '.', 'ar.txt', b'ar'],
+ [2, '.', 'fo.txt', b'fo'],
+ [2, '.', 'ob.txt', b'ob'],
+ [0, '.', 'zero.txt', b'']])
def test_collection_empty_file(self):
cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('zero.txt')
- cw.write('')
+ cw.write(b'')
self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n")
self.check_manifest_file_sizes(cw.manifest_text(), [0])
cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('zero.txt')
- cw.write('')
+ cw.write(b'')
cw.start_new_file('one.txt')
- cw.write('1')
+ cw.write(b'1')
cw.start_new_stream('foo')
cw.start_new_file('zero.txt')
- cw.write('')
+ cw.write(b'')
self.check_manifest_file_sizes(cw.manifest_text(), [0,1,0])
def test_no_implicit_normalize(self):
cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('b')
- cw.write('b')
+ cw.write(b'b')
cw.start_new_file('a')
- cw.write('')
+ cw.write(b'')
self.check_manifest_file_sizes(cw.manifest_text(), [1,0])
self.check_manifest_file_sizes(
arvados.CollectionReader(
@@ -313,14 +313,16 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
return self.content[locator]
def test_stream_reader(self):
- keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij',
- 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy',
- 'cccccccccccccccccccccccccccccccc+5': 'z0123'}
+ keepblocks = {
+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': b'abcdefghij',
+ 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': b'klmnopqrstuvwxy',
+ 'cccccccccccccccccccccccccccccccc+5': b'z0123',
+ }
mk = self.MockKeep(keepblocks)
sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk)
- content = 'abcdefghijklmnopqrstuvwxyz0123456789'
+ content = b'abcdefghijklmnopqrstuvwxyz0123456789'
self.assertEqual(sr.readfrom(0, 30), content[0:30])
self.assertEqual(sr.readfrom(2, 30), content[2:30])
@@ -334,7 +336,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
self.assertEqual(sr.readfrom(15, 5), content[15:20])
self.assertEqual(sr.readfrom(20, 5), content[20:25])
self.assertEqual(sr.readfrom(25, 5), content[25:30])
- self.assertEqual(sr.readfrom(30, 5), '')
+ self.assertEqual(sr.readfrom(30, 5), b'')
def test_extract_file(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
@@ -423,7 +425,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
def test_write_multiple_files(self):
cwriter = arvados.CollectionWriter(self.api_client)
for letter in 'ABC':
- with self.make_test_file(letter) as testfile:
+ with self.make_test_file(letter.encode()) as testfile:
cwriter.write_file(testfile.name, letter)
self.assertEqual(
cwriter.manifest_text(),
@@ -466,7 +468,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
with self.make_test_file() as testfile:
cwriter.write_file(testfile.name, 'test')
orig_mtime = os.fstat(testfile.fileno()).st_mtime
- testfile.write('extra')
+ testfile.write(b'extra')
testfile.flush()
os.utime(testfile.name, (orig_mtime, orig_mtime))
self.assertRaises(arvados.errors.StaleWriterStateError,
@@ -592,8 +594,8 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
num_retries=3)
with tutil.mock_keep_responses('foo', 500, 500, 200):
- self.assertEqual('foo',
- ''.join(f.read(9) for f in reader.all_files()))
+ self.assertEqual(b'foo',
+ b''.join(f.read(9) for f in reader.all_files()))
def test_read_nonnormalized_manifest_with_collection_reader(self):
# client should be able to use CollectionReader on a manifest without normalizing it
@@ -680,7 +682,7 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
kwargs.setdefault('api_client', self.api_client_mock())
writer = arvados.CollectionWriter(**kwargs)
writer.start_new_file('foo')
- writer.write('foo')
+ writer.write(b'foo')
return writer
def test_write_whole_collection(self):
@@ -739,7 +741,7 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
with writer.open('out') as out_file:
self.assertEqual('.', writer.current_stream_name())
self.assertEqual('out', writer.current_file_name())
- out_file.write('test data')
+ out_file.write(b'test data')
data_loc = tutil.str_keep_locator('test data')
self.assertTrue(out_file.closed, "writer file not closed after context")
self.assertRaises(ValueError, out_file.write, 'extra text')
@@ -764,9 +766,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
writer = arvados.CollectionWriter(client)
with writer.open('flush_test') as out_file:
- out_file.write('flush1')
+ out_file.write(b'flush1')
out_file.flush()
- out_file.write('flush2')
+ out_file.write(b'flush2')
self.assertEqual(". {} {} 0:12:flush_test\n".format(data_loc1,
data_loc2),
writer.manifest_text())
@@ -775,9 +777,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
client = self.api_client_mock()
writer = arvados.CollectionWriter(client)
with writer.open('.', '1') as out_file:
- out_file.write('1st')
+ out_file.write(b'1st')
with writer.open('.', '2') as out_file:
- out_file.write('2nd')
+ out_file.write(b'2nd')
data_loc = tutil.str_keep_locator('1st2nd')
with self.mock_keep(data_loc, 200) as keep_mock:
self.assertEqual(". {} 0:3:1 3:3:2\n".format(data_loc),
@@ -790,9 +792,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
writer = arvados.CollectionWriter(client)
with writer.open('file') as out_file:
- out_file.write('file')
+ out_file.write(b'file')
with writer.open('./dir', 'indir') as out_file:
- out_file.write('indir')
+ out_file.write(b'indir')
expected = ". {} 0:4:file\n./dir {} 0:5:indir\n".format(
data_loc1, data_loc2)
self.assertEqual(expected, writer.manifest_text())
@@ -1030,7 +1032,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
f = c1.open("count1.txt", "w")
- f.write("zzzzz")
+ f.write(b"zzzzz")
# c1 changed, so it should not be deleted.
c1.apply(d)
@@ -1042,7 +1044,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
f = c1.open("count1.txt", "w")
- f.write("zzzzz")
+ f.write(b"zzzzz")
# c1 changed, so c2 mod will go to a conflict file
c1.apply(d)
@@ -1055,7 +1057,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
('add', './count1.txt', c2["count1.txt"])])
f = c1.open("count1.txt", "w")
- f.write("zzzzz")
+ f.write(b"zzzzz")
# c1 added count1.txt, so c2 add will go to a conflict file
c1.apply(d)
@@ -1092,9 +1094,9 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
with c.open("count.txt", "w") as f:
# One file committed
with c.open("foo.txt", "w") as foo:
- foo.write("foo")
+ foo.write(b"foo")
foo.flush() # Force block commit
- f.write("0123456789")
+ f.write(b"0123456789")
# Other file not committed. Block not written to keep yet.
self.assertEqual(
c._get_manifest_text(".",
@@ -1115,14 +1117,14 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
c = Collection()
# Write a couple of small files,
f = c.open("count.txt", "w")
- f.write("0123456789")
+ f.write(b"0123456789")
f.close(flush=False)
foo = c.open("foo.txt", "w")
- foo.write("foo")
+ foo.write(b"foo")
foo.close(flush=False)
# Then, write a big file, it shouldn't be packed with the ones above
big = c.open("bigfile.txt", "w")
- big.write("x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2
+ big.write(b"x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2
big.close(flush=False)
self.assertEqual(
c.manifest_text("."),
@@ -1143,7 +1145,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertEqual(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
with c.open("count.txt", "w") as f:
- f.write("0123456789")
+ f.write(b"0123456789")
self.assertEqual(c.portable_manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
@@ -1165,7 +1167,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c2 = Collection(c1.manifest_locator())
with c2.open("count.txt", "w") as f:
- f.write("abcdefg")
+ f.write(b"abcdefg")
diff = c1.diff(c2)
@@ -1193,7 +1195,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c2 = arvados.collection.Collection(c1.manifest_locator())
with c2.open("count.txt", "w") as f:
- f.write("abcdefg")
+ f.write(b"abcdefg")
c2.save()
@@ -1207,11 +1209,11 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c1.save()
with c1.open("count.txt", "w") as f:
- f.write("XYZ")
+ f.write(b"XYZ")
c2 = arvados.collection.Collection(c1.manifest_locator())
with c2.open("count.txt", "w") as f:
- f.write("abcdefg")
+ f.write(b"abcdefg")
c2.save()
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 4596b6c..84b586f 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -5,24 +5,25 @@ from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import object
-import arvados
-import io
import logging
import mock
import queue
-from . import run_test_server
+import sys
import threading
import time
import unittest
-from . import arvados_testutil
+import arvados
+from . import arvados_testutil as tutil
+from . import run_test_server
+
class WebsocketTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
TIME_PAST = time.time()-3600
TIME_FUTURE = time.time()+3600
- MOCK_WS_URL = 'wss://[{}]/'.format(arvados_testutil.TEST_HOST)
+ MOCK_WS_URL = 'wss://[{}]/'.format(tutil.TEST_HOST)
TEST_TIMEOUT = 10.0
@@ -96,10 +97,12 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
error_mock = mock.MagicMock()
error_mock.resp.status = 0
error_mock._get_reason.return_value = "testing"
- api_mock.logs().list().execute.side_effect = (arvados.errors.ApiError(error_mock, ""),
- {"items": [{"id": 1}], "items_available": 1},
- arvados.errors.ApiError(error_mock, ""),
- {"items": [{"id": 1}], "items_available": 1})
+ api_mock.logs().list().execute.side_effect = (
+ arvados.errors.ApiError(error_mock, b""),
+ {"items": [{"id": 1}], "items_available": 1},
+ arvados.errors.ApiError(error_mock, b""),
+ {"items": [{"id": 1}], "items_available": 1},
+ )
pc = arvados.events.PollClient(api_mock, [], on_ev, 15, None)
pc.start()
while len(n) < 2:
@@ -161,7 +164,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
run_test_server.authorize_with('active')
events = queue.Queue(100)
- logstream = io.BytesIO()
+ logstream = tutil.StringIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
@@ -229,7 +232,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
def test_websocket_reconnect_retry(self, event_client_connect):
event_client_connect.side_effect = [None, Exception('EventClient.connect error'), None]
- logstream = io.BytesIO()
+ logstream = tutil.StringIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 1b0b627..5ea2cc1 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -12,6 +12,7 @@ import pycurl
import random
import re
import socket
+import sys
import threading
import time
import unittest
@@ -48,12 +49,12 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
self.assertEqual(0, self.keep_client.download_counter.get())
self.assertEqual(self.keep_client.get(foo_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
self.assertEqual(3, self.keep_client.download_counter.get())
def test_KeepBinaryRWTest(self):
- blob_str = '\xff\xfe\xf7\x00\x01\x02'
+ blob_str = b'\xff\xfe\xf7\x00\x01\x02'
blob_locator = self.keep_client.put(blob_str)
self.assertRegexpMatches(
blob_locator,
@@ -64,28 +65,28 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'wrong content from Keep.get(md5(<binarydata>))')
def test_KeepLongBinaryRWTest(self):
- blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
+ blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03'
for i in range(0,23):
- blob_str = blob_str + blob_str
- blob_locator = self.keep_client.put(blob_str)
+ blob_data = blob_data + blob_data
+ blob_locator = self.keep_client.put(blob_data)
self.assertRegexpMatches(
blob_locator,
'^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
- blob_str,
+ blob_data,
'wrong content from Keep.get(md5(<binarydata>))')
@unittest.skip("unreliable test - please fix and close #8752")
def test_KeepSingleCopyRWTest(self):
- blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
- blob_locator = self.keep_client.put(blob_str, copies=1)
+ blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03'
+ blob_locator = self.keep_client.put(blob_data, copies=1)
self.assertRegexpMatches(
blob_locator,
'^c902006bc98a3eb4a3663b65ab4a6fab\+8',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
- blob_str,
+ blob_data,
'wrong content from Keep.get(md5(<binarydata>))')
def test_KeepEmptyCollectionTest(self):
@@ -103,9 +104,10 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
- with self.assertRaises(UnicodeEncodeError):
- # Error if it is not ASCII
- self.keep_client.put(u'\xe2')
+ if sys.version_info < (3, 0):
+ with self.assertRaises(UnicodeEncodeError):
+ # Error if it is not ASCII
+ self.keep_client.put(u'\xe2')
with self.assertRaises(AttributeError):
# Must be bytes or have an encode() method
@@ -119,7 +121,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'wrong md5 hash from Keep.put for "test_head": ' + locator)
self.assertEqual(True, self.keep_client.head(locator))
self.assertEqual(self.keep_client.get(locator),
- 'test_head',
+ b'test_head',
'wrong content from Keep.get for "test_head"')
class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
@@ -136,7 +138,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
self.assertEqual(keep_client.get(foo_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
# GET with an unsigned locator => NotFound
@@ -203,13 +205,13 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
def test_KeepAuthenticatedSignedTest(self):
signed_locator = self._put_foo_and_check()
self.assertEqual(self.keep_client.get(signed_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepAuthenticatedUnsignedTest(self):
signed_locator = self._put_foo_and_check()
self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedSignedTest(self):
@@ -218,7 +220,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
signed_locator = self._put_foo_and_check()
self.keep_client.api_token = ''
self.assertEqual(self.keep_client.get(signed_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedUnsignedTest(self):
@@ -227,7 +229,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
signed_locator = self._put_foo_and_check()
self.keep_client.api_token = ''
self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
@@ -257,7 +259,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
'^73feffa4b7f6bb68e44cf984c85f6e88\+3',
'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
self.assertEqual(keep_client.get(baz_locator),
- 'baz',
+ b'baz',
'wrong content from Keep.get(md5("baz"))')
self.assertTrue(keep_client.using_proxy)
@@ -273,7 +275,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
'^91f372a266fe2bf2823cb8ec7fda31ce\+4',
'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
self.assertEqual(keep_client.get(baz_locator),
- 'baz2',
+ b'baz2',
'wrong content from Keep.get(md5("baz2"))')
self.assertTrue(keep_client.using_proxy)
@@ -341,7 +343,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepWriteError):
- keep_client.put('foo')
+ keep_client.put(b'foo')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000))
@@ -482,7 +484,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertEqual(0, len(exc_check.exception.request_errors()))
def test_oddball_service_get(self):
- body = 'oddball service get'
+ body = b'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
keep_client = arvados.KeepClient(api_client=api_client)
@@ -490,7 +492,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertEqual(body, actual)
def test_oddball_service_put(self):
- body = 'oddball service put'
+ body = b'oddball service put'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
@@ -499,7 +501,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertEqual(pdh, actual)
def test_oddball_service_writer_count(self):
- body = 'oddball service writer count'
+ body = b'oddball service writer count'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore',
count=4)
@@ -530,7 +532,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
list('9d81c02e76a3bf54'),
]
self.blocks = [
- "{:064x}".format(x)
+ "{:064x}".format(x).encode()
for x in range(len(self.expected_order))]
self.hashes = [
hashlib.md5(self.blocks[x]).hexdigest()
@@ -567,7 +569,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertRaises(arvados.errors.KeepRequestError):
op(i)
got_order = [
- re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1)
+ re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL).decode()).group(1)
for resp in mock.responses]
self.assertEqual(self.expected_order[i]*2, got_order)
@@ -578,7 +580,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put(self.blocks[i], num_retries=2, copies=copies)
got_order = [
- re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1)
+ re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL).decode()).group(1)
for resp in mock.responses]
# With T threads racing to make requests, the position
# of a given server in the sequence of HTTP requests
@@ -606,7 +608,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
def test_probe_waste_adding_one_server(self):
hashes = [
- hashlib.md5("{:064x}".format(x)).hexdigest() for x in range(100)]
+ hashlib.md5("{:064x}".format(x).encode()).hexdigest() for x in range(100)]
initial_services = 12
self.api_client = self.mock_keep_services(count=initial_services)
self.keep_client = arvados.KeepClient(api_client=self.api_client)
@@ -644,7 +646,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
max_penalty))
def check_64_zeros_error_order(self, verb, exc_class):
- data = '0' * 64
+ data = b'0' * 64
if verb == 'get':
data = tutil.str_keep_locator(data)
# Arbitrary port number:
@@ -653,7 +655,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
keep_client = arvados.KeepClient(api_client=api_client)
with mock.patch('pycurl.Curl') as curl_mock, \
self.assertRaises(exc_class) as err_check:
- curl_mock.return_value.side_effect = socket.timeout
+ curl_mock.return_value = tutil.FakeCurl.make(code=500, body=b'')
getattr(keep_client, verb)(data)
urls = [urllib.parse.urlparse(url)
for url in err_check.exception.request_errors()]
@@ -671,7 +673,7 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
# BANDWIDTH_LOW_LIM must be less than len(DATA) so we can transfer
# 1s worth of data and then trigger bandwidth errors before running
# out of data.
- DATA = 'x'*2**11
+ DATA = b'x'*2**11
BANDWIDTH_LOW_LIM = 1024
TIMEOUT_TIME = 1.0
@@ -847,9 +849,9 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
code=200, body='foo', headers={'Content-Length': 3})
self.mock_disks_and_gateways()
locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K@' + self.gateways[0]['uuid']
- self.assertEqual('foo', self.keepClient.get(locator))
+ self.assertEqual(b'foo', self.keepClient.get(locator))
self.assertEqual(self.gateway_roots[0]+locator,
- MockCurl.return_value.getopt(pycurl.URL))
+ MockCurl.return_value.getopt(pycurl.URL).decode())
self.assertEqual(True, self.keepClient.head(locator))
@mock.patch('pycurl.Curl')
@@ -869,11 +871,11 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
# Gateways are tried first, in the order given.
for i, root in enumerate(self.gateway_roots):
self.assertEqual(root+locator,
- mocks[i].getopt(pycurl.URL))
+ mocks[i].getopt(pycurl.URL).decode())
# Disk services are tried next.
for i in range(gateways, gateways+disks):
self.assertRegexpMatches(
- mocks[i].getopt(pycurl.URL),
+ mocks[i].getopt(pycurl.URL).decode(),
r'keep0x')
@mock.patch('pycurl.Curl')
@@ -881,7 +883,7 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
gateways = 4
disks = 3
mocks = [
- tutil.FakeCurl.make(code=404, body='')
+ tutil.FakeCurl.make(code=404, body=b'')
for _ in range(gateways+disks)
]
MockCurl.side_effect = tutil.queue_with(mocks)
@@ -893,32 +895,32 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
# Gateways are tried first, in the order given.
for i, root in enumerate(self.gateway_roots):
self.assertEqual(root+locator,
- mocks[i].getopt(pycurl.URL))
+ mocks[i].getopt(pycurl.URL).decode())
# Disk services are tried next.
for i in range(gateways, gateways+disks):
self.assertRegexpMatches(
- mocks[i].getopt(pycurl.URL),
+ mocks[i].getopt(pycurl.URL).decode(),
r'keep0x')
@mock.patch('pycurl.Curl')
def test_get_with_remote_proxy_hint(self, MockCurl):
MockCurl.return_value = tutil.FakeCurl.make(
- code=200, body='foo', headers={'Content-Length': 3})
+ code=200, body=b'foo', headers={'Content-Length': 3})
self.mock_disks_and_gateways()
locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K at xyzzy'
- self.assertEqual('foo', self.keepClient.get(locator))
+ self.assertEqual(b'foo', self.keepClient.get(locator))
self.assertEqual('https://keep.xyzzy.arvadosapi.com/'+locator,
- MockCurl.return_value.getopt(pycurl.URL))
+ MockCurl.return_value.getopt(pycurl.URL).decode())
@mock.patch('pycurl.Curl')
def test_head_with_remote_proxy_hint(self, MockCurl):
MockCurl.return_value = tutil.FakeCurl.make(
- code=200, body='foo', headers={'Content-Length': 3})
+ code=200, body=b'foo', headers={'Content-Length': 3})
self.mock_disks_and_gateways()
locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K at xyzzy'
self.assertEqual(True, self.keepClient.head(locator))
self.assertEqual('https://keep.xyzzy.arvadosapi.com/'+locator,
- MockCurl.return_value.getopt(pycurl.URL))
+ MockCurl.return_value.getopt(pycurl.URL).decode())
class KeepClientRetryTestMixin(object):
@@ -937,7 +939,7 @@ class KeepClientRetryTestMixin(object):
# out appropriate methods in the client.
PROXY_ADDR = 'http://[%s]:65535/' % (tutil.TEST_HOST,)
- TEST_DATA = 'testdata'
+ TEST_DATA = b'testdata'
TEST_LOCATOR = 'ef654c40ab4f1747fc699915d4f70902+8'
def setUp(self):
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 7277628..37cdbf2 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -26,28 +26,28 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_read_block_crossing_behavior(self):
# read() calls will be aligned on block boundaries - see #3663.
sfile = self.make_count_reader()
- self.assertEqual('123', sfile.read(10))
+ self.assertEqual(b'123', sfile.read(10))
def test_small_read(self):
sfile = self.make_count_reader()
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
def test_successive_reads(self):
sfile = self.make_count_reader()
- for expect in ['123', '456', '789', '']:
+ for expect in [b'123', b'456', b'789', b'']:
self.assertEqual(expect, sfile.read(10))
def test_readfrom_spans_blocks(self):
sfile = self.make_count_reader()
- self.assertEqual('6789', sfile.readfrom(5, 12))
+ self.assertEqual(b'6789', sfile.readfrom(5, 12))
def test_small_readfrom_spanning_blocks(self):
sfile = self.make_count_reader()
- self.assertEqual('2345', sfile.readfrom(1, 4))
+ self.assertEqual(b'2345', sfile.readfrom(1, 4))
def test_readall(self):
sfile = self.make_count_reader()
- self.assertEqual('123456789', ''.join(sfile.readall()))
+ self.assertEqual(b'123456789', b''.join(sfile.readall()))
def test_one_arg_seek(self):
self.test_absolute_seek([])
@@ -55,20 +55,20 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_absolute_seek(self, args=[os.SEEK_SET]):
sfile = self.make_count_reader()
sfile.seek(6, *args)
- self.assertEqual('78', sfile.read(2))
+ self.assertEqual(b'78', sfile.read(2))
sfile.seek(4, *args)
- self.assertEqual('56', sfile.read(2))
+ self.assertEqual(b'56', sfile.read(2))
def test_relative_seek(self, args=[os.SEEK_CUR]):
sfile = self.make_count_reader()
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
sfile.seek(2, *args)
- self.assertEqual('56', sfile.read(2))
+ self.assertEqual(b'56', sfile.read(2))
def test_end_seek(self):
sfile = self.make_count_reader()
sfile.seek(-6, os.SEEK_END)
- self.assertEqual('45', sfile.read(2))
+ self.assertEqual(b'45', sfile.read(2))
def test_seek_min_zero(self):
sfile = self.make_count_reader()
@@ -101,7 +101,7 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_context(self):
with self.make_count_reader() as sfile:
self.assertFalse(sfile.closed, "reader is closed inside context")
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
self.assertTrue(sfile.closed, "reader is open after context")
def make_newlines_reader(self):
@@ -163,12 +163,12 @@ class StreamFileReaderTestCase(unittest.TestCase):
self.check_decompressed_name('test.log.bz2', 'test.log')
def check_decompression(self, compress_ext, compress_func):
- test_text = 'decompression\ntest\n'
+ test_text = b'decompression\ntest\n'
test_data = compress_func(test_text)
stream = tutil.MockStreamReader('.', test_data)
reader = StreamFileReader(stream, [Range(0, 0, len(test_data))],
'test.' + compress_ext)
- self.assertEqual(test_text, ''.join(reader.readall_decompressed()))
+ self.assertEqual(test_text, b''.join(reader.readall_decompressed()))
@staticmethod
def gzip_compress(data):
@@ -197,7 +197,7 @@ class StreamFileReaderTestCase(unittest.TestCase):
reader = self.make_newlines_reader()
data = reader.readline()
self.assertEqual('one\n', data)
- self.assertEqual(''.join(['two\n', '\n', 'three\n', 'four\n', '\n']), ''.join(reader.readall()))
+ self.assertEqual(b''.join([b'two\n', b'\n', b'three\n', b'four\n', b'\n']), b''.join(reader.readall()))
class StreamRetryTestMixin(object):
@@ -216,7 +216,7 @@ class StreamRetryTestMixin(object):
def test_success_without_retries(self):
with tutil.mock_keep_responses('bar', 200):
reader = self.reader_for('bar_file')
- self.assertEqual('bar', self.read_for_test(reader, 3))
+ self.assertEqual(b'bar', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_no_default_retry(self):
@@ -229,13 +229,13 @@ class StreamRetryTestMixin(object):
def test_read_with_instance_retries(self):
with tutil.mock_keep_responses('foo', 500, 200):
reader = self.reader_for('foo_file', num_retries=3)
- self.assertEqual('foo', self.read_for_test(reader, 3))
+ self.assertEqual(b'foo', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_with_method_retries(self):
with tutil.mock_keep_responses('foo', 500, 200):
reader = self.reader_for('foo_file')
- self.assertEqual('foo',
+ self.assertEqual(b'foo',
self.read_for_test(reader, 3, num_retries=3))
@tutil.skip_sleep
@@ -291,17 +291,17 @@ class StreamFileReadFromTestCase(StreamFileReadTestCase):
class StreamFileReadAllTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall(**kwargs))
+ return b''.join(reader.readall(**kwargs))
class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall_decompressed(**kwargs))
+ return b''.join(reader.readall_decompressed(**kwargs))
class StreamFileReadlinesTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readlines(**kwargs))
+ return ''.join(reader.readlines(**kwargs)).encode()
if __name__ == '__main__':
unittest.main()
diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py
index 41739a9..bf59d0d 100644
--- a/sdk/python/tests/test_util.py
+++ b/sdk/python/tests/test_util.py
@@ -20,7 +20,7 @@ class MkdirDashPTest(unittest.TestCase):
def runTest(self):
arvados.util.mkdir_dash_p('./tmp/foo')
with open('./tmp/bar', 'wb') as f:
- f.write('bar')
+ f.write(b'bar')
self.assertRaises(OSError, arvados.util.mkdir_dash_p, './tmp/bar')
@@ -28,8 +28,8 @@ class RunCommandTestCase(unittest.TestCase):
def test_success(self):
stdout, stderr = arvados.util.run_command(['echo', 'test'],
stderr=subprocess.PIPE)
- self.assertEqual("test\n", stdout)
- self.assertEqual("", stderr)
+ self.assertEqual("test\n".encode(), stdout)
+ self.assertEqual("".encode(), stderr)
def test_failure(self):
with self.assertRaises(arvados.errors.CommandFailedError):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list