[ARVADOS] created: 76d9365ab731c9c8a2645887bad5173a399e3e11
Git user
git at public.curoverse.com
Sun Apr 2 01:26:57 EDT 2017
at 76d9365ab731c9c8a2645887bad5173a399e3e11 (commit)
commit 76d9365ab731c9c8a2645887bad5173a399e3e11
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Apr 2 01:26:36 2017 -0400
more str vs bytes, for py3
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index c6cb1c9..8b8c2ce 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -112,7 +112,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
def readall(self, size=2**20, num_retries=None):
while True:
data = self.read(size, num_retries=num_retries)
- if data == '':
+ if len(data) == 0:
break
yield data
@@ -124,23 +124,23 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
data = [cache_data]
self._filepos += len(cache_data)
else:
- data = ['']
+ data = [b'']
data_size = len(data[-1])
- while (data_size < size) and ('\n' not in data[-1]):
+ while (data_size < size) and (b'\n' not in data[-1]):
next_read = self.read(2 ** 20, num_retries=num_retries)
if not next_read:
break
data.append(next_read)
data_size += len(next_read)
- data = ''.join(data)
+ data = b''.join(data)
try:
- nextline_index = data.index('\n') + 1
+ nextline_index = data.index(b'\n') + 1
except ValueError:
nextline_index = len(data)
nextline_index = min(nextline_index, size)
self._filepos -= len(data) - nextline_index
self._readline_cache = (self.tell(), data[nextline_index:])
- return data[:nextline_index]
+ return data[:nextline_index].decode()
@_FileLikeObjectBase._before_close
@retry_method
@@ -175,7 +175,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
data_size += len(s)
if data_size >= sizehint:
break
- return ''.join(data).splitlines(True)
+ return b''.join(data).decode().splitlines(True)
def size(self):
raise NotImplementedError()
@@ -212,9 +212,9 @@ class StreamFileReader(ArvadosFileReaderBase):
def read(self, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at the current file position"""
if size == 0:
- return ''
+ return b''
- data = ''
+ data = b''
available_chunks = locators_and_ranges(self.segments, self._filepos, size)
if available_chunks:
lr = available_chunks[0]
@@ -230,13 +230,13 @@ class StreamFileReader(ArvadosFileReaderBase):
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
if size == 0:
- return ''
+ return b''
data = []
for lr in locators_and_ranges(self.segments, start, size):
data.append(self._stream.readfrom(lr.locator+lr.segment_offset, lr.segment_size,
num_retries=num_retries))
- return ''.join(data)
+ return b''.join(data)
def as_manifest(self):
segs = []
@@ -316,6 +316,8 @@ class _BufferBlock(object):
"""
if self._state == _BufferBlock.WRITABLE:
+ if not isinstance(data, bytes):
+ data = data.encode()
while (self.write_pointer+len(data)) > len(self.buffer_block):
new_buffer_block = bytearray(len(self.buffer_block) * 2)
new_buffer_block[0:self.write_pointer] = self.buffer_block[0:self.write_pointer]
@@ -944,7 +946,7 @@ class ArvadosFile(object):
with self.lock:
if size == 0 or offset >= self.size():
- return ''
+ return b''
readsegs = locators_and_ranges(self._segments, offset, size)
prefetch = locators_and_ranges(self._segments, offset + size, config.KEEP_BLOCK_SIZE, limit=32)
@@ -964,7 +966,7 @@ class ArvadosFile(object):
self.parent._my_block_manager().block_prefetch(lr.locator)
locs.add(lr.locator)
- return ''.join(data)
+ return b''.join(data)
def _repack_writes(self, num_retries):
"""Test if the buffer block has more data than actual segments.
@@ -1001,6 +1003,8 @@ class ArvadosFile(object):
necessary.
"""
+ if not isinstance(data, bytes):
+ data = data.encode()
if len(data) == 0:
return
@@ -1157,7 +1161,7 @@ class ArvadosFileReader(ArvadosFileReaderBase):
data.append(rd)
self._filepos += len(rd)
rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
- return ''.join(data)
+ return b''.join(data)
else:
data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
self._filepos += len(data)
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index 868b478..f59d92f 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -33,7 +33,7 @@ class SafeHTTPCache(object):
return self._dir
def _filename(self, url):
- return os.path.join(self._dir, hashlib.md5(url).hexdigest()+'.tmp')
+ return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
def get(self, url):
filename = self._filename(url)
@@ -50,7 +50,7 @@ class SafeHTTPCache(object):
return None
try:
try:
- f = os.fdopen(fd, 'w')
+ f = os.fdopen(fd, 'wb')
except:
os.close(fd)
raise
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 0d88084..6d19ff2 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -220,7 +220,11 @@ class CollectionWriter(CollectionBase):
self.do_queued_work()
def write(self, newdata):
- if hasattr(newdata, '__iter__'):
+ if isinstance(newdata, bytes):
+ pass
+ elif isinstance(newdata, str):
+ newdata = newdata.encode()
+ elif hasattr(newdata, '__iter__'):
for s in newdata:
self.write(s)
return
@@ -260,7 +264,7 @@ class CollectionWriter(CollectionBase):
return self._last_open
def flush_data(self):
- data_buffer = ''.join(self._data_buffer)
+ data_buffer = b''.join(self._data_buffer)
if data_buffer:
self._current_stream_locators.append(
self._my_keep().put(
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index ee91491..cdc3b33 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -442,7 +442,7 @@ class KeepClient(object):
raise arvados.errors.HttpError(0, str(e))
self._result = {
'status_code': curl.getinfo(pycurl.RESPONSE_CODE),
- 'body': response_body.getvalue(),
+ 'body': response_body.getvalue().decode('utf-8'),
'headers': self._headers,
'error': False,
}
@@ -851,7 +851,7 @@ class KeepClient(object):
The weight is md5(h + u) where u is the last 15 characters of
the service endpoint's UUID.
"""
- return hashlib.md5(data_hash + service_uuid[-15:]).hexdigest()
+ return hashlib.md5((data_hash + service_uuid[-15:]).encode()).hexdigest()
def weighted_service_roots(self, locator, force_rebuild=False, need_writable=False):
"""Return an array of Keep service endpoints, in the order in
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 5955816..1fe5d35 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -80,13 +80,13 @@ class StreamReader(object):
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
if size == 0:
- return ''
+ return b''
if self._keep is None:
self._keep = KeepClient(num_retries=self.num_retries)
data = []
for lr in locators_and_ranges(self._data_locators, start, size):
data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size])
- return ''.join(data)
+ return b''.join(data)
def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 5992371..5bcb8e4 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -55,7 +55,7 @@ def mock_api_responses(api_client, body, codes, headers={}):
(fake_httplib2_response(code, **headers), body) for code in codes)))
def str_keep_locator(s):
- return '{}+{}'.format(hashlib.md5(s).hexdigest(), len(s))
+ return '{}+{}'.format(hashlib.md5(s if isinstance(s, bytes) else s.encode()).hexdigest(), len(s))
@contextlib.contextmanager
def redirected_streams(stdout=None, stderr=None):
@@ -146,7 +146,9 @@ def mock_keep_responses(body, *codes, **headers):
class MockStreamReader(object):
def __init__(self, name='.', *data):
self._name = name
- self._data = ''.join(data)
+ self._data = b''.join([
+ b if isinstance(b, bytes) else b.encode()
+ for b in data])
self._data_locators = [str_keep_locator(d) for d in data]
self.num_retries = 0
diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py
index c8eb5d1..e9ee34d 100644
--- a/sdk/python/tests/test_arv_copy.py
+++ b/sdk/python/tests/test_arv_copy.py
@@ -2,12 +2,16 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
-import io
import os
import sys
import tempfile
import unittest
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
import arvados.commands.arv_copy as arv_copy
from . import arvados_testutil as tutil
@@ -21,8 +25,8 @@ class ArvCopyTestCase(unittest.TestCase):
self.run_copy(['-x=unknown'])
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with tutil.redirected_streams(stdout=out, stderr=err):
with self.assertRaises(SystemExit):
self.run_copy(['--version'])
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index 2257227..153ce7f 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -4,7 +4,6 @@
from __future__ import absolute_import
import arvados
import hashlib
-import io
import mock
import os
import subprocess
@@ -17,6 +16,11 @@ import arvados.commands.keepdocker as arv_keepdocker
from . import arvados_testutil as tutil
from . import run_test_server
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
class StopTest(Exception):
pass
@@ -37,13 +41,17 @@ class ArvKeepdockerTestCase(unittest.TestCase):
self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with tutil.redirected_streams(stdout=out, stderr=err):
with self.assertRaises(SystemExit):
self.run_arv_keepdocker(['--version'], sys.stderr)
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ if sys.version >= (3, 0):
+ self.assertEqual(err.getvalue(), '')
+ self.assertRegexpMatches(out.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ else:
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
@mock.patch('arvados.commands.keepdocker.find_image_hashes',
return_value=['abc123'])
@@ -68,8 +76,8 @@ class ArvKeepdockerTestCase(unittest.TestCase):
else:
fakeDD['dockerImageFormats'] = supported
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with tutil.redirected_streams(stdout=out), \
mock.patch('arvados.api') as api, \
@@ -101,8 +109,8 @@ class ArvKeepdockerTestCase(unittest.TestCase):
fakeDD = arvados.api('v1')._rootDesc
fakeDD['dockerImageFormats'] = ['v1']
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with tutil.redirected_streams(stdout=out), \
mock.patch('arvados.api') as api, \
mock.patch('arvados.commands.keepdocker.popen_docker',
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index ae26ae7..84fbbdf 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -4,13 +4,17 @@
from __future__ import absolute_import
from builtins import str
from builtins import range
-import io
import os
import random
import sys
import mock
import tempfile
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
import arvados.errors as arv_error
import arvados.commands.ls as arv_ls
from . import run_test_server
@@ -39,8 +43,8 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
return coll_info, api_client
def run_ls(self, args, api_client):
- self.stdout = io.BytesIO()
- self.stderr = io.BytesIO()
+ self.stdout = StringIO()
+ self.stderr = StringIO()
return arv_ls.main(args, self.stdout, self.stderr, api_client)
def test_plain_listing(self):
@@ -85,10 +89,15 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
self.assertNotEqual('', self.stderr.getvalue())
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with redirected_streams(stdout=out, stderr=err):
with self.assertRaises(SystemExit):
self.run_ls(['--version'], None)
- self.assertEqual(out.getvalue(), '')
- self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ if sys.version_info >= (3, 0):
+ self.assertEqual(err.getvalue(), '')
+ self.assertRegexpMatches(out.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ else:
+ # Python 2 writes version info on stderr.
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_normalize.py b/sdk/python/tests/test_arv_normalize.py
index 426d41a..a57097a 100644
--- a/sdk/python/tests/test_arv_normalize.py
+++ b/sdk/python/tests/test_arv_normalize.py
@@ -21,7 +21,11 @@ class ArvNormalizeTestCase(unittest.TestCase):
def test_version_argument(self):
returncode, out, err = self.run_arv_normalize(['--version'])
- self.assertEqual(b'', out)
- self.assertNotEqual(b'', err)
- self.assertRegexpMatches(err.decode(), "^bin/arv-normalize [0-9]+\.[0-9]+\.[0-9]+$")
+ if sys.version_info >= (3, 0):
+ self.assertEqual(b'', err)
+ v = out.decode()
+ else:
+ self.assertEqual(b'', out)
+ v = err.decode()
+ self.assertRegexpMatches(v, "^bin/arv-normalize [0-9]+\.[0-9]+\.[0-9]+$")
self.assertEqual(0, returncode)
diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py
index 0f33dfc..35cb43a 100644
--- a/sdk/python/tests/test_arv_run.py
+++ b/sdk/python/tests/test_arv_run.py
@@ -2,12 +2,16 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
-import io
import os
import sys
import tempfile
import unittest
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
import arvados.commands.run as arv_run
from . import arvados_testutil as tutil
@@ -21,8 +25,8 @@ class ArvRunTestCase(unittest.TestCase):
self.run_arv_run(['-x=unknown'])
def test_version_argument(self):
- err = io.BytesIO()
- out = io.BytesIO()
+ err = StringIO()
+ out = StringIO()
with tutil.redirected_streams(stdout=out, stderr=err):
with self.assertRaises(SystemExit):
self.run_arv_run(['--version'])
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 0b2b741..ae510fb 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -68,27 +68,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_truncate(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_truncate",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.read(12))
+ self.assertEqual(b"0123456789", writer.read(12))
writer.truncate(8)
# Make sure reading off the end doesn't break
- self.assertEqual("", writer.read(12))
+ self.assertEqual(b"", writer.read(12))
self.assertEqual(writer.size(), 8)
writer.seek(0, os.SEEK_SET)
- self.assertEqual("01234567", writer.read(12))
+ self.assertEqual(b"01234567", writer.read(12))
self.assertIsNone(c.manifest_locator())
self.assertTrue(c.modified())
@@ -97,27 +102,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertFalse(c.modified())
def test_write_to_end(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_append",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
writer.seek(5, os.SEEK_SET)
- self.assertEqual("56789", writer.read(8))
+ self.assertEqual(b"56789", writer.read(8))
writer.seek(10, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 13)
writer.seek(5, os.SEEK_SET)
- self.assertEqual("56789foo", writer.read(8))
+ self.assertEqual(b"56789foo", writer.read(8))
self.assertIsNone(c.manifest_locator())
self.assertTrue(c.modified())
@@ -126,35 +136,39 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_append")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
+ self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
def test_append(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "a+")
- self.assertEqual(writer.read(20), "0123456789")
+ self.assertEqual(writer.read(20), b"0123456789")
writer.seek(0, os.SEEK_SET)
writer.write("hello")
- self.assertEqual(writer.read(20), "0123456789hello")
+ self.assertEqual(writer.read(20), b"0123456789hello")
writer.seek(0, os.SEEK_SET)
writer.write("world")
- self.assertEqual(writer.read(20), "0123456789helloworld")
+ self.assertEqual(writer.read(20), b"0123456789helloworld")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
def test_write_at_beginning(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(0, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("foo3456789", writer.readfrom(0, 13))
+ self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
def test_write_empty(self):
@@ -168,7 +182,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection(keep_client=keep) as c:
writer = c.open("count.txt", "w")
- writer.write("0123456789")
+ writer.write(b"0123456789")
self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
@@ -187,51 +201,51 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_in_middle(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(3, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("012foo6789", writer.readfrom(0, 13))
+ self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
def test_write_at_end(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("0123456789", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(7, os.SEEK_SET)
writer.write("foo")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456foo", writer.readfrom(0, 13))
+ self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
def test_write_across_segment_boundary(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("012345678901234", writer.readfrom(0, 15))
+ self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
writer.seek(7, os.SEEK_SET)
writer.write("foobar")
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
+ self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
def test_write_across_several_segments(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- self.assertEqual("012301230123", writer.readfrom(0, 15))
+ self.assertEqual(b"012301230123", writer.readfrom(0, 15))
writer.seek(2, os.SEEK_SET)
writer.write("abcdefg")
self.assertEqual(writer.size(), 12)
- self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
+ self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
def test_write_large(self):
@@ -283,15 +297,17 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("0123456789")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 20))
self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
writer.flush()
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
def test_rewrite_append_existing_file(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
@@ -300,16 +316,18 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("abcdefghij")
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
writer.arvadosfile.flush()
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
def test_rewrite_over_existing_file(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
@@ -318,13 +336,13 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("abcdefghij")
self.assertEqual(writer.size(), 15)
- self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
writer.arvadosfile.flush()
self.assertEqual(writer.size(), 15)
- self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
+ self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
def test_write_large_rewrite(self):
@@ -338,7 +356,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- text = ''.join(["0123456789" for a in range(0, 100)])
+ text = b''.join([b"0123456789" for a in range(0, 100)])
for b in range(0, 100000):
writer.write(text)
writer.seek(0, os.SEEK_SET)
@@ -353,12 +371,15 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_create(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
- "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
- "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name":"test_create",
+ "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
+ "replication_desired":None,
+ }, {
+ "uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
+ "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
+ })
with Collection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
@@ -371,7 +392,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_create")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
+ self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
def test_create_subdir(self):
@@ -444,7 +465,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c.save_new("test_create_multiple")
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
- self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
+ self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
@@ -478,7 +499,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
stream = []
n = 0
blocks = {}
- for d in ['01234', '34567', '67890']:
+ for d in [b'01234', b'34567', b'67890']:
loc = tutil.str_keep_locator(d)
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
@@ -490,27 +511,30 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
# read() needs to return all the data requested if possible, even if it
# crosses uncached blocks: https://arvados.org/issues/5856
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('12345678', sfile.read(8))
+ self.assertEqual(b'12345678', sfile.read(8))
def test_successive_reads(self):
# Override StreamFileReaderTestCase.test_successive_reads
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('1234', sfile.read(4))
- self.assertEqual('5678', sfile.read(4))
- self.assertEqual('9', sfile.read(4))
- self.assertEqual('', sfile.read(4))
+ self.assertEqual(b'1234', sfile.read(4))
+ self.assertEqual(b'5678', sfile.read(4))
+ self.assertEqual(b'9', sfile.read(4))
+ self.assertEqual(b'', sfile.read(4))
def test_tell_after_block_read(self):
# Override StreamFileReaderTestCase.test_tell_after_block_read
sfile = self.make_count_reader(nocache=True)
- self.assertEqual('12345678', sfile.read(8))
+ self.assertEqual(b'12345678', sfile.read(8))
self.assertEqual(8, sfile.tell())
def test_prefetch(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
+ "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
+ })
with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
- self.assertEqual("0123", r.read(4))
+ self.assertEqual(b"0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
@@ -572,12 +596,12 @@ class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall(**kwargs))
+ return b''.join(reader.readall(**kwargs))
class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall_decompressed(**kwargs))
+ return b''.join(reader.readall_decompressed(**kwargs))
class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
@@ -612,13 +636,13 @@ class BlockManagerTest(unittest.TestCase):
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
bufferblock.append("bar")
self.assertEqual(bufferblock.size(), 6)
- self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
+ self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
@@ -632,7 +656,7 @@ class BlockManagerTest(unittest.TestCase):
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
@@ -642,21 +666,23 @@ class BlockManagerTest(unittest.TestCase):
bufferblock2.append("bar")
self.assertEqual(bufferblock2.size(), 6)
- self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
+ self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
self.assertEqual(bufferblock.size(), 3)
- self.assertEqual(bufferblock.buffer_view[0:3], "foo")
+ self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
def test_bufferblock_get(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
with arvados.arvfile._BlockManager(keep) as blockmanager:
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
- self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
- self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
+ self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
+ self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
def test_bufferblock_commit(self):
mockkeep = mock.MagicMock()
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index 1c00551..6adab7b 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -34,15 +34,17 @@ class CacheTestThread(threading.Thread):
for x in range(16):
try:
data_in = _random(128)
- data_in = hashlib.md5(data_in).hexdigest() + bytes("\n") + bytes(data_in)
+ data_in = bytes(hashlib.md5(data_in).hexdigest()) + bytes("\n") + bytes(data_in)
c.set(url, data_in)
data_out = c.get(url)
digest, _, content = data_out.partition("\n")
- if digest != hashlib.md5(content).hexdigest():
+ if digest != bytes(hashlib.md5(content).hexdigest()):
self.ok = False
- except Exception as err:
- self.ok = False
- print("cache failed: {}".format(err), file=sys.stderr)
+ finally:
+ pass
+ #except Exception as err:
+ # self.ok = False
+ # print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
class CacheTest(unittest.TestCase):
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 4596b6c..c7f681e 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -5,17 +5,23 @@ from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import object
-import arvados
-import io
import logging
import mock
import queue
-from . import run_test_server
+import sys
import threading
import time
import unittest
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
+
+import arvados
from . import arvados_testutil
+from . import run_test_server
+
class WebsocketTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
@@ -96,10 +102,12 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
error_mock = mock.MagicMock()
error_mock.resp.status = 0
error_mock._get_reason.return_value = "testing"
- api_mock.logs().list().execute.side_effect = (arvados.errors.ApiError(error_mock, ""),
- {"items": [{"id": 1}], "items_available": 1},
- arvados.errors.ApiError(error_mock, ""),
- {"items": [{"id": 1}], "items_available": 1})
+ api_mock.logs().list().execute.side_effect = (
+ arvados.errors.ApiError(error_mock, ""),
+ {"items": [{"id": 1}], "items_available": 1},
+ arvados.errors.ApiError(error_mock, ""),
+ {"items": [{"id": 1}], "items_available": 1},
+ )
pc = arvados.events.PollClient(api_mock, [], on_ev, 15, None)
pc.start()
while len(n) < 2:
@@ -161,7 +169,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
run_test_server.authorize_with('active')
events = queue.Queue(100)
- logstream = io.BytesIO()
+ logstream = StringIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
@@ -229,7 +237,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
def test_websocket_reconnect_retry(self, event_client_connect):
event_client_connect.side_effect = [None, Exception('EventClient.connect error'), None]
- logstream = io.BytesIO()
+ logstream = StringIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 1b0b627..8266639 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -12,6 +12,7 @@ import pycurl
import random
import re
import socket
+import sys
import threading
import time
import unittest
@@ -48,7 +49,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
self.assertEqual(0, self.keep_client.download_counter.get())
self.assertEqual(self.keep_client.get(foo_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
self.assertEqual(3, self.keep_client.download_counter.get())
@@ -64,28 +65,28 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'wrong content from Keep.get(md5(<binarydata>))')
def test_KeepLongBinaryRWTest(self):
- blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
+ blob_data = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
for i in range(0,23):
- blob_str = blob_str + blob_str
- blob_locator = self.keep_client.put(blob_str)
+ blob_data = blob_data + blob_data
+ blob_locator = self.keep_client.put(blob_data)
self.assertRegexpMatches(
blob_locator,
'^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
- blob_str,
+ blob_data,
'wrong content from Keep.get(md5(<binarydata>))')
@unittest.skip("unreliable test - please fix and close #8752")
def test_KeepSingleCopyRWTest(self):
- blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
- blob_locator = self.keep_client.put(blob_str, copies=1)
+ blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03'
+ blob_locator = self.keep_client.put(blob_data, copies=1)
self.assertRegexpMatches(
blob_locator,
'^c902006bc98a3eb4a3663b65ab4a6fab\+8',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
- blob_str,
+ blob_data,
'wrong content from Keep.get(md5(<binarydata>))')
def test_KeepEmptyCollectionTest(self):
@@ -103,9 +104,10 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
- with self.assertRaises(UnicodeEncodeError):
- # Error if it is not ASCII
- self.keep_client.put(u'\xe2')
+ if sys.version_info < (3, 0):
+ with self.assertRaises(UnicodeEncodeError):
+ # Error if it is not ASCII
+ self.keep_client.put(u'\xe2')
with self.assertRaises(AttributeError):
# Must be bytes or have an encode() method
@@ -119,7 +121,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'wrong md5 hash from Keep.put for "test_head": ' + locator)
self.assertEqual(True, self.keep_client.head(locator))
self.assertEqual(self.keep_client.get(locator),
- 'test_head',
+ b'test_head',
'wrong content from Keep.get for "test_head"')
class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
@@ -136,7 +138,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
self.assertEqual(keep_client.get(foo_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
# GET with an unsigned locator => NotFound
@@ -203,13 +205,13 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
def test_KeepAuthenticatedSignedTest(self):
signed_locator = self._put_foo_and_check()
self.assertEqual(self.keep_client.get(signed_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepAuthenticatedUnsignedTest(self):
signed_locator = self._put_foo_and_check()
self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedSignedTest(self):
@@ -218,7 +220,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
signed_locator = self._put_foo_and_check()
self.keep_client.api_token = ''
self.assertEqual(self.keep_client.get(signed_locator),
- 'foo',
+ b'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedUnsignedTest(self):
@@ -257,7 +259,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
'^73feffa4b7f6bb68e44cf984c85f6e88\+3',
'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
self.assertEqual(keep_client.get(baz_locator),
- 'baz',
+ b'baz',
'wrong content from Keep.get(md5("baz"))')
self.assertTrue(keep_client.using_proxy)
@@ -273,7 +275,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
'^91f372a266fe2bf2823cb8ec7fda31ce\+4',
'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
self.assertEqual(keep_client.get(baz_locator),
- 'baz2',
+ b'baz2',
'wrong content from Keep.get(md5("baz2"))')
self.assertTrue(keep_client.using_proxy)
@@ -530,7 +532,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
list('9d81c02e76a3bf54'),
]
self.blocks = [
- "{:064x}".format(x)
+ "{:064x}".format(x).encode('utf-8')
for x in range(len(self.expected_order))]
self.hashes = [
hashlib.md5(self.blocks[x]).hexdigest()
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 7277628..9c1cacd 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -26,28 +26,28 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_read_block_crossing_behavior(self):
# read() calls will be aligned on block boundaries - see #3663.
sfile = self.make_count_reader()
- self.assertEqual('123', sfile.read(10))
+ self.assertEqual(b'123', sfile.read(10))
def test_small_read(self):
sfile = self.make_count_reader()
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
def test_successive_reads(self):
sfile = self.make_count_reader()
- for expect in ['123', '456', '789', '']:
+ for expect in [b'123', b'456', b'789', b'']:
self.assertEqual(expect, sfile.read(10))
def test_readfrom_spans_blocks(self):
sfile = self.make_count_reader()
- self.assertEqual('6789', sfile.readfrom(5, 12))
+ self.assertEqual(b'6789', sfile.readfrom(5, 12))
def test_small_readfrom_spanning_blocks(self):
sfile = self.make_count_reader()
- self.assertEqual('2345', sfile.readfrom(1, 4))
+ self.assertEqual(b'2345', sfile.readfrom(1, 4))
def test_readall(self):
sfile = self.make_count_reader()
- self.assertEqual('123456789', ''.join(sfile.readall()))
+ self.assertEqual(b'123456789', b''.join(sfile.readall()))
def test_one_arg_seek(self):
self.test_absolute_seek([])
@@ -55,20 +55,20 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_absolute_seek(self, args=[os.SEEK_SET]):
sfile = self.make_count_reader()
sfile.seek(6, *args)
- self.assertEqual('78', sfile.read(2))
+ self.assertEqual(b'78', sfile.read(2))
sfile.seek(4, *args)
- self.assertEqual('56', sfile.read(2))
+ self.assertEqual(b'56', sfile.read(2))
def test_relative_seek(self, args=[os.SEEK_CUR]):
sfile = self.make_count_reader()
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
sfile.seek(2, *args)
- self.assertEqual('56', sfile.read(2))
+ self.assertEqual(b'56', sfile.read(2))
def test_end_seek(self):
sfile = self.make_count_reader()
sfile.seek(-6, os.SEEK_END)
- self.assertEqual('45', sfile.read(2))
+ self.assertEqual(b'45', sfile.read(2))
def test_seek_min_zero(self):
sfile = self.make_count_reader()
@@ -101,7 +101,7 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_context(self):
with self.make_count_reader() as sfile:
self.assertFalse(sfile.closed, "reader is closed inside context")
- self.assertEqual('12', sfile.read(2))
+ self.assertEqual(b'12', sfile.read(2))
self.assertTrue(sfile.closed, "reader is open after context")
def make_newlines_reader(self):
@@ -163,12 +163,12 @@ class StreamFileReaderTestCase(unittest.TestCase):
self.check_decompressed_name('test.log.bz2', 'test.log')
def check_decompression(self, compress_ext, compress_func):
- test_text = 'decompression\ntest\n'
+ test_text = b'decompression\ntest\n'
test_data = compress_func(test_text)
stream = tutil.MockStreamReader('.', test_data)
reader = StreamFileReader(stream, [Range(0, 0, len(test_data))],
'test.' + compress_ext)
- self.assertEqual(test_text, ''.join(reader.readall_decompressed()))
+ self.assertEqual(test_text, b''.join(reader.readall_decompressed()))
@staticmethod
def gzip_compress(data):
@@ -197,7 +197,7 @@ class StreamFileReaderTestCase(unittest.TestCase):
reader = self.make_newlines_reader()
data = reader.readline()
self.assertEqual('one\n', data)
- self.assertEqual(''.join(['two\n', '\n', 'three\n', 'four\n', '\n']), ''.join(reader.readall()))
+ self.assertEqual(b''.join([b'two\n', b'\n', b'three\n', b'four\n', b'\n']), b''.join(reader.readall()))
class StreamRetryTestMixin(object):
@@ -291,17 +291,17 @@ class StreamFileReadFromTestCase(StreamFileReadTestCase):
class StreamFileReadAllTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall(**kwargs))
+ return b''.join(reader.readall(**kwargs))
class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readall_decompressed(**kwargs))
+ return b''.join(reader.readall_decompressed(**kwargs))
class StreamFileReadlinesTestCase(StreamFileReadTestCase):
def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readlines(**kwargs))
+ return b''.join(reader.readlines(**kwargs))
if __name__ == '__main__':
unittest.main()
diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py
index 41739a9..bf59d0d 100644
--- a/sdk/python/tests/test_util.py
+++ b/sdk/python/tests/test_util.py
@@ -20,7 +20,7 @@ class MkdirDashPTest(unittest.TestCase):
def runTest(self):
arvados.util.mkdir_dash_p('./tmp/foo')
with open('./tmp/bar', 'wb') as f:
- f.write('bar')
+ f.write(b'bar')
self.assertRaises(OSError, arvados.util.mkdir_dash_p, './tmp/bar')
@@ -28,8 +28,8 @@ class RunCommandTestCase(unittest.TestCase):
def test_success(self):
stdout, stderr = arvados.util.run_command(['echo', 'test'],
stderr=subprocess.PIPE)
- self.assertEqual("test\n", stdout)
- self.assertEqual("", stderr)
+ self.assertEqual("test\n".encode(), stdout)
+ self.assertEqual("".encode(), stderr)
def test_failure(self):
with self.assertRaises(arvados.errors.CommandFailedError):
commit 41e9f18dc873c0260b5d32523e9b34ea41040f22
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 18:00:33 2017 -0400
11308: Fix deprecated logger.warn -> logger.warning.
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index ac24224..0eca6d8 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -804,7 +804,7 @@ def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_
if git_url.startswith("http:"):
if allow_insecure_http:
- logger.warn("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
+ logger.warning("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
else:
raise Exception("Refusing to use insecure git url %s, use %s if you really want this." % (git_url, allow_insecure_http_opt))
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 0c49138..a7ef0be 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -112,7 +112,7 @@ def docker_image_format(image_hash):
def docker_image_compatible(api, image_hash):
supported = api._rootDesc.get('dockerImageFormats', [])
if not supported:
- logger.warn("server does not specify supported image formats (see docker_image_formats in server config).")
+ logger.warning("server does not specify supported image formats (see docker_image_formats in server config).")
return False
fmt = docker_image_format(image_hash)
@@ -369,7 +369,7 @@ def main(arguments=None, stdout=sys.stdout):
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
- logger.warn("forcing incompatible image")
+ logger.warning("forcing incompatible image")
else:
logger.error("refusing to store " \
"incompatible format (use --force-image-format to override)")
diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index c58abe5..6b3b21f 100644
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -124,14 +124,14 @@ class EventClient(object):
def on_closed(self):
if not self.is_closed.is_set():
- _logger.warn("Unexpected close. Reconnecting.")
+ _logger.warning("Unexpected close. Reconnecting.")
for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=15):
try:
self._setup_event_client()
- _logger.warn("Reconnect successful.")
+ _logger.warning("Reconnect successful.")
break
except Exception as e:
- _logger.warn("Error '%s' during websocket reconnect.", e)
+ _logger.warning("Error '%s' during websocket reconnect.", e)
if tries_left == 0:
_logger.exception("EventClient thread could not contact websocket server.")
self.is_closed.set()
@@ -293,7 +293,7 @@ def _subscribe_websocket(api, filters, on_event, last_log_id=None):
try:
client = EventClient(uri_with_token, filters, on_event, last_log_id)
except Exception:
- _logger.warn("Failed to connect to websockets on %s" % endpoint)
+ _logger.warning("Failed to connect to websockets on %s" % endpoint)
raise
else:
return client
@@ -322,7 +322,7 @@ def subscribe(api, filters, on_event, poll_fallback=15, last_log_id=None):
else:
_logger.info("Using polling because ARVADOS_DISABLE_WEBSOCKETS is true")
except Exception as e:
- _logger.warn("Falling back to polling after websocket error: %s" % e)
+ _logger.warning("Falling back to polling after websocket error: %s" % e)
p = PollClient(api, filters, on_event, poll_fallback, last_log_id)
p.start()
return p
commit a81a5f35357a099750f0ba3f65885d14a757349d
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 17:52:05 2017 -0400
11308: md5 -> hashlib.
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index a1fbc27..1c00551 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -3,7 +3,7 @@ from __future__ import absolute_import
from builtins import str
from builtins import range
-import md5
+import hashlib
import mock
import os
import random
@@ -34,11 +34,11 @@ class CacheTestThread(threading.Thread):
for x in range(16):
try:
data_in = _random(128)
- data_in = md5.new(data_in).hexdigest() + bytes("\n") + bytes(data_in)
+ data_in = hashlib.md5(data_in).hexdigest() + bytes("\n") + bytes(data_in)
c.set(url, data_in)
data_out = c.get(url)
digest, _, content = data_out.partition("\n")
- if digest != md5.new(content).hexdigest():
+ if digest != hashlib.md5(content).hexdigest():
self.ok = False
except Exception as err:
self.ok = False
commit 0e0c1400b57d5de8aa8c18dd4897527f905a4b42
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 02:36:46 2017 -0400
11308: Fix bytes vs. str problems.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index f680d6c..ee91491 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,6 +22,11 @@ import threading
from . import timer
import urllib.parse
+if sys.version_info >= (3, 0):
+ from io import BytesIO
+else:
+ from cStringIO import StringIO as BytesIO
+
import arvados
import arvados.config as config
import arvados.errors
@@ -335,7 +340,7 @@ class KeepClient(object):
try:
with timer.Timer() as t:
self._headers = {}
- response_body = io.StringIO()
+ response_body = BytesIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
@@ -412,8 +417,8 @@ class KeepClient(object):
try:
with timer.Timer() as t:
self._headers = {}
- body_reader = io.StringIO(body)
- response_body = io.StringIO()
+ body_reader = BytesIO(body)
+ response_body = BytesIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
@@ -486,7 +491,8 @@ class KeepClient(object):
curl.setopt(pycurl.LOW_SPEED_LIMIT, int(math.ceil(bandwidth_bps)))
def _headerfunction(self, header_line):
- header_line = header_line.decode('iso-8859-1')
+ if isinstance(header_line, bytes):
+ header_line = header_line.decode('iso-8859-1')
if ':' in header_line:
name, value = header_line.split(':', 1)
name = name.strip().lower()
@@ -1066,10 +1072,8 @@ class KeepClient(object):
KeepClient is initialized.
"""
- if isinstance(data, str):
- data = data.encode("ascii")
- elif not isinstance(data, str):
- raise arvados.errors.ArgumentError("Argument 'data' to KeepClient.put is not type 'str'")
+ if not isinstance(data, bytes):
+ data = data.encode()
self.put_counter.add(1)
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 667abbe..05e5e06 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -24,7 +24,10 @@ import threading
import hashlib
import random
-from io import StringIO
+if sys.version_info >= (3, 0):
+ from io import StringIO
+else:
+ from cStringIO import StringIO
import arvados
import arvados.commands.put as arv_put
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index f4bcd8a..0b2b741 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -36,7 +36,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
return self.blocks.get(locator)
def put(self, data, num_retries=None, copies=None):
pdh = tutil.str_keep_locator(data)
- self.blocks[pdh] = str(data)
+ self.blocks[pdh] = bytes(data)
return pdh
class MockApi(object):
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index ea34c80..a1fbc27 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -34,10 +34,10 @@ class CacheTestThread(threading.Thread):
for x in range(16):
try:
data_in = _random(128)
- data_in = md5.new(data_in).hexdigest() + "\n" + str(data_in)
+ data_in = md5.new(data_in).hexdigest() + bytes("\n") + bytes(data_in)
c.set(url, data_in)
data_out = c.get(url)
- digest, content = data_out.split("\n", 1)
+ digest, _, content = data_out.partition("\n")
if digest != md5.new(content).hexdigest():
self.ok = False
except Exception as err:
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 8c2a671..1b0b627 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -107,8 +107,8 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
# Error if it is not ASCII
self.keep_client.put(u'\xe2')
- with self.assertRaises(arvados.errors.ArgumentError):
- # Must be a string type
+ with self.assertRaises(AttributeError):
+ # Must be bytes or have an encode() method
self.keep_client.put({})
def test_KeepHeadTest(self):
commit 0199da877b222c751a31d70a6617b92c2a56fe70
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 02:36:08 2017 -0400
11308: Fix variable name conflict.
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 51e85c8..5992371 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -34,10 +34,10 @@ def queue_with(items):
given, it will be consumed to fill the queue before queue_with()
returns.
"""
- queue = queue.Queue()
+ q = queue.Queue()
for val in items:
- queue.put(val)
- return lambda *args, **kwargs: queue.get(block=False)
+ q.put(val)
+ return lambda *args, **kwargs: q.get(block=False)
# fake_httplib2_response and mock_responses
# mock calls to httplib2.Http.request()
commit 7fde9a0a090e5f092b946f7569f1350778a47574
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 02:11:08 2017 -0400
11308: Eliminate old_div().
diff --git a/sdk/python/arvados/_ranges.py b/sdk/python/arvados/_ranges.py
index 5c8b00f..6d7c112 100644
--- a/sdk/python/arvados/_ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -1,5 +1,4 @@
from __future__ import division
-from past.utils import old_div
from builtins import object
import logging
@@ -34,7 +33,7 @@ def first_block(data_locators, range_start):
hi = len(data_locators)
lo = 0
- i = int(old_div((hi + lo), 2))
+ i = (hi + lo) // 2
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
@@ -50,7 +49,7 @@ def first_block(data_locators, range_start):
lo = i
else:
hi = i
- i = int(old_div((hi + lo), 2))
+ i = (hi + lo) // 2
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index ab1c645..c6cb1c9 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -3,7 +3,6 @@ from __future__ import division
from future import standard_library
standard_library.install_aliases()
from builtins import range
-from past.utils import old_div
from builtins import object
import functools
import os
@@ -890,7 +889,7 @@ class ArvadosFile(object):
"""
self._writers.remove(writer)
- if flush or self.size() > old_div(config.KEEP_BLOCK_SIZE, 2):
+ if flush or self.size() > config.KEEP_BLOCK_SIZE // 2:
# File writer closed, not small enough for repacking
self.flush()
elif self.closed():
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index c5d74ef..ac24224 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -21,7 +21,6 @@ from future import standard_library
standard_library.install_aliases()
from past.builtins import basestring
from builtins import object
-from past.utils import old_div
import argparse
import contextlib
import getpass
@@ -954,7 +953,7 @@ def human_progress(obj_uuid, bytes_written, bytes_expected):
return "\r{}: {}M / {}M {:.1%} ".format(
obj_uuid,
bytes_written >> 20, bytes_expected >> 20,
- old_div(float(bytes_written), bytes_expected))
+ float(bytes_written) / bytes_expected)
else:
return "\r{}: {} ".format(obj_uuid, bytes_written)
diff --git a/sdk/python/arvados/commands/ls.py b/sdk/python/arvados/commands/ls.py
index ea93eff..ca2d6d7 100755
--- a/sdk/python/arvados/commands/ls.py
+++ b/sdk/python/arvados/commands/ls.py
@@ -3,7 +3,6 @@
from __future__ import print_function
from __future__ import division
-from past.utils import old_div
import argparse
import sys
@@ -28,7 +27,7 @@ def parse_args(args):
return parser.parse_args(args)
def size_formatter(coll_file):
- return "{:>10}".format(old_div((coll_file.size() + 1023), 1024))
+ return "{:>10}".format((coll_file.size() + 1023) // 1024)
def name_formatter(coll_file):
return "{}/{}".format(coll_file.stream_name(), coll_file.name)
diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index 349e57b..724de7b 100644
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -1,6 +1,5 @@
from __future__ import print_function
from __future__ import division
-from past.utils import old_div
import argparse
import time
import sys
@@ -136,7 +135,7 @@ def main(arguments=None):
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", old_div(biggest,(2**20)), old_div((biggest*2),(2**20)))
+ logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest>>20, biggest>>19)
if args.dry_run:
return
@@ -152,7 +151,7 @@ def main(arguments=None):
tarfile = list(oldcol.keys())[0]
logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
- old_image["tag"], old_image["collection"], old_div(list(oldcol.values())[0].size(),(2**20)))
+ old_image["tag"], old_image["collection"], list(oldcol.values())[0].size()>>20)
count += 1
start = time.time()
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 681e78e..ef86fef 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -5,7 +5,6 @@
from __future__ import division
from builtins import str
-from past.utils import old_div
from builtins import object
import argparse
import arvados
@@ -821,7 +820,7 @@ def human_progress(bytes_written, bytes_expected):
if bytes_expected:
return "\r{}M / {}M {:.1%} ".format(
bytes_written >> 20, bytes_expected >> 20,
- old_div(float(bytes_written), bytes_expected))
+ float(bytes_written) / bytes_expected)
else:
return "\r{} ".format(bytes_written)
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index b709473..f680d6c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -5,7 +5,6 @@ standard_library.install_aliases()
from builtins import next
from builtins import str
from builtins import range
-from past.utils import old_div
from builtins import object
import io
import datetime
@@ -392,7 +391,7 @@ class KeepClient(object):
self._result['status_code'],
len(self._result['body']),
t.msecs,
- old_div((old_div(len(self._result['body']),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
+ 1.0*len(self._result['body'])/2**20/t.secs if t.secs > 0 else 0)
if self.download_counter:
self.download_counter.add(len(self._result['body']))
@@ -465,7 +464,7 @@ class KeepClient(object):
self._result['status_code'],
len(body),
t.msecs,
- old_div((old_div(len(body),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
+ 1.0*len(body)/2**20/t.secs if t.secs > 0 else 0)
if self.upload_counter:
self.upload_counter.add(len(body))
return True
@@ -568,7 +567,7 @@ class KeepClient(object):
if (not max_service_replicas) or (max_service_replicas >= copies):
num_threads = 1
else:
- num_threads = int(math.ceil(old_div(float(copies), max_service_replicas)))
+ num_threads = int(math.ceil(1.0*copies/max_service_replicas))
_logger.debug("Pool max threads is %d", num_threads)
self.workers = []
self.queue = KeepClient.KeepWriterQueue(copies)
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index 965bf29..28bd483 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -2,7 +2,6 @@ from __future__ import division
from future import standard_library
standard_library.install_aliases()
from builtins import str
-from past.utils import old_div
import http.server
import hashlib
import os
@@ -64,7 +63,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
if self.server.bandwidth == None and self.server.delays['mid_write'] == 0:
self.wfile.write(data_to_write)
else:
- BYTES_PER_WRITE = int(old_div(self.server.bandwidth,4.0)) or 32768
+ BYTES_PER_WRITE = int(self.server.bandwidth/4) or 32768
outage_happened = False
num_bytes = len(data_to_write)
num_sent_bytes = 0
@@ -80,7 +79,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
num_sent_bytes:num_sent_bytes+num_write_bytes])
num_sent_bytes += num_write_bytes
if self.server.bandwidth is not None:
- target_time += old_div(num_write_bytes, self.server.bandwidth)
+ target_time += num_write_bytes / self.server.bandwidth
self.server._sleep_at_least(target_time - time.time())
return None
@@ -88,7 +87,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
if self.server.bandwidth == None and self.server.delays['mid_read'] == 0:
return self.rfile.read(bytes_to_read)
else:
- BYTES_PER_READ = int(old_div(self.server.bandwidth,4.0)) or 32768
+ BYTES_PER_READ = int(self.server.bandwidth/4) or 32768
data = ''
outage_happened = False
bytes_read = 0
@@ -103,7 +102,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object):
data += self.rfile.read(next_bytes_to_read)
bytes_read += next_bytes_to_read
if self.server.bandwidth is not None:
- target_time += old_div(next_bytes_to_read, self.server.bandwidth)
+ target_time += next_bytes_to_read / self.server.bandwidth
self.server._sleep_at_least(target_time - time.time())
return data
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d966126..7dcd647 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -4,7 +4,6 @@ from __future__ import print_function
from __future__ import division
from builtins import str
from builtins import range
-from past.utils import old_div
import argparse
import atexit
import errno
@@ -100,7 +99,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
# Use up to half of the +wait+ period waiting for "passenger
# stop" to work. If the process hasn't exited by then, start
# sending TERM signals.
- startTERM += old_div(wait,2)
+ startTERM += wait//2
server_pid = None
while now <= deadline and server_pid is None:
diff --git a/sdk/python/tests/test_arv_normalize.py b/sdk/python/tests/test_arv_normalize.py
index 8bce7e3..426d41a 100644
--- a/sdk/python/tests/test_arv_normalize.py
+++ b/sdk/python/tests/test_arv_normalize.py
@@ -21,7 +21,7 @@ class ArvNormalizeTestCase(unittest.TestCase):
def test_version_argument(self):
returncode, out, err = self.run_arv_normalize(['--version'])
+ self.assertEqual(b'', out)
+ self.assertNotEqual(b'', err)
+ self.assertRegexpMatches(err.decode(), "^bin/arv-normalize [0-9]+\.[0-9]+\.[0-9]+$")
self.assertEqual(0, returncode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 5abf388..667abbe 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -7,7 +7,6 @@ from future import standard_library
standard_library.install_aliases()
from builtins import str
from builtins import range
-from past.utils import old_div
import apiclient
import io
import mock
@@ -264,8 +263,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
_, self.large_file_name = tempfile.mkstemp()
fileobj = open(self.large_file_name, 'w')
# Make sure to write just a little more than one block
- for _ in range((old_div(arvados.config.KEEP_BLOCK_SIZE,(1024*1024)))+1):
- data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB
+ for _ in range((arvados.config.KEEP_BLOCK_SIZE>>20)+1):
+ data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MiB
fileobj.write(data)
fileobj.close()
# Temp dir containing small files to be repacked
@@ -531,7 +530,7 @@ class ArvadosPutReportTest(ArvadosBaseTestCase):
def test_known_human_progress(self):
for count, total in [(0, 1), (2, 4), (45, 60)]:
- expect = '{:.1%}'.format(old_div(float(count), total))
+ expect = '{:.1%}'.format(1.0*count/total)
actual = arv_put.human_progress(count, total)
self.assertTrue(actual.startswith('\r'))
self.assertIn(expect, actual)
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 2cca773..4596b6c 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -5,7 +5,6 @@ from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import object
-from past.utils import old_div
import arvados
import io
import logging
@@ -151,11 +150,11 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t))
def localiso(self, t):
- return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(old_div(-time.timezone,60))
+ return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(-time.timezone//60)
def isotz(self, offset):
"""Convert minutes-east-of-UTC to RFC3339- and ISO-compatible time zone designator"""
- return '{:+03d}:{:02d}'.format(old_div(offset,60), offset%60)
+ return '{:+03d}:{:02d}'.format(offset//60, offset%60)
# Test websocket reconnection on (un)execpted close
def _test_websocket_reconnect(self, close_unexpected):
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index b69563f..8c2a671 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -4,7 +4,6 @@ from future import standard_library
standard_library.install_aliases()
from builtins import str
from builtins import range
-from past.utils import old_div
from builtins import object
import hashlib
import mock
@@ -1125,35 +1124,35 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock):
def test_only_write_enough_on_success(self):
for i in range(10):
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_on_partial_success(self):
for i in range(5):
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=False)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=False)
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_when_some_crash(self):
for i in range(5):
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
+ ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_fail_when_too_many_crash(self):
for i in range(self.copies+1):
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
+ ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
self.pool.add_task(ks, None)
for i in range(self.copies-1):
- ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies-1)
commit d1af87de56519cad7ae3c95d636c3c54f37c260b
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 17:37:23 2017 -0400
11308: Import future module.
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 99ea04d..9380a03 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -53,7 +53,8 @@ setup(name='arvados-python-client',
'python-gflags >=3',
'setuptools',
'ws4py <0.4',
- 'ruamel.yaml >=0.13.7'
+ 'ruamel.yaml >=0.13.7',
+ 'future',
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
commit 8e48567442d2d40bd9f6230f243f75cdbbe03bb1
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 01:49:09 2017 -0400
11308: Update python-gflags.
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index ed380e9..99ea04d 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -50,10 +50,10 @@ setup(name='arvados-python-client',
'ciso8601',
'httplib2',
'pycurl >=7.19.5.1, <7.21.5',
- 'python-gflags<3.0',
+ 'python-gflags >=3',
'setuptools',
- 'ws4py<0.4',
- 'ruamel.yaml==0.13.7'
+ 'ws4py <0.4',
+ 'ruamel.yaml >=0.13.7'
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
commit de259910871b7a01ecc199722ea26e6b335ca55e
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 01:48:37 2017 -0400
11308: md5 to hashlib.
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index ee1c51f..868b478 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -1,6 +1,6 @@
from builtins import object
import errno
-import md5
+import hashlib
import os
import tempfile
import time
@@ -33,7 +33,7 @@ class SafeHTTPCache(object):
return self._dir
def _filename(self, url):
- return os.path.join(self._dir, md5.new(url).hexdigest()+'.tmp')
+ return os.path.join(self._dir, hashlib.md5(url).hexdigest()+'.tmp')
def get(self, url):
filename = self._filename(url)
commit fbce7b14ac2ed443a3054262c16689cbebc16f61
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 01:48:02 2017 -0400
11308: UserDict to Python 3.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 5acb961..c018d56 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -3,25 +3,29 @@ from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import object
+import bz2
+import fcntl
import gflags
+import hashlib
import http.client
import httplib2
+import json
import logging
import os
import pprint
-import sys
-import types
-import subprocess
-import json
-import UserDict
import re
-import hashlib
import string
-import bz2
-import zlib
-import fcntl
-import time
+import subprocess
+import sys
import threading
+import time
+import types
+import zlib
+
+if sys.version_info >= (3, 0):
+ from collections import UserDict
+else:
+ from UserDict import UserDict
from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
@@ -68,7 +72,7 @@ def current_task(num_retries=5):
for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
task = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
- task = UserDict.UserDict(task)
+ task = UserDict(task)
task.set_output = types.MethodType(task_set_output, task)
task.tmpdir = os.environ['TASK_WORK']
_current_task = task
@@ -88,7 +92,7 @@ def current_job(num_retries=5):
for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
- job = UserDict.UserDict(job)
+ job = UserDict(job)
job.tmpdir = os.environ['JOB_WORK']
_current_job = job
return job
commit e61df48303291900581ef4a64dcf97864598c5f4
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 01:41:10 2017 -0400
11308: Futurize stage2.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 36d54e5..5acb961 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -1,7 +1,10 @@
from __future__ import print_function
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
import gflags
-import httplib
+import http.client
import httplib2
import logging
import os
@@ -108,7 +111,7 @@ class JobTask(object):
def __init__(self, parameters=dict(), runtime_constraints=dict()):
print("init jobtask %s %s" % (parameters, runtime_constraints))
-class job_setup:
+class job_setup(object):
@staticmethod
def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None):
if if_sequence != current_task()['sequence']:
diff --git a/sdk/python/arvados/_ranges.py b/sdk/python/arvados/_ranges.py
index e0fe615..5c8b00f 100644
--- a/sdk/python/arvados/_ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -1,3 +1,6 @@
+from __future__ import division
+from past.utils import old_div
+from builtins import object
import logging
_logger = logging.getLogger('arvados.ranges')
@@ -31,7 +34,7 @@ def first_block(data_locators, range_start):
hi = len(data_locators)
lo = 0
- i = int((hi + lo) / 2)
+ i = int(old_div((hi + lo), 2))
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
@@ -47,7 +50,7 @@ def first_block(data_locators, range_start):
lo = i
else:
hi = i
- i = int((hi + lo) / 2)
+ i = int(old_div((hi + lo), 2))
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 65aadbd..59a73b4 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -1,6 +1,9 @@
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
import collections
-import httplib
+import http.client
import httplib2
import json
import logging
@@ -68,7 +71,7 @@ def _intercept_http_request(self, uri, **kwargs):
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
- for conn in self.connections.itervalues():
+ for conn in self.connections.values():
conn.close()
self.connections.clear()
@@ -77,7 +80,7 @@ def _intercept_http_request(self, uri, **kwargs):
self._last_request_time = time.time()
try:
return self.orig_http_request(uri, **kwargs)
- except httplib.HTTPException:
+ except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
except socket.error:
@@ -88,7 +91,7 @@ def _intercept_http_request(self, uri, **kwargs):
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay, exc_info=True)
- for conn in self.connections.itervalues():
+ for conn in self.connections.values():
conn.close()
time.sleep(delay)
delay = delay * self._retry_delay_backoff
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 2c44c34..ab1c645 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -1,4 +1,10 @@
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from past.utils import old_div
+from builtins import object
import functools
import os
import zlib
@@ -6,7 +12,7 @@ import bz2
from . import config
import hashlib
import threading
-import Queue
+import queue
import copy
import errno
import re
@@ -508,10 +514,10 @@ class _BlockManager(object):
# blocks pending. If they are full 64 MiB blocks, that means up to
# 256 MiB of internal buffering, which is the same size as the
# default download block cache in KeepClient.
- self._put_queue = Queue.Queue(maxsize=2)
+ self._put_queue = queue.Queue(maxsize=2)
self._put_threads = []
- for i in xrange(0, self.num_put_threads):
+ for i in range(0, self.num_put_threads):
thread = threading.Thread(target=self._commit_bufferblock_worker)
self._put_threads.append(thread)
thread.daemon = True
@@ -531,9 +537,9 @@ class _BlockManager(object):
@synchronized
def start_get_threads(self):
if self._prefetch_threads is None:
- self._prefetch_queue = Queue.Queue()
+ self._prefetch_queue = queue.Queue()
self._prefetch_threads = []
- for i in xrange(0, self.num_get_threads):
+ for i in range(0, self.num_get_threads):
thread = threading.Thread(target=self._block_prefetch_worker)
self._prefetch_threads.append(thread)
thread.daemon = True
@@ -579,7 +585,7 @@ class _BlockManager(object):
# A WRITABLE block with its owner.closed() implies that it's
# size is <= KEEP_BLOCK_SIZE/2.
try:
- small_blocks = [b for b in self._bufferblocks.values() if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
+ small_blocks = [b for b in list(self._bufferblocks.values()) if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
except AttributeError:
# Writable blocks without owner shouldn't exist.
raise UnownedBlockError()
@@ -692,7 +698,7 @@ class _BlockManager(object):
self.repack_small_blocks(force=True, sync=True)
with self.lock:
- items = self._bufferblocks.items()
+ items = list(self._bufferblocks.items())
for k,v in items:
if v.state() != _BufferBlock.COMMITTED and v.owner:
@@ -824,7 +830,7 @@ class ArvadosFile(object):
with self.lock:
if len(self._segments) != len(othersegs):
return False
- for i in xrange(0, len(othersegs)):
+ for i in range(0, len(othersegs)):
seg1 = self._segments[i]
seg2 = othersegs[i]
loc1 = seg1.locator
@@ -884,7 +890,7 @@ class ArvadosFile(object):
"""
self._writers.remove(writer)
- if flush or self.size() > config.KEEP_BLOCK_SIZE / 2:
+ if flush or self.size() > old_div(config.KEEP_BLOCK_SIZE, 2):
# File writer closed, not small enough for repacking
self.flush()
elif self.closed():
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index ac6d184..ee1c51f 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -1,3 +1,4 @@
+from builtins import object
import errno
import md5
import os
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 1a42781..0d88084 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,4 +1,7 @@
from __future__ import absolute_import
+from builtins import str
+from past.builtins import basestring
+from builtins import object
import functools
import logging
import os
@@ -419,7 +422,7 @@ class ResumableCollectionWriter(CollectionWriter):
return writer
def check_dependencies(self):
- for path, orig_stat in self._dependencies.items():
+ for path, orig_stat in list(self._dependencies.items()):
if not S_ISREG(orig_stat[ST_MODE]):
raise errors.StaleWriterStateError("{} not file".format(path))
try:
@@ -673,7 +676,7 @@ class RichCollectionBase(CollectionBase):
if value == self._committed:
return
if value:
- for k,v in self._items.items():
+ for k,v in list(self._items.items()):
v.set_committed(True)
self._committed = True
else:
@@ -684,7 +687,7 @@ class RichCollectionBase(CollectionBase):
@synchronized
def __iter__(self):
"""Iterate over names of files and collections contained in this collection."""
- return iter(self._items.keys())
+ return iter(list(self._items.keys()))
@synchronized
def __getitem__(self, k):
@@ -716,17 +719,17 @@ class RichCollectionBase(CollectionBase):
@synchronized
def keys(self):
"""Get a list of names of files and collections directly contained in this collection."""
- return self._items.keys()
+ return list(self._items.keys())
@synchronized
def values(self):
"""Get a list of files and collection objects directly contained in this collection."""
- return self._items.values()
+ return list(self._items.values())
@synchronized
def items(self):
"""Get a list of (name, object) tuples directly contained in this collection."""
- return self._items.items()
+ return list(self._items.items())
def exists(self, path):
"""Test if there is a file or collection at `path`."""
@@ -759,7 +762,7 @@ class RichCollectionBase(CollectionBase):
item.remove(pathcomponents[1])
def _clonefrom(self, source):
- for k,v in source.items():
+ for k,v in list(source.items()):
self._items[k] = v.clone(self, k)
def clone(self):
@@ -1117,7 +1120,7 @@ class RichCollectionBase(CollectionBase):
@synchronized
def flush(self):
"""Flush bufferblocks to Keep."""
- for e in self.values():
+ for e in list(self.values()):
e.flush()
@@ -1584,7 +1587,7 @@ class Collection(RichCollectionBase):
if state == BLOCKS:
block_locator = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
if block_locator:
- blocksize = long(block_locator.group(1))
+ blocksize = int(block_locator.group(1))
blocks.append(Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
else:
@@ -1593,8 +1596,8 @@ class Collection(RichCollectionBase):
if state == SEGMENTS:
file_segment = re.search(r'^(\d+):(\d+):(\S+)', tok)
if file_segment:
- pos = long(file_segment.group(1))
- size = long(file_segment.group(2))
+ pos = int(file_segment.group(1))
+ size = int(file_segment.group(2))
name = file_segment.group(3).replace('\\040', ' ')
filepath = os.path.join(stream_name, name)
afile = self.find_or_create(filepath, FILE)
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 5c51928..c5d74ef 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -16,6 +16,12 @@
# instances src and dst. If either of these files is not found,
# arv-copy will issue an error.
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from past.builtins import basestring
+from builtins import object
+from past.utils import old_div
import argparse
import contextlib
import getpass
@@ -25,7 +31,7 @@ import shutil
import sys
import logging
import tempfile
-import urlparse
+import urllib.parse
import arvados
import arvados.config
@@ -152,7 +158,7 @@ def main():
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
# Clean up any outstanding temp git repositories.
- for d in local_repo_dir.values():
+ for d in list(local_repo_dir.values()):
shutil.rmtree(d, ignore_errors=True)
# If no exception was thrown and the response does not have an
@@ -344,7 +350,7 @@ def migrate_components_filters(template_components, dst_git_repo):
be None if that is not known.
"""
errors = []
- for cname, cspec in template_components.iteritems():
+ for cname, cspec in template_components.items():
def add_error(errmsg):
errors.append("{}: {}".format(cname, errmsg))
if not isinstance(cspec, dict):
@@ -553,7 +559,7 @@ def migrate_jobspec(jobspec, src, dst, dst_repo, args):
# names. The return value is undefined.
#
def copy_git_repos(p, src, dst, dst_repo, args):
- for component in p['components'].itervalues():
+ for component in p['components'].values():
migrate_jobspec(component, src, dst, dst_repo, args)
if 'job' in component:
migrate_jobspec(component['job'], src, dst, dst_repo, args)
@@ -774,8 +780,8 @@ def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_
git_url = None
for url in priority:
if url.startswith("http"):
- u = urlparse.urlsplit(url)
- baseurl = urlparse.urlunsplit((u.scheme, u.netloc, "", "", ""))
+ u = urllib.parse.urlsplit(url)
+ baseurl = urllib.parse.urlunsplit((u.scheme, u.netloc, "", "", ""))
git_config = ["-c", "credential.%s/.username=none" % baseurl,
"-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
else:
@@ -859,7 +865,7 @@ def copy_docker_images(pipeline, src, dst, args):
runtime_constraints field from src to dst."""
logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
- for c_name, c_info in pipeline['components'].iteritems():
+ for c_name, c_info in pipeline['components'].items():
if ('runtime_constraints' in c_info and
'docker_image' in c_info['runtime_constraints']):
copy_docker_image(
@@ -948,7 +954,7 @@ def human_progress(obj_uuid, bytes_written, bytes_expected):
return "\r{}: {}M / {}M {:.1%} ".format(
obj_uuid,
bytes_written >> 20, bytes_expected >> 20,
- float(bytes_written) / bytes_expected)
+ old_div(float(bytes_written), bytes_expected))
else:
return "\r{}: {} ".format(obj_uuid, bytes_written)
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 5783248..0c49138 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from builtins import next
import argparse
import collections
import datetime
@@ -315,7 +316,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
# and add image listings for them, retaining the API server preference
# sorting.
images_start_size = len(images)
- for collection_uuid, link in hash_link_map.iteritems():
+ for collection_uuid, link in hash_link_map.items():
if not seen_image_names[collection_uuid]:
images.append(_new_image_listing(link, link['name']))
if len(images) > images_start_size:
diff --git a/sdk/python/arvados/commands/ls.py b/sdk/python/arvados/commands/ls.py
index a2f2e54..ea93eff 100755
--- a/sdk/python/arvados/commands/ls.py
+++ b/sdk/python/arvados/commands/ls.py
@@ -1,7 +1,9 @@
#!/usr/bin/env python
from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
import argparse
import sys
@@ -26,7 +28,7 @@ def parse_args(args):
return parser.parse_args(args)
def size_formatter(coll_file):
- return "{:>10}".format((coll_file.size() + 1023) / 1024)
+ return "{:>10}".format(old_div((coll_file.size() + 1023), 1024))
def name_formatter(coll_file):
return "{}/{}".format(coll_file.stream_name(), coll_file.name)
diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index 802744b..349e57b 100644
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -1,4 +1,6 @@
from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
import argparse
import time
import sys
@@ -122,8 +124,8 @@ def main(arguments=None):
if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
need_migrate[pdh] = img
with CollectionReader(i["manifest_text"]) as c:
- if c.values()[0].size() > biggest:
- biggest = c.values()[0].size()
+ if list(c.values())[0].size() > biggest:
+ biggest = list(c.values())[0].size()
if args.print_unmigrated:
only_migrate = set()
@@ -134,7 +136,7 @@ def main(arguments=None):
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20))
+ logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", old_div(biggest,(2**20)), old_div((biggest*2),(2**20)))
if args.dry_run:
return
@@ -142,15 +144,15 @@ def main(arguments=None):
success = []
failures = []
count = 1
- for old_image in need_migrate.values():
+ for old_image in list(need_migrate.values()):
if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
continue
oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
- tarfile = oldcol.keys()[0]
+ tarfile = list(oldcol.keys())[0]
logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
- old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
+ old_image["tag"], old_image["collection"], old_div(list(oldcol.values())[0].size(),(2**20)))
count += 1
start = time.time()
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 32d5fef..681e78e 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -3,6 +3,10 @@
# TODO:
# --md5sum - display md5 of each file as read from disk
+from __future__ import division
+from builtins import str
+from past.utils import old_div
+from builtins import object
import argparse
import arvados
import arvados.collection
@@ -205,7 +209,7 @@ def parse_arguments(arguments):
if len(args.paths) == 0:
args.paths = ['-']
- args.paths = map(lambda x: "-" if x == "/dev/stdin" else x, args.paths)
+ args.paths = ["-" if x == "/dev/stdin" else x for x in args.paths]
if len(args.paths) != 1 or os.path.isdir(args.paths[0]):
if args.filename:
@@ -509,7 +513,7 @@ class ArvPutUploadJob(object):
Recursively get the total size of the collection
"""
size = 0
- for item in collection.values():
+ for item in list(collection.values()):
if isinstance(item, arvados.collection.Collection) or isinstance(item, arvados.collection.Subcollection):
size += self._collection_size(item)
else:
@@ -701,7 +705,7 @@ class ArvPutUploadJob(object):
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
file_paths = []
- for name, item in col.items():
+ for name, item in list(col.items()):
if isinstance(item, arvados.arvfile.ArvadosFile):
file_paths.append(os.path.join(path_prefix, name))
elif isinstance(item, arvados.collection.Subcollection):
@@ -778,7 +782,7 @@ class ArvPutUploadJob(object):
locators.append(loc)
return locators
elif isinstance(item, arvados.collection.Collection):
- l = [self._datablocks_on_item(x) for x in item.values()]
+ l = [self._datablocks_on_item(x) for x in list(item.values())]
# Fast list flattener method taken from:
# http://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
return [loc for sublist in l for loc in sublist]
@@ -817,7 +821,7 @@ def human_progress(bytes_written, bytes_expected):
if bytes_expected:
return "\r{}M / {}M {:.1%} ".format(
bytes_written >> 20, bytes_expected >> 20,
- float(bytes_written) / bytes_expected)
+ old_div(float(bytes_written), bytes_expected))
else:
return "\r{} ".format(bytes_written)
@@ -982,7 +986,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
if not output.endswith('\n'):
stdout.write('\n')
- for sigcode, orig_handler in orig_signal_handlers.items():
+ for sigcode, orig_handler in list(orig_signal_handlers.items()):
signal.signal(sigcode, orig_handler)
if status != 0:
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 2f3e042..01a18e5 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -2,6 +2,9 @@
from __future__ import print_function
from __future__ import absolute_import
+from builtins import range
+from past.builtins import basestring
+from builtins import object
import arvados
import arvados.commands.ws as ws
import argparse
@@ -301,7 +304,7 @@ def main(arguments=None):
if files:
uploadfiles(files, api, dry_run=args.dry_run, num_retries=args.retries, project=project)
- for i in xrange(1, len(slots)):
+ for i in range(1, len(slots)):
slots[i] = [("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c for c in slots[i]]
component = {
@@ -321,8 +324,8 @@ def main(arguments=None):
group_parser.add_argument('-b', '--batch-size', type=int)
group_parser.add_argument('args', nargs=argparse.REMAINDER)
- for s in xrange(2, len(slots)):
- for i in xrange(0, len(slots[s])):
+ for s in range(2, len(slots)):
+ for i in range(0, len(slots[s])):
if slots[s][i] == '--':
inp = "input%i" % (s-2)
groupargs = group_parser.parse_args(slots[2][i+1:])
diff --git a/sdk/python/arvados/crunch.py b/sdk/python/arvados/crunch.py
index c184e6a..f4651cd 100644
--- a/sdk/python/arvados/crunch.py
+++ b/sdk/python/arvados/crunch.py
@@ -1,3 +1,4 @@
+from builtins import object
import json
import os
diff --git a/sdk/python/arvados/errors.py b/sdk/python/arvados/errors.py
index bfd471b..c9eda2d 100644
--- a/sdk/python/arvados/errors.py
+++ b/sdk/python/arvados/errors.py
@@ -37,7 +37,7 @@ class KeepRequestError(Exception):
self._request_errors = OrderedDict(request_errors)
if self._request_errors:
exc_reports = [self._format_error(*err_pair)
- for err_pair in self._request_errors.iteritems()]
+ for err_pair in self._request_errors.items()]
base_msg = "{}: {}".format(message, "; ".join(exc_reports))
else:
base_msg = message
diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index b385761..c58abe5 100644
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -1,4 +1,8 @@
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import object
import arvados
from . import config
from . import errors
@@ -6,7 +10,7 @@ from .retry import RetryLoop
import logging
import json
-import thread
+import _thread
import threading
import time
import os
@@ -116,7 +120,7 @@ class EventClient(object):
self.on_event_cb(m)
except Exception as e:
_logger.exception("Unexpected exception from event callback.")
- thread.interrupt_main()
+ _thread.interrupt_main()
def on_closed(self):
if not self.is_closed.is_set():
@@ -131,7 +135,7 @@ class EventClient(object):
if tries_left == 0:
_logger.exception("EventClient thread could not contact websocket server.")
self.is_closed.set()
- thread.interrupt_main()
+ _thread.interrupt_main()
return
def run_forever(self):
@@ -226,7 +230,7 @@ class PollClient(threading.Thread):
_logger.exception("PollClient thread could not contact API server.")
with self._closing_lock:
self._closing.set()
- thread.interrupt_main()
+ _thread.interrupt_main()
return
for i in items["items"]:
skip_old_events = [["id", ">", str(i["id"])]]
@@ -237,7 +241,7 @@ class PollClient(threading.Thread):
self.on_event(i)
except Exception as e:
_logger.exception("Unexpected exception from event callback.")
- thread.interrupt_main()
+ _thread.interrupt_main()
if items["items_available"] > len(items["items"]):
moreitems = True
if not moreitems:
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 4efa698..b709473 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -1,19 +1,27 @@
from __future__ import absolute_import
-import cStringIO
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import str
+from builtins import range
+from past.utils import old_div
+from builtins import object
+import io
import datetime
import hashlib
import logging
import math
import os
import pycurl
-import Queue
+import queue
import re
import socket
import ssl
import sys
import threading
from . import timer
-import urlparse
+import urllib.parse
import arvados
import arvados.config as config
@@ -191,7 +199,7 @@ class KeepBlockCache(object):
self._cache = [c for c in self._cache if not (c.ready.is_set() and c.content is None)]
sm = sum([slot.size() for slot in self._cache])
while len(self._cache) > 0 and sm > self.cache_max:
- for i in xrange(len(self._cache)-1, -1, -1):
+ for i in range(len(self._cache)-1, -1, -1):
if self._cache[i].ready.is_set():
del self._cache[i]
break
@@ -199,7 +207,7 @@ class KeepBlockCache(object):
def _get(self, locator):
# Test if the locator is already in the cache
- for i in xrange(0, len(self._cache)):
+ for i in range(0, len(self._cache)):
if self._cache[i].locator == locator:
n = self._cache[i]
if i != 0:
@@ -270,7 +278,7 @@ class KeepClient(object):
arvados.errors.HttpError,
)
- def __init__(self, root, user_agent_pool=Queue.LifoQueue(),
+ def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
download_counter=None, **headers):
self.root = root
@@ -298,7 +306,7 @@ class KeepClient(object):
def _get_user_agent(self):
try:
return self._user_agent_pool.get(block=False)
- except Queue.Empty:
+ except queue.Empty:
return pycurl.Curl()
def _put_user_agent(self, ua):
@@ -328,12 +336,12 @@ class KeepClient(object):
try:
with timer.Timer() as t:
self._headers = {}
- response_body = cStringIO.StringIO()
+ response_body = io.StringIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
curl.setopt(pycurl.HTTPHEADER, [
- '{}: {}'.format(k,v) for k,v in self.get_headers.iteritems()])
+ '{}: {}'.format(k,v) for k,v in self.get_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if method == "HEAD":
@@ -384,7 +392,7 @@ class KeepClient(object):
self._result['status_code'],
len(self._result['body']),
t.msecs,
- (len(self._result['body'])/(1024.0*1024))/t.secs if t.secs > 0 else 0)
+ old_div((old_div(len(self._result['body']),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
if self.download_counter:
self.download_counter.add(len(self._result['body']))
@@ -405,8 +413,8 @@ class KeepClient(object):
try:
with timer.Timer() as t:
self._headers = {}
- body_reader = cStringIO.StringIO(body)
- response_body = cStringIO.StringIO()
+ body_reader = io.StringIO(body)
+ response_body = io.StringIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
@@ -420,7 +428,7 @@ class KeepClient(object):
curl.setopt(pycurl.INFILESIZE, len(body))
curl.setopt(pycurl.READFUNCTION, body_reader.read)
curl.setopt(pycurl.HTTPHEADER, [
- '{}: {}'.format(k,v) for k,v in self.put_headers.iteritems()])
+ '{}: {}'.format(k,v) for k,v in self.put_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
self._setcurltimeouts(curl, timeout)
@@ -457,7 +465,7 @@ class KeepClient(object):
self._result['status_code'],
len(body),
t.msecs,
- (len(body)/(1024.0*1024))/t.secs if t.secs > 0 else 0)
+ old_div((old_div(len(body),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
if self.upload_counter:
self.upload_counter.add(len(body))
return True
@@ -498,9 +506,9 @@ class KeepClient(object):
# Returning None implies all bytes were written
- class KeepWriterQueue(Queue.Queue):
+ class KeepWriterQueue(queue.Queue):
def __init__(self, copies):
- Queue.Queue.__init__(self) # Old-style superclass
+ queue.Queue.__init__(self) # Old-style superclass
self.wanted_copies = copies
self.successful_copies = 0
self.response = None
@@ -548,7 +556,7 @@ class KeepClient(object):
return service, service_root
elif self.empty():
self.pending_tries_notification.notify_all()
- raise Queue.Empty
+ raise queue.Empty
else:
self.pending_tries_notification.wait()
@@ -560,7 +568,7 @@ class KeepClient(object):
if (not max_service_replicas) or (max_service_replicas >= copies):
num_threads = 1
else:
- num_threads = int(math.ceil(float(copies) / max_service_replicas))
+ num_threads = int(math.ceil(old_div(float(copies), max_service_replicas)))
_logger.debug("Pool max threads is %d", num_threads)
self.workers = []
self.queue = KeepClient.KeepWriterQueue(copies)
@@ -602,7 +610,7 @@ class KeepClient(object):
while True:
try:
service, service_root = self.queue.get_next_task()
- except Queue.Empty:
+ except queue.Empty:
return
try:
locator, copies = self.do_task(service, service_root)
@@ -720,7 +728,7 @@ class KeepClient(object):
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
- self._user_agent_pool = Queue.LifoQueue()
+ self._user_agent_pool = queue.LifoQueue()
self.upload_counter = Counter()
self.download_counter = Counter()
self.put_counter = Counter()
@@ -741,7 +749,7 @@ class KeepClient(object):
if not proxy_uris[i].endswith('/'):
proxy_uris[i] += '/'
# URL validation
- url = urlparse.urlparse(proxy_uris[i])
+ url = urllib.parse.urlparse(proxy_uris[i])
if not (url.scheme and url.netloc):
raise arvados.errors.ArgumentError("Invalid proxy URI: {}".format(proxy_uris[i]))
self.api_token = api_token
@@ -807,7 +815,7 @@ class KeepClient(object):
raise arvados.errors.NoKeepServersError()
# Precompute the base URI for each service.
- for r in self._gateway_services.itervalues():
+ for r in self._gateway_services.values():
host = r['service_host']
if not host.startswith('[') and host.find(':') >= 0:
# IPv6 URIs must be formatted like http://[::1]:80/...
@@ -819,7 +827,7 @@ class KeepClient(object):
_logger.debug(str(self._gateway_services))
self._keep_services = [
- ks for ks in self._gateway_services.itervalues()
+ ks for ks in self._gateway_services.values()
if not ks.get('service_type', '').startswith('gateway:')]
self._writable_services = [ks for ks in self._keep_services
if not ks.get('read_only')]
@@ -1059,7 +1067,7 @@ class KeepClient(object):
KeepClient is initialized.
"""
- if isinstance(data, unicode):
+ if isinstance(data, str):
data = data.encode("ascii")
elif not isinstance(data, str):
raise arvados.errors.ArgumentError("Argument 'data' to KeepClient.put is not type 'str'")
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index 5ba4f4e..168bd39 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -1,5 +1,7 @@
#!/usr/bin/env python
+from builtins import range
+from builtins import object
import functools
import inspect
import pycurl
@@ -9,7 +11,7 @@ from collections import deque
import arvados.errors
-_HTTP_SUCCESSES = set(xrange(200, 300))
+_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 422, 423, 500, 502, 503, 504])
class RetryLoop(object):
@@ -69,7 +71,7 @@ class RetryLoop(object):
def running(self):
return self._running and (self._success is None)
- def next(self):
+ def __next__(self):
if self._running is None:
self._running = True
if (self.tries_left < 1) or not self.running():
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index a9ca978..488b758 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -1,5 +1,6 @@
from __future__ import absolute_import
+from builtins import object
import copy
import threading
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 042ed47..5955816 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -1,5 +1,6 @@
from __future__ import print_function
from __future__ import absolute_import
+from builtins import object
import collections
import hashlib
import os
@@ -36,15 +37,15 @@ class StreamReader(object):
s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
if s:
- blocksize = long(s.group(1))
+ blocksize = int(s.group(1))
self._data_locators.append(Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
continue
s = re.search(r'^(\d+):(\d+):(\S+)', tok)
if s:
- pos = long(s.group(1))
- size = long(s.group(2))
+ pos = int(s.group(1))
+ size = int(s.group(2))
name = s.group(3).replace('\\040', ' ')
if name not in self._files:
self._files[name] = StreamFileReader(self, [Range(pos, 0, size, 0)], name)
@@ -62,7 +63,7 @@ class StreamReader(object):
return self._files
def all_files(self):
- return self._files.values()
+ return list(self._files.values())
def size(self):
n = self._data_locators[-1]
@@ -97,5 +98,5 @@ class StreamReader(object):
manifest_text.extend([d.locator for d in self._data_locators])
manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
for seg in f.segments])
- for f in self._files.values()])
+ for f in list(self._files.values())])
return ' '.join(manifest_text) + '\n'
diff --git a/sdk/python/arvados/timer.py b/sdk/python/arvados/timer.py
index 6d4a73f..f3bf839 100644
--- a/sdk/python/arvados/timer.py
+++ b/sdk/python/arvados/timer.py
@@ -1,4 +1,5 @@
from __future__ import print_function
+from builtins import object
import time
class Timer(object):
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 522a95e..3f50553 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -124,7 +124,7 @@ def tarball_extract(tarball, path):
raise arvados.errors.CommandFailedError(
"tar exited %d" % p.returncode)
os.symlink(tarball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts = [f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
@@ -190,7 +190,7 @@ def zipball_extract(zipball, path):
"unzip exited %d" % p.returncode)
os.unlink(zip_filename)
os.symlink(zipball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts = [f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index dae3dd3..51e85c8 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -1,16 +1,21 @@
#!/usr/bin/env python
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from builtins import object
import arvados
import contextlib
import errno
import hashlib
-import httplib
+import http.client
import httplib2
import io
import mock
import os
import pycurl
-import Queue
+import queue
import shutil
import sys
import tempfile
@@ -29,7 +34,7 @@ def queue_with(items):
given, it will be consumed to fill the queue before queue_with()
returns.
"""
- queue = Queue.Queue()
+ queue = queue.Queue()
for val in items:
queue.put(val)
return lambda *args, **kwargs: queue.get(block=False)
@@ -38,7 +43,7 @@ def queue_with(items):
# mock calls to httplib2.Http.request()
def fake_httplib2_response(code, **headers):
headers.update(status=str(code),
- reason=httplib.responses.get(code, "Unknown Response"))
+ reason=http.client.responses.get(code, "Unknown Response"))
return httplib2.Response(headers)
def mock_responses(body, *codes, **headers):
@@ -63,7 +68,7 @@ def redirected_streams(stdout=None, stderr=None):
sys.stderr = orig_stderr
-class FakeCurl:
+class FakeCurl(object):
@classmethod
def make(cls, code, body='', headers={}):
return mock.Mock(spec=cls, wraps=cls(code, body, headers))
@@ -96,7 +101,7 @@ class FakeCurl:
raise ValueError
if self._headerfunction:
self._headerfunction("HTTP/1.1 {} Status".format(self._resp_code))
- for k, v in self._resp_headers.iteritems():
+ for k, v in self._resp_headers.items():
self._headerfunction(k + ': ' + str(v))
if type(self._resp_body) is not bool:
self._writer(self._resp_body)
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index d79788c..965bf29 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -1,11 +1,16 @@
-import BaseHTTPServer
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.utils import old_div
+import http.server
import hashlib
import os
import re
-import SocketServer
+import socketserver
import time
-class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
+class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
allow_reuse_address = 1
@@ -32,7 +37,7 @@ class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
def setdelays(self, **kwargs):
"""In future requests, induce delays at the given checkpoints."""
- for (k, v) in kwargs.iteritems():
+ for (k, v) in kwargs.items():
self.delays.get(k) # NameError if unknown key
self.delays[k] = v
@@ -54,12 +59,12 @@ class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
self._sleep_at_least(self.delays[k])
-class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+class Handler(http.server.BaseHTTPRequestHandler, object):
def wfile_bandwidth_write(self, data_to_write):
if self.server.bandwidth == None and self.server.delays['mid_write'] == 0:
self.wfile.write(data_to_write)
else:
- BYTES_PER_WRITE = int(self.server.bandwidth/4.0) or 32768
+ BYTES_PER_WRITE = int(old_div(self.server.bandwidth,4.0)) or 32768
outage_happened = False
num_bytes = len(data_to_write)
num_sent_bytes = 0
@@ -75,7 +80,7 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
num_sent_bytes:num_sent_bytes+num_write_bytes])
num_sent_bytes += num_write_bytes
if self.server.bandwidth is not None:
- target_time += num_write_bytes / self.server.bandwidth
+ target_time += old_div(num_write_bytes, self.server.bandwidth)
self.server._sleep_at_least(target_time - time.time())
return None
@@ -83,7 +88,7 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
if self.server.bandwidth == None and self.server.delays['mid_read'] == 0:
return self.rfile.read(bytes_to_read)
else:
- BYTES_PER_READ = int(self.server.bandwidth/4.0) or 32768
+ BYTES_PER_READ = int(old_div(self.server.bandwidth,4.0)) or 32768
data = ''
outage_happened = False
bytes_read = 0
@@ -98,7 +103,7 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
data += self.rfile.read(next_bytes_to_read)
bytes_read += next_bytes_to_read
if self.server.bandwidth is not None:
- target_time += next_bytes_to_read / self.server.bandwidth
+ target_time += old_div(next_bytes_to_read, self.server.bandwidth)
self.server._sleep_at_least(target_time - time.time())
return data
diff --git a/sdk/python/tests/manifest_examples.py b/sdk/python/tests/manifest_examples.py
index f20d909..91c37e6 100644
--- a/sdk/python/tests/manifest_examples.py
+++ b/sdk/python/tests/manifest_examples.py
@@ -1,4 +1,6 @@
from __future__ import absolute_import
+from builtins import range
+from builtins import object
import arvados
from . import arvados_testutil as tutil
diff --git a/sdk/python/tests/performance/test_a_sample.py b/sdk/python/tests/performance/test_a_sample.py
index b99ca64..0c5fe3a 100644
--- a/sdk/python/tests/performance/test_a_sample.py
+++ b/sdk/python/tests/performance/test_a_sample.py
@@ -1,5 +1,6 @@
from __future__ import print_function
from __future__ import absolute_import
+from builtins import range
import unittest
from .performance_profiler import profiled
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 8d7e708..d966126 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,6 +1,10 @@
#!/usr/bin/env python
from __future__ import print_function
+from __future__ import division
+from builtins import str
+from builtins import range
+from past.utils import old_div
import argparse
import atexit
import errno
@@ -96,7 +100,7 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
# Use up to half of the +wait+ period waiting for "passenger
# stop" to work. If the process hasn't exited by then, start
# sending TERM signals.
- startTERM += wait/2
+ startTERM += old_div(wait,2)
server_pid = None
while now <= deadline and server_pid is None:
@@ -439,7 +443,7 @@ def _start_keep(n, keep_args):
"-listen=:{}".format(port),
"-pid="+_pidfile('keep{}'.format(n))]
- for arg, val in keep_args.iteritems():
+ for arg, val in keep_args.items():
keep_cmd.append("{}={}".format(arg, val))
logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
@@ -736,7 +740,7 @@ class TestCaseWithServers(unittest.TestCase):
@staticmethod
def _restore_dict(src, dest):
- for key in dest.keys():
+ for key in list(dest.keys()):
if key not in src:
del dest[key]
dest.update(src)
diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py
index b6b2b56..a2dcaa0 100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@ -1,6 +1,8 @@
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import arvados
import collections
import httplib2
@@ -106,7 +108,7 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers):
api = arvados.api('v1',
requestBuilder=req_builder, model=OrderedJsonModel())
result = api.humans().get(uuid='test').execute()
- self.assertEqual(string.hexdigits, ''.join(result.keys()))
+ self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
class RetryREST(unittest.TestCase):
@@ -167,7 +169,7 @@ class RetryREST(unittest.TestCase):
mock_conns = {str(i): mock.MagicMock() for i in range(2)}
self.api._http.connections = mock_conns.copy()
self.api.users().create(body={}).execute()
- for c in mock_conns.itervalues():
+ for c in mock_conns.values():
self.assertEqual(c.close.call_count, expect)
@mock.patch('time.sleep')
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index 8e5c5ad..ae26ae7 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -2,6 +2,8 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import io
import os
import random
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 5f314bd..5abf388 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -2,6 +2,12 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from past.utils import old_div
import apiclient
import io
import mock
@@ -19,7 +25,7 @@ import threading
import hashlib
import random
-from cStringIO import StringIO
+from io import StringIO
import arvados
import arvados.commands.put as arv_put
@@ -258,7 +264,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
_, self.large_file_name = tempfile.mkstemp()
fileobj = open(self.large_file_name, 'w')
# Make sure to write just a little more than one block
- for _ in range((arvados.config.KEEP_BLOCK_SIZE/(1024*1024))+1):
+ for _ in range((old_div(arvados.config.KEEP_BLOCK_SIZE,(1024*1024)))+1):
data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB
fileobj.write(data)
fileobj.close()
@@ -525,7 +531,7 @@ class ArvadosPutReportTest(ArvadosBaseTestCase):
def test_known_human_progress(self):
for count, total in [(0, 1), (2, 4), (45, 60)]:
- expect = '{:.1%}'.format(float(count) / total)
+ expect = '{:.1%}'.format(old_div(float(count), total))
actual = arv_put.human_progress(count, total)
self.assertTrue(actual.startswith('\r'))
self.assertIn(expect, actual)
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 6c4976e..f4bcd8a 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -1,6 +1,10 @@
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import hex
+from builtins import str
+from builtins import range
+from builtins import object
import bz2
import datetime
import gzip
@@ -242,7 +246,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = "0123456789" * 100
- for b in xrange(0, 100000):
+ for b in range(0, 100000):
writer.write(text)
self.assertEqual(writer.size(), 100000000)
@@ -274,7 +278,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(0, os.SEEK_SET)
writer.write("0123456789")
@@ -291,7 +295,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(10, os.SEEK_SET)
writer.write("abcdefghij")
@@ -309,7 +313,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(5, os.SEEK_SET)
writer.write("abcdefghij")
@@ -334,8 +338,8 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- text = ''.join(["0123456789" for a in xrange(0, 100)])
- for b in xrange(0, 100000):
+ text = ''.join(["0123456789" for a in range(0, 100)])
+ for b in range(0, 100000):
writer.write(text)
writer.seek(0, os.SEEK_SET)
writer.write("foo")
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index 93388e6..ea34c80 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -1,6 +1,8 @@
from __future__ import print_function
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import md5
import mock
import os
@@ -17,7 +19,7 @@ from . import run_test_server
def _random(n):
- return bytearray(random.getrandbits(8) for _ in xrange(n))
+ return bytearray(random.getrandbits(8) for _ in range(n))
class CacheTestThread(threading.Thread):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 46682de..259c5aa 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
+from builtins import object
import arvados
import copy
import mock
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 73b9f22..2cca773 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -1,10 +1,16 @@
from __future__ import print_function
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
+from past.utils import old_div
import arvados
import io
import logging
import mock
-import Queue
+import queue
from . import run_test_server
import threading
import time
@@ -35,7 +41,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
def _test_subscribe(self, poll_fallback, expect_type, start_time=None, expected=1):
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
# Create ancestor before subscribing.
# When listening with start_time in the past, this should also be retrieved.
@@ -65,7 +71,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
log_object_uuids.append(events.get(True, 5)['object_uuid'])
if expected < 2:
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
# assertEqual just serves to show us what unexpected
# thing comes out of the queue when the assertRaises
# fails; when the test passes, this assertEqual
@@ -145,16 +151,16 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t))
def localiso(self, t):
- return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(-time.timezone/60)
+ return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(old_div(-time.timezone,60))
def isotz(self, offset):
"""Convert minutes-east-of-UTC to RFC3339- and ISO-compatible time zone designator"""
- return '{:+03d}:{:02d}'.format(offset/60, offset%60)
+ return '{:+03d}:{:02d}'.format(old_div(offset,60), offset%60)
# Test websocket reconnection on (un)execpted close
def _test_websocket_reconnect(self, close_unexpected):
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
logstream = io.BytesIO()
rootLogger = logging.getLogger()
@@ -176,7 +182,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
# expect an event
self.assertIn(human['uuid'], events.get(True, 5)['object_uuid'])
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
# close (im)properly
@@ -195,12 +201,12 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
event = events.get(True, 5)
if event.get('object_uuid') != None:
log_object_uuids.append(event['object_uuid'])
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
self.assertNotIn(human['uuid'], log_object_uuids)
self.assertIn(human2['uuid'], log_object_uuids)
else:
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
# verify log message to ensure that an (un)expected close
@@ -230,7 +236,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
rootLogger.addHandler(streamHandler)
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
filters = [['object_uuid', 'is_a', 'arvados#human']]
self.ws = arvados.events.subscribe(
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index bb6e983..b69563f 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -1,4 +1,11 @@
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from past.utils import old_div
+from builtins import object
import hashlib
import mock
import os
@@ -9,7 +16,7 @@ import socket
import threading
import time
import unittest
-import urlparse
+import urllib.parse
import arvados
import arvados.retry
@@ -292,7 +299,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
def get_service_roots(self, api_client):
keep_client = arvados.KeepClient(api_client=api_client)
services = keep_client.weighted_service_roots(arvados.KeepLocator('0'*32))
- return [urlparse.urlparse(url) for url in sorted(services)]
+ return [urllib.parse.urlparse(url) for url in sorted(services)]
def test_ssl_flag_respected_in_roots(self):
for ssl_flag in [False, True]:
@@ -443,7 +450,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
num_retries=3)
self.assertEqual([403, 403], [
getattr(error, 'status_code', None)
- for error in err_check.exception.request_errors().itervalues()])
+ for error in err_check.exception.request_errors().values()])
def test_get_error_reflects_last_retry(self):
self.check_errors_from_last_retry('get', arvados.errors.KeepReadError)
@@ -649,7 +656,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
self.assertRaises(exc_class) as err_check:
curl_mock.return_value.side_effect = socket.timeout
getattr(keep_client, verb)(data)
- urls = [urlparse.urlparse(url)
+ urls = [urllib.parse.urlparse(url)
for url in err_check.exception.request_errors()]
self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'],
[(url.hostname, url.port) for url in urls])
@@ -1118,35 +1125,35 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock):
def test_only_write_enough_on_success(self):
for i in range(10):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_on_partial_success(self):
for i in range(5):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=False)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=False)
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_when_some_crash(self):
for i in range(5):
- ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_fail_when_too_many_crash(self):
for i in range(self.copies+1):
- ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
self.pool.add_task(ks, None)
for i in range(self.copies-1):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies-1)
diff --git a/sdk/python/tests/test_keep_locator.py b/sdk/python/tests/test_keep_locator.py
index 273992a..2a90316 100644
--- a/sdk/python/tests/test_keep_locator.py
+++ b/sdk/python/tests/test_keep_locator.py
@@ -1,6 +1,10 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from builtins import next
+from builtins import zip
+from builtins import str
+from builtins import range
import datetime
import itertools
import random
@@ -14,7 +18,7 @@ class ArvadosKeepLocatorTest(unittest.TestCase):
def numstrs(fmtstr, base, exponent):
def genstrs(self, count=None):
return (fmtstr.format(random.randint(0, base ** exponent))
- for c in xrange(count or self.DEFAULT_TEST_COUNT))
+ for c in range(count or self.DEFAULT_TEST_COUNT))
return genstrs
checksums = numstrs('{:032x}', 16, 32)
@@ -24,17 +28,17 @@ class ArvadosKeepLocatorTest(unittest.TestCase):
def base_locators(self, count=DEFAULT_TEST_COUNT):
return ('+'.join(pair) for pair in
- itertools.izip(self.checksums(count), self.sizes(count)))
+ zip(self.checksums(count), self.sizes(count)))
def perm_hints(self, count=DEFAULT_TEST_COUNT):
- for sig, ts in itertools.izip(self.signatures(count),
+ for sig, ts in zip(self.signatures(count),
self.timestamps(count)):
yield 'A{}@{}'.format(sig, ts)
def test_good_locators_returned(self):
for hint_gens in [(), (self.sizes(),),
(self.sizes(), self.perm_hints())]:
- for loc_data in itertools.izip(self.checksums(), *hint_gens):
+ for loc_data in zip(self.checksums(), *hint_gens):
locator = '+'.join(loc_data)
self.assertEqual(locator, str(KeepLocator(locator)))
diff --git a/sdk/python/tests/test_retry.py b/sdk/python/tests/test_retry.py
index cc12f39..82725c2 100644
--- a/sdk/python/tests/test_retry.py
+++ b/sdk/python/tests/test_retry.py
@@ -1,5 +1,8 @@
#!/usr/bin/env python
+from builtins import zip
+from builtins import range
+from builtins import object
import itertools
import unittest
@@ -25,7 +28,7 @@ class RetryLoopTestMixin(object):
responses = itertools.chain(results, itertools.repeat(None))
retrier = arv_retry.RetryLoop(num_retries, self.loop_success,
**kwargs)
- for tries_left, response in itertools.izip(retrier, responses):
+ for tries_left, response in zip(retrier, responses):
retrier.save_result(response)
return retrier
@@ -166,11 +169,11 @@ class CheckHTTPResponseSuccessTestCase(unittest.TestCase):
check_is_not = check('assertIsNot')
def test_obvious_successes(self):
- self.check_is(True, *range(200, 207))
+ self.check_is(True, *list(range(200, 207)))
def test_obvious_stops(self):
self.check_is(False, 424, 426, 428, 431,
- *range(400, 408) + range(410, 420))
+ *list(range(400, 408)) + list(range(410, 420)))
def test_obvious_retries(self):
self.check_is(None, 500, 502, 503, 504)
@@ -179,13 +182,13 @@ class CheckHTTPResponseSuccessTestCase(unittest.TestCase):
self.check_is(None, 408, 409, 422, 423)
def test_5xx_failures(self):
- self.check_is(False, 501, *range(505, 512))
+ self.check_is(False, 501, *list(range(505, 512)))
def test_1xx_not_retried(self):
self.check_is_not(None, 100, 101)
def test_redirects_not_retried(self):
- self.check_is_not(None, *range(300, 309))
+ self.check_is_not(None, *list(range(300, 309)))
def test_wacky_code_retries(self):
self.check_is(None, 0, 99, 600, -200)
diff --git a/sdk/python/tests/test_retry_job_helpers.py b/sdk/python/tests/test_retry_job_helpers.py
index 3ccaa37..9ad957a 100644
--- a/sdk/python/tests/test_retry_job_helpers.py
+++ b/sdk/python/tests/test_retry_job_helpers.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import object
import mock
import os
import unittest
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 9e35bcf..7277628 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import object
import bz2
import gzip
import io
commit 5e46c19fa6f037d41bd196a5418066f9fbe3bdb3
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Mar 31 22:55:14 2017 -0400
11308: Fix futurize.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 4537077..36d54e5 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -20,7 +20,7 @@ import fcntl
import time
import threading
-from .api import api, http_cache
+from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
from .keep import *
from .stream import *
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 095187e..1a42781 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -17,10 +17,10 @@ from .stream import StreamReader
from ._normalize_stream import normalize_stream
from ._ranges import Range, LocatorAndRange
from .safeapi import ThreadSafeApiCache
-from . import config
-from . import errors
-from . import util
-from . import events
+import arvados.config as config
+import arvados.errors as errors
+import arvados.util
+import arvados.events as events
from arvados.retry import retry_method
_logger = logging.getLogger('arvados.collection')
@@ -52,7 +52,7 @@ class CollectionBase(object):
if fields:
clean_fields = fields[:1] + [
(re.sub(r'\+[^\d][^\+]*', '', x)
- if re.match(util.keep_locator_pattern, x)
+ if re.match(arvados.util.keep_locator_pattern, x)
else x)
for x in fields[1:]]
clean += [' '.join(clean_fields), "\n"]
@@ -181,7 +181,7 @@ class CollectionWriter(CollectionBase):
def _work_trees(self):
path, stream_name, max_manifest_depth = self._queued_trees[0]
- d = util.listdir_recursive(
+ d = arvados.util.listdir_recursive(
path, max_depth = (None if max_manifest_depth == 0 else 0))
if d:
self._queue_dirents(stream_name, d)
@@ -1223,11 +1223,11 @@ class Collection(RichCollectionBase):
self.events = None
if manifest_locator_or_text:
- if re.match(util.keep_locator_pattern, manifest_locator_or_text):
+ if re.match(arvados.util.keep_locator_pattern, manifest_locator_or_text):
self._manifest_locator = manifest_locator_or_text
- elif re.match(util.collection_uuid_pattern, manifest_locator_or_text):
+ elif re.match(arvados.util.collection_uuid_pattern, manifest_locator_or_text):
self._manifest_locator = manifest_locator_or_text
- elif re.match(util.manifest_pattern, manifest_locator_or_text):
+ elif re.match(arvados.util.manifest_pattern, manifest_locator_or_text):
self._manifest_text = manifest_locator_or_text
else:
raise errors.ArgumentError(
@@ -1343,10 +1343,10 @@ class Collection(RichCollectionBase):
error_via_api = None
error_via_keep = None
should_try_keep = ((self._manifest_text is None) and
- util.keep_locator_pattern.match(
+ arvados.util.keep_locator_pattern.match(
self._manifest_locator))
if ((self._manifest_text is None) and
- util.signed_locator_pattern.match(self._manifest_locator)):
+ arvados.util.signed_locator_pattern.match(self._manifest_locator)):
error_via_keep = self._populate_from_keep()
if self._manifest_text is None:
error_via_api = self._populate_from_api_server()
@@ -1372,7 +1372,7 @@ class Collection(RichCollectionBase):
def _has_collection_uuid(self):
- return self._manifest_locator is not None and re.match(util.collection_uuid_pattern, self._manifest_locator)
+ return self._manifest_locator is not None and re.match(arvados.util.collection_uuid_pattern, self._manifest_locator)
def __enter__(self):
return self
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index bc87536..a9ca978 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -1,9 +1,11 @@
from __future__ import absolute_import
-import threading
-from . import api
-from . import keep
-from . import config
+
import copy
+import threading
+
+import arvados
+import arvados.keep as keep
+import arvados.config as config
class ThreadSafeApiCache(object):
"""Threadsafe wrapper for API objects.
@@ -22,7 +24,7 @@ class ThreadSafeApiCache(object):
def localapi(self):
if 'api' not in self.local.__dict__:
- self.local.api = api.api_from_config('v1', apiconfig=self.apiconfig)
+ self.local.api = arvados.api_from_config('v1', apiconfig=self.apiconfig)
return self.local.api
def __getattr__(self, name):
commit f69e114cf6dbe51525e98ca30a52ef3dd4341167
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Mar 31 22:36:21 2017 -0400
11308: Futurize.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index b74f828..4537077 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+from __future__ import absolute_import
import gflags
import httplib
import httplib2
@@ -19,13 +21,13 @@ import time
import threading
from .api import api, http_cache
-from collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
-from keep import *
-from stream import *
-from arvfile import StreamFileReader
-from retry import RetryLoop
-import errors
-import util
+from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
+from .keep import *
+from .stream import *
+from .arvfile import StreamFileReader
+from .retry import RetryLoop
+from . import errors
+from . import util
# Set up Arvados logging based on the user's configuration.
# All Arvados code should log under the arvados hierarchy.
@@ -104,7 +106,7 @@ def get_task_param_mount(*args):
class JobTask(object):
def __init__(self, parameters=dict(), runtime_constraints=dict()):
- print "init jobtask %s %s" % (parameters, runtime_constraints)
+ print("init jobtask %s %s" % (parameters, runtime_constraints))
class job_setup:
@staticmethod
diff --git a/sdk/python/arvados/_normalize_stream.py b/sdk/python/arvados/_normalize_stream.py
index e0831b1..babcabc 100644
--- a/sdk/python/arvados/_normalize_stream.py
+++ b/sdk/python/arvados/_normalize_stream.py
@@ -1,4 +1,5 @@
-import config
+from __future__ import absolute_import
+from . import config
def normalize_stream(stream_name, stream):
"""Take manifest stream and return a list of tokens in normalized format.
@@ -17,7 +18,7 @@ def normalize_stream(stream_name, stream):
sortedfiles.sort()
blocks = {}
- streamoffset = 0L
+ streamoffset = 0
# Go through each file and add each referenced block exactly once.
for streamfile in sortedfiles:
for segment in stream[streamfile]:
diff --git a/sdk/python/arvados/_ranges.py b/sdk/python/arvados/_ranges.py
index 8387416..e0fe615 100644
--- a/sdk/python/arvados/_ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -24,7 +24,7 @@ class Range(object):
self.segment_offset == other.segment_offset)
def first_block(data_locators, range_start):
- block_start = 0L
+ block_start = 0
# range_start/block_start is the inclusive lower bound
# range_end/block_end is the exclusive upper bound
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 1af50b3..65aadbd 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import collections
import httplib
import httplib2
@@ -12,10 +13,10 @@ import types
import apiclient
from apiclient import discovery as apiclient_discovery
from apiclient import errors as apiclient_errors
-import config
-import errors
-import util
-import cache
+from . import config
+from . import errors
+from . import util
+from . import cache
_logger = logging.getLogger('arvados.api')
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index ca45cf2..2c44c34 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -1,8 +1,9 @@
+from __future__ import absolute_import
import functools
import os
import zlib
import bz2
-import config
+from . import config
import hashlib
import threading
import Queue
@@ -76,7 +77,7 @@ class _FileLikeObjectBase(object):
class ArvadosFileReaderBase(_FileLikeObjectBase):
def __init__(self, name, mode, num_retries=None):
super(ArvadosFileReaderBase, self).__init__(name, mode)
- self._filepos = 0L
+ self._filepos = 0
self.num_retries = num_retries
self._readline_cache = (None, None)
@@ -96,7 +97,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
pos += self._filepos
elif whence == os.SEEK_END:
pos += self.size()
- self._filepos = min(max(pos, 0L), self.size())
+ self._filepos = min(max(pos, 0), self.size())
def tell(self):
return self._filepos
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index 08c19e4..ac6d184 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -39,7 +39,7 @@ class SafeHTTPCache(object):
try:
with open(filename, 'rb') as f:
return f.read()
- except IOError, OSError:
+ except IOError as OSError:
return None
def set(self, url, content):
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index f26d3a3..095187e 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import functools
import logging
import os
@@ -11,15 +12,15 @@ from collections import deque
from stat import *
from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, NoopLock
-from keep import KeepLocator, KeepClient
+from .keep import KeepLocator, KeepClient
from .stream import StreamReader
from ._normalize_stream import normalize_stream
from ._ranges import Range, LocatorAndRange
from .safeapi import ThreadSafeApiCache
-import config
-import errors
-import util
-import events
+from . import config
+from . import errors
+from . import util
+from . import events
from arvados.retry import retry_method
_logger = logging.getLogger('arvados.collection')
@@ -1575,7 +1576,7 @@ class Collection(RichCollectionBase):
stream_name = tok.replace('\\040', ' ')
blocks = []
segments = []
- streamoffset = 0L
+ streamoffset = 0
state = BLOCKS
self.find_or_create(stream_name, COLLECTION)
continue
diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index 4be9996..802744b 100644
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
import argparse
import time
import sys
@@ -127,7 +128,7 @@ def main(arguments=None):
if args.print_unmigrated:
only_migrate = set()
for pdh in need_migrate:
- print pdh
+ print(pdh)
return
logger.info("Already migrated %i images", len(already_migrated))
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 46a56ca..2f3e042 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -1,5 +1,7 @@
#!/usr/bin/env python
+from __future__ import print_function
+from __future__ import absolute_import
import arvados
import arvados.commands.ws as ws
import argparse
@@ -7,7 +9,7 @@ import json
import re
import os
import stat
-import put
+from . import put
import time
import subprocess
import logging
diff --git a/sdk/python/arvados/commands/ws.py b/sdk/python/arvados/commands/ws.py
index 72ef1be..cd67b73 100644
--- a/sdk/python/arvados/commands/ws.py
+++ b/sdk/python/arvados/commands/ws.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from __future__ import print_function
import sys
import logging
import argparse
@@ -97,7 +98,7 @@ def main(arguments=None):
elif 'status' in ev and ev['status'] == 200:
pass
else:
- print json.dumps(ev)
+ print(json.dumps(ev))
try:
ws = subscribe(arvados.api('v1'), filters, on_message, poll_fallback=args.poll_interval, last_log_id=last_log_id)
diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index cf26f9e..b385761 100644
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -1,7 +1,8 @@
+from __future__ import absolute_import
import arvados
-import config
-import errors
-from retry import RetryLoop
+from . import config
+from . import errors
+from .retry import RetryLoop
import logging
import json
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 38f332b..4efa698 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import cStringIO
import datetime
import hashlib
@@ -11,7 +12,7 @@ import socket
import ssl
import sys
import threading
-import timer
+from . import timer
import urlparse
import arvados
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index 5c5c872..bc87536 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -1,7 +1,8 @@
+from __future__ import absolute_import
import threading
-import api
-import keep
-import config
+from . import api
+from . import keep
+from . import config
import copy
class ThreadSafeApiCache(object):
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index afc202e..042ed47 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+from __future__ import absolute_import
import collections
import hashlib
import os
@@ -9,10 +11,10 @@ import copy
from ._ranges import locators_and_ranges, Range
from .arvfile import StreamFileReader
from arvados.retry import retry_method
-from keep import *
-import config
-import errors
-from _normalize_stream import normalize_stream
+from .keep import *
+from . import config
+from . import errors
+from ._normalize_stream import normalize_stream
class StreamReader(object):
def __init__(self, tokens, keep=None, debug=False, _empty=False,
@@ -23,11 +25,11 @@ class StreamReader(object):
self._keep = keep
self.num_retries = num_retries
- streamoffset = 0L
+ streamoffset = 0
# parse stream
for tok in tokens:
- if debug: print 'tok', tok
+ if debug: print('tok', tok)
if self._stream_name is None:
self._stream_name = tok.replace('\\040', ' ')
continue
diff --git a/sdk/python/arvados/timer.py b/sdk/python/arvados/timer.py
index 739d0d5..6d4a73f 100644
--- a/sdk/python/arvados/timer.py
+++ b/sdk/python/arvados/timer.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
import time
class Timer(object):
@@ -13,4 +14,4 @@ class Timer(object):
self.secs = self.end - self.start
self.msecs = self.secs * 1000 # millisecs
if self.verbose:
- print 'elapsed time: %f ms' % self.msecs
+ print('elapsed time: %f ms' % self.msecs)
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index e2692b7..522a95e 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -364,10 +364,10 @@ def is_hex(s, *length_args):
def list_all(fn, num_retries=0, **kwargs):
# Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxint)
+ kwargs.setdefault('limit', sys.maxsize)
items = []
offset = 0
- items_available = sys.maxint
+ items_available = sys.maxsize
while len(items) < items_available:
c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
items += c['items']
diff --git a/sdk/python/tests/manifest_examples.py b/sdk/python/tests/manifest_examples.py
index 8f0abd2..f20d909 100644
--- a/sdk/python/tests/manifest_examples.py
+++ b/sdk/python/tests/manifest_examples.py
@@ -1,5 +1,6 @@
+from __future__ import absolute_import
import arvados
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class ManifestExamples(object):
def make_manifest(self,
diff --git a/sdk/python/tests/performance/test_a_sample.py b/sdk/python/tests/performance/test_a_sample.py
index dff0984..b99ca64 100644
--- a/sdk/python/tests/performance/test_a_sample.py
+++ b/sdk/python/tests/performance/test_a_sample.py
@@ -1,6 +1,8 @@
+from __future__ import print_function
+from __future__ import absolute_import
import unittest
-from performance_profiler import profiled
+from .performance_profiler import profiled
class PerformanceTestSample(unittest.TestCase):
def foo(self):
@@ -12,4 +14,4 @@ class PerformanceTestSample(unittest.TestCase):
for i in range(0,2**20):
j += i
self.foo()
- print 'Hello'
+ print('Hello')
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d10e60c..8d7e708 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -211,7 +211,7 @@ def _fifo2stderr(label):
except OSError as error:
if error.errno != errno.ENOENT:
raise
- os.mkfifo(fifo, 0700)
+ os.mkfifo(fifo, 0o700)
subprocess.Popen(
['stdbuf', '-i0', '-oL', '-eL', 'sed', '-e', 's/^/['+label+'] /', fifo],
stdout=sys.stderr)
diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py
index 6c379e1..b6b2b56 100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import arvados
import collections
import httplib2
@@ -12,12 +13,12 @@ import string
import unittest
import mock
-import run_test_server
+from . import run_test_server
from apiclient import errors as apiclient_errors
from apiclient import http as apiclient_http
from arvados.api import OrderedJsonModel, RETRY_DELAY_INITIAL, RETRY_DELAY_BACKOFF, RETRY_COUNT
-from arvados_testutil import fake_httplib2_response, queue_with
+from .arvados_testutil import fake_httplib2_response, queue_with
if not mimetypes.inited:
mimetypes.init()
diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py
index e291ee0..c8eb5d1 100644
--- a/sdk/python/tests/test_arv_copy.py
+++ b/sdk/python/tests/test_arv_copy.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
import io
import os
import sys
@@ -8,7 +9,7 @@ import tempfile
import unittest
import arvados.commands.arv_copy as arv_copy
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class ArvCopyTestCase(unittest.TestCase):
def run_copy(self, args):
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index a25ad16..2257227 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
import arvados
import hashlib
import io
@@ -13,8 +14,8 @@ import unittest
import logging
import arvados.commands.keepdocker as arv_keepdocker
-import arvados_testutil as tutil
-import run_test_server
+from . import arvados_testutil as tutil
+from . import run_test_server
class StopTest(Exception):
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index 5064f07..8e5c5ad 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
import io
import os
import random
@@ -10,9 +11,9 @@ import tempfile
import arvados.errors as arv_error
import arvados.commands.ls as arv_ls
-import run_test_server
+from . import run_test_server
-from arvados_testutil import str_keep_locator, redirected_streams
+from .arvados_testutil import str_keep_locator, redirected_streams
class ArvLsTestCase(run_test_server.TestCaseWithServers):
FAKE_UUID = 'zzzzz-4zz18-12345abcde12345'
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 286a22e..5f314bd 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
import apiclient
import io
import mock
@@ -22,10 +23,10 @@ from cStringIO import StringIO
import arvados
import arvados.commands.put as arv_put
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
-from arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response
-import run_test_server
+from .arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response
+from . import run_test_server
class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
CACHE_ARGSET = [
diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py
index 3d04d27..0f33dfc 100644
--- a/sdk/python/tests/test_arv_run.py
+++ b/sdk/python/tests/test_arv_run.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
import io
import os
import sys
@@ -8,7 +9,7 @@ import tempfile
import unittest
import arvados.commands.run as arv_run
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class ArvRunTestCase(unittest.TestCase):
def run_arv_run(self, args):
diff --git a/sdk/python/tests/test_arv_ws.py b/sdk/python/tests/test_arv_ws.py
index 2a85e04..4a0b823 100644
--- a/sdk/python/tests/test_arv_ws.py
+++ b/sdk/python/tests/test_arv_ws.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import io
import os
import sys
@@ -8,7 +9,7 @@ import unittest
import arvados.errors as arv_error
import arvados.commands.ws as arv_ws
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class ArvWsTestCase(unittest.TestCase):
def run_ws(self, args):
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 1b66935..6c4976e 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import bz2
import datetime
import gzip
@@ -15,8 +16,8 @@ from arvados.keep import KeepLocator
from arvados.collection import Collection, CollectionReader
from arvados.arvfile import ArvadosFile, ArvadosFileReader
-import arvados_testutil as tutil
-from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
+from . import arvados_testutil as tutil
+from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
class ArvadosFileWriterTestCase(unittest.TestCase):
class MockKeep(object):
diff --git a/sdk/python/tests/test_benchmark_collections.py b/sdk/python/tests/test_benchmark_collections.py
index d75ad47..273011e 100644
--- a/sdk/python/tests/test_benchmark_collections.py
+++ b/sdk/python/tests/test_benchmark_collections.py
@@ -1,10 +1,11 @@
+from __future__ import absolute_import
import arvados
import sys
-import run_test_server
-import arvados_testutil as tutil
-import manifest_examples
-from performance.performance_profiler import profiled
+from . import run_test_server
+from . import arvados_testutil as tutil
+from . import manifest_examples
+from .performance.performance_profiler import profiled
class CollectionBenchmark(run_test_server.TestCaseWithServers,
tutil.ArvadosBaseTestCase,
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index 4d68b40..93388e6 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -1,4 +1,5 @@
from __future__ import print_function
+from __future__ import absolute_import
import md5
import mock
@@ -10,9 +11,9 @@ import tempfile
import threading
import unittest
-import arvados.cache
import arvados
-import run_test_server
+import arvados.cache
+from . import run_test_server
def _random(n):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 0e3d5e1..46682de 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# usage example:
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
@@ -11,10 +12,10 @@ import re
import tempfile
import unittest
-import run_test_server
+from . import run_test_server
from arvados._ranges import Range, LocatorAndRange
from arvados.collection import Collection, CollectionReader
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
diff --git a/sdk/python/tests/test_errors.py b/sdk/python/tests/test_errors.py
index db875dc..2ccd594 100644
--- a/sdk/python/tests/test_errors.py
+++ b/sdk/python/tests/test_errors.py
@@ -1,10 +1,11 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import traceback
import unittest
import arvados.errors as arv_error
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
class KeepRequestErrorTestCase(unittest.TestCase):
REQUEST_ERRORS = [
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 88e5283..73b9f22 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -1,14 +1,16 @@
+from __future__ import print_function
+from __future__ import absolute_import
import arvados
import io
import logging
import mock
import Queue
-import run_test_server
+from . import run_test_server
import threading
import time
import unittest
-import arvados_testutil
+from . import arvados_testutil
class WebsocketTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
@@ -27,7 +29,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
if self.ws:
self.ws.close()
except Exception as e:
- print("Error in teardown: ", e)
+ print(("Error in teardown: ", e))
super(WebsocketTest, self).tearDown()
run_test_server.reset()
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 85b5bc8..bb6e983 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import hashlib
import mock
import os
@@ -12,9 +13,9 @@ import urlparse
import arvados
import arvados.retry
-import arvados_testutil as tutil
-import keepstub
-import run_test_server
+from . import arvados_testutil as tutil
+from . import keepstub
+from . import run_test_server
class KeepTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
diff --git a/sdk/python/tests/test_pipeline_template.py b/sdk/python/tests/test_pipeline_template.py
index fa9fef2..4c0059a 100644
--- a/sdk/python/tests/test_pipeline_template.py
+++ b/sdk/python/tests/test_pipeline_template.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# usage example:
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
@@ -5,7 +6,7 @@
import unittest
import arvados
import apiclient
-import run_test_server
+from . import run_test_server
class PipelineTemplateTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
diff --git a/sdk/python/tests/test_retry_job_helpers.py b/sdk/python/tests/test_retry_job_helpers.py
index 6e562a0..3ccaa37 100644
--- a/sdk/python/tests/test_retry_job_helpers.py
+++ b/sdk/python/tests/test_retry_job_helpers.py
@@ -1,13 +1,14 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import mock
import os
import unittest
import hashlib
-import run_test_server
+from . import run_test_server
import json
import arvados
-import arvados_testutil as tutil
+from . import arvados_testutil as tutil
from apiclient import http as apiclient_http
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 624f1b8..9e35bcf 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+from __future__ import absolute_import
import bz2
import gzip
import io
@@ -12,8 +13,8 @@ import arvados
from arvados import StreamReader, StreamFileReader
from arvados._ranges import Range
-import arvados_testutil as tutil
-import run_test_server
+from . import arvados_testutil as tutil
+from . import run_test_server
class StreamFileReaderTestCase(unittest.TestCase):
def make_count_reader(self):
commit 9b823dc4b56a3bda5d095a4b3549ea2c0b024fd8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Mar 31 22:35:25 2017 -0400
11308: Test sdk/python with Python 3.
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 44f9a30..f21c2e6 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -464,20 +464,36 @@ fi
# Deactivate Python 2 virtualenv
deactivate
+declare -a pythonstuff
+pythonstuff=(
+ sdk/pam
+ sdk/python
+ sdk/cwl
+ services/fuse
+ services/nodemanager
+ tools/crunchstat-summary
+ )
+
+declare -a python3stuff
+python3stuff=(
+ sdk/python
+ services/dockercleaner
+)
+
# If Python 3 is available, set up its virtualenv in $VENV3DIR.
# Otherwise, skip dependent tests.
PYTHON3=$(which python3)
-if [ "0" = "$?" ]; then
+if [[ ${?} = 0 ]]; then
setup_virtualenv "$VENV3DIR" --python python3
else
- PYTHON3=
- skip[services/dockercleaner]=1
cat >&2 <<EOF
-Warning: python3 could not be found
-services/dockercleaner install and tests will be skipped
+Warning: python3 could not be found. The following tests will be skipped:
+
+ ${python3stuff}
EOF
+ python3stuff=()
fi
# Reactivate Python 2 virtualenv
@@ -698,22 +714,15 @@ do_install services/login-sync login-sync
# module. We can't actually *test* the Python SDK yet though, because
# its own test suite brings up some of those other programs (like
# keepproxy).
-declare -a pythonstuff
-pythonstuff=(
- sdk/pam
- sdk/python
- sdk/cwl
- services/fuse
- services/nodemanager
- tools/crunchstat-summary
- )
for p in "${pythonstuff[@]}"
do
do_install "$p" pip
done
-if [ -n "$PYTHON3" ]; then
- do_install services/dockercleaner pip "$VENV3DIR/bin/"
-fi
+
+for p in "${python3stuff[@]}"
+do
+ do_install "$p" pip "$VENV3DIR/bin/"
+done
install_apiserver() {
cd "$WORKSPACE/services/api" \
@@ -857,7 +866,11 @@ for p in "${pythonstuff[@]}"
do
do_test "$p" pip
done
-do_test services/dockercleaner pip "$VENV3DIR/bin/"
+
+for p in "${python3stuff[@]}"
+do
+ do_test "$p" pip "$VENV3DIR/bin/"
+done
for g in "${gostuff[@]}"
do
commit d8fef944b2691e8d77df4185f297f205c4975b21
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 1 17:36:52 2017 -0400
11308: fmt
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 9db19b0..ca45cf2 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -213,8 +213,8 @@ class StreamFileReader(ArvadosFileReaderBase):
if available_chunks:
lr = available_chunks[0]
data = self._stream.readfrom(lr.locator+lr.segment_offset,
- lr.segment_size,
- num_retries=num_retries)
+ lr.segment_size,
+ num_retries=num_retries)
self._filepos += len(data)
return data
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list