[ARVADOS] updated: 1.2.0-459-g48332b69a
Git user
git at public.curoverse.com
Thu Nov 29 16:10:17 EST 2018
Summary of changes:
sdk/python/arvados/collection.py | 8 ++------
sdk/python/arvados/commands/run.py | 2 +-
2 files changed, 3 insertions(+), 7 deletions(-)
discards 3138f9673f6cdd557800e6252b6e9d605ddcf2b8 (commit)
via 48332b69a4bd4a184803698b9cd059dcf1b8e01a (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (3138f9673f6cdd557800e6252b6e9d605ddcf2b8)
\
N -- N -- N (48332b69a4bd4a184803698b9cd059dcf1b8e01a)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 48332b69a4bd4a184803698b9cd059dcf1b8e01a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Nov 29 16:08:59 2018 -0500
11419: Support text modes in Collection.open().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 3281d78e2..ef3cd5273 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -15,6 +15,7 @@ import copy
import errno
import functools
import hashlib
+import locale
import logging
import os
import queue
@@ -57,10 +58,13 @@ class UnownedBlockError(Exception):
class _FileLikeObjectBase(object):
- def __init__(self, name, mode):
+ def __init__(self, name, mode, encoding=None):
self.name = name
self.mode = mode
self.closed = False
+ if encoding is None:
+ encoding = locale.getpreferredencoding(False)
+ self.encoding = encoding
@staticmethod
def _before_close(orig_func):
@@ -86,11 +90,8 @@ class _FileLikeObjectBase(object):
class ArvadosFileReaderBase(_FileLikeObjectBase):
- def __init__(self, name, mode, num_retries=None):
- super(ArvadosFileReaderBase, self).__init__(name, mode)
- self._binary = 'b' in mode
- if sys.version_info >= (3, 0) and not self._binary:
- raise NotImplementedError("text mode {!r} is not implemented".format(mode))
+ def __init__(self, name, mode, num_retries=None, encoding=None):
+ super(ArvadosFileReaderBase, self).__init__(name, mode, encoding=encoding)
self._filepos = 0
self.num_retries = num_retries
self._readline_cache = (None, None)
@@ -1268,8 +1269,8 @@ class ArvadosFileReader(ArvadosFileReaderBase):
"""
- def __init__(self, arvadosfile, mode="r", num_retries=None):
- super(ArvadosFileReader, self).__init__(arvadosfile.name, mode=mode, num_retries=num_retries)
+ def __init__(self, arvadosfile, mode="r", num_retries=None, encoding=None):
+ super(ArvadosFileReader, self).__init__(arvadosfile.name, mode=mode, num_retries=num_retries, encoding=encoding)
self.arvadosfile = arvadosfile
def size(self):
@@ -1278,6 +1279,11 @@ class ArvadosFileReader(ArvadosFileReaderBase):
def stream_name(self):
return self.arvadosfile.parent.stream_name()
+ def readinto(self, b):
+ data = self.read(len(b))
+ b[:len(data)] = data
+ return len(data)
+
@_FileLikeObjectBase._before_close
@retry_method
def read(self, size=None, num_retries=None):
@@ -1320,8 +1326,8 @@ class ArvadosFileWriter(ArvadosFileReader):
"""
- def __init__(self, arvadosfile, mode, num_retries=None):
- super(ArvadosFileWriter, self).__init__(arvadosfile, mode=mode, num_retries=num_retries)
+ def __init__(self, arvadosfile, mode, num_retries=None, encoding=None):
+ super(ArvadosFileWriter, self).__init__(arvadosfile, mode=mode, num_retries=num_retries, encoding=encoding)
self.arvadosfile.add_writer(self)
def writable(self):
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 48fdaf03e..bc344abc0 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -7,16 +7,18 @@ from future.utils import listitems, listvalues, viewkeys
from builtins import str
from past.builtins import basestring
from builtins import object
+import ciso8601
+import datetime
+import errno
import functools
+import hashlib
+import io
import logging
import os
import re
-import errno
-import hashlib
-import datetime
-import ciso8601
-import time
+import sys
import threading
+import time
from collections import deque
from stat import *
@@ -35,6 +37,21 @@ from arvados.retry import retry_method
_logger = logging.getLogger('arvados.collection')
+
+if sys.version_info >= (3, 0):
+ TextIOWrapper = io.TextIOWrapper
+else:
+ class TextIOWrapper(io.TextIOWrapper):
+ """To maintain backward compatibility, cast str to unicode in
+ write('foo').
+
+ """
+ def write(self, data):
+ if isinstance(data, basestring):
+ data = unicode(data)
+ return super(TextIOWrapper, self).write(data)
+
+
class CollectionBase(object):
"""Abstract base class for Collection classes."""
@@ -654,7 +671,7 @@ class RichCollectionBase(CollectionBase):
return self.find_or_create(path, COLLECTION)
- def open(self, path, mode="r"):
+ def open(self, path, mode="r", encoding=None):
"""Open a file-like object for access.
:path:
@@ -676,17 +693,20 @@ class RichCollectionBase(CollectionBase):
opens for reading and writing. All writes are appended to
the end of the file. Writing does not affect the file pointer for
reading.
+
"""
if not re.search(r'^[rwa][bt]?\+?$', mode):
raise errors.ArgumentError("Invalid mode {!r}".format(mode))
if mode[0] == 'r' and '+' not in mode:
+ writing = False
fclass = ArvadosFileReader
arvfile = self.find(path)
elif not self.writable():
raise IOError(errno.EROFS, "Collection is read only")
else:
+ writing = True
fclass = ArvadosFileWriter
arvfile = self.find_or_create(path, FILE)
@@ -698,7 +718,14 @@ class RichCollectionBase(CollectionBase):
if mode[0] == 'w':
arvfile.truncate(0)
- return fclass(arvfile, mode=mode, num_retries=self.num_retries)
+ f = fclass(arvfile, mode='b'+mode, num_retries=self.num_retries, encoding=encoding)
+ if 'b' not in mode:
+ if writing:
+ f = io.BufferedRandom(raw=f)
+ else:
+ f = io.BufferedReader(raw=f)
+ f = TextIOWrapper(buffer=f)
+ return f
def modified(self):
"""Determine if the collection has been modified since last commited."""
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 96f5bdd44..c7aacffa1 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -152,7 +152,7 @@ def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)",
def write_file(collection, pathprefix, fn, flush=False):
with open(os.path.join(pathprefix, fn)) as src:
- dst = collection.open(fn, "w")
+ dst = collection.open(fn, "wb")
r = src.read(1024*128)
while r:
dst.write(r)
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index ac18c44c6..de0100674 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -836,17 +836,58 @@ class CollectionOpenModes(run_test_server.TestCaseWithServers):
with c.open('foo', 'wb') as f:
f.write('foo')
for mode in ['r', 'rt', 'r+', 'rt+', 'w', 'wt', 'a', 'at']:
- if sys.version_info >= (3, 0):
- with self.assertRaises(NotImplementedError):
- c.open('foo', mode)
- else:
- with c.open('foo', mode) as f:
- if mode[0] == 'r' and '+' not in mode:
- self.assertEqual('foo', f.read(3))
- else:
- f.write('bar')
- f.seek(-3, os.SEEK_CUR)
- self.assertEqual('bar', f.read(3))
+ with c.open('foo', mode) as f:
+ if mode[0] == 'r' and '+' not in mode:
+ self.assertEqual('foo', f.read(3))
+ else:
+ f.write('bar')
+ f.seek(0, os.SEEK_SET)
+ self.assertEqual('bar', f.read(3))
+
+
+class TextModes(run_test_server.TestCaseWithServers):
+
+ def setUp(self):
+ arvados.config.KEEP_BLOCK_SIZE = 4
+ if sys.version_info < (3, 0):
+ import unicodedata
+ self.sailboat = unicodedata.lookup('SAILBOAT')
+ self.snowman = unicodedata.lookup('SNOWMAN')
+ else:
+ self.sailboat = '\N{SAILBOAT}'
+ self.snowman = '\N{SNOWMAN}'
+
+ def tearDown(self):
+ arvados.config.KEEP_BLOCK_SIZE = 2 ** 26
+
+ def test_read_sailboat_across_block_boundary(self):
+ c = Collection()
+ f = c.open('sailboats', 'wb')
+ data = self.sailboat.encode('utf-8')
+ f.write(data)
+ f.write(data[:1])
+ f.write(data[1:])
+ f.write(b'\n')
+ f.close()
+ self.assertRegex(c.portable_manifest_text(), r'\+4 .*\+3 ')
+
+ f = c.open('sailboats', 'r')
+ string = f.readline()
+ self.assertEqual(string, self.sailboat+self.sailboat+'\n')
+ f.close()
+
+ def test_write_snowman_across_block_boundary(self):
+ c = Collection()
+ f = c.open('snowmany', 'w')
+ data = self.snowman
+ f.write(data+data+'\n'+data+'\n')
+ f.close()
+ self.assertRegex(c.portable_manifest_text(), r'\+4 .*\+4 .*\+3 ')
+
+ f = c.open('snowmany', 'r')
+ self.assertEqual(f.readline(), self.snowman+self.snowman+'\n')
+ self.assertEqual(f.readline(), self.snowman+'\n')
+ f.close()
class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list