[ARVADOS] updated: 5180238a10bd15302a1c15b9a428f2fdeeabdf4e
Git user
git at public.curoverse.com
Thu Apr 20 13:26:01 EDT 2017
Summary of changes:
sdk/python/arvados/arvfile.py | 19 ++++++++------
sdk/python/arvados/config.py | 3 ---
sdk/python/tests/test_arvfile.py | 53 ++++++++++++++++++++++++++++++++++++++--
sdk/python/tests/test_stream.py | 4 ++-
4 files changed, 65 insertions(+), 14 deletions(-)
via 5180238a10bd15302a1c15b9a428f2fdeeabdf4e (commit)
from bfc9660a8b2467893baf131b20e83e76c41ae438 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5180238a10bd15302a1c15b9a428f2fdeeabdf4e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Apr 20 13:25:56 2017 -0400
11510: Fix truncate() to not change file pointer. Additional tests.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index ebefc5b..cd70c02 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -659,7 +659,12 @@ class _BlockManager(object):
@synchronized
def get_padding_block(self):
"""Get a bufferblock 64 MB in size consisting of all zeros, used as padding
- when using truncate() to extend the size of a file."""
+ when using truncate() to extend the size of a file.
+
+ For reference (and possible future optimization), the md5sum of the
+ padding block is: 7f614da9329cd3aebf59b91aadc30bf0+67108864
+
+ """
if self.padding_block is None:
self.padding_block = self._alloc_bufferblock(starting_capacity=config.KEEP_BLOCK_SIZE)
@@ -998,12 +1003,12 @@ class ArvadosFile(object):
# Collect the segments that reference the buffer block.
bufferblock_segs = [s for s in segs if s.locator == self._current_bblock.blockid]
- if len(bufferblock_segs) > 1:
- # Collect total data referenced by segments (could be smaller than
- # bufferblock size if a portion of the file was written and
- # then overwritten).
- write_total = sum([s.range_size for s in bufferblock_segs])
+ # Collect total data referenced by segments (could be smaller than
+ # bufferblock size if a portion of the file was written and
+ # then overwritten).
+ write_total = sum([s.range_size for s in bufferblock_segs])
+ if write_total < self._current_bblock.size() or len(bufferblock_segs) > 1:
# If there's more than one segment referencing this block, it is
# due to out-of-order writes and will produce a fragmented
# manifest, so try to optimize by re-packing into a new buffer.
@@ -1233,8 +1238,6 @@ class ArvadosFileWriter(ArvadosFileReader):
if size is None:
size = self._filepos
self.arvadosfile.truncate(size)
- if self._filepos > self.size():
- self._filepos = self.size()
@_FileLikeObjectBase._before_close
def flush(self):
diff --git a/sdk/python/arvados/config.py b/sdk/python/arvados/config.py
index 0c4ccb0..8f2b265 100644
--- a/sdk/python/arvados/config.py
+++ b/sdk/python/arvados/config.py
@@ -15,9 +15,6 @@ else:
KEEP_BLOCK_SIZE = 2**26
EMPTY_BLOCK_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
-# This is the 64 MiB block consisting entirely of zeros
-PADDING_BLOCK_LOCATOR = '7f614da9329cd3aebf59b91aadc30bf0+67108864'
-
def initialize(config_file=default_config_file):
global _settings
_settings = {}
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 1a2c034..d45e4d4 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -119,6 +119,40 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
+ def test_truncate3(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
+ "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
+ api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired":None},
+ {"uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 20)
+ self.assertEqual("0123456789ab", writer.read(12))
+ self.assertEqual(12, writer.tell())
+
+ writer.truncate(8)
+
+ # Make sure reading off the end doesn't break
+ self.assertEqual(12, writer.tell())
+ self.assertEqual("", writer.read(12))
+
+ self.assertEqual(writer.size(), 8)
+ writer.seek(-12, os.SEEK_CUR)
+ self.assertEqual("01234567", writer.read(12))
+
+ self.assertIsNone(c.manifest_locator())
+ self.assertTrue(c.modified())
+ c.save_new("test_truncate")
+ self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+ self.assertFalse(c.modified())
+
+
+
def test_write_to_end(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
@@ -346,6 +380,21 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.seek(0)
self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444")
+ def test_sparse_write4(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({})
+ api = ArvadosFileWriterTestCase.MockApi({}, {})
+ for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
+ with Collection() as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 0)
+
+ for i in r:
+ w = ("%s" % i) * 10
+ writer.seek(i*10)
+ writer.write(w)
+ writer.seek(0)
+ self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
+
def test_rewrite_on_empty_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
@@ -404,10 +453,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_large_rewrite(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
- "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
+ "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
"replication_desired":None},
{"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
+ "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
"portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index d4b2207..082af1b 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -70,7 +70,9 @@ class StreamFileReaderTestCase(unittest.TestCase):
def test_seek_min_zero(self):
sfile = self.make_count_reader()
- sfile.seek(-2, os.SEEK_SET)
+ self.assertEqual(0, sfile.tell())
+ with self.assertRaises(IOError):
+ sfile.seek(-2, os.SEEK_SET)
self.assertEqual(0, sfile.tell())
def test_seek_max_size(self):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list