[ARVADOS] created: 299c01cb4d4a96a94ece77db417ceb7af8e9ba69

[git at public.curoverse.com]

        at  299c01cb4d4a96a94ece77db417ceb7af8e9ba69 (commit)


commit 299c01cb4d4a96a94ece77db417ceb7af8e9ba69
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri May 29 16:28:37 2015 -0400

    6194: Chunk large ArvadosFile writes automatically instead of raising an error.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 2e0eaa8..2d38866 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -10,7 +10,7 @@ import copy
 import errno
 import re
 
-from .errors import KeepWriteError, AssertionError
+from .errors import KeepWriteError, AssertionError, ArgumentError
 from .keep import KeepLocator
 from ._normalize_stream import normalize_stream
 from ._ranges import locators_and_ranges, replace_range, Range
@@ -785,7 +785,14 @@ class ArvadosFile(object):
             raise ArgumentError("Offset is past the end of the file")
 
         if len(data) > config.KEEP_BLOCK_SIZE:
-            raise ArgumentError("Please append data in chunks smaller than %i bytes (config.KEEP_BLOCK_SIZE)" % (config.KEEP_BLOCK_SIZE))
+            # Chunk it up into smaller writes
+            n = 0
+            while (n + config.KEEP_BLOCK_SIZE) < len(data):
+                self.writeto(self, offset+n, data[n:n + config.KEEP_BLOCK_SIZE], num_retries)
+                n += config.KEEP_BLOCK_SIZE
+            if n < len(data):
+                self.writeto(self, offset+n, data[n:], num_retries)
+            return
 
         self._modified = True
 
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 1701aa4..96553bd 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -228,7 +228,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
                              api_client=api, keep_client=keep) as c:
             writer = c.open("count.txt", "r+")
-            text = ''.join(["0123456789" for a in xrange(0, 100)])
+            text = "0123456789" * 100
             for b in xrange(0, 100000):
                 writer.write(text)
             self.assertEqual(writer.size(), 100000000)
@@ -239,6 +239,26 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
 
+
+    def test_large_write(self):
+        keep = ArvadosFileWriterTestCase.MockKeep({})
+        api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
+                                                 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
+                                                {"uuid":"zzzzz-4zz18-mockcollection0",
+                                                 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
+        with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+                             api_client=api, keep_client=keep) as c:
+            writer = c.open("count.txt", "r+")
+            text = "0123456789" * 10000000
+            writer.write(text)
+            self.assertEqual(writer.size(), 100000000)
+
+            self.assertEqual(None, c.manifest_locator())
+            self.assertEqual(True, c.modified())
+            c.save_new("test_write_large")
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+            self.assertEqual(False, c.modified())
+
     def test_rewrite_on_empty_file(self):
         keep = ArvadosFileWriterTestCase.MockKeep({})
         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',

-----------------------------------------------------------------------


hooks/post-receive
--