[ARVADOS] created: c70fbca19235ebff6ab560ced462496cea7d2f72

[Git user]

        at  c70fbca19235ebff6ab560ced462496cea7d2f72 (commit)


commit c70fbca19235ebff6ab560ced462496cea7d2f72
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Fri May 12 11:00:55 2017 -0300

    11647: When repacking small blocks into big ones, commit them synchronously.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 2fc9c73..9ba4349 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -680,7 +680,7 @@ class _BlockManager(object):
             new_bb.append(bb.buffer_view[0:bb.write_pointer].tobytes())
             files.append((bb, new_bb.write_pointer - bb.size()))
 
-        self.commit_bufferblock(new_bb, sync=sync)
+        self.commit_bufferblock(new_bb, sync=True)
 
         for bb, new_bb_segment_offset in files:
             newsegs = bb.owner.segments()

commit 86c4a20bcf282407f92cb3f3e0a022de79107aca
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Fri May 12 10:57:58 2017 -0300

    11647: Test proving that when small blocks get repacked and the resulting big block is
    committed asynchronously, it doesn't get the permission token on the manifest.

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 77ec7bb..c94118c 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -6,9 +6,11 @@ import copy
 import mock
 import os
 import pprint
+import random
 import re
 import sys
 import tempfile
+import time
 import unittest
 
 from . import run_test_server
@@ -1169,6 +1171,38 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         self.assertEqual(c1["count1.txt"].size(), 0)
 
 
+class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {}
+    KEEP_SERVER = {}
+
+    def setUp(self):
+        self.keep_put = getattr(arvados.keep.KeepClient, 'put')
+
+    def test_repacked_block_sumbmission_get_permission_token(self):
+        '''
+        Make sure that those blocks that are committed after repacking small ones,
+        get their permission tokens assigned on the collection manifest.
+        '''
+        def wrapped_keep_put(*args, **kwargs):
+            # Simulate slow put operations
+            time.sleep(1)
+            return self.keep_put(*args, **kwargs)
+
+        re_locator = "[0-9a-f]{32}\+\d+\+A[a-f0-9]{40}@[a-f0-9]{8}"
+
+        with mock.patch('arvados.keep.KeepClient.put', autospec=True) as mocked_put:
+            mocked_put.side_effect = wrapped_keep_put
+            c = Collection()
+            # Write 70 files ~1MiB each so we force to produce 1 big block by repacking
+            # small ones before finishing the upload.
+            for i in range(70):
+                f = c.open("file_{}.txt".format(i), 'wb')
+                f.write(random.choice('abcdefghijklmnopqrstuvwxyz') * (2**20+i))
+                f.close(flush=False)
+            # We should get 2 blocks with their tokens
+            self.assertEqual(len(re.findall(re_locator, c.manifest_text())), 2)
+
+
 class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
     def test_get_manifest_text_only_committed(self):
         c = Collection()

-----------------------------------------------------------------------


hooks/post-receive
--