[ARVADOS] updated: d91658ea2bc1973597cd0cbd803ed2a66561958b

git at public.curoverse.com git at public.curoverse.com
Mon Dec 1 19:30:58 EST 2014


Summary of changes:

  discards  492074f1c328c7057b4673f84ae9371ad3e8fdf2 (commit)
       via  d91658ea2bc1973597cd0cbd803ed2a66561958b (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (492074f1c328c7057b4673f84ae9371ad3e8fdf2)
            \
             N -- N -- N (d91658ea2bc1973597cd0cbd803ed2a66561958b)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d91658ea2bc1973597cd0cbd803ed2a66561958b
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Dec 1 19:31:03 2014 -0500

    4696: Bring back StreamReader.tokens() method used by one_task_per_input_stream().

diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index c263dd8..cabf1ec 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -319,15 +319,21 @@ class StreamReader(object):
             data.append(self._keep.get(locator, num_retries=num_retries)[segmentoffset:segmentoffset+segmentsize])
         return ''.join(data)
 
-    def manifest_text(self, strip=False):
-        manifest_text = [self.name().replace(' ', '\\040')]
+    def tokens(self, strip=False):
+        tokens = [self.name().replace(' ', '\\040')]
         if strip:
             for d in self._data_locators:
                 m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR])
-                manifest_text.append(m.group(0))
+                tokens.append(m.group(0))
         else:
-            manifest_text.extend([d[LOCATOR] for d in self._data_locators])
-        manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
-                                        for seg in f.segments])
-                              for f in self._files.values()])
-        return ' '.join(manifest_text) + '\n'
+            tokens.extend([d[LOCATOR] for d in self._data_locators])
+        for f in self._files.values():
+            for seg in f.segments:
+                tokens.append("{}:{}:{}".format(
+                    seg[LOCATOR],
+                    seg[BLOCKSIZE],
+                    f.name().replace(' ', '\\040')))
+        return tokens
+
+    def manifest_text(self, strip=False):
+        return ' '.join(self.tokens(strip=strip)) + '\n'
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 08a3d28..d73dbc1 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -243,6 +243,12 @@ class StreamReaderTestCase(unittest.TestCase, StreamRetryTestMixin):
             reader = StreamReader(line.split())
             self.assertEqual(line + '\n', reader.manifest_text())
 
+    def test_stream_tokens_without_keep_client(self):
+        mtext = self.manifest_for('multilevel_collection_1')
+        for line in mtext.rstrip('\n').split('\n'):
+            reader = StreamReader(line.split())
+            self.assertEqual(line.split(), reader.tokens())
+
 
 class StreamFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
     def reader_for(self, coll_name, **kwargs):

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list