[ARVADOS] updated: 8a2645e668815d6af0e7a61165585a440bc55a93

git at public.curoverse.com git at public.curoverse.com
Thu Jan 21 11:30:32 EST 2016


Summary of changes:
 sdk/python/arvados/collection.py         | 13 ++++++++---
 sdk/python/tests/run_test_server.py      |  5 +++-
 services/fuse/arvados_fuse/__init__.py   |  8 ++++---
 services/fuse/tests/mount_test_base.py   | 14 +++++++----
 services/fuse/tests/test_mount.py        | 37 +++++++++++++++++++++++++++++
 services/fuse/tests/test_token_expiry.py | 40 --------------------------------
 6 files changed, 65 insertions(+), 52 deletions(-)
 delete mode 100644 services/fuse/tests/test_token_expiry.py

       via  8a2645e668815d6af0e7a61165585a440bc55a93 (commit)
       via  2037d864f6ff8f75a83ceea28e5406336f682f61 (commit)
      from  ef5692e0152b70fc379fcff3b26a10b08938723f (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8a2645e668815d6af0e7a61165585a440bc55a93
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jan 21 11:30:23 2016 -0500

    6833: Test to confirm that enabling polling on CollectionDirectory causes
    tokens to be refreshed.

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d6febdd..155bced 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -422,7 +422,10 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
     # keepstore services.
     proxypidfile = _pidfile('keepproxy')
     if os.path.exists(proxypidfile):
-        os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+        try:
+            os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+        except OSError:
+            os.remove(proxypidfile)
 
 def _stop_keep(n):
     kill_server_pid(_pidfile('keep{}'.format(n)))
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 65f117b..f85af20 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -459,6 +459,7 @@ class Operations(llfuse.Operations):
         else:
             if parent_inode in self.inodes:
                 p = self.inodes[parent_inode]
+                self.inodes.touch(p)
                 if name == '..':
                     inode = p.parent_inode
                 elif isinstance(p, Directory) and name in p:
@@ -500,11 +501,14 @@ class Operations(llfuse.Operations):
         fh = next(self._filehandles_counter)
         self._filehandles[fh] = FileHandle(fh, p)
         self.inodes.touch(p)
+
+        _logger.debug("arv-mount open inode %i flags %x fh %i", inode, flags, fh)
+
         return fh
 
     @catch_exceptions
     def read(self, fh, off, size):
-        _logger.debug("arv-mount read %i %i %i", fh, off, size)
+        _logger.debug("arv-mount read fh %i off %i size %i", fh, off, size)
         self.read_ops_counter.add(1)
 
         if fh in self._filehandles:
@@ -587,8 +591,6 @@ class Operations(llfuse.Operations):
         else:
             raise llfuse.FUSEError(errno.EBADF)
 
-        _logger.debug("arv-mount handle.dirobj %s", handle.obj)
-
         e = off
         while e < len(handle.entries):
             if handle.entries[e][1].inode in self.inodes:
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index f3c020c..44ec199 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -17,7 +17,7 @@ import run_test_server
 logger = logging.getLogger('arvados.arv-mount')
 
 class MountTestBase(unittest.TestCase):
-    def setUp(self, api=None):
+    def setUp(self, api=None, local_store=True):
         # The underlying C implementation of open() makes a fstat() syscall
         # with the GIL still held.  When the GETATTR message comes back to
         # llfuse (which in these tests is in the same interpreter process) it
@@ -27,8 +27,11 @@ class MountTestBase(unittest.TestCase):
         # relatively easy.
         self.pool = multiprocessing.Pool(1)
 
-        self.keeptmp = tempfile.mkdtemp()
-        os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+        if local_store:
+            self.keeptmp = tempfile.mkdtemp()
+            os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+        else:
+            self.keeptmp = None
         self.mounttmp = tempfile.mkdtemp()
         run_test_server.run()
         run_test_server.authorize_with("admin")
@@ -65,8 +68,9 @@ class MountTestBase(unittest.TestCase):
         self.operations.destroy()
 
         os.rmdir(self.mounttmp)
-        shutil.rmtree(self.keeptmp)
-        os.environ.pop('KEEP_LOCAL_STORE')
+        if self.keeptmp:
+            shutil.rmtree(self.keeptmp)
+            os.environ.pop('KEEP_LOCAL_STORE')
         run_test_server.reset()
 
     def assertDirContents(self, subdir, expect_content):
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 05c8685..48158a7 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -16,6 +16,7 @@ import logging
 import multiprocessing
 import run_test_server
 import mock
+import re
 
 from mount_test_base import MountTestBase
 
@@ -1127,3 +1128,39 @@ class FuseMagicTestPDHOnly(MountTestBase):
 
     def test_with_default_by_id(self):
         self.verify_pdh_only(skip_pdh_only=True)
+
+def _test_refresh_old_manifest(zzz):
+    fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
+    os.listdir(os.path.join(zzz))
+    time.sleep(3)
+    with open(os.path.join(zzz, fnm)) as f:
+        f.read()
+
+class TokenExpiryTest(MountTestBase):
+    def setUp(self):
+        super(TokenExpiryTest, self).setUp(local_store=False)
+
+    @mock.patch('arvados.keep.KeepClient.get')
+    def runTest(self, mocked_get):
+        logging.getLogger('arvados.arvados_fuse').setLevel(logging.DEBUG)
+        mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
+        mocked_get.return_value = 'fake data'
+
+        mnt._poll = True
+        mnt._poll_time = 1
+
+        old_exp = int(time.time()) + 86400*14
+        self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
+        want_exp = int(time.time()) + 86400*14
+
+        got_loc = mocked_get.call_args[0][0]
+        got_exp = int(
+            re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
+            16)
+        self.assertGreaterEqual(
+            got_exp, want_exp-1,
+            msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
+                want_exp, got_loc, old_exp))
+        self.assertLessEqual(
+            got_exp, want_exp,
+            msg='server is not using the expected 2w TTL; test is ineffective')
diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py
deleted file mode 100644
index df839e3..0000000
--- a/services/fuse/tests/test_token_expiry.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import arvados
-import logging
-import mock
-import os
-import re
-import time
-import unittest
-
-from .integration_test import IntegrationTest
-from .mount_test_base import MountTestBase
-
-logger = logging.getLogger('arvados.arv-mount')
-
-class TokenExpiryTest(IntegrationTest):
-    @mock.patch('arvados.keep.KeepClient.get')
-    @IntegrationTest.mount(argv=['--mount-by-id', 'zzz'])
-    def test_refresh_old_manifest(self, mocked_get):
-        mocked_get.return_value = 'fake data'
-        # TODO: mock something in arvados_fuse here so it thinks
-        # manifests/signatures expire in 1 second
-        self.pool_test(os.path.join(self.mnt, 'zzz'))
-        want_exp = int(time.time()) + 86400*14
-        got_loc = mocked_get.call_args[0][0]
-        got_exp = int(
-            re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
-            16)
-        self.assertGreaterEqual(
-            got_exp, want_exp-1,
-            msg='now+2w = {:x}, but fuse fetched old locator {}'.format(
-                want_exp, got_loc))
-        self.assertLessEqual(
-            got_exp, want_exp,
-            msg='server is not using the expected 2w TTL; test is ineffective')
-    @staticmethod
-    def _test_refresh_old_manifest(self, zzz):
-        uuid = 'zzzzz-4zz18-op4e2lbej01tcvu'
-        fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
-        os.listdir(os.path.join(zzz, uuid))
-        time.sleep(3)
-        open(os.path.join(zzz, uuid, fnm)).read()

commit 2037d864f6ff8f75a83ceea28e5406336f682f61
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jan 21 11:29:40 2016 -0500

    6833: Collection update file block list (to get most recent tokens) even when actual
    content hasn't changed.

diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 38e794c..8450bd1 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -474,6 +474,7 @@ class ResumableCollectionWriter(CollectionWriter):
 ADD = "add"
 DEL = "del"
 MOD = "mod"
+TOK = "tok"
 FILE = "file"
 COLLECTION = "collection"
 
@@ -986,6 +987,8 @@ class RichCollectionBase(CollectionBase):
                     changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k), holding_collection))
                 elif end_collection[k] != self[k]:
                     changes.append((MOD, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
+                else:
+                    changes.append((TOK, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
             else:
                 changes.append((ADD, os.path.join(prefix, k), end_collection[k].clone(holding_collection, "")))
         return changes
@@ -1016,7 +1019,7 @@ class RichCollectionBase(CollectionBase):
                     # There is already local file and it is different:
                     # save change to conflict file.
                     self.copy(initial, conflictpath)
-            elif event_type == MOD:
+            elif event_type == MOD or event_type == TOK:
                 final = change[3]
                 if local == initial:
                     # Local matches the "initial" item so it has not
@@ -1211,8 +1214,12 @@ class Collection(RichCollectionBase):
             if self._manifest_locator is None:
                 raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
             response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
-            if self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))):
-                # We've merged this record this before.  Don't do anything.
+            if (self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))) and
+                response.get("portable_data_hash") != self.portable_data_hash()):
+                # The record on the server is different from our current one, but we've seen it before,
+                # so ignore it because it's already been merged.
+                # However, if it's the same as our current record, proceed with the update, because we want to update
+                # our tokens.
                 return
             else:
                 self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list