[ARVADOS] created: f82d809aad0833f231765042aaee8ee43f3358ee
git at public.curoverse.com
git at public.curoverse.com
Thu Jan 21 15:47:26 EST 2016
at f82d809aad0833f231765042aaee8ee43f3358ee (commit)
commit f82d809aad0833f231765042aaee8ee43f3358ee
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 21 15:46:53 2016 -0500
6833: Test setting small blobSignatureTtl. Fix earlier fix.
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 00efab7..abad22d 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -327,12 +327,12 @@ class CollectionDirectory(CollectionDirectoryBase):
self.num_retries = num_retries
self.collection_record_file = None
self.collection_record = None
- self.poll = True
+ self._poll = True
try:
- self.poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
+ self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
except:
_logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
- self.poll_time = 60*60
+ self._poll_time = 60*60
if isinstance(collection_record, dict):
self.collection_locator = collection_record['uuid']
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 48158a7..fa48849 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -1143,12 +1143,10 @@ class TokenExpiryTest(MountTestBase):
@mock.patch('arvados.keep.KeepClient.get')
def runTest(self, mocked_get):
logging.getLogger('arvados.arvados_fuse').setLevel(logging.DEBUG)
+ self.api._rootDesc = {"blobSignatureTtl": 2}
mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
mocked_get.return_value = 'fake data'
- mnt._poll = True
- mnt._poll_time = 1
-
old_exp = int(time.time()) + 86400*14
self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
want_exp = int(time.time()) + 86400*14
commit d3313e652e392b1c0067bcddd65903ed0af19b44
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 21 11:30:23 2016 -0500
6833: Test to confirm that enabling polling on CollectionDirectory causes
tokens to be refreshed.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d6febdd..155bced 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -422,7 +422,10 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
# keepstore services.
proxypidfile = _pidfile('keepproxy')
if os.path.exists(proxypidfile):
- os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+ try:
+ os.kill(int(open(proxypidfile).read()), signal.SIGHUP)
+ except OSError:
+ os.remove(proxypidfile)
def _stop_keep(n):
kill_server_pid(_pidfile('keep{}'.format(n)))
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 65f117b..f85af20 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -459,6 +459,7 @@ class Operations(llfuse.Operations):
else:
if parent_inode in self.inodes:
p = self.inodes[parent_inode]
+ self.inodes.touch(p)
if name == '..':
inode = p.parent_inode
elif isinstance(p, Directory) and name in p:
@@ -500,11 +501,14 @@ class Operations(llfuse.Operations):
fh = next(self._filehandles_counter)
self._filehandles[fh] = FileHandle(fh, p)
self.inodes.touch(p)
+
+ _logger.debug("arv-mount open inode %i flags %x fh %i", inode, flags, fh)
+
return fh
@catch_exceptions
def read(self, fh, off, size):
- _logger.debug("arv-mount read %i %i %i", fh, off, size)
+ _logger.debug("arv-mount read fh %i off %i size %i", fh, off, size)
self.read_ops_counter.add(1)
if fh in self._filehandles:
@@ -587,8 +591,6 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.EBADF)
- _logger.debug("arv-mount handle.dirobj %s", handle.obj)
-
e = off
while e < len(handle.entries):
if handle.entries[e][1].inode in self.inodes:
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index f3c020c..44ec199 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -17,7 +17,7 @@ import run_test_server
logger = logging.getLogger('arvados.arv-mount')
class MountTestBase(unittest.TestCase):
- def setUp(self, api=None):
+ def setUp(self, api=None, local_store=True):
# The underlying C implementation of open() makes a fstat() syscall
# with the GIL still held. When the GETATTR message comes back to
# llfuse (which in these tests is in the same interpreter process) it
@@ -27,8 +27,11 @@ class MountTestBase(unittest.TestCase):
# relatively easy.
self.pool = multiprocessing.Pool(1)
- self.keeptmp = tempfile.mkdtemp()
- os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ if local_store:
+ self.keeptmp = tempfile.mkdtemp()
+ os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ else:
+ self.keeptmp = None
self.mounttmp = tempfile.mkdtemp()
run_test_server.run()
run_test_server.authorize_with("admin")
@@ -65,8 +68,9 @@ class MountTestBase(unittest.TestCase):
self.operations.destroy()
os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
- os.environ.pop('KEEP_LOCAL_STORE')
+ if self.keeptmp:
+ shutil.rmtree(self.keeptmp)
+ os.environ.pop('KEEP_LOCAL_STORE')
run_test_server.reset()
def assertDirContents(self, subdir, expect_content):
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 05c8685..48158a7 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -16,6 +16,7 @@ import logging
import multiprocessing
import run_test_server
import mock
+import re
from mount_test_base import MountTestBase
@@ -1127,3 +1128,39 @@ class FuseMagicTestPDHOnly(MountTestBase):
def test_with_default_by_id(self):
self.verify_pdh_only(skip_pdh_only=True)
+
+def _test_refresh_old_manifest(zzz):
+ fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
+ os.listdir(os.path.join(zzz))
+ time.sleep(3)
+ with open(os.path.join(zzz, fnm)) as f:
+ f.read()
+
+class TokenExpiryTest(MountTestBase):
+ def setUp(self):
+ super(TokenExpiryTest, self).setUp(local_store=False)
+
+ @mock.patch('arvados.keep.KeepClient.get')
+ def runTest(self, mocked_get):
+ logging.getLogger('arvados.arvados_fuse').setLevel(logging.DEBUG)
+ mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
+ mocked_get.return_value = 'fake data'
+
+ mnt._poll = True
+ mnt._poll_time = 1
+
+ old_exp = int(time.time()) + 86400*14
+ self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
+ want_exp = int(time.time()) + 86400*14
+
+ got_loc = mocked_get.call_args[0][0]
+ got_exp = int(
+ re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
+ 16)
+ self.assertGreaterEqual(
+ got_exp, want_exp-1,
+ msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
+ want_exp, got_loc, old_exp))
+ self.assertLessEqual(
+ got_exp, want_exp,
+ msg='server is not using the expected 2w TTL; test is ineffective')
diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py
deleted file mode 100644
index df839e3..0000000
--- a/services/fuse/tests/test_token_expiry.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import arvados
-import logging
-import mock
-import os
-import re
-import time
-import unittest
-
-from .integration_test import IntegrationTest
-from .mount_test_base import MountTestBase
-
-logger = logging.getLogger('arvados.arv-mount')
-
-class TokenExpiryTest(IntegrationTest):
- @mock.patch('arvados.keep.KeepClient.get')
- @IntegrationTest.mount(argv=['--mount-by-id', 'zzz'])
- def test_refresh_old_manifest(self, mocked_get):
- mocked_get.return_value = 'fake data'
- # TODO: mock something in arvados_fuse here so it thinks
- # manifests/signatures expire in 1 second
- self.pool_test(os.path.join(self.mnt, 'zzz'))
- want_exp = int(time.time()) + 86400*14
- got_loc = mocked_get.call_args[0][0]
- got_exp = int(
- re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
- 16)
- self.assertGreaterEqual(
- got_exp, want_exp-1,
- msg='now+2w = {:x}, but fuse fetched old locator {}'.format(
- want_exp, got_loc))
- self.assertLessEqual(
- got_exp, want_exp,
- msg='server is not using the expected 2w TTL; test is ineffective')
- @staticmethod
- def _test_refresh_old_manifest(self, zzz):
- uuid = 'zzzzz-4zz18-op4e2lbej01tcvu'
- fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
- os.listdir(os.path.join(zzz, uuid))
- time.sleep(3)
- open(os.path.join(zzz, uuid, fnm)).read()
commit e40d18172103a45acccc4ae5aebc1b522c270e68
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 21 11:29:40 2016 -0500
6833: Collection update file block list (to get most recent tokens) even when actual
content hasn't changed.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 38e794c..8450bd1 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -474,6 +474,7 @@ class ResumableCollectionWriter(CollectionWriter):
ADD = "add"
DEL = "del"
MOD = "mod"
+TOK = "tok"
FILE = "file"
COLLECTION = "collection"
@@ -986,6 +987,8 @@ class RichCollectionBase(CollectionBase):
changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k), holding_collection))
elif end_collection[k] != self[k]:
changes.append((MOD, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
+ else:
+ changes.append((TOK, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
else:
changes.append((ADD, os.path.join(prefix, k), end_collection[k].clone(holding_collection, "")))
return changes
@@ -1016,7 +1019,7 @@ class RichCollectionBase(CollectionBase):
# There is already local file and it is different:
# save change to conflict file.
self.copy(initial, conflictpath)
- elif event_type == MOD:
+ elif event_type == MOD or event_type == TOK:
final = change[3]
if local == initial:
# Local matches the "initial" item so it has not
@@ -1211,8 +1214,12 @@ class Collection(RichCollectionBase):
if self._manifest_locator is None:
raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
- if self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))):
- # We've merged this record this before. Don't do anything.
+ if (self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))) and
+ response.get("portable_data_hash") != self.portable_data_hash()):
+ # The record on the server is different from our current one, but we've seen it before,
+ # so ignore it because it's already been merged.
+ # However, if it's the same as our current record, proceed with the update, because we want to update
+ # our tokens.
return
else:
self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
commit 73448d12b20fd56321ed556d5b4ad3665564740e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 15 16:20:49 2016 -0500
6833: Add (most of) a test case for token expiry.
diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py
new file mode 100644
index 0000000..df839e3
--- /dev/null
+++ b/services/fuse/tests/test_token_expiry.py
@@ -0,0 +1,40 @@
+import arvados
+import logging
+import mock
+import os
+import re
+import time
+import unittest
+
+from .integration_test import IntegrationTest
+from .mount_test_base import MountTestBase
+
+logger = logging.getLogger('arvados.arv-mount')
+
+class TokenExpiryTest(IntegrationTest):
+ @mock.patch('arvados.keep.KeepClient.get')
+ @IntegrationTest.mount(argv=['--mount-by-id', 'zzz'])
+ def test_refresh_old_manifest(self, mocked_get):
+ mocked_get.return_value = 'fake data'
+ # TODO: mock something in arvados_fuse here so it thinks
+ # manifests/signatures expire in 1 second
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ want_exp = int(time.time()) + 86400*14
+ got_loc = mocked_get.call_args[0][0]
+ got_exp = int(
+ re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
+ 16)
+ self.assertGreaterEqual(
+ got_exp, want_exp-1,
+ msg='now+2w = {:x}, but fuse fetched old locator {}'.format(
+ want_exp, got_loc))
+ self.assertLessEqual(
+ got_exp, want_exp,
+ msg='server is not using the expected 2w TTL; test is ineffective')
+ @staticmethod
+ def _test_refresh_old_manifest(self, zzz):
+ uuid = 'zzzzz-4zz18-op4e2lbej01tcvu'
+ fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
+ os.listdir(os.path.join(zzz, uuid))
+ time.sleep(3)
+ open(os.path.join(zzz, uuid, fnm)).read()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list