[ARVADOS] updated: c333c47a1334a36876017ab616a3646cbf1468cd
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 24 13:42:12 EST 2015
Summary of changes:
services/fuse/arvados_fuse/fusedir.py | 63 ++++++++++++++++--------------
services/fuse/arvados_fuse/fusefile.py | 32 ++++++++++++++-
services/fuse/tests/test_tmp_collection.py | 19 +++++----
3 files changed, 74 insertions(+), 40 deletions(-)
via c333c47a1334a36876017ab616a3646cbf1468cd (commit)
via 128f2f47bb8d6ae97d73188fcfff337f960c299a (commit)
from 4574dbbacdd10d24cf5ed312d7b8e91ec9d9c14a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c333c47a1334a36876017ab616a3646cbf1468cd
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 24 13:45:02 2015 -0500
7751: Fix race by invalidating name entry (not just inode)
when .arvados#collection content is stale.
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index a2135b3..97a06ba 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -505,6 +505,7 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
with llfuse.lock:
self.collection_record_file.invalidate()
self.inodes.invalidate_inode(self.collection_record_file.inode)
+ self.inodes.invalidate_entry(self.inode, '.arvados#collection')
_logger.debug("%s invalidated collection record", self)
def collection_record(self):
diff --git a/services/fuse/tests/test_tmp_collection.py b/services/fuse/tests/test_tmp_collection.py
index e403a2c..60eba1b 100644
--- a/services/fuse/tests/test_tmp_collection.py
+++ b/services/fuse/tests/test_tmp_collection.py
@@ -110,12 +110,15 @@ class TmpCollectionTest(IntegrationTest):
r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:bar 3:3:foo\n$'),
('foo', None,
r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:bar\n$'),
+ ('bar', None,
+ r'^$'),
]
- for fn, content, expect in ops:
- path = os.path.join(tmpdir, fn)
- if content is None:
- os.unlink(path)
- else:
- with open(path, 'w') as f:
- f.write(content)
- self.assertRegexpMatches(current_manifest(tmpdir), expect)
+ for _ in range(10):
+ for fn, content, expect in ops:
+ path = os.path.join(tmpdir, fn)
+ if content is None:
+ os.unlink(path)
+ else:
+ with open(path, 'w') as f:
+ f.write(content)
+ self.assertRegexpMatches(current_manifest(tmpdir), expect)
commit 128f2f47bb8d6ae97d73188fcfff337f960c299a
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 24 13:43:42 2015 -0500
7751: Refactor TmpCollectionDirectory: generate .arvados#collection less often.
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 04c2d50..a2135b3 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -8,8 +8,9 @@ import functools
import threading
from apiclient import errors as apiclient_errors
import errno
+import time
-from fusefile import StringFile, ObjectFile, FuseArvadosFile
+from fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
from fresh import FreshBase, convertTime, use_counter, check_update
import arvados.collection
@@ -483,43 +484,36 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
job output.
"""
+ class UnsaveableCollection(arvados.collection.Collection):
+ def save(self):
+ pass
+ def save_new(self):
+ pass
+
def __init__(self, parent_inode, inodes, api_client, num_retries):
- collection = arvados.collection.Collection(
+ collection = self.UnsaveableCollection(
api_client=api_client,
keep_client=api_client.keep)
- collection.save = self._commit_collection
- collection.save_new = self._commit_collection
super(TmpCollectionDirectory, self).__init__(
parent_inode, inodes, collection)
self.collection_record_file = None
- self._subscribed = False
- self._update_collection_record()
-
- def update(self, *args, **kwargs):
- if not self._subscribed:
- with llfuse.lock_released:
- self.populate(self.mtime())
- self._subscribed = True
-
- @use_counter
- def _commit_collection(self):
- """Commit the data blocks, but don't save the collection to API.
+ self.populate(self.mtime())
- Update the content of the special .arvados#collection file, if
- it has been instantiated.
- """
- self.collection.flush()
- self._update_collection_record()
- if self.collection_record_file is not None:
- self.collection_record_file.update(self.collection_record)
+ def on_event(self, *args, **kwargs):
+ super(TmpCollectionDirectory, self).on_event(*args, **kwargs)
+ if self.collection_record_file:
+ with llfuse.lock:
+ self.collection_record_file.invalidate()
self.inodes.invalidate_inode(self.collection_record_file.inode)
+ _logger.debug("%s invalidated collection record", self)
- def _update_collection_record(self):
- self.collection_record = {
- "uuid": None,
- "manifest_text": self.collection.manifest_text(),
- "portable_data_hash": self.collection.portable_data_hash(),
- }
+ def collection_record(self):
+ with llfuse.lock_released:
+ return {
+ "uuid": None,
+ "manifest_text": self.collection.manifest_text(),
+ "portable_data_hash": self.collection.portable_data_hash(),
+ }
def __contains__(self, k):
return (k == '.arvados#collection' or
@@ -529,18 +523,26 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
def __getitem__(self, item):
if item == '.arvados#collection':
if self.collection_record_file is None:
- self.collection_record_file = ObjectFile(
+ self.collection_record_file = FuncToJSONFile(
self.inode, self.collection_record)
self.inodes.add_entry(self.collection_record_file)
return self.collection_record_file
return super(TmpCollectionDirectory, self).__getitem__(item)
+ def persisted(self):
+ return False
+
def writable(self):
return True
def finalize(self):
self.collection.stop_threads()
+ def invalidate(self):
+ if self.collection_record_file:
+ self.collection_record_file.invalidate()
+ super(TmpCollectionDirectory, self).invalidate()
+
class MagicDirectory(Directory):
"""A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index 4d472cf..1eff08f 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -1,7 +1,8 @@
-import logging
-import re
import json
import llfuse
+import logging
+import re
+import time
from fresh import FreshBase, convertTime
@@ -99,3 +100,30 @@ class ObjectFile(StringFile):
def persisted(self):
return True
+
+
+class FuncToJSONFile(StringFile):
+ """File content is the return value of a given function, encoded as JSON.
+
+ The function is called, with llfuse lock released, at the time the
+ file is read. The result is cached until invalidate() is called.
+ """
+ def __init__(self, parent_inode, func):
+ super(FuncToJSONFile, self).__init__(parent_inode, "", 0)
+ self.func = func
+
+ def size(self):
+ self._update()
+ return super(FuncToJSONFile, self).size()
+
+ def readfrom(self, *args, **kwargs):
+ self._update()
+ return super(FuncToJSONFile, self).readfrom(*args, **kwargs)
+
+ def _update(self):
+ if not self.stale():
+ return
+ self._mtime = time.time()
+ obj = self.func()
+ self.contents = json.dumps(obj, indent=4, sort_keys=True) + "\n"
+ self.fresh()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list