[ARVADOS] updated: 2.1.0-1878-gd702b6f48
Git user
git at public.arvados.org
Sun Feb 6 19:59:40 UTC 2022
Summary of changes:
sdk/python/arvados/arvfile.py | 7 ++--
services/fuse/arvados_fuse/__init__.py | 2 +-
services/fuse/arvados_fuse/fusedir.py | 73 +++++++++++++++-------------------
3 files changed, 37 insertions(+), 45 deletions(-)
via d702b6f48fd4463084b7e0654520e6b319a19d21 (commit)
via 08acb72dd180391007554783a08d9213e5d6d6c0 (commit)
from 7439eed0806fd8096f61fc751224f9696adf615e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d702b6f48fd4463084b7e0654520e6b319a19d21
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Sun Feb 6 14:58:13 2022 -0500
18719: Don't try to apply incremental updates to parent inode
Just invalidate it and update it.
Don't crash commit_all race.
Running "aws s3 sync" seems to be working now.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index e915ff2ac..0fcdc1e63 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -760,9 +760,10 @@ class _BlockManager(object):
self._delete_bufferblock(locator)
def _delete_bufferblock(self, locator):
- bb = self._bufferblocks[locator]
- bb.clear()
- del self._bufferblocks[locator]
+ if locator in self._bufferblocks:
+ bb = self._bufferblocks[locator]
+ bb.clear()
+ del self._bufferblocks[locator]
def get_block_contents(self, locator, num_retries, cache_only=False):
"""Fetch a block.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 2b282af44..9aabe85a2 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -492,7 +492,7 @@ class Operations(llfuse.Operations):
for parent in (
self.inodes.inode_cache.find_by_uuid(oldowner) +
self.inodes.inode_cache.find_by_uuid(newowner)):
- parent.child_event(ev)
+ parent.invalidate()
@getattr_time.time()
@catch_exceptions
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 3c262fa96..704dbe439 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -270,7 +270,7 @@ class CollectionDirectoryBase(Directory):
"""
- def __init__(self, parent_inode, inodes, apiconfig, enable_write, collection, collectionRoot):
+ def __init__(self, parent_inode, inodes, apiconfig, enable_write, collection, collection_root):
super(CollectionDirectoryBase, self).__init__(parent_inode, inodes, apiconfig, enable_write)
self.apiconfig = apiconfig
self.collection = collection
@@ -293,6 +293,10 @@ class CollectionDirectoryBase(Directory):
item.fuse_entry = self._entries[name]
def on_event(self, event, collection, name, item):
+ # These are events from the Collection object (ADD/DEL/MOD)
+ # emitted by operations on the Collection object (like
+ # "mkdirs" or "remove"), and by "update", which we need to
+ # synchronize with our FUSE objects that are assigned inodes.
if collection == self.collection:
name = self.sanitize_filename(name)
@@ -465,16 +469,6 @@ class CollectionDirectory(CollectionDirectoryBase):
def want_event_subscribe(self):
return (uuid_pattern.match(self.collection_locator) is not None)
- # Used by arv-web.py to switch the contents of the CollectionDirectory
- def change_collection(self, new_locator):
- """Switch the contents of the CollectionDirectory.
-
- Must be called with llfuse.lock held.
- """
-
- self.collection_locator = new_locator
- self.update()
-
def new_collection(self, new_collection_record, coll_reader):
if self.inode:
self.clear()
@@ -484,21 +478,21 @@ class CollectionDirectory(CollectionDirectoryBase):
def new_collection_record(self, new_collection_record):
if not new_collection_record:
- self.collection_record_file = None
- self._mtime = 0
- return
+ raise Exception("invalid new_collection_record")
self._mtime = convertTime(new_collection_record.get('modified_at'))
- self._manifest_size = len(self.collection.manifest_text())
+ self._manifest_size = len(new_collection_record["manifest_text"])
self.collection_locator = new_collection_record["uuid"]
if self.collection_record_file is not None:
+ self.collection_record_file.invalidate()
self.inodes.invalidate_inode(self.collection_record_file)
_logger.debug("%s invalidated collection record file", self)
+ self.fresh()
def uuid(self):
return self.collection_locator
@use_counter
- def update(self, to_record_version=None):
+ def update(self):
try:
if self.collection is not None and portable_data_hash_pattern.match(self.collection_locator):
# It's immutable, nothing to update
@@ -514,17 +508,14 @@ class CollectionDirectory(CollectionDirectoryBase):
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
- return
+ return True
- _logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
+ _logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode)
coll_reader = None
if self.collection is not None:
# Already have a collection object
- if self.collection.known_past_version(to_record_version):
- _logger.debug("%s already processed %s", self.collection_locator, to_record_version)
- else:
- self.collection.update()
- new_collection_record = self.collection.api_response()
+ self.collection.update()
+ new_collection_record = self.collection.api_response()
else:
# Create a new collection object
if uuid_pattern.match(self.collection_locator):
@@ -553,9 +544,8 @@ class CollectionDirectory(CollectionDirectoryBase):
if coll_reader is not None:
self.new_collection(new_collection_record, coll_reader)
else:
- self.new_collection_record(new_collection_record, coll_reader)
+ self.new_collection_record(new_collection_record)
- self.fresh()
return True
finally:
self._updating_lock.release()
@@ -568,7 +558,7 @@ class CollectionDirectory(CollectionDirectoryBase):
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
if new_collection_record is not None and "manifest_text" in new_collection_record:
- _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
+ _logger.error("arv-mount manifest_text is: %s", new_collection_record["manifest_text"])
self.invalidate()
return False
@@ -597,6 +587,7 @@ class CollectionDirectory(CollectionDirectoryBase):
def invalidate(self):
if self.collection_record_file is not None:
+ self.collection_record_file.invalidate()
self.inodes.invalidate_inode(self.collection_record_file)
super(CollectionDirectory, self).invalidate()
commit 08acb72dd180391007554783a08d9213e5d6d6c0
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Sat Feb 5 22:05:19 2022 -0500
18719: Handle collection_record differently
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index c60215333..3c262fa96 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -430,7 +430,6 @@ class CollectionDirectory(CollectionDirectoryBase):
self.api = api
self.num_retries = num_retries
self.collection_record_file = None
- self.collection_record = None
self._poll = True
try:
self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2)
@@ -474,7 +473,6 @@ class CollectionDirectory(CollectionDirectoryBase):
"""
self.collection_locator = new_locator
- self.collection_record = None
self.update()
def new_collection(self, new_collection_record, coll_reader):
@@ -485,16 +483,14 @@ class CollectionDirectory(CollectionDirectoryBase):
self.populate(self.mtime())
def new_collection_record(self, new_collection_record):
- self.collection_record = new_collection_record
- if not self.collection_record:
+ if not new_collection_record:
self.collection_record_file = None
self._mtime = 0
return
- self._mtime = convertTime(self.collection_record.get('modified_at'))
+ self._mtime = convertTime(new_collection_record.get('modified_at'))
self._manifest_size = len(self.collection.manifest_text())
- self.collection_locator = self.collection_record["uuid"]
+ self.collection_locator = new_collection_record["uuid"]
if self.collection_record_file is not None:
- self.collection_record_file.update(self.collection_record)
self.inodes.invalidate_inode(self.collection_record_file)
_logger.debug("%s invalidated collection record file", self)
@@ -504,7 +500,7 @@ class CollectionDirectory(CollectionDirectoryBase):
@use_counter
def update(self, to_record_version=None):
try:
- if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
+ if self.collection is not None and portable_data_hash_pattern.match(self.collection_locator):
# It's immutable, nothing to update
return True
@@ -513,6 +509,7 @@ class CollectionDirectory(CollectionDirectoryBase):
self.fresh()
return True
+ new_collection_record = None
try:
with llfuse.lock_released:
self._updating_lock.acquire()
@@ -520,7 +517,6 @@ class CollectionDirectory(CollectionDirectoryBase):
return
_logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
- new_collection_record = None
coll_reader = None
if self.collection is not None:
# Already have a collection object
@@ -567,21 +563,27 @@ class CollectionDirectory(CollectionDirectoryBase):
_logger.error("Error fetching collection '%s': %s", self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
- if self.collection_record is not None and "manifest_text" in self.collection_record:
- _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
+ if new_collection_record is not None and "manifest_text" in new_collection_record:
+ _logger.warning("arv-mount manifest_text is: %s", new_collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
- if self.collection_record is not None and "manifest_text" in self.collection_record:
+ if new_collection_record is not None and "manifest_text" in new_collection_record:
_logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
self.invalidate()
return False
+ @use_counter
+ @check_update
+ def collection_record(self):
+ return self.collection.api_response()
+
@use_counter
@check_update
def __getitem__(self, item):
if item == '.arvados#collection':
if self.collection_record_file is None:
- self.collection_record_file = ObjectFile(self.inode, self.collection_record)
+ self.collection_record_file = FuncToJSONFile(
+ self.inode, self.collection_record)
self.inodes.add_entry(self.collection_record_file)
return self.collection_record_file
else:
@@ -594,10 +596,8 @@ class CollectionDirectory(CollectionDirectoryBase):
return super(CollectionDirectory, self).__contains__(k)
def invalidate(self):
- self.collection_record = None
if self.collection_record_file is not None:
self.inodes.invalidate_inode(self.collection_record_file)
- self.collection_record_file = None
super(CollectionDirectory, self).invalidate()
def persisted(self):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list