[arvados] created: 2.7.0-6104-ged32d41e28
git repository hosting
git at public.arvados.org
Wed Mar 6 20:17:50 UTC 2024
at ed32d41e28c05a83496cf05f7ec956ca5fe10d4f (commit)
commit ed32d41e28c05a83496cf05f7ec956ca5fe10d4f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Mar 6 15:03:38 2024 -0500
21541: inodes are marked as "in use" in directory list
When updating directory contents (using merge()) and when the client
is reading a directory using opendir(), the inodes are marked as "in
use" so they don't get immediately reclaimed in a memory pressure
situation.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 31afcda8d1..b075c9085e 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -133,6 +133,13 @@ class DirectoryHandle(Handle):
def __init__(self, fh, dirobj, entries):
super(DirectoryHandle, self).__init__(fh, dirobj)
self.entries = entries
+ for ent in self.entries:
+ ent[1].inc_use()
+
+ def release(self):
+ for ent in self.entries:
+ ent[1].dec_use()
+ super(DirectoryHandle, self).release()
class InodeCache(object):
@@ -176,11 +183,11 @@ class InodeCache(object):
# has_ref() check first.
if obj.has_ref(True):
- _logger.debug("InodeCache cannot clear inode %i, still referenced", obj.inode)
+ #_logger.debug("InodeCache cannot clear inode %i, still referenced", obj.inode)
return
if obj.in_use():
- _logger.debug("InodeCache cannot clear inode %i, in use", obj.inode)
+ #_logger.debug("InodeCache cannot clear inode %i, in use", obj.inode)
return
obj.kernel_invalidate()
@@ -564,14 +571,13 @@ class Operations(llfuse.Operations):
if name == '.':
inode = parent_inode
- else:
- if parent_inode in self.inodes:
- p = self.inodes[parent_inode]
- self.inodes.touch(p)
- if name == '..':
- inode = p.parent_inode
- elif isinstance(p, Directory) and name in p:
- inode = p[name].inode
+ elif parent_inode in self.inodes:
+ p = self.inodes[parent_inode]
+ self.inodes.touch(p)
+ if name == '..':
+ inode = p.parent_inode
+ elif isinstance(p, Directory) and name in p:
+ inode = p[name].inode
if inode != None:
_logger.debug("arv-mount lookup: parent_inode %i name '%s' inode %i",
@@ -703,8 +709,11 @@ class Operations(llfuse.Operations):
if p.parent_inode in self.inodes:
parent = self.inodes[p.parent_inode]
else:
+ _logger.warning("arv-mount opendir: parent inode %i of %i is missing", p.parent_inode, inode)
raise llfuse.FUSEError(errno.EIO)
+ _logger.debug("arv-mount opendir: inode %i fh %i ", inode, fh)
+
# update atime
self.inodes.touch(p)
self._filehandles[fh] = DirectoryHandle(fh, p, [('.', p), ('..', parent)] + listitems(p))
@@ -722,8 +731,9 @@ class Operations(llfuse.Operations):
e = off
while e < len(handle.entries):
- if handle.entries[e][1].inode in self.inodes:
- yield (handle.entries[e][0].encode(self.inodes.encoding), self.getattr(handle.entries[e][1].inode), e+1)
+ ent = handle.entries[e]
+ if ent[1].inode in self.inodes:
+ yield (ent[0].encode(self.inodes.encoding), self.getattr(ent[1].inode), e+1)
e += 1
@statfs_time.time()
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index e3b8dd4c2c..eeb96bceb3 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -176,30 +176,44 @@ class Directory(FreshBase):
changed = False
for i in items:
name = self.sanitize_filename(fn(i))
- if name:
- if name in oldentries and same(oldentries[name], i):
+ if not name:
+ continue
+ if name in oldentries:
+ ent = oldentries[name]
+ ent.inc_use()
+ if same(ent, i):
# move existing directory entry over
- self._entries[name] = oldentries[name]
+ self._entries[name] = ent
del oldentries[name]
- else:
- _logger.debug("Adding entry '%s' to inode %i", name, self.inode)
- # create new directory entry
- ent = new_entry(i)
- if ent is not None:
- self._entries[name] = self.inodes.add_entry(ent)
- changed = True
+
+ for i in items:
+ name = self.sanitize_filename(fn(i))
+ if not name:
+ continue
+ if name not in self._entries:
+ _logger.debug("Adding entry '%s' to inode %i", name, self.inode)
+ # create new directory entry
+ ent = new_entry(i)
+ if ent is not None:
+ ent.inc_use()
+ self._entries[name] = self.inodes.add_entry(ent)
+ changed = True
# delete any other directory entries that were not in found in 'items'
for i in oldentries:
_logger.debug("Forgetting about entry '%s' on inode %i", i, self.inode)
self.inodes.invalidate_entry(self, i)
self.inodes.del_entry(oldentries[i])
+ ent.dec_use()
changed = True
if changed:
self.inodes.invalidate_inode(self)
self._mtime = time.time()
+ for ent in self._entries.values():
+ ent.dec_use()
+
self.fresh()
def in_use(self):
@@ -1060,10 +1074,14 @@ class ProjectDirectory(Directory):
try:
with llfuse.lock_released:
+ _logger.debug("Getting lock to update %s", self.project_uuid)
self._updating_lock.acquire()
if not self.stale():
+ _logger.debug("%s was updated already", self.project_uuid)
return
+ _logger.debug("Requesting update of %s", self.project_uuid)
+
if group_uuid_pattern.match(self.project_uuid):
self.project_object = self.api.groups().get(
uuid=self.project_uuid).execute(num_retries=self.num_retries)
@@ -1092,7 +1110,6 @@ class ProjectDirectory(Directory):
*self._filters_for('collections', qualified=True),
],
) if obj['current_version_uuid'] == obj['uuid'])
-
# end with llfuse.lock_released, re-acquire lock
self.merge(contents,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list