[arvados] updated: 2.7.0-6107-g32e56d60fd

[git repository hosting]

Summary of changes:
 services/fuse/arvados_fuse/__init__.py | 12 +++++++++++-
 services/fuse/arvados_fuse/command.py  |  2 +-
 services/fuse/arvados_fuse/fusedir.py  | 10 +++++++---
 3 files changed, 19 insertions(+), 5 deletions(-)

       via  32e56d60fd965260662b5c8d8aaafc0d793e33bd (commit)
      from  66e7b6244a81e8b335d1da7ce8de0105f60b798c (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 32e56d60fd965260662b5c8d8aaafc0d793e33bd
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Thu Mar 7 09:39:35 2024 -0500

    21541: Manage memory related to project directories
    
    The existing cache management only related to collection directories.
    This adds an estimate for other kinds of directories so they will also
    contribute to cache size and be candidates to be removed from cache.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index c04a880122..a70c99fde9 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -165,6 +165,8 @@ class InodeCache(object):
         return self._total
 
     def _remove(self, obj, clear):
+        if obj.inode is None:
+            return
         if clear:
             # Kernel behavior seems to be that if a file is
             # referenced, its parents remain referenced too. This
@@ -212,11 +214,13 @@ class InodeCache(object):
             _logger.debug("InodeCache cleared inode %i total now %i", obj.inode, self._total)
 
     def cap_cache(self):
+        _logger.debug("in cap_cache %i, %i", self._total, self.cap)
         if self._total > self.cap:
             for ent in listvalues(self._entries):
                 if self._total < self.cap or len(self._entries) < self.min_entries:
                     break
                 self._remove(ent, True)
+            _logger.debug("end cap_cache %i, %i", self._total, self.cap)
 
     def manage(self, obj):
         if obj.persisted():
@@ -229,10 +233,16 @@ class InodeCache(object):
                 else:
                     if obj not in self._by_uuid[obj.cache_uuid]:
                         self._by_uuid[obj.cache_uuid].append(obj)
-            self._total += obj.objsize()
+            self._total += obj.cache_size
             _logger.debug("InodeCache touched inode %i (size %i) (uuid %s) total now %i (%i entries)",
                           obj.inode, obj.objsize(), obj.cache_uuid, self._total, len(self._entries))
 
+    def update_cache_size(self, obj):
+        if obj.inode in self._entries:
+            self._total -= obj.cache_size
+            obj.cache_size = obj.objsize()
+            self._total += obj.cache_size
+
     def touch(self, obj):
         if obj.persisted():
             if obj.inode in self._entries:
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index 74d28a9e8a..8c24467aeb 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -472,7 +472,7 @@ From here, the following directories are available:
 
     def _llfuse_main(self):
         try:
-            llfuse.main()
+            llfuse.main(workers=1)
         except:
             llfuse.close(unmount=False)
             raise
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 00b9d8b547..0671670c9d 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -150,6 +150,9 @@ class Directory(FreshBase):
         self.inodes.touch(self)
         super(Directory, self).fresh()
 
+    def objsize(self):
+        return len(self._entries) * 64
+
     def merge(self, items, fn, same, new_entry):
         """Helper method for updating the contents of the directory.
 
@@ -210,8 +213,8 @@ class Directory(FreshBase):
         if changed:
             self.inodes.invalidate_inode(self)
             self._mtime = time.time()
-
-        self.inodes.inode_cache.cap_cache()
+            self.inodes.inode_cache.update_cache_size(self)
+            self.inodes.inode_cache.cap_cache()
 
         for ent in self._entries.values():
            ent.dec_use()
@@ -239,7 +242,6 @@ class Directory(FreshBase):
         oldentries = self._entries
         self._entries = {}
         for n in oldentries:
-            oldentries[n].clear()
             self.inodes.del_entry(oldentries[n])
         self.invalidate()
 
@@ -531,6 +533,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection_record_file.invalidate()
             self.inodes.invalidate_inode(self.collection_record_file)
             _logger.debug("%s invalidated collection record file", self)
+        self.inodes.inode_cache.update_cache_size(self)
         self.fresh()
 
     def uuid(self):
@@ -657,6 +660,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection.stop_threads()
         super(CollectionDirectory, self).clear()
         self._manifest_size = 0
+        self.inodes.inode_cache.update_cache_size(self)
 
 
 class TmpCollectionDirectory(CollectionDirectoryBase):

-----------------------------------------------------------------------


hooks/post-receive
--