[arvados] updated: 2.7.0-6110-gf1a8112ffc
git repository hosting
git at public.arvados.org
Sat Mar 9 02:46:52 UTC 2024
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 36 +++++++++++-----------------------
services/fuse/arvados_fuse/fusedir.py | 17 ++++------------
2 files changed, 15 insertions(+), 38 deletions(-)
via f1a8112ffc7940e3ab4b39c38086ed63d2d87459 (commit)
from eb986fe200a4998052fd08323ff0a67eb05490fd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f1a8112ffc7940e3ab4b39c38086ed63d2d87459
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Mar 8 21:44:50 2024 -0500
21541: Yank the "total" lock and inc_use during merge
Added because I thought they might address certain issues, but on
further consideration I think they just add extra work for no benefit.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 0370237c1b..5f06502051 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -150,22 +150,17 @@ class InodeCache(object):
self.cap = cap
self._total = 0
self.min_entries = min_entries
- self._total_lock = threading.Lock()
def total(self):
return self._total
def evict_candidates(self):
- with self._total_lock:
- total = self._total
- if total <= self.cap:
+ if self._total <= self.cap:
return
_logger.debug("InodeCache evict_candidates total %i cap %i entries %i", self._total, self.cap, len(self._cache_entries))
for ent in listvalues(self._cache_entries):
- with self._total_lock:
- total = self._total
- if total < self.cap or len(self._cache_entries) < self.min_entries:
+ if self._total < self.cap or len(self._cache_entries) < self.min_entries:
break
yield ent
@@ -174,12 +169,7 @@ class InodeCache(object):
return
obj.cache_size = obj.objsize()
-
- with self._total_lock:
- _logger.debug("InodeCache b4 cache_size %i total %i", obj.cache_size, self._total)
- self._total += obj.cache_size
- _logger.debug("InodeCache after cache_size %i total %i", obj.cache_size, self._total)
- total = self._total
+ self._total += obj.cache_size
self._cache_entries[obj.inode] = obj
@@ -192,16 +182,15 @@ class InodeCache(object):
self._by_uuid[obj.cache_uuid].append(obj)
_logger.debug("InodeCache managing inode %i (size %i) (uuid %s) total now %i (%i entries)",
- obj.inode, obj.cache_size, obj.cache_uuid, total, len(self._cache_entries))
+ obj.inode, obj.cache_size, obj.cache_uuid, self._total, len(self._cache_entries))
def unmanage(self, entry):
if entry.inode not in self._cache_entries:
return
# manage cache size running sum
- # with self._total_lock:
- # self._total -= entry.cache_size
- # entry.cache_size = 0
+ self._total -= entry.cache_size
+ entry.cache_size = 0
# manage the mapping of uuid to object
if entry.cache_uuid:
@@ -214,14 +203,10 @@ class InodeCache(object):
del self._cache_entries[entry.inode]
def update_cache_size(self, obj):
- pass
- # if obj.inode in self._cache_entries:
- # with self._total_lock:
- # _logger.debug("update_cache_size b4 cache_size %i total %i", obj.cache_size, self._total)
- # self._total -= obj.cache_size
- # obj.cache_size = obj.objsize()
- # self._total += obj.cache_size
- # _logger.debug("update_cache_size after cache_size %i total %i", obj.cache_size, self._total)
+ if obj.inode in self._cache_entries:
+ self._total -= obj.cache_size
+ obj.cache_size = obj.objsize()
+ self._total += obj.cache_size
def touch(self, obj):
if obj.inode in self._cache_entries:
@@ -371,6 +356,7 @@ class Inodes(object):
_logger.debug("InodeCache clearing inode %i, total %i, forget_inode %s",
entry.inode, self.inode_cache.total(), forget_inode)
if forget_inode:
+ del self._entries[entry.inode]
entry.inode = None
# stop anything else
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 25e611e67d..fe1db416f1 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -151,7 +151,7 @@ class Directory(FreshBase):
super(Directory, self).fresh()
def objsize(self):
- return len(self._entries) * 64
+ return len(self._entries) * 128
def merge(self, items, fn, same, new_entry):
"""Helper method for updating the contents of the directory.
@@ -183,11 +183,11 @@ class Directory(FreshBase):
continue
if name in oldentries:
ent = oldentries[name]
- ent.inc_use()
if same(ent, i):
# move existing directory entry over
self._entries[name] = ent
del oldentries[name]
+ self.inodes.inode_cache.touch(ent)
for i in items:
name = self.sanitize_filename(fn(i))
@@ -197,7 +197,6 @@ class Directory(FreshBase):
# create new directory entry
ent = new_entry(i)
if ent is not None:
- ent.inc_use()
self._entries[name] = self.inodes.add_entry(ent)
changed = True
_logger.debug("Added entry '%s' as inode %i to parent inode %i", name, ent.inode, self.inode)
@@ -207,7 +206,6 @@ class Directory(FreshBase):
_logger.debug("Forgetting about entry '%s' on inode %i", i, self.inode)
self.inodes.invalidate_entry(self, i)
self.inodes.del_entry(oldentries[i])
- ent.dec_use()
changed = True
if changed:
@@ -215,9 +213,6 @@ class Directory(FreshBase):
self._mtime = time.time()
self.inodes.inode_cache.update_cache_size(self)
- for ent in self._entries.values():
- ent.dec_use()
-
self.fresh()
def in_use(self):
@@ -644,10 +639,8 @@ class CollectionDirectory(CollectionDirectoryBase):
return (self.collection_locator is not None)
def objsize(self):
- # This is an empirically-derived heuristic to estimate the memory used
- # to store this collection's metadata. Calculating the memory
- # footprint directly would be more accurate, but also more complicated.
- return self._manifest_size * 128
+ # Very rough estimate of memory footprint
+ return self._manifest_size * 4
def finalize(self):
if self.collection is not None:
@@ -1131,9 +1124,7 @@ class ProjectDirectory(Directory):
def _add_entry(self, i, name):
ent = self.createDirectory(i)
- ent.inc_use()
self._entries[name] = self.inodes.add_entry(ent)
- ent.dec_use()
return self._entries[name]
@use_counter
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list