[ARVADOS] updated: 024a666e751e4463f5c9cabed771a3bd8833c589
Git user
git at public.curoverse.com
Fri Jun 30 14:53:11 EDT 2017
Summary of changes:
services/fuse/arvados_fuse/fusedir.py | 90 +++++++++++++++++++----------------
1 file changed, 48 insertions(+), 42 deletions(-)
via 024a666e751e4463f5c9cabed771a3bd8833c589 (commit)
from cecd50ebf523bbb25dbee52efe83471820e0d0d4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 024a666e751e4463f5c9cabed771a3bd8833c589
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Jun 30 14:52:10 2017 -0400
11158: Restore paging to get full list of collections and projects (but still
don't load other object types). Defer full listing load until first call to
items() to support efficient access of individual collections/subprojects by
name.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index fc5a9be..89da451 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -742,7 +742,7 @@ class ProjectDirectory(Directory):
self._poll_time = poll_time
self._updating_lock = threading.Lock()
self._current_user = None
- self._extra = set()
+ self._full_listing = False
def want_event_subscribe(self):
return True
@@ -765,27 +765,35 @@ class ProjectDirectory(Directory):
def uuid(self):
return self.project_uuid
+ def items(self):
+ self._full_listing = True
+ return super(ProjectDirectory, self).items()
+
+ def namefn(self, i):
+ if 'name' in i:
+ if i['name'] is None or len(i['name']) == 0:
+ return None
+ elif collection_uuid_pattern.match(i['uuid']) or group_uuid_pattern.match(i['uuid']):
+ # collection or subproject
+ return i['name']
+ elif link_uuid_pattern.match(i['uuid']) and i['head_kind'] == 'arvados#collection':
+ # name link
+ return i['name']
+ elif 'kind' in i and i['kind'].startswith('arvados#'):
+ # something else
+ return "{}.{}".format(i['name'], i['kind'][8:])
+ else:
+ return None
+
+
@use_counter
def update(self):
if self.project_object_file == None:
self.project_object_file = ObjectFile(self.inode, self.project_object)
self.inodes.add_entry(self.project_object_file)
- def namefn(i):
- if 'name' in i:
- if i['name'] is None or len(i['name']) == 0:
- return None
- elif collection_uuid_pattern.match(i['uuid']) or group_uuid_pattern.match(i['uuid']):
- # collection or subproject
- return i['name']
- elif link_uuid_pattern.match(i['uuid']) and i['head_kind'] == 'arvados#collection':
- # name link
- return i['name']
- elif 'kind' in i and i['kind'].startswith('arvados#'):
- # something else
- return "{}.{}".format(i['name'], i['kind'][8:])
- else:
- return None
+ if not self._full_listing:
+ return
def samefn(a, i):
if isinstance(a, CollectionDirectory) or isinstance(a, ProjectDirectory):
@@ -807,27 +815,24 @@ class ProjectDirectory(Directory):
self.project_object = self.api.users().get(
uuid=self.project_uuid).execute(num_retries=self.num_retries)
- contents = self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["group_class", "=", "project"]],
- limit=1000, order="modified_at desc").execute(num_retries=self.num_retries)["items"]
- contents.extend(self.api.collections().list(filters=[["owner_uuid", "=", self.project_uuid]],
- limit=1000, order="modified_at desc").execute(num_retries=self.num_retries)["items"])
- if self._extra:
- contents.extend(self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["group_class", "=", "project"],
- ["uuid", "in", list(self._extra)]],
- limit=1000).execute(num_retries=self.num_retries)["items"])
- contents.extend(self.api.collections().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["uuid", "in", list(self._extra)]],
- limit=1000).execute(num_retries=self.num_retries)["items"])
+ contents = arvados.util.list_all(self.api.groups().list,
+ self.num_retries,
+ filters=[["owner_uuid", "=", self.project_uuid],
+ ["group_class", "=", "project"]],
+ limit=1000, order="modified_at desc")
+ contents.extend(arvados.util.list_all(self.api.collections().list,
+ self.num_retries,
+ filters=[["owner_uuid", "=", self.project_uuid]],
+ limit=1000, order="modified_at desc"))
# end with llfuse.lock_released, re-acquire lock
self.merge(contents,
- namefn,
+ self.namefn,
samefn,
self.createDirectory)
finally:
+ self._full_listing = False
self._updating_lock.release()
@use_counter
@@ -844,22 +849,23 @@ class ProjectDirectory(Directory):
else:
if super(ProjectDirectory, self).__contains__(k):
return True
- else:
+ elif not self._full_listing:
with llfuse.lock_released:
- contents = self.api.collections().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["name", "=", k]],
- limit=1).execute(num_retries=self.num_retries)["items"]
+ contents = self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
+ ["group_class", "=", "project"],
+ ["name", "=", k]],
+ limit=1).execute(num_retries=self.num_retries)["items"]
if not contents:
- contents = self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["group_class", "=", "project"],
- ["name", "=", k]],
- limit=1).execute(num_retries=self.num_retries)["items"]
+ contents = self.api.collections().list(filters=[["owner_uuid", "=", self.project_uuid],
+ ["name", "=", k]],
+ limit=1).execute(num_retries=self.num_retries)["items"]
if contents:
- self._extra.add(contents[0]["uuid"])
- self.invalidate()
+ i = contents[0]
+ name = sanitize_filename(self.namefn(i))
+ ent = self.createDirectory(i)
+ self._entries[name] = self.inodes.add_entry(ent)
return True
- else:
- return False
+ return False
@use_counter
@check_update
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list