[ARVADOS] updated: 6a0610e67965f2ba9ff9df2df36d92d3fa2450a2
git at public.curoverse.com
git at public.curoverse.com
Thu Sep 4 14:22:28 EDT 2014
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 172 ++++++++++++++++++++-------------
services/fuse/bin/arv-mount | 26 ++++-
2 files changed, 127 insertions(+), 71 deletions(-)
via 6a0610e67965f2ba9ff9df2df36d92d3fa2450a2 (commit)
via 0576030b3181b72f8395c73e0f3562582b59c2aa (commit)
from 6b4405e36bb59ebd4714690eeb8518c4a2fa019b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6a0610e67965f2ba9ff9df2df36d92d3fa2450a2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Sep 4 14:22:25 2014 -0400
3644: Added choose-your-own-adventure README files to the --all and --by-id
directories. Regular expressions are now compiled once at the top for
efficiency and readability.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index f026d4e..607a941 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -22,6 +22,13 @@ import threading
_logger = logging.getLogger('arvados.arvados_fuse')
+portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+\d+')
+uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+
class SafeApi(object):
'''Threadsafe wrapper for API object. This stores and returns a different api
object per thread, because httplib2 which underlies apiclient is not
@@ -314,7 +321,7 @@ class CollectionDirectory(Directory):
def update(self):
try:
- if self.collection_object is not None and re.match(r'^[a-f0-9]{32}', self.collection_locator):
+ if self.collection_object is not None and portable_data_hash_pattern.match(self.collection_locator):
return True
with llfuse.lock_released:
@@ -359,11 +366,11 @@ class CollectionDirectory(Directory):
def ctime(self):
self.checkupdate()
- return convertTime(self.collection_object["created_at"]) if self.collection_object is not None else 0
+ return convertTime(self.collection_object["created_at"]) if self.collection_object is not None and 'created_at' in self.collection_object else 0
def mtime(self):
self.checkupdate()
- return convertTime(self.collection_object["modified_at"]) if self.collection_object is not None else 0
+ return convertTime(self.collection_object["modified_at"]) if self.collection_object is not None and 'modified_at' in self.collection_object else 0
class MagicDirectory(Directory):
@@ -380,10 +387,28 @@ class MagicDirectory(Directory):
super(MagicDirectory, self).__init__(parent_inode)
self.inodes = inodes
self.api = api
+ self.readme_file = None
def __contains__(self, k):
+ if self.readme_file is None:
+ text = '''This directory provides access to Arvados collections as subdirectories listed
+by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
+the form '1234567890abcdefghijklmnopqrstuv+123').
+
+Note that this directory will appear empty until you attempt to access a
+specific collection subdirectory (such as trying to 'cd' into it), at which
+point the collection will actually be looked up on the server and the directory
+will appear if it exists.
+'''
+ self.readme_file = self.inodes.add_entry(StringFile(self.inode, text, 0, 0))
+ self._entries["README"] = self.readme_file
+
if k in self._entries:
return True
+
+ if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(k):
+ return False
+
try:
e = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, self.api, k))
if e.update():
@@ -478,18 +503,16 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
self.uuid = project_object['uuid']
def createDirectory(self, i):
- if re.match(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}', i['uuid']):
+ if collection_uuid_pattern.match(i['uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, i)
- elif re.match(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}', i['uuid']):
+ elif group_uuid_pattern.match(i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, i, self._poll, self._poll_time)
- elif re.match(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}', i['uuid']):
- if i['head_kind'] == 'arvados#collection' or re.match('[0-9a-f]{32}\+\d+', i['head_uuid']):
+ elif link_uuid_pattern.match(i['uuid']):
+ if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
return CollectionDirectory(self.inode, self.inodes, self.api, i['head_uuid'])
else:
return None
- #elif re.match(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}', i['uuid']):
- # return None
- elif re.match(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}', i['uuid']):
+ elif uuid_pattern.match(i['uuid']):
return ObjectFile(self.parent_inode, i)
else:
return None
@@ -499,10 +522,10 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
if 'name' in i:
if i['name'] is None or len(i['name']) == 0:
return None
- elif re.match(r'[a-z0-9]{5}-(4zz18|j7d0g)-[a-z0-9]{15}', i['uuid']):
+ elif collection_uuid_pattern.match(i['uuid']) or group_uuid_pattern.match(i['uuid']):
# collection or subproject
return i['name']
- elif re.match(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}', i['uuid']) and i['head_kind'] == 'arvados#collection':
+ elif link_uuid_pattern.match(i['uuid']) and i['head_kind'] == 'arvados#collection':
# name link
return i['name']
elif 'kind' in i and i['kind'].startswith('arvados#'):
@@ -521,9 +544,9 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
return False
with llfuse.lock_released:
- if re.match(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}', self.uuid):
+ if group_uuid_pattern.match(self.uuid):
self.project_object = self.api.groups().get(uuid=self.uuid).execute()
- elif re.match(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}', self.uuid):
+ elif user_uuid_pattern.match(self.uuid):
self.project_object = self.api.users().get(uuid=self.uuid).execute()
contents = arvados.util.list_all(self.api.groups().contents, uuid=self.uuid)
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 39fdf1d..00cfbde 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -110,6 +110,22 @@ with "--".
e._entries['by_tag'] = operations.inodes.add_entry(TagsDirectory(e.inode, operations.inodes, api))
e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(e.inode, operations.inodes, api))
+ text = '''
+Welcome to Arvados! This directory provides file system access to files and objects
+available on the Arvados installation located at '{}'
+using credentials for user '{}'.
+
+From here, the following directories are available:
+
+ by_id/ Access to Keep collections by uuid or portable data hash (see by_id/README for details).
+ by_tag/ Access to Keep collections organized by tag.
+ home/ The contents of your home project.
+ shared/ Projects shared with you.
+'''.format(arvados.config.get('ARVADOS_API_HOST'), usr['email'])
+
+ e._entries["README"] = operations.inodes.add_entry(StringFile(e.inode, text, 0, 0))
+
+
except Exception:
logger.exception("arv-mount: exception during API setup")
exit(1)
commit 0576030b3181b72f8395c73e0f3562582b59c2aa
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Sep 4 13:31:26 2014 -0400
3644: Change metafiles to .arvados#collection and .arvados#project. Change
--by-hash to be --by-id since it works with both uuids and portable data
hashes.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index adf6186..f026d4e 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -159,15 +159,19 @@ class StringFile(File):
def readfrom(self, off, size):
return self.contents[off:(off+size)]
+
class ObjectFile(StringFile):
'''Wrap a dict as a serialized json object.'''
- def __init__(self, parent_inode, contents):
- _ctime = convertTime(contents['created_at']) if 'created_at' in contents else 0
- _mtime = convertTime(contents['modified_at']) if 'modified_at' in contents else 0
- super(ObjectFile, self).__init__(parent_inode, json.dumps(contents, indent=4, sort_keys=True)+"\n", _ctime, _mtime)
- self.contentsdict = contents
- self.uuid = self.contentsdict['uuid']
+ def __init__(self, parent_inode, obj):
+ super(ObjectFile, self).__init__(parent_inode, "", 0, 0)
+ self.uuid = obj['uuid']
+ self.update(obj)
+
+ def update(self, obj):
+ self._ctime = convertTime(obj['created_at']) if 'created_at' in obj else 0
+ self._mtime = convertTime(obj['modified_at']) if 'modified_at' in obj else 0
+ self.contents = json.dumps(obj, indent=4, sort_keys=True) + "\n"
class Directory(FreshBase):
@@ -275,18 +279,39 @@ class Directory(FreshBase):
class CollectionDirectory(Directory):
'''Represents the root of a directory tree holding a collection.'''
- def __init__(self, parent_inode, inodes, api, collection_locator):
+ def __init__(self, parent_inode, inodes, api, collection):
super(CollectionDirectory, self).__init__(parent_inode)
self.inodes = inodes
self.api = api
- self.collection_locator = collection_locator
- self.manifest_text_file = None
- self.pdh_file = None
+ self.collection_object_file = None
self.collection_object = None
+ if isinstance(collection, dict):
+ self.collection_locator = collection['uuid']
+ else:
+ self.collection_locator = collection
def same(self, i):
return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
+ def new_collection(self, new_collection_object):
+ self.collection_object = new_collection_object
+
+ if self.collection_object_file is not None:
+ self.collection_object_file.update(self.collection_object)
+
+ self.clear()
+ collection = arvados.CollectionReader(self.collection_object["manifest_text"], self.api)
+ for s in collection.all_streams():
+ cwd = self
+ for part in s.name().split('/'):
+ if part != '' and part != '.':
+ partname = sanitize_filename(part)
+ if partname not in cwd._entries:
+ cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode))
+ cwd = cwd._entries[partname]
+ for k, v in s.files().items():
+ cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.ctime(), self.mtime()))
+
def update(self):
try:
if self.collection_object is not None and re.match(r'^[a-f0-9]{32}', self.collection_locator):
@@ -299,75 +324,47 @@ class CollectionDirectory(Directory):
# end with llfuse.lock_released, re-acquire lock
if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]:
- self.collection_object = new_collection_object
-
- if self.manifest_text_file is not None:
- self.manifest_text_file.contents = self.collection_object["manifest_text"]
- self.manifest_text_file._ctime = self.ctime()
- self.manifest_text_file._mtime = self.mtime()
- if self.pdh_file is not None:
- self.pdh_file.contents = self.collection_object["portable_data_hash"]
- self.pdh_file._ctime = self.ctime()
- self.pdh_file._mtime = self.mtime()
-
- self.clear()
- collection = arvados.CollectionReader(self.collection_object["manifest_text"], self.api)
- for s in collection.all_streams():
- cwd = self
- for part in s.name().split('/'):
- if part != '' and part != '.':
- partname = sanitize_filename(part)
- if partname not in cwd._entries:
- cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode))
- cwd = cwd._entries[partname]
- for k, v in s.files().items():
- cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.ctime(), self.mtime()))
+ self.new_collection(new_collection_object)
+
self.fresh()
return True
except apiclient.errors.HttpError as e:
- import pprint
- pprint.pprint(self.resp.status)
- if self.resp.status == 404:
+ if e.resp.status == 404:
_logger.warn("arv-mount %s: not found", self.collection_locator)
else:
_logger.error("arv-mount %s: error", self.collection_locator)
_logger.exception(detail)
- return False
except Exception as detail:
_logger.error("arv-mount %s: error", self.collection_locator)
if "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
_logger.exception(detail)
- return False
+ return False
def __getitem__(self, item):
self.checkupdate()
- if item == '.arvados#collection.manifest_text':
- if self.manifest_text_file is None:
- self.manifest_text_file = StringFile(self.inode, self.collection_object["manifest_text"], self.ctime(), self.mtime())
- self.inodes.add_entry(self.manifest_text_file)
- return self.manifest_text_file
- elif item == '.arvados#collection.portable_data_hash':
- if self.pdh_file is None:
- self.pdh_file = StringFile(self.inode, self.collection_object["portable_data_hash"], self.ctime(), self.mtime())
- self.inodes.add_entry(self.pdh_file)
- return self.pdh_file
+ if item == '.arvados#collection':
+ if self.collection_object_file is None:
+ self.collection_object_file = ObjectFile(self.inode, self.collection_object)
+ self.inodes.add_entry(self.collection_object_file)
+ return self.collection_object_file
else:
return super(CollectionDirectory, self).__getitem__(item)
def __contains__(self, k):
- if k in ('.arvados#collection.manifest_text', '.arvados#collection.portable_data_hash'):
+ if k == '.arvados#collection':
return True
else:
return super(CollectionDirectory, self).__contains__(k)
def ctime(self):
self.checkupdate()
- return convertTime(self.collection_object["created_at"])
+ return convertTime(self.collection_object["created_at"]) if self.collection_object is not None else 0
def mtime(self):
self.checkupdate()
- return convertTime(self.collection_object["modified_at"])
+ return convertTime(self.collection_object["modified_at"]) if self.collection_object is not None else 0
+
class MagicDirectory(Directory):
'''A special directory that logically contains the set of all extant keep
@@ -404,6 +401,7 @@ class MagicDirectory(Directory):
else:
raise KeyError("No collection with id " + item)
+
class RecursiveInvalidateDirectory(Directory):
def invalidate(self):
if self.inode == llfuse.ROOT_INODE:
@@ -418,6 +416,7 @@ class RecursiveInvalidateDirectory(Directory):
if self.inode == llfuse.ROOT_INODE:
llfuse.lock.release()
+
class TagsDirectory(RecursiveInvalidateDirectory):
'''A special directory that contains as subdirectories all tags visible to the user.'''
@@ -440,6 +439,7 @@ class TagsDirectory(RecursiveInvalidateDirectory):
lambda a, i: a.tag == i,
lambda i: TagDirectory(self.inode, self.inodes, self.api, i['name'], poll=self._poll, poll_time=self._poll_time))
+
class TagDirectory(Directory):
'''A special directory that contains as subdirectories all collections visible
to the user that are tagged with a particular tag.
@@ -473,15 +473,20 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
self.inodes = inodes
self.api = api
self.project_object = project_object
+ self.project_object_file = ObjectFile(self.inode, self.project_object)
+ self.inodes.add_entry(self.project_object_file)
self.uuid = project_object['uuid']
def createDirectory(self, i):
if re.match(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}', i['uuid']):
- return CollectionDirectory(self.inode, self.inodes, self.api, i['uuid'])
+ return CollectionDirectory(self.inode, self.inodes, self.api, i)
elif re.match(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}', i['uuid']):
return ProjectDirectory(self.inode, self.inodes, self.api, i, self._poll, self._poll_time)
- elif re.match(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}', i['uuid']) and i['head_kind'] == 'arvados#collection':
- return CollectionDirectory(self.inode, self.inodes, self.api, i['head_uuid'])
+ elif re.match(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}', i['uuid']):
+ if i['head_kind'] == 'arvados#collection' or re.match('[0-9a-f]{32}\+\d+', i['head_uuid']):
+ return CollectionDirectory(self.inode, self.inodes, self.api, i['head_uuid'])
+ else:
+ return None
#elif re.match(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}', i['uuid']):
# return None
elif re.match(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}', i['uuid']):
@@ -492,7 +497,7 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
def update(self):
def namefn(i):
if 'name' in i:
- if i['name'] is None:
+ if i['name'] is None or len(i['name']) == 0:
return None
elif re.match(r'[a-z0-9]{5}-(4zz18|j7d0g)-[a-z0-9]{15}', i['uuid']):
# collection or subproject
@@ -532,6 +537,19 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
samefn,
self.createDirectory)
+ def __getitem__(self, item):
+ self.checkupdate()
+ if item == '.arvados#project':
+ return self.project_object_file
+ else:
+ return super(ProjectDirectory, self).__getitem__(item)
+
+ def __contains__(self, k):
+ if k == '.arvados#project':
+ return True
+ else:
+ return super(ProjectDirectory, self).__contains__(k)
+
def ctime(self):
return convertTime(self.project_object["created_at"]) if "created_at" in self.project_object else 0
@@ -539,7 +557,6 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
return convertTime(self.project_object["modified_at"]) if "modified_at" in self.project_object else 0
-
class SharedDirectory(RecursiveInvalidateDirectory):
'''A special directory that represents users or groups who have shared projects with me.'''
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 8fbb45e..39fdf1d 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -27,13 +27,13 @@ with "--".
mount_mode = parser.add_mutually_exclusive_group()
- mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, tags, portable data hash (default).""")
+ mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default).""")
mount_mode.add_argument('--home', action='store_true', help="""Mount only the user's home project.""")
mount_mode.add_argument('--shared', action='store_true', help="""Mount only list of projects shared with the user.""")
mount_mode.add_argument('--by-tag', action='store_true',
help="""Mount subdirectories listed by tag.""")
- mount_mode.add_argument('--by-hash', action='store_true',
- help="""Mount subdirectories listed by portable data hash.""")
+ mount_mode.add_argument('--by-id', action='store_true',
+ help="""Mount subdirectories listed by portable data hash or uuid.""")
mount_mode.add_argument('--project', type=str, help="""Mount a specific project.""")
mount_mode.add_argument('--collection', type=str, help="""Mount only the specified collection.""")
@@ -89,7 +89,7 @@ with "--".
api = SafeApi(arvados.config)
usr = api.users().current().execute()
- if args.by_hash:
+ if args.by_id:
# Set up the request handler with the 'magic directory' at the root
operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes, api))
elif args.by_tag:
@@ -108,7 +108,7 @@ with "--".
e._entries['home'] = operations.inodes.add_entry(ProjectDirectory(e.inode, operations.inodes, api, usr))
e._entries['shared'] = operations.inodes.add_entry(SharedDirectory(e.inode, operations.inodes, api, usr))
e._entries['by_tag'] = operations.inodes.add_entry(TagsDirectory(e.inode, operations.inodes, api))
- e._entries['by_hash'] = operations.inodes.add_entry(MagicDirectory(e.inode, operations.inodes, api))
+ e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(e.inode, operations.inodes, api))
except Exception:
logger.exception("arv-mount: exception during API setup")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list