[ARVADOS] updated: be4c2feb3d9ba3c2631d78e68bc5ed2945c17cab
git at public.curoverse.com
git at public.curoverse.com
Mon Sep 8 15:06:41 EDT 2014
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 54 +++++--
services/fuse/tests/test_mount.py | 273 +++++++++++++++------------------
2 files changed, 162 insertions(+), 165 deletions(-)
via be4c2feb3d9ba3c2631d78e68bc5ed2945c17cab (commit)
from 52e25652032ccf70f6cade4e575102840b212aa1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit be4c2feb3d9ba3c2631d78e68bc5ed2945c17cab
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Sep 8 15:06:33 2014 -0400
3644: Restore 'tag' tests and add 'SharedDirectory' and 'HomeDirectory' tests.
All tests should pass now.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 710457a..c3f913e 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -57,10 +57,13 @@ class SafeApi(object):
def users(self):
return self.localapi().users()
-
+
def convertTime(t):
'''Parse Arvados timestamp to unix time.'''
- return calendar.timegm(time.strptime(t, "%Y-%m-%dT%H:%M:%SZ"))
+ try:
+ return calendar.timegm(time.strptime(t, "%Y-%m-%dT%H:%M:%SZ"))
+ except (TypeError, ValueError):
+ return 0
def sanitize_filename(dirty):
'''Remove troublesome characters from filenames.'''
@@ -96,9 +99,8 @@ class FreshBase(object):
def invalidate(self):
self._stale = True
- # Test if the entries dict is stale. Also updates atime.
+ # Test if the entries dict is stale.
def stale(self):
- self._atime = time.time()
if self._stale:
return True
if self._poll:
@@ -158,7 +160,7 @@ class StringFile(File):
return len(self.contents)
def readfrom(self, off, size):
- return self.contents[off:(off+size)]
+ return self.contents[off:(off+size)]
class ObjectFile(StringFile):
@@ -262,7 +264,7 @@ class Directory(FreshBase):
changed = True
# delete any other directory entries that were not in found in 'items'
- for i in oldentries:
+ for i in oldentries:
llfuse.invalidate_entry(self.inode, str(i))
self.inodes.del_entry(oldentries[i])
changed = True
@@ -321,7 +323,7 @@ class CollectionDirectory(Directory):
cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode))
cwd = cwd._entries[partname]
for k, v in s.files().items():
- cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.mtime()))
+ cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.mtime()))
def update(self):
try:
@@ -347,9 +349,9 @@ class CollectionDirectory(Directory):
_logger.exception(detail)
except Exception as detail:
_logger.error("arv-mount %s: error", self.collection_locator)
- if "manifest_text" in self.collection_object:
+ if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- _logger.exception(detail)
+ _logger.exception(detail)
return False
def __getitem__(self, item):
@@ -387,9 +389,11 @@ class MagicDirectory(Directory):
super(MagicDirectory, self).__init__(parent_inode)
self.inodes = inodes
self.api = api
+ # Have to defer creating readme_file because at this point we don't
+ # yet have an inode assigned.
self.readme_file = None
- def __contains__(self, k):
+ def create_readme(self):
if self.readme_file is None:
text = '''This directory provides access to Arvados collections as subdirectories listed
by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
@@ -403,6 +407,9 @@ will appear if it exists.
self.readme_file = self.inodes.add_entry(StringFile(self.inode, text, time.time()))
self._entries["README"] = self.readme_file
+ def __contains__(self, k):
+ self.create_readme()
+
if k in self._entries:
return True
@@ -420,6 +427,10 @@ will appear if it exists.
_logger.debug('arv-mount exception keep %s', e)
return False
+ def items(self):
+ self.create_readme()
+ return self._entries.items()
+
def __getitem__(self, item):
if item in self:
return self._entries[item]
@@ -487,7 +498,7 @@ class TagDirectory(Directory):
lambda i: CollectionDirectory(self.inode, self.inodes, self.api, i['head_uuid']))
-class ProjectDirectory(RecursiveInvalidateDirectory):
+class ProjectDirectory(Directory):
'''A special directory that contains the contents of a project.'''
def __init__(self, parent_inode, inodes, api, project_object, poll=False, poll_time=60):
@@ -495,8 +506,7 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
self.inodes = inodes
self.api = api
self.project_object = project_object
- self.project_object_file = ObjectFile(self.inode, self.project_object)
- self.inodes.add_entry(self.project_object_file)
+ self.project_object_file = None
self.uuid = project_object['uuid']
def createDirectory(self, i):
@@ -515,6 +525,10 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
return None
def update(self):
+ if self.project_object_file == None:
+ self.project_object_file = ObjectFile(self.inode, self.project_object)
+ self.inodes.add_entry(self.project_object_file)
+
def namefn(i):
if 'name' in i:
if i['name'] is None or len(i['name']) == 0:
@@ -527,7 +541,7 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
return i['name']
elif 'kind' in i and i['kind'].startswith('arvados#'):
# something else
- return "{}.{}".format(i['name'], i['kind'][8:])
+ return "{}.{}".format(i['name'], i['kind'][8:])
else:
return None
@@ -549,7 +563,7 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
contents = arvados.util.list_all(self.api.groups().contents, uuid=self.uuid)
# Name links will be obsolete soon, take this out when there are no more pre-#3036 in use.
contents += arvados.util.list_all(self.api.links().list, filters=[['tail_uuid', '=', self.uuid], ['link_class', '=', 'name']])
-
+
# end with llfuse.lock_released, re-acquire lock
self.merge(contents,
@@ -571,7 +585,7 @@ class ProjectDirectory(RecursiveInvalidateDirectory):
return super(ProjectDirectory, self).__contains__(k)
-class SharedDirectory(RecursiveInvalidateDirectory):
+class SharedDirectory(Directory):
'''A special directory that represents users or groups who have shared projects with me.'''
def __init__(self, parent_inode, inodes, api, exclude, poll=False, poll_time=60):
@@ -750,7 +764,7 @@ class Operations(llfuse.Operations):
p = self.inodes[parent_inode]
if name == '..':
inode = p.parent_inode
- elif name in p:
+ elif isinstance(p, Directory) and name in p:
inode = p[name].inode
if inode != None:
@@ -782,6 +796,9 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.EBADF)
+ # update atime
+ handle.entry._atime = time.time()
+
try:
with llfuse.lock_released:
return handle.entry.readfrom(off, size)
@@ -810,6 +827,9 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.EIO)
+ # update atime
+ p._atime = time.time()
+
self._filehandles[fh] = FileHandle(fh, [('.', p), ('..', parent)] + list(p.items()))
return fh
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 5474061..8287a19 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -17,8 +17,13 @@ class MountTestBase(unittest.TestCase):
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
+ run_test_server.run(False)
+ run_test_server.authorize_with("admin")
+ self.api = api = fuse.SafeApi(arvados.config)
def tearDown(self):
+ run_test_server.stop()
+
# llfuse.close is buggy, so use fusermount instead.
#llfuse.close(unmount=True)
count = 0
@@ -63,11 +68,12 @@ class FuseMountTest(MountTestBase):
cw.write("data 8")
self.testcollection = cw.finish()
+ self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
# Create the request handler
operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, None, self.testcollection))
+ e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, self.testcollection))
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
@@ -117,11 +123,12 @@ class FuseMagicTest(MountTestBase):
cw.write("data 1")
self.testcollection = cw.finish()
+ self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
# Create the request handler
operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, None))
+ e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api))
self.mounttmp = tempfile.mkdtemp()
@@ -143,7 +150,7 @@ class FuseMagicTest(MountTestBase):
d3 = os.listdir(self.mounttmp)
d3.sort()
- self.assertEqual(['README', self.testcollection], d3)
+ self.assertEqual([self.testcollection, 'README'], d3)
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
@@ -152,189 +159,159 @@ class FuseMagicTest(MountTestBase):
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
-#
-# Restore these tests when working on issue #3644
-#
-# class FuseTagsTest(MountTestBase):
-# def setUp(self):
-# super(FuseTagsTest, self).setUp()
-
-# cw = arvados.CollectionWriter()
-
-# cw.start_new_file('foo')
-# cw.write("foo")
-
-# self.testcollection = cw.finish()
-
-# run_test_server.run()
-
-# def runTest(self):
-# run_test_server.authorize_with("admin")
-# api = arvados.api('v1', cache=False)
-
-# operations = fuse.Operations(os.getuid(), os.getgid())
-# e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
-
-# llfuse.init(operations, self.mounttmp, [])
-# t = threading.Thread(None, lambda: llfuse.main())
-# t.start()
-# # wait until the driver is finished initializing
-# operations.initlock.wait()
-
-# d1 = os.listdir(self.mounttmp)
-# d1.sort()
-# self.assertEqual(['foo_tag'], d1)
-
-# d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
-# d2.sort()
-# self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
-
-# d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
-# d3.sort()
-# self.assertEqual(['foo'], d3)
+class FuseTagsTest(MountTestBase):
+ def setUp(self):
+ super(FuseTagsTest, self).setUp()
-# files = {}
-# files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
+ cw = arvados.CollectionWriter()
-# for k, v in files.items():
-# with open(os.path.join(self.mounttmp, k)) as f:
-# self.assertEqual(v, f.read())
+ cw.start_new_file('foo')
+ cw.write("foo")
+ self.testcollection = cw.finish()
-# def tearDown(self):
-# run_test_server.stop()
+ def runTest(self):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api))
-# super(FuseTagsTest, self).tearDown()
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
-# class FuseTagsUpdateTestBase(MountTestBase):
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
-# def runRealTest(self):
-# run_test_server.authorize_with("admin")
-# api = arvados.api('v1', cache=False)
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(['foo_tag'], d1)
-# operations = fuse.Operations(os.getuid(), os.getgid())
-# e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
+ d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
+ d2.sort()
+ self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
-# llfuse.init(operations, self.mounttmp, [])
-# t = threading.Thread(None, lambda: llfuse.main())
-# t.start()
+ d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
+ d3.sort()
+ self.assertEqual(['foo'], d3)
-# # wait until the driver is finished initializing
-# operations.initlock.wait()
+ files = {}
+ files[os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7', 'foo')] = 'foo'
-# d1 = os.listdir(self.mounttmp)
-# d1.sort()
-# self.assertEqual(['foo_tag'], d1)
+ for k, v in files.items():
+ with open(os.path.join(self.mounttmp, k)) as f:
+ self.assertEqual(v, f.read())
-# api.links().create(body={'link': {
-# 'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
-# 'link_class': 'tag',
-# 'name': 'bar_tag'
-# }}).execute()
-# time.sleep(1)
+class FuseTagsUpdateTest(MountTestBase):
+ def runRealTest(self):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, poll_time=1))
-# d2 = os.listdir(self.mounttmp)
-# d2.sort()
-# self.assertEqual(['bar_tag', 'foo_tag'], d2)
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
-# d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-# d3.sort()
-# self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
-# l = api.links().create(body={'link': {
-# 'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
-# 'link_class': 'tag',
-# 'name': 'bar_tag'
-# }}).execute()
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(['foo_tag'], d1)
-# time.sleep(1)
+ self.api.links().create(body={'link': {
+ 'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
-# d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-# d4.sort()
-# self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
+ time.sleep(1)
-# api.links().delete(uuid=l['uuid']).execute()
+ d2 = os.listdir(self.mounttmp)
+ d2.sort()
+ self.assertEqual(['bar_tag', 'foo_tag'], d2)
-# time.sleep(1)
+ d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d3.sort()
+ self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
-# d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-# d5.sort()
-# self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
+ l = self.api.links().create(body={'link': {
+ 'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
+ time.sleep(1)
-# class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
-# def setUp(self):
-# super(FuseTagsUpdateTestWebsockets, self).setUp()
-# run_test_server.run(True)
+ d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d4.sort()
+ self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
-# def runTest(self):
-# self.runRealTest()
+ self.api.links().delete(uuid=l['uuid']).execute()
-# def tearDown(self):
-# run_test_server.stop()
-# super(FuseTagsUpdateTestWebsockets, self).tearDown()
+ time.sleep(1)
+ d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d5.sort()
+ self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
-# class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
-# def setUp(self):
-# super(FuseTagsUpdateTestPoll, self).setUp()
-# run_test_server.run(False)
-# def runTest(self):
-# self.runRealTest()
+class FuseSharedTest(MountTestBase):
+ def runTest(self):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.SharedDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, self.api.users().current().execute()['uuid']))
-# def tearDown(self):
-# run_test_server.stop()
-# super(FuseTagsUpdateTestPoll, self).tearDown()
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
-# class FuseGroupsTest(MountTestBase):
-# def setUp(self):
-# super(FuseGroupsTest, self).setUp()
-# run_test_server.run()
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertIn('Active User', d1)
-# def runTest(self):
-# run_test_server.authorize_with("admin")
-# api = arvados.api('v1', cache=False)
+ d2 = os.listdir(os.path.join(self.mounttmp, 'Active User'))
+ d2.sort()
+ self.assertEqual(['A Project',
+ "Empty collection",
+ "Empty collection.link",
+ "Pipeline Template with Jobspec Components.pipelineTemplate",
+ "pipeline_with_job.pipelineInstance"
+ ], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'Active User', 'A Project'))
+ d3.sort()
+ self.assertEqual(["A Subproject",
+ "Two Part Pipeline Template.pipelineTemplate",
+ "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
+ ], d3)
-# operations = fuse.Operations(os.getuid(), os.getgid())
-# e = operations.inodes.add_entry(fuse.GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+ with open(os.path.join(self.mounttmp, 'Active User', "A Project", "Two Part Pipeline Template.pipelineTemplate")) as f:
+ j = json.load(f)
+ self.assertEqual("Two Part Pipeline Template", j['name'])
-# llfuse.init(operations, self.mounttmp, [])
-# t = threading.Thread(None, lambda: llfuse.main())
-# t.start()
-# # wait until the driver is finished initializing
-# operations.initlock.wait()
+class FuseHomeTest(MountTestBase):
+ def runTest(self):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.ProjectDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, self.api.users().current().execute()))
-# d1 = os.listdir(self.mounttmp)
-# d1.sort()
-# self.assertIn('zzzzz-j7d0g-v955i6s2oi1cbso', d1)
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
-# d2 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso'))
-# d2.sort()
-# self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime',
-# "I'm a job in a project",
-# "I'm a template in a project",
-# "zzzzz-j58dm-5gid26432uujf79",
-# "zzzzz-j58dm-7r18rnd5nzhg5yk",
-# "zzzzz-j58dm-ypsjlol9dofwijz",
-# "zzzzz-j7d0g-axqo7eu9pwvna1x"
-# ], d2)
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
-# d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
-# d3.sort()
-# self.assertEqual(["I'm in a subproject, too",
-# "ea10d51bcf88862dbcc36eb292017dfd+45 added sometime",
-# "zzzzz-j58dm-c40lddwcqqr1ffs"
-# ], d3)
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertIn('Unrestricted public data', d1)
-# with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a project")) as f:
-# j = json.load(f)
-# self.assertEqual("Two Part Pipeline Template", j['name'])
+ d2 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
+ d2.sort()
+ self.assertEqual(['GNU General Public License, version 3'], d2)
-# def tearDown(self):
-# run_test_server.stop()
-# super(FuseGroupsTest, self).tearDown()
+ d3 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
+ d3.sort()
+ self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list