[ARVADOS] created: 7c2a8888585e2b87347295915f7d7045b61d02bc
git at public.curoverse.com
git at public.curoverse.com
Wed Dec 3 10:16:33 EST 2014
at 7c2a8888585e2b87347295915f7d7045b61d02bc (commit)
commit 7c2a8888585e2b87347295915f7d7045b61d02bc
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 10:07:26 2014 -0500
4705: FUSE driver can read Collections that only exist in Keep.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 65f6e8b..80ad6b3 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -304,18 +304,14 @@ class CollectionDirectory(Directory):
def same(self, i):
return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
- def new_collection(self, new_collection_object):
+ def new_collection(self, new_collection_object, coll_reader):
self.collection_object = new_collection_object
if self.collection_object_file is not None:
self.collection_object_file.update(self.collection_object)
self.clear()
- collection = arvados.CollectionReader(
- self.collection_object["manifest_text"], self.api,
- self.api.localkeep(), num_retries=self.num_retries)
- collection.normalize()
- for s in collection.all_streams():
+ for s in coll_reader.all_streams():
cwd = self
for part in s.name().split('/'):
if part != '' and part != '.':
@@ -332,23 +328,28 @@ class CollectionDirectory(Directory):
return True
with llfuse.lock_released:
- new_collection_object = self.api.collections().get(
- uuid=self.collection_locator
- ).execute(num_retries=self.num_retries)
+ coll_reader = arvados.CollectionReader(
+ self.collection_locator, self.api, self.api.localkeep(),
+ num_retries=self.num_retries)
+ new_collection_object = coll_reader.api_response() or {}
+ # If the Collection only exists in Keep, there will be no API
+ # response. Fill in the fields we need.
+ if 'uuid' not in new_collection_object:
+ new_collection_object['uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_object:
new_collection_object["portable_data_hash"] = new_collection_object["uuid"]
+ if 'manifest_text' not in new_collection_object:
+ new_collection_object['manifest_text'] = coll_reader.manifest_text()
+ coll_reader.normalize()
# end with llfuse.lock_released, re-acquire lock
if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]:
- self.new_collection(new_collection_object)
+ self.new_collection(new_collection_object, coll_reader)
self.fresh()
return True
- except apiclient.errors.HttpError as e:
- if e.resp.status == 404:
- _logger.warn("arv-mount %s: not found", self.collection_locator)
- else:
- _logger.exception("arv-mount %s: error", self.collection_locator)
+ except apiclient.errors.NotFoundError:
+ _logger.exception("arv-mount %s: error", self.collection_locator)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 9c1d63b..84dceee 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -122,6 +122,24 @@ class FuseMountTest(MountTestBase):
self.assertEqual(v, f.read())
+class FuseNoAPITest(MountTestBase):
+ def setUp(self):
+ super(FuseNoAPITest, self).setUp()
+ keep = arvados.keep.KeepClient(local_store=self.keeptmp)
+ self.file_data = "API-free text\n"
+ self.file_loc = keep.put(self.file_data)
+ self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
+ self.file_loc, len(self.file_data)))
+
+ def runTest(self):
+ self.make_mount(fuse.MagicDirectory)
+ self.assertDirContents(self.coll_loc, ['api-free.txt'])
+ with open(os.path.join(
+ self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
+ actual = keep_file.read(-1)
+ self.assertEqual(self.file_data, actual)
+
+
class FuseMagicTest(MountTestBase):
def setUp(self):
super(FuseMagicTest, self).setUp()
commit 5bcdf152a287ea4e929508b3cdcc3aaff49e9753
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 10:12:40 2014 -0500
4705: Fix FUSE exception logging.
logger.exception() doesn't take the exception as an argument, it takes
a message like all the other logger methods. It gets the exception
information from sys.exc_info().
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b2a3b2e..65f6e8b 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -348,17 +348,15 @@ class CollectionDirectory(Directory):
if e.resp.status == 404:
_logger.warn("arv-mount %s: not found", self.collection_locator)
else:
- _logger.error("arv-mount %s: error", self.collection_locator)
- _logger.exception(detail)
+ _logger.exception("arv-mount %s: error", self.collection_locator)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- except Exception as detail:
- _logger.error("arv-mount %s: error", self.collection_locator)
+ except Exception:
+ _logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- _logger.exception(detail)
return False
def __getitem__(self, item):
@@ -455,8 +453,8 @@ class RecursiveInvalidateDirectory(Directory):
super(RecursiveInvalidateDirectory, self).invalidate()
for a in self._entries:
self._entries[a].invalidate()
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
finally:
if self.inode == llfuse.ROOT_INODE:
llfuse.lock.release()
@@ -674,8 +672,8 @@ class SharedDirectory(Directory):
lambda i: i[0],
lambda a, i: a.uuid == i[1]['uuid'],
lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
class FileHandle(object):
@@ -839,8 +837,8 @@ class Operations(llfuse.Operations):
except arvados.errors.NotFoundError as e:
_logger.warning("Block not found: " + str(e))
raise llfuse.FUSEError(errno.EIO)
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
raise llfuse.FUSEError(errno.EIO)
def release(self, fh):
commit abadcf3ace9afbaa2dca2110bfca0e5d31a36e30
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 08:59:22 2014 -0500
4705: Refactor mount creation in FUSE tests.
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index f0f375c..9c1d63b 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -21,6 +21,15 @@ class MountTestBase(unittest.TestCase):
run_test_server.authorize_with("admin")
self.api = api = fuse.SafeApi(arvados.config)
+ def make_mount(self, root_class, *root_args):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ operations.inodes.add_entry(root_class(
+ llfuse.ROOT_INODE, operations.inodes, self.api, 0, *root_args))
+ llfuse.init(operations, self.mounttmp, [])
+ threading.Thread(None, llfuse.main).start()
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
def tearDown(self):
run_test_server.stop()
@@ -87,18 +96,8 @@ class FuseMountTest(MountTestBase):
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.testcollection))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.CollectionDirectory, self.testcollection)
- # now check some stuff
self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
'edgecases', 'dir1', 'dir2'])
self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
@@ -136,18 +135,8 @@ class FuseMagicTest(MountTestBase):
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
+ self.make_mount(fuse.MagicDirectory)
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- # now check some stuff
mount_ls = os.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
@@ -173,15 +162,7 @@ class FuseMagicTest(MountTestBase):
class FuseTagsTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.TagsDirectory)
d1 = os.listdir(self.mounttmp)
d1.sort()
@@ -234,15 +215,8 @@ class FuseTagsUpdateTest(MountTestBase):
class FuseSharedTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.SharedDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.api.users().current().execute()['uuid']))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.SharedDirectory,
+ self.api.users().current().execute()['uuid'])
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
@@ -284,26 +258,16 @@ class FuseSharedTest(MountTestBase):
class FuseHomeTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.ProjectDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.api.users().current().execute()))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.ProjectDirectory,
+ self.api.users().current().execute())
d1 = os.listdir(self.mounttmp)
- d1.sort()
self.assertIn('Unrestricted public data', d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
- d2.sort()
self.assertEqual(['GNU General Public License, version 3'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
- d3.sort()
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
commit 277b3c719955d63a7d5994d3973640b7bcca1e7f
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 09:45:44 2014 -0500
4705: Add CollectionReader.api_response() to PySDK.
The FUSE driver wants this information.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 0f49438..d530f58 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,3 +1,4 @@
+import functools
import logging
import os
import re
@@ -124,6 +125,7 @@ class CollectionReader(CollectionBase):
else:
raise errors.ArgumentError(
"Argument to CollectionReader must be a manifest or a collection UUID")
+ self._api_response = None
self._streams = None
def _populate_from_api_server(self):
@@ -138,10 +140,10 @@ class CollectionReader(CollectionBase):
if self._api_client is None:
self._api_client = arvados.api('v1')
self._keep_client = None # Make a new one with the new api.
- c = self._api_client.collections().get(
+ self._api_response = self._api_client.collections().get(
uuid=self._manifest_locator).execute(
num_retries=self.num_retries)
- self._manifest_text = c['manifest_text']
+ self._manifest_text = self._api_response['manifest_text']
return None
except Exception as e:
return e
@@ -158,8 +160,6 @@ class CollectionReader(CollectionBase):
return e
def _populate(self):
- if self._streams is not None:
- return
error_via_api = None
error_via_keep = None
should_try_keep = ((self._manifest_text is None) and
@@ -190,9 +190,27 @@ class CollectionReader(CollectionBase):
for sline in self._manifest_text.split("\n")
if sline]
- def normalize(self):
- self._populate()
+ def _populate_first(orig_func):
+ # Decorator for methods that read actual Collection data.
+ @functools.wraps(orig_func)
+ def wrapper(self, *args, **kwargs):
+ if self._streams is None:
+ self._populate()
+ return orig_func(self, *args, **kwargs)
+ return wrapper
+
+ @_populate_first
+ def api_response(self):
+ """api_response() -> dict or None
+
+ Returns information about this Collection fetched from the API server.
+ If the Collection exists in Keep but not the API server, currently
+ returns None. Future versions may provide a synthetic response.
+ """
+ return self._api_response
+ @_populate_first
+ def normalize(self):
# Rearrange streams
streams = {}
for s in self.all_streams():
@@ -213,6 +231,7 @@ class CollectionReader(CollectionBase):
[StreamReader(stream, keep=self._my_keep()).manifest_text()
for stream in self._streams])
+ @_populate_first
def open(self, streampath, filename=None):
"""open(streampath[, filename]) -> file-like object
@@ -220,7 +239,6 @@ class CollectionReader(CollectionBase):
single string or as two separate stream name and file name arguments.
This method returns a file-like object to read that file.
"""
- self._populate()
if filename is None:
streampath, filename = split(streampath)
keep_client = self._my_keep()
@@ -238,8 +256,8 @@ class CollectionReader(CollectionBase):
raise ValueError("file '{}' not found in Collection stream '{}'".
format(filename, streampath))
+ @_populate_first
def all_streams(self):
- self._populate()
return [StreamReader(s, self._my_keep(), num_retries=self.num_retries)
for s in self._streams]
@@ -248,6 +266,7 @@ class CollectionReader(CollectionBase):
for f in s.all_files():
yield f
+ @_populate_first
def manifest_text(self, strip=False, normalize=False):
if normalize:
cr = CollectionReader(self.manifest_text())
@@ -256,7 +275,6 @@ class CollectionReader(CollectionBase):
elif strip:
return self.stripped_manifest()
else:
- self._populate()
return self._manifest_text
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 254a29f..c991154 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -643,6 +643,20 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
api_client=client)
self.assertEqual('', reader.manifest_text())
+ def test_api_response(self):
+ client = self.api_client_mock()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
+ self.assertEqual(self.DEFAULT_COLLECTION, reader.api_response())
+
+ def test_api_response_with_collection_from_keep(self):
+ client = self.api_client_mock()
+ self.mock_get_collection(client, 404, 'foo')
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
+ reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
+ api_client=client)
+ api_response = reader.api_response()
+ self.assertIsNone(api_response)
+
def check_open_file(self, coll_file, stream_name, file_name, file_size):
self.assertFalse(coll_file.closed, "returned file is not open")
self.assertEqual(stream_name, coll_file.stream_name())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list