[ARVADOS] updated: 0cc93f8d08a343e61575eace0e76e1a243412504
git at public.curoverse.com
git at public.curoverse.com
Wed Dec 3 11:11:08 EST 2014
Summary of changes:
sdk/python/arvados/collection.py | 36 ++++++++++----
sdk/python/tests/test_collections.py | 14 ++++++
services/fuse/arvados_fuse/__init__.py | 49 ++++++++++---------
services/fuse/setup.py | 2 +-
services/fuse/tests/test_mount.py | 86 ++++++++++++++--------------------
5 files changed, 100 insertions(+), 87 deletions(-)
via 0cc93f8d08a343e61575eace0e76e1a243412504 (commit)
via aacc78d5403686ea326eda6bae3896f9725f9506 (commit)
via 19e02c2bbb7f9d8db25d0b670124e29a08145843 (commit)
via 39b166626a00b68780526ae4607bbbcf34d24a04 (commit)
via e7439285d7a7b8d5deff066264b1207c311331d8 (commit)
from 8f5e45c6617beb7151b6f10779713290c4d05c1a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0cc93f8d08a343e61575eace0e76e1a243412504
Merge: 8f5e45c aacc78d
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 11:10:23 2014 -0500
Merge branch '4705-fuse-collections-from-keep-wip'
Closes #4705, #4708.
commit aacc78d5403686ea326eda6bae3896f9725f9506
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 10:07:26 2014 -0500
4705: FUSE driver can read Collections that only exist in Keep.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 65f6e8b..80ad6b3 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -304,18 +304,14 @@ class CollectionDirectory(Directory):
def same(self, i):
return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
- def new_collection(self, new_collection_object):
+ def new_collection(self, new_collection_object, coll_reader):
self.collection_object = new_collection_object
if self.collection_object_file is not None:
self.collection_object_file.update(self.collection_object)
self.clear()
- collection = arvados.CollectionReader(
- self.collection_object["manifest_text"], self.api,
- self.api.localkeep(), num_retries=self.num_retries)
- collection.normalize()
- for s in collection.all_streams():
+ for s in coll_reader.all_streams():
cwd = self
for part in s.name().split('/'):
if part != '' and part != '.':
@@ -332,23 +328,28 @@ class CollectionDirectory(Directory):
return True
with llfuse.lock_released:
- new_collection_object = self.api.collections().get(
- uuid=self.collection_locator
- ).execute(num_retries=self.num_retries)
+ coll_reader = arvados.CollectionReader(
+ self.collection_locator, self.api, self.api.localkeep(),
+ num_retries=self.num_retries)
+ new_collection_object = coll_reader.api_response() or {}
+ # If the Collection only exists in Keep, there will be no API
+ # response. Fill in the fields we need.
+ if 'uuid' not in new_collection_object:
+ new_collection_object['uuid'] = self.collection_locator
if "portable_data_hash" not in new_collection_object:
new_collection_object["portable_data_hash"] = new_collection_object["uuid"]
+ if 'manifest_text' not in new_collection_object:
+ new_collection_object['manifest_text'] = coll_reader.manifest_text()
+ coll_reader.normalize()
# end with llfuse.lock_released, re-acquire lock
if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]:
- self.new_collection(new_collection_object)
+ self.new_collection(new_collection_object, coll_reader)
self.fresh()
return True
- except apiclient.errors.HttpError as e:
- if e.resp.status == 404:
- _logger.warn("arv-mount %s: not found", self.collection_locator)
- else:
- _logger.exception("arv-mount %s: error", self.collection_locator)
+ except apiclient.errors.NotFoundError:
+ _logger.exception("arv-mount %s: error", self.collection_locator)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index a1b1dca..c9ea98c 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -42,7 +42,7 @@ setup(name='arvados_fuse',
'bin/arv-mount'
],
install_requires=[
- 'arvados-python-client>=0.1.20141103223015.68dae83',
+ 'arvados-python-client>=0.1.20141203150737.277b3c7',
'llfuse',
'python-daemon'
],
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 9c1d63b..84dceee 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -122,6 +122,24 @@ class FuseMountTest(MountTestBase):
self.assertEqual(v, f.read())
+class FuseNoAPITest(MountTestBase):
+ def setUp(self):
+ super(FuseNoAPITest, self).setUp()
+ keep = arvados.keep.KeepClient(local_store=self.keeptmp)
+ self.file_data = "API-free text\n"
+ self.file_loc = keep.put(self.file_data)
+ self.coll_loc = keep.put(". {} 0:{}:api-free.txt\n".format(
+ self.file_loc, len(self.file_data)))
+
+ def runTest(self):
+ self.make_mount(fuse.MagicDirectory)
+ self.assertDirContents(self.coll_loc, ['api-free.txt'])
+ with open(os.path.join(
+ self.mounttmp, self.coll_loc, 'api-free.txt')) as keep_file:
+ actual = keep_file.read(-1)
+ self.assertEqual(self.file_data, actual)
+
+
class FuseMagicTest(MountTestBase):
def setUp(self):
super(FuseMagicTest, self).setUp()
commit 19e02c2bbb7f9d8db25d0b670124e29a08145843
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 10:12:40 2014 -0500
4705: Fix FUSE exception logging.
logger.exception() doesn't take the exception as an argument, it takes
a message like all the other logger methods. It gets the exception
information from sys.exc_info().
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b2a3b2e..65f6e8b 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -348,17 +348,15 @@ class CollectionDirectory(Directory):
if e.resp.status == 404:
_logger.warn("arv-mount %s: not found", self.collection_locator)
else:
- _logger.error("arv-mount %s: error", self.collection_locator)
- _logger.exception(detail)
+ _logger.exception("arv-mount %s: error", self.collection_locator)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- except Exception as detail:
- _logger.error("arv-mount %s: error", self.collection_locator)
+ except Exception:
+ _logger.exception("arv-mount %s: error", self.collection_locator)
if self.collection_object is not None and "manifest_text" in self.collection_object:
_logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"])
- _logger.exception(detail)
return False
def __getitem__(self, item):
@@ -455,8 +453,8 @@ class RecursiveInvalidateDirectory(Directory):
super(RecursiveInvalidateDirectory, self).invalidate()
for a in self._entries:
self._entries[a].invalidate()
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
finally:
if self.inode == llfuse.ROOT_INODE:
llfuse.lock.release()
@@ -674,8 +672,8 @@ class SharedDirectory(Directory):
lambda i: i[0],
lambda a, i: a.uuid == i[1]['uuid'],
lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
class FileHandle(object):
@@ -839,8 +837,8 @@ class Operations(llfuse.Operations):
except arvados.errors.NotFoundError as e:
_logger.warning("Block not found: " + str(e))
raise llfuse.FUSEError(errno.EIO)
- except Exception as e:
- _logger.exception(e)
+ except Exception:
+ _logger.exception()
raise llfuse.FUSEError(errno.EIO)
def release(self, fh):
commit 39b166626a00b68780526ae4607bbbcf34d24a04
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 08:59:22 2014 -0500
4705: Refactor mount creation in FUSE tests.
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index f0f375c..9c1d63b 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -21,6 +21,15 @@ class MountTestBase(unittest.TestCase):
run_test_server.authorize_with("admin")
self.api = api = fuse.SafeApi(arvados.config)
+ def make_mount(self, root_class, *root_args):
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ operations.inodes.add_entry(root_class(
+ llfuse.ROOT_INODE, operations.inodes, self.api, 0, *root_args))
+ llfuse.init(operations, self.mounttmp, [])
+ threading.Thread(None, llfuse.main).start()
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
def tearDown(self):
run_test_server.stop()
@@ -87,18 +96,8 @@ class FuseMountTest(MountTestBase):
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.testcollection))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.CollectionDirectory, self.testcollection)
- # now check some stuff
self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
'edgecases', 'dir1', 'dir2'])
self.assertDirContents('dir1', ['thing3.txt', 'thing4.txt'])
@@ -136,18 +135,8 @@ class FuseMagicTest(MountTestBase):
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
+ self.make_mount(fuse.MagicDirectory)
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- # now check some stuff
mount_ls = os.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
@@ -173,15 +162,7 @@ class FuseMagicTest(MountTestBase):
class FuseTagsTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.TagsDirectory)
d1 = os.listdir(self.mounttmp)
d1.sort()
@@ -234,15 +215,8 @@ class FuseTagsUpdateTest(MountTestBase):
class FuseSharedTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.SharedDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.api.users().current().execute()['uuid']))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.SharedDirectory,
+ self.api.users().current().execute()['uuid'])
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
@@ -284,26 +258,16 @@ class FuseSharedTest(MountTestBase):
class FuseHomeTest(MountTestBase):
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.ProjectDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, self.api.users().current().execute()))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
+ self.make_mount(fuse.ProjectDirectory,
+ self.api.users().current().execute())
d1 = os.listdir(self.mounttmp)
- d1.sort()
self.assertIn('Unrestricted public data', d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
- d2.sort()
self.assertEqual(['GNU General Public License, version 3'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
- d3.sort()
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
commit e7439285d7a7b8d5deff066264b1207c311331d8
Author: Brett Smith <brett at curoverse.com>
Date: Wed Dec 3 09:45:44 2014 -0500
4705: Add CollectionReader.api_response() to PySDK.
The FUSE driver wants this information.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 0f49438..d530f58 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,3 +1,4 @@
+import functools
import logging
import os
import re
@@ -124,6 +125,7 @@ class CollectionReader(CollectionBase):
else:
raise errors.ArgumentError(
"Argument to CollectionReader must be a manifest or a collection UUID")
+ self._api_response = None
self._streams = None
def _populate_from_api_server(self):
@@ -138,10 +140,10 @@ class CollectionReader(CollectionBase):
if self._api_client is None:
self._api_client = arvados.api('v1')
self._keep_client = None # Make a new one with the new api.
- c = self._api_client.collections().get(
+ self._api_response = self._api_client.collections().get(
uuid=self._manifest_locator).execute(
num_retries=self.num_retries)
- self._manifest_text = c['manifest_text']
+ self._manifest_text = self._api_response['manifest_text']
return None
except Exception as e:
return e
@@ -158,8 +160,6 @@ class CollectionReader(CollectionBase):
return e
def _populate(self):
- if self._streams is not None:
- return
error_via_api = None
error_via_keep = None
should_try_keep = ((self._manifest_text is None) and
@@ -190,9 +190,27 @@ class CollectionReader(CollectionBase):
for sline in self._manifest_text.split("\n")
if sline]
- def normalize(self):
- self._populate()
+ def _populate_first(orig_func):
+ # Decorator for methods that read actual Collection data.
+ @functools.wraps(orig_func)
+ def wrapper(self, *args, **kwargs):
+ if self._streams is None:
+ self._populate()
+ return orig_func(self, *args, **kwargs)
+ return wrapper
+
+ @_populate_first
+ def api_response(self):
+ """api_response() -> dict or None
+
+ Returns information about this Collection fetched from the API server.
+ If the Collection exists in Keep but not the API server, currently
+ returns None. Future versions may provide a synthetic response.
+ """
+ return self._api_response
+ @_populate_first
+ def normalize(self):
# Rearrange streams
streams = {}
for s in self.all_streams():
@@ -213,6 +231,7 @@ class CollectionReader(CollectionBase):
[StreamReader(stream, keep=self._my_keep()).manifest_text()
for stream in self._streams])
+ @_populate_first
def open(self, streampath, filename=None):
"""open(streampath[, filename]) -> file-like object
@@ -220,7 +239,6 @@ class CollectionReader(CollectionBase):
single string or as two separate stream name and file name arguments.
This method returns a file-like object to read that file.
"""
- self._populate()
if filename is None:
streampath, filename = split(streampath)
keep_client = self._my_keep()
@@ -238,8 +256,8 @@ class CollectionReader(CollectionBase):
raise ValueError("file '{}' not found in Collection stream '{}'".
format(filename, streampath))
+ @_populate_first
def all_streams(self):
- self._populate()
return [StreamReader(s, self._my_keep(), num_retries=self.num_retries)
for s in self._streams]
@@ -248,6 +266,7 @@ class CollectionReader(CollectionBase):
for f in s.all_files():
yield f
+ @_populate_first
def manifest_text(self, strip=False, normalize=False):
if normalize:
cr = CollectionReader(self.manifest_text())
@@ -256,7 +275,6 @@ class CollectionReader(CollectionBase):
elif strip:
return self.stripped_manifest()
else:
- self._populate()
return self._manifest_text
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 254a29f..c991154 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -643,6 +643,20 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
api_client=client)
self.assertEqual('', reader.manifest_text())
+ def test_api_response(self):
+ client = self.api_client_mock()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
+ self.assertEqual(self.DEFAULT_COLLECTION, reader.api_response())
+
+ def test_api_response_with_collection_from_keep(self):
+ client = self.api_client_mock()
+ self.mock_get_collection(client, 404, 'foo')
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
+ reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
+ api_client=client)
+ api_response = reader.api_response()
+ self.assertIsNone(api_response)
+
def check_open_file(self, coll_file, stream_name, file_name, file_size):
self.assertFalse(coll_file.closed, "returned file is not open")
self.assertEqual(stream_name, coll_file.stream_name())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list