[ARVADOS] updated: b5f193aa657134822d4df67cabc25c631926395b
git at public.curoverse.com
git at public.curoverse.com
Mon May 5 22:43:03 EDT 2014
Summary of changes:
.../api_client_authorizations_controller.rb | 2 +-
apps/workbench/app/models/arvados_api_client.rb | 12 ++-
apps/workbench/app/models/arvados_resource_list.rb | 36 +++++-
apps/workbench/app/models/group.rb | 10 +-
apps/workbench/app/models/user.rb | 5 -
.../test/unit/arvados_resource_list_test.rb | 55 ++++++++
apps/workbench/test/unit/group_test.rb | 29 ++++-
apps/workbench/test/unit/user_test.rb | 12 --
doc/api/methods/groups.html.textile.liquid | 26 ++--
doc/api/methods/users.html.textile.liquid | 12 --
doc/install/client.html.textile.liquid | 43 +------
doc/sdk/python/sdk-python.html.textile.liquid | 4 +
sdk/python/arvados/fuse/__init__.py | 139 ++++++++++++++++++--
sdk/python/bin/arv-mount | 6 +-
sdk/python/requirements.txt | 1 +
services/api/Gemfile | 3 +
services/api/Gemfile.lock | 4 +
.../api/app/controllers/application_controller.rb | 32 ++---
services/api/app/models/arvados_model.rb | 104 ++++++++++-----
services/api/app/models/link.rb | 11 ++
services/api/app/models/node.rb | 2 +-
services/api/config/routes.rb | 3 +-
...0140501165548_add_unique_name_index_to_links.rb | 13 ++
services/api/db/schema.rb | 6 +-
services/api/lib/current_api_client.rb | 4 +-
services/api/lib/load_param.rb | 2 +-
services/api/test/fixtures/api_clients.yml | 2 +
services/api/test/fixtures/groups.yml | 22 +++-
services/api/test/fixtures/links.yml | 52 +++++++-
services/api/test/fixtures/repositories.yml | 4 +-
services/api/test/fixtures/specimens.yml | 12 +-
.../arvados/v1/groups_controller_test.rb | 92 ++++++++++++-
.../functional/arvados/v1/links_controller_test.rb | 13 ++
.../functional/arvados/v1/users_controller_test.rb | 76 -----------
services/api/test/unit/group_test.rb | 61 ++++++++-
services/api/test/unit/link_test.rb | 47 ++++++-
36 files changed, 693 insertions(+), 264 deletions(-)
create mode 100644 apps/workbench/test/unit/arvados_resource_list_test.rb
create mode 100644 services/api/db/migrate/20140501165548_add_unique_name_index_to_links.rb
via b5f193aa657134822d4df67cabc25c631926395b (commit)
via ff1e29f702d4f3036f5af014ad6e423b886ed0bb (commit)
via f8ba79b88683be984913e28677c4522ca21019aa (commit)
via 66f1ae7a9be590461592199c6d3646dbc4eba786 (commit)
via 4bd14ccfa216fe1d115772508ac7d7c64ec59a48 (commit)
via 06c00e72091d7c058cbc607ed863a9444d26156c (commit)
via 7fc412308bcb4d853e6f9aeb9e7d45e5ab76b11f (commit)
via 58242491cbdb85f19342a2ca04b499b9bed98a2d (commit)
via 5845d196105cc4676847695833b7ef3658c8a180 (commit)
via d8de76141cbe9a9af10dfc408b6d656cc727bbe5 (commit)
via 6b21bd450e8b8ed968a965290e641a75660cd695 (commit)
via 8370640cef5110b65ba6c3a2bf9d7f56541b4c7f (commit)
via e76d4b556f07bbeab0998ca0332dad187bb119ee (commit)
via 5af51b564ca9d805f8e61b4bb3c37ed813b3bd16 (commit)
via 546687a7c95109a57efdf7cecbc3f795bc3e72f1 (commit)
via 76e20e694fbe708e17e57a2e0b1c36aca6e8d7d0 (commit)
via 4e05647f7a5b3971771c5a928634c6b2a41aa591 (commit)
via 10d03fd3b09ec9f2eaea62cc665a1022f3744b89 (commit)
via 3ea6744bcf9cc4812ab24913286c3bc8e123b76a (commit)
via af3a38d81374f70d15134813563b7371e8e4127c (commit)
via bcc477ed88c04b3200a70d16f1fc99da11c670f8 (commit)
via 2bb2789ff129b4112d81ba343fadfb7ad362ecd1 (commit)
via 1c9ae7e8c8d7920602250958333751b769fb3a72 (commit)
via b93c1cd880d8367d11bd3fd7574e55f4545a34c5 (commit)
via ccce5e8ad71ed2e50ecd57c0f73f1aaafc468539 (commit)
via 70fddd1f7cacaa0db92b3b76a0baeeddf6f0d9b6 (commit)
via 83dcc2c95eeb38c6b0f6f7509338aa729f14d762 (commit)
via 3bd5f6c3f353059c15eeea51b16b2fdee694d504 (commit)
via 6a53a3c5d97d33586567f8df082a725f7d229327 (commit)
via da67c81ff6cc5fd540c0725de1c8208c2d8933be (commit)
via 9cfbcdf804974812b1a4ec514f5dc7902fce6137 (commit)
via 34350a8b802a8c48b534673a712614d36a5b97ac (commit)
via d649a716392760cd394e18a628dc23aaec5fa3b3 (commit)
via 50b746d9246c19c1ad2cf506bb18a0eb8ddd0755 (commit)
via 520d80f58ab4358dfce0233fe6880794c819760c (commit)
via f8cc86219281026b2867c543524f8e7fa23da291 (commit)
from cbf0429a8a5edd2f8a8d9b01ce39334e07e56bf8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b5f193aa657134822d4df67cabc25c631926395b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 22:42:55 2014 -0400
Adding support for groups and viewing arvados objects as files (serialized
json). Work in progress.
diff --git a/sdk/python/arvados/fuse/__init__.py b/sdk/python/arvados/fuse/__init__.py
index cc35d58..da1582c 100644
--- a/sdk/python/arvados/fuse/__init__.py
+++ b/sdk/python/arvados/fuse/__init__.py
@@ -12,10 +12,53 @@ import threading
import arvados
import pprint
import arvados.events
+import re
from time import time
from llfuse import FUSEError
+class File(object):
+ '''Wraps a StreamFileReader for use by Directory.'''
+
+ def __init__(self, parent_inode):
+ self.inode = None
+ self.parent_inode = parent_inode
+
+ def size(self):
+ return 0
+
+ def readfrom(self, off, size)):
+ return ''
+
+
+class StreamReaderFile(File):
+ '''Wraps a StreamFileReader as a file.'''
+
+ def __init__(self, parent_inode, reader):
+ super(StreamReaderFile, self).__init__(parent_inode, reader)
+ self.reader = reader
+
+ def size(self):
+ return self.reader.size()
+
+ def readfrom(self, off, size)):
+ return self.reader.readfrom(off, size)
+
+
+class ObjectFile(File):
+ '''Wraps a serialized object as a file.'''
+
+ def __init__(self, parent_inode, contents):
+ super(ObjectFile, self).__init__(parent_inode, reader)
+ self.contents = contents
+
+ def size(self):
+ return len(self.contents)
+
+ def readfrom(self, off, size):
+ return self.contents[off:(off+size)]
+
+
class Directory(object):
'''Generic directory object, backed by a dict.
Consists of a set of entries with the key representing the filename
@@ -99,9 +142,10 @@ class CollectionDirectory(Directory):
cwd._entries[part] = self.inodes.add_entry(Directory(cwd.inode))
cwd = cwd._entries[part]
for k, v in s.files().items():
- cwd._entries[k] = self.inodes.add_entry(File(cwd.inode, v))
+ cwd._entries[k] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v))
self.fresh()
+
class MagicDirectory(Directory):
'''A special directory that logically contains the set of all extant keep
locators. When a file is referenced by lookup(), it is tested to see if it
@@ -154,7 +198,7 @@ class TagsDirectory(Directory):
self._entries[a].invalidate()
def update(self):
- tags = self.api.links().list(filters=[['link_class', '=', 'tag']], select=['name'], distinct = 'name').execute()
+ tags = self.api.links().list(filters=[['link_class', '=', 'tag']], select=['name'], distinct = True).execute()
oldentries = self._entries
self._entries = {}
for n in tags['items']:
@@ -194,16 +238,89 @@ class TagDirectory(Directory):
self._entries[n] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, n))
self.fresh()
-class File(object):
- '''Wraps a StreamFileReader for use by Directory.'''
- def __init__(self, parent_inode, reader):
- self.inode = None
- self.parent_inode = parent_inode
- self.reader = reader
+class GroupsDirectory(Directory):
+ '''A special directory that contains as subdirectories all groups visible to the user.'''
- def size(self):
- return self.reader.size()
+ def __init__(self, parent_inode, inodes, api, poll_time=60):
+ super(GroupsDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ try:
+ arvados.events.subscribe(self.api, [], lambda ev: self.invalidate())
+ except:
+ self._poll = True
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(GroupsDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
+
+ def update(self):
+ groups = self.api.groups().list().execute()
+ oldentries = self._entries
+ self._entries = {}
+ for n in groups['items']:
+ id = n['name']
+ if id in oldentries and oldentries[id].uuid == n['uuid']:
+ self._entries[id] = oldentries[id]
+ else:
+ self._entries[id] = self.inodes.add_entry(GroupDirectory(self.inode, self.inodes, self.api,
+ n['uuid'], poll=self._poll, poll_time=self._poll_time))
+ self.fresh()
+
+
+class GroupDirectory(Directory):
+ '''A special directory that contains the contents of a group.'''
+
+ def __init__(self, parent_inode, inodes, api, uuid, poll=False, poll_time=60):
+ super(GroupDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ self.uuid = uuid
+ self._poll = poll
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(GroupDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
+
+ def createDirectory(self, parent_inode, inodes, api, uuid, poll, poll_time):
+ print uuid
+ if re.match(r'[0-9a-f]{32}\+\d+', i['uuid']):
+ return CollectionDirectory(parent_inode, inodes, i['uuid'])
+ if re.match(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}', i['uuid']):
+ return ObjectFile(parent_inode, inodes, json.dumps(i))
+ return None
+
+ def update(self):
+ contents = self.api.groups().contents(uuid=self.uuid).execute()
+ links = {}
+ for a in contents['links']:
+ links[a['head_uuid']] = a['name']
+
+ oldentries = self._entries
+ self._entries = {}
+
+ for i in contents['items']:
+ if i['uuid'] in links:
+ n = links[i['uuid']]
+ elif 'name' in i and len(i['name']) > 0:
+ n = i['name']
+ else:
+ n = i['uuid']
+
+ if n in oldentries and oldentries[n].uuid == i['uuid']:
+ self._entries[n] = oldentries[n]
+ else:
+ d = self.createDirectory(self.inode, self.inodes, self.api, i, self._poll, self._poll_time)
+ if d != None:
+ self._entries[n] = self.inodes.add_entry(d)
+ self.fresh()
class FileHandle(object):
@@ -353,7 +470,7 @@ class Operations(llfuse.Operations):
try:
with llfuse.lock_released:
- return handle.entry.reader.readfrom(off, size)
+ return handle.entry.readfrom(off, size)
except:
raise llfuse.FUSEError(errno.EIO)
diff --git a/sdk/python/bin/arv-mount b/sdk/python/bin/arv-mount
index 79b9148..849af3b 100755
--- a/sdk/python/bin/arv-mount
+++ b/sdk/python/bin/arv-mount
@@ -22,6 +22,7 @@ with "--".
parser.add_argument('--collection', type=str, help="""Mount only the specified collection at the mount point.""")
parser.add_argument('--tags', action='store_true', help="""Mount as a virtual directory consisting of subdirectories representing tagged
collections on the server.""")
+ parser.add_argument('--groups', action='store_true', help="""Mount as a virtual directory consisting of subdirectories representing groups on the server.""")
parser.add_argument('--debug', action='store_true', help="""Debug mode""")
parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
dest="exec_args", metavar=('command', 'args', '...', '--'),
@@ -32,7 +33,10 @@ collections on the server.""")
# Create the request handler
operations = Operations(os.getuid(), os.getgid())
- if args.tags:
+ if args.groups:
+ api = arvados.api('v1')
+ e = operations.inodes.add_entry(GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+ elif args.tags:
api = arvados.api('v1')
e = operations.inodes.add_entry(TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
elif args.collection != None:
commit ff1e29f702d4f3036f5af014ad6e423b886ed0bb
Merge: 66f1ae7 f8ba79b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 21:25:20 2014 -0400
Merge branch 'master' into origin-2035-arv-mount-tags-folders
commit 66f1ae7a9be590461592199c6d3646dbc4eba786
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 21:15:47 2014 -0400
Added PyYAML to requirements.txt
diff --git a/sdk/python/requirements.txt b/sdk/python/requirements.txt
index 652e3ce..1a8219b 100644
--- a/sdk/python/requirements.txt
+++ b/sdk/python/requirements.txt
@@ -4,3 +4,4 @@ python-gflags==2.0
urllib3==1.7.1
llfuse==0.40
ws4py==0.3.4
+PyYAML==3.11
commit 4bd14ccfa216fe1d115772508ac7d7c64ec59a48
Merge: cbf0429 06c00e7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 20:41:45 2014 -0400
Merge branch 'master' into origin-2035-arv-mount-tags-folders
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list