[ARVADOS] updated: 121fb6bdcdf3731903ce3d21c0a284c20e6a6fca
git at public.curoverse.com
git at public.curoverse.com
Mon May 5 16:49:23 EDT 2014
Summary of changes:
sdk/python/arvados/events.py | 33 ++++
sdk/python/arvados/fuse/__init__.py | 65 ++++++--
sdk/python/bin/arv-mount | 10 +-
sdk/python/requirements.txt | 1 +
sdk/python/run_test_server.py | 54 +++++--
sdk/python/setup.py.src | 1 +
sdk/python/test_mount.py | 175 ++++++++++++++++----
sdk/python/test_websockets.py | 32 ++++
.../controllers/arvados/v1/schema_controller.rb | 4 +-
9 files changed, 309 insertions(+), 66 deletions(-)
create mode 100644 sdk/python/arvados/events.py
create mode 100644 sdk/python/test_websockets.py
via 121fb6bdcdf3731903ce3d21c0a284c20e6a6fca (commit)
via 6abcf1f6bdb82b176a9fbb48b6c7cefa64970a9d (commit)
via 2b03b05de753509c026b4c67857cd49a11f44dcf (commit)
from 603bccd66ac868c2accf9dff9c3ca6b7feeed962 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 121fb6bdcdf3731903ce3d21c0a284c20e6a6fca
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 16:49:08 2014 -0400
Refactored freshness/updating to be more generic for different virtual
directory types. Added polling based on timeout as an alternative to
websockets. Refactored tests a bit.
diff --git a/sdk/python/arvados/fuse/__init__.py b/sdk/python/arvados/fuse/__init__.py
index c5d6a18..cc35d58 100644
--- a/sdk/python/arvados/fuse/__init__.py
+++ b/sdk/python/arvados/fuse/__init__.py
@@ -11,6 +11,7 @@ import stat
import threading
import arvados
import pprint
+import arvados.events
from time import time
from llfuse import FUSEError
@@ -28,7 +29,10 @@ class Directory(object):
raise Exception("parent_inode should be an int")
self.parent_inode = parent_inode
self._entries = {}
- self.stale = True
+ self._stale = True
+ self._poll = False
+ self._last_update = time()
+ self._poll_time = 60
# Overriden by subclasses to implement logic to update the entries dict
# when the directory is stale
@@ -37,7 +41,19 @@ class Directory(object):
# Mark the entries dict as stale
def invalidate(self):
- self.stale = True
+ self._stale = True
+
+ # Test if the entries dict is stale
+ def stale(self):
+ if self._stale:
+ return True
+ if self._poll:
+ return (self._last_update + self._poll_time) < time()
+ return False
+
+ def fresh(self):
+ self._stale = False
+ self._last_update = time()
# Only used when computing the size of the disk footprint of the directory
# (stub)
@@ -45,22 +61,22 @@ class Directory(object):
return 0
def __getitem__(self, item):
- if self.stale:
+ if self.stale():
self.update()
return self._entries[item]
def items(self):
- if self.stale:
+ if self.stale():
self.update()
return self._entries.items()
def __iter__(self):
- if self.stale:
+ if self.stale():
self.update()
return self._entries.iterkeys()
def __contains__(self, k):
- if self.stale:
+ if self.stale():
self.update()
return k in self._entries
@@ -84,8 +100,7 @@ class CollectionDirectory(Directory):
cwd = cwd._entries[part]
for k, v in s.files().items():
cwd._entries[k] = self.inodes.add_entry(File(cwd.inode, v))
- self.stale = False
-
+ self.fresh()
class MagicDirectory(Directory):
'''A special directory that logically contains the set of all extant keep
@@ -122,10 +137,21 @@ class MagicDirectory(Directory):
class TagsDirectory(Directory):
'''A special directory that contains as subdirectories all tags visible to the user.'''
- def __init__(self, parent_inode, inodes, api):
+ def __init__(self, parent_inode, inodes, api, poll_time=60):
super(TagsDirectory, self).__init__(parent_inode)
self.inodes = inodes
self.api = api
+ try:
+ arvados.events.subscribe(self.api, [['object_uuid', 'is_a', 'arvados#link']], lambda ev: self.invalidate())
+ except:
+ self._poll = True
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(TagsDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
def update(self):
tags = self.api.links().list(filters=[['link_class', '=', 'tag']], select=['name'], distinct = 'name').execute()
@@ -136,8 +162,8 @@ class TagsDirectory(Directory):
if n in oldentries:
self._entries[n] = oldentries[n]
else:
- self._entries[n] = self.inodes.add_entry(TagDirectory(self.inode, self.inodes, self.api, n))
- self.stale = False
+ self._entries[n] = self.inodes.add_entry(TagDirectory(self.inode, self.inodes, self.api, n, poll=self._poll, poll_time=self._poll_time))
+ self.fresh()
class TagDirectory(Directory):
@@ -145,11 +171,13 @@ class TagDirectory(Directory):
to the user that are tagged with a particular tag.
'''
- def __init__(self, parent_inode, inodes, api, tag):
+ def __init__(self, parent_inode, inodes, api, tag, poll=False, poll_time=60):
super(TagDirectory, self).__init__(parent_inode)
self.inodes = inodes
self.api = api
self.tag = tag
+ self._poll = poll
+ self._poll_time = poll_time
def update(self):
collections = self.api.links().list(filters=[['link_class', '=', 'tag'],
@@ -164,8 +192,7 @@ class TagDirectory(Directory):
self._entries[n] = oldentries[n]
else:
self._entries[n] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, n))
- self.stale = False
-
+ self.fresh()
class File(object):
'''Wraps a StreamFileReader for use by Directory.'''
diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index 6892459..511f2e6 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -5,6 +5,7 @@ import signal
import yaml
import sys
import argparse
+import arvados.config
ARV_API_SERVER_DIR = '../../services/api'
SERVER_PID_PATH = 'tmp/pids/server.pid'
@@ -71,7 +72,7 @@ def fixture(fix):
def authorize_with(token):
'''token is the symbolic name of the token from the api_client_authorizations fixture'''
- os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
+ arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
if __name__ == "__main__":
parser = argparse.ArgumentParser()
diff --git a/sdk/python/test_mount.py b/sdk/python/test_mount.py
index 9344ea7..9251f69 100644
--- a/sdk/python/test_mount.py
+++ b/sdk/python/test_mount.py
@@ -74,19 +74,19 @@ class FuseMountTest(MountTestBase):
# now check some stuff
d1 = os.listdir(self.mounttmp)
d1.sort()
- self.assertEqual(d1, ['dir1', 'dir2', 'thing1.txt', 'thing2.txt'])
+ self.assertEqual(['dir1', 'dir2', 'thing1.txt', 'thing2.txt'], d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'dir1'))
d2.sort()
- self.assertEqual(d2, ['thing3.txt', 'thing4.txt'])
+ self.assertEqual(['thing3.txt', 'thing4.txt'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'dir2'))
d3.sort()
- self.assertEqual(d3, ['dir3', 'thing5.txt', 'thing6.txt'])
+ self.assertEqual(['dir3', 'thing5.txt', 'thing6.txt'], d3)
d4 = os.listdir(os.path.join(self.mounttmp, 'dir2/dir3'))
d4.sort()
- self.assertEqual(d4, ['thing7.txt', 'thing8.txt'])
+ self.assertEqual(['thing7.txt', 'thing8.txt'], d4)
files = {'thing1.txt': 'data 1',
'thing2.txt': 'data 2',
@@ -99,7 +99,7 @@ class FuseMountTest(MountTestBase):
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
- self.assertEqual(f.read(), v)
+ self.assertEqual(v, f.read())
class FuseMagicTest(MountTestBase):
@@ -130,22 +130,22 @@ class FuseMagicTest(MountTestBase):
# now check some stuff
d1 = os.listdir(self.mounttmp)
d1.sort()
- self.assertEqual(d1, [])
+ self.assertEqual([], d1)
d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
d2.sort()
- self.assertEqual(d2, ['thing1.txt'])
+ self.assertEqual(['thing1.txt'], d2)
d3 = os.listdir(self.mounttmp)
d3.sort()
- self.assertEqual(d3, [self.testcollection])
+ self.assertEqual([self.testcollection], d3)
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
- self.assertEqual(f.read(), v)
+ self.assertEqual(v, f.read())
class FuseTagsTest(MountTestBase):
@@ -177,25 +177,99 @@ class FuseTagsTest(MountTestBase):
d1 = os.listdir(self.mounttmp)
d1.sort()
- self.assertEqual(d1, ['foo_tag'])
+ self.assertEqual(['foo_tag'], d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
d2.sort()
- self.assertEqual(d2, ['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'])
+ self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
d3.sort()
- self.assertEqual(d3, ['foo'])
+ self.assertEqual(['foo'], d3)
files = {}
files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
- self.assertEqual(f.read(), v)
+ self.assertEqual(v, f.read())
def tearDown(self):
run_test_server.stop()
super(FuseTagsTest, self).tearDown()
+
+class FuseTagsUpdateTestBase(MountTestBase):
+
+ def runRealTest(self):
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1')
+
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(d1, ['foo_tag'])
+
+ api.links().create(body={'link': {
+ 'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
+
+ time.sleep(1)
+
+ d2 = os.listdir(self.mounttmp)
+ d2.sort()
+ self.assertEqual(['bar_tag', 'foo_tag'], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d3.sort()
+ self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
+
+ api.links().create(body={'link': {
+ 'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
+
+ time.sleep(1)
+
+ d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d4.sort()
+ self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
+
+
+class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
+ def setUp(self):
+ super(FuseTagsUpdateTestWebsockets, self).setUp()
+ run_test_server.run(True)
+
+ def runTest(self):
+ self.runRealTest()
+
+ def tearDown(self):
+ run_test_server.stop(True)
+ super(FuseTagsUpdateTestWebsockets, self).tearDown()
+
+
+class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
+ def setUp(self):
+ super(FuseTagsUpdateTestPoll, self).setUp()
+ run_test_server.run(False)
+
+ def runTest(self):
+ self.runRealTest()
+
+ def tearDown(self):
+ run_test_server.stop(False)
+ super(FuseTagsUpdateTestPoll, self).tearDown()
commit 6abcf1f6bdb82b176a9fbb48b6c7cefa64970a9d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 15:27:31 2014 -0400
Added websockets support to Python SDK, with integration test. Also built out
runner script for running test server from python.
diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
new file mode 100644
index 0000000..e61b20c
--- /dev/null
+++ b/sdk/python/arvados/events.py
@@ -0,0 +1,33 @@
+from ws4py.client.threadedclient import WebSocketClient
+import thread
+import json
+import os
+import time
+import ssl
+import re
+import config
+
+class EventClient(WebSocketClient):
+ def __init__(self, url, filters, on_event):
+ ssl_options = None
+ if re.match(r'(?i)^(true|1|yes)$',
+ config.get('ARVADOS_API_HOST_INSECURE', 'no')):
+ ssl_options={'cert_reqs': ssl.CERT_NONE}
+ else:
+ ssl_options={'cert_reqs': ssl.CERT_REQUIRED}
+
+ super(EventClient, self).__init__(url, ssl_options)
+ self.filters = filters
+ self.on_event = on_event
+
+ def opened(self):
+ self.send(json.dumps({"method": "subscribe", "filters": self.filters}))
+
+ def received_message(self, m):
+ self.on_event(json.loads(str(m)))
+
+def subscribe(api, filters, on_event):
+ url = "{}?api_token={}".format(api._rootDesc['websocketUrl'], config.get('ARVADOS_API_TOKEN'))
+ ws = EventClient(url, filters, on_event)
+ ws.connect()
+ return ws
diff --git a/sdk/python/requirements.txt b/sdk/python/requirements.txt
index 16dcffe..652e3ce 100644
--- a/sdk/python/requirements.txt
+++ b/sdk/python/requirements.txt
@@ -3,3 +3,4 @@ httplib2==0.8
python-gflags==2.0
urllib3==1.7.1
llfuse==0.40
+ws4py==0.3.4
diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index 356f473..6892459 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -3,17 +3,20 @@ import time
import os
import signal
import yaml
+import sys
+import argparse
ARV_API_SERVER_DIR = '../../services/api'
SERVER_PID_PATH = 'tmp/pids/server.pid'
+WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger.3001.pid'
-def find_server_pid():
+def find_server_pid(PID_PATH):
timeout = time.time() + 10
good_pid = False
while (not good_pid) and (time.time() < timeout):
time.sleep(0.2)
try:
- with open(SERVER_PID_PATH, 'r') as f:
+ with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
good_pid = (server_pid > 0) and (os.kill(server_pid, 0) == None)
except:
@@ -24,22 +27,40 @@ def find_server_pid():
return server_pid
-def run():
+def run(websockets=False):
cwd = os.getcwd()
os.chdir(ARV_API_SERVER_DIR)
os.environ["RAILS_ENV"] = "test"
subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
- find_server_pid()
- os.environ["ARVADOS_API_HOST"] = "localhost:3001"
+ if websockets:
+ os.environ["ARVADOS_WEBSOCKETS"] = "true"
+ subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
+ '-out', './self-signed.pem',
+ '-keyout', './self-signed.key',
+ '-days', '3650',
+ '-subj', '/CN=localhost'])
+ subprocess.call(['passenger', 'start', '-d', '-p3001', '--ssl',
+ '--ssl-certificate', 'self-signed.pem',
+ '--ssl-certificate-key', 'self-signed.key'])
+ find_server_pid(WEBSOCKETS_SERVER_PID_PATH)
+ else:
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
+ find_server_pid(SERVER_PID_PATH)
+ #os.environ["ARVADOS_API_HOST"] = "localhost:3001"
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
os.chdir(cwd)
-def stop():
+def stop(websockets=False):
cwd = os.getcwd()
os.chdir(ARV_API_SERVER_DIR)
- os.kill(find_server_pid(), signal.SIGTERM)
+ if websockets:
+ os.kill(find_server_pid(WEBSOCKETS_SERVER_PID_PATH), signal.SIGTERM)
+ os.unlink('self-signed.pem')
+ os.unlink('self-signed.key')
+ else:
+ os.kill(find_server_pid(SERVER_PID_PATH), signal.SIGTERM)
os.chdir(cwd)
def fixture(fix):
@@ -51,3 +72,14 @@ def fixture(fix):
def authorize_with(token):
'''token is the symbolic name of the token from the api_client_authorizations fixture'''
os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument('action', type=str, help='''one of "start" or "stop"''')
+ parser.add_argument('--websockets', action='store_true', default=False)
+ args = parser.parse_args()
+
+ if args.action == 'start':
+ run(args.websockets)
+ elif args.action == 'stop':
+ stop(args.websockets)
diff --git a/sdk/python/setup.py.src b/sdk/python/setup.py.src
index 9468525..807327c 100644
--- a/sdk/python/setup.py.src
+++ b/sdk/python/setup.py.src
@@ -27,5 +27,6 @@ setup(name='arvados-python-client',
'google-api-python-client',
'httplib2',
'urllib3',
+ 'ws4py'
],
zip_safe=False)
diff --git a/sdk/python/test_websockets.py b/sdk/python/test_websockets.py
new file mode 100644
index 0000000..b9f6502
--- /dev/null
+++ b/sdk/python/test_websockets.py
@@ -0,0 +1,32 @@
+import run_test_server
+import unittest
+import arvados
+import arvados.events
+import time
+
+class WebsocketTest(unittest.TestCase):
+ def setUp(self):
+ run_test_server.run(True)
+
+ def on_event(self, ev):
+ if self.state == 1:
+ self.assertEqual(200, ev['status'])
+ self.state = 2
+ elif self.state == 2:
+ self.assertEqual(self.h[u'uuid'], ev[u'object_uuid'])
+ self.state = 3
+ elif self.state == 3:
+ self.fail()
+
+ def runTest(self):
+ self.state = 1
+
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1')
+ arvados.events.subscribe(api, [['object_uuid', 'is_a', 'arvados#human']], lambda ev: self.on_event(ev))
+ time.sleep(1)
+ self.h = api.humans().create(body={}).execute()
+ time.sleep(1)
+
+ def tearDown(self):
+ run_test_server.stop(True)
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 1db5eff..3d12b62 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -20,7 +20,7 @@ class Arvados::V1::SchemaController < ApplicationController
description: "The API to interact with Arvados.",
documentationLink: "http://doc.arvados.org/api/index.html",
protocol: "rest",
- baseUrl: root_url + "/arvados/v1/",
+ baseUrl: root_url + "arvados/v1/",
basePath: "/arvados/v1/",
rootUrl: root_url,
servicePath: "arvados/v1/",
@@ -73,7 +73,7 @@ class Arvados::V1::SchemaController < ApplicationController
if Rails.application.config.websocket_address
discovery[:websocketUrl] = Rails.application.config.websocket_address
elsif ENV['ARVADOS_WEBSOCKETS']
- discovery[:websocketUrl] = (root_url.sub /^http/, 'ws') + "/websocket"
+ discovery[:websocketUrl] = (root_url.sub /^http/, 'ws') + "websocket"
end
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
commit 2b03b05de753509c026b4c67857cd49a11f44dcf
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 5 10:28:46 2014 -0400
Finished implementing FuseTagsTest, fixed bugs in arv-mount, test passes.
diff --git a/sdk/python/arvados/fuse/__init__.py b/sdk/python/arvados/fuse/__init__.py
index b3723f0..c5d6a18 100644
--- a/sdk/python/arvados/fuse/__init__.py
+++ b/sdk/python/arvados/fuse/__init__.py
@@ -22,7 +22,10 @@ class Directory(object):
'''
def __init__(self, parent_inode):
+ '''parent_inode is the integer inode number'''
self.inode = None
+ if not isinstance(parent_inode, int):
+ raise Exception("parent_inode should be an int")
self.parent_inode = parent_inode
self._entries = {}
self.stale = True
@@ -129,10 +132,11 @@ class TagsDirectory(Directory):
oldentries = self._entries
self._entries = {}
for n in tags['items']:
+ n = n['name']
if n in oldentries:
self._entries[n] = oldentries[n]
else:
- self._entries[n] = self.inodes.add_entry(TagDirectory(self, inodes, api, n))
+ self._entries[n] = self.inodes.add_entry(TagDirectory(self.inode, self.inodes, self.api, n))
self.stale = False
@@ -155,10 +159,11 @@ class TagDirectory(Directory):
oldentries = self._entries
self._entries = {}
for c in collections['items']:
+ n = c['head_uuid']
if n in oldentries:
self._entries[n] = oldentries[n]
else:
- self._entries[n] = self.inodes.add_entry(CollectionDirectory(self, inodes, api, n['head_uuid']))
+ self._entries[n] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, n))
self.stale = False
@@ -345,7 +350,8 @@ class Operations(llfuse.Operations):
if p.parent_inode in self.inodes:
parent = self.inodes[p.parent_inode]
else:
- parent = None
+ raise llfuse.FUSEError(errno.EIO)
+
self._filehandles[fh] = FileHandle(fh, [('.', p), ('..', parent)] + list(p.items()))
return fh
diff --git a/sdk/python/bin/arv-mount b/sdk/python/bin/arv-mount
index 991c260..79b9148 100755
--- a/sdk/python/bin/arv-mount
+++ b/sdk/python/bin/arv-mount
@@ -20,7 +20,7 @@ with "--".
""")
parser.add_argument('mountpoint', type=str, help="""Mount point.""")
parser.add_argument('--collection', type=str, help="""Mount only the specified collection at the mount point.""")
- parser.add_argument('--tags', type=str, help="""Mount as a virtual directory consisting of subdirectories representing tagged
+ parser.add_argument('--tags', action='store_true', help="""Mount as a virtual directory consisting of subdirectories representing tagged
collections on the server.""")
parser.add_argument('--debug', action='store_true', help="""Debug mode""")
parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
@@ -32,10 +32,12 @@ collections on the server.""")
# Create the request handler
operations = Operations(os.getuid(), os.getgid())
- if args.collection != None:
+ if args.tags:
+ api = arvados.api('v1')
+ e = operations.inodes.add_entry(TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+ elif args.collection != None:
# Set up the request handler with the collection at the root
- e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE))
- operations.inodes.load_collection(e, arvados.CollectionReader(arvados.Keep.get(args.collection)))
+ e = operations.inodes.add_entry(CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, args.collection))
else:
# Set up the request handler with the 'magic directory' at the root
operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index 78ad061..356f473 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -22,9 +22,6 @@ def find_server_pid():
if not good_pid:
raise Exception("could not find API server Rails pid")
- os.environ["ARVADOS_API_HOST"] = "localhost:3001"
- os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
-
return server_pid
def run():
@@ -33,8 +30,10 @@ def run():
os.environ["RAILS_ENV"] = "test"
subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d'])
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
find_server_pid()
+ os.environ["ARVADOS_API_HOST"] = "localhost:3001"
+ os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
os.chdir(cwd)
def stop():
diff --git a/sdk/python/test_mount.py b/sdk/python/test_mount.py
index 253dc7b..9344ea7 100644
--- a/sdk/python/test_mount.py
+++ b/sdk/python/test_mount.py
@@ -12,10 +12,24 @@ import glob
import run_test_server
-class FuseMountTest(unittest.TestCase):
+class MountTestBase(unittest.TestCase):
def setUp(self):
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ self.mounttmp = tempfile.mkdtemp()
+
+ def tearDown(self):
+ # llfuse.close is buggy, so use fusermount instead.
+ #llfuse.close(unmount=True)
+ subprocess.call(["fusermount", "-u", self.mounttmp])
+
+ os.rmdir(self.mounttmp)
+ shutil.rmtree(self.keeptmp)
+
+
+class FuseMountTest(MountTestBase):
+ def setUp(self):
+ super(FuseMountTest, self).setUp()
cw = arvados.CollectionWriter()
@@ -48,12 +62,8 @@ class FuseMountTest(unittest.TestCase):
def runTest(self):
# Create the request handler
operations = fuse.Operations(os.getuid(), os.getgid())
- #e = operations.inodes.add_entry(fuse.Directory(llfuse.ROOT_INODE))
- #operations.inodes.load_collection(e, arvados.CollectionReader(arvados.Keep.get(self.testcollection)))
e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.testcollection))
- self.mounttmp = tempfile.mkdtemp()
-
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
t.start()
@@ -92,18 +102,9 @@ class FuseMountTest(unittest.TestCase):
self.assertEqual(f.read(), v)
- def tearDown(self):
- # llfuse.close is buggy, so use fusermount instead.
- #llfuse.close(unmount=True)
- subprocess.call(["fusermount", "-u", self.mounttmp])
-
- os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
-
-class FuseMagicTest(unittest.TestCase):
+class FuseMagicTest(MountTestBase):
def setUp(self):
- self.keeptmp = tempfile.mkdtemp()
- os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ super(FuseMagicTest, self).setUp()
cw = arvados.CollectionWriter()
@@ -147,20 +148,54 @@ class FuseMagicTest(unittest.TestCase):
self.assertEqual(f.read(), v)
- def tearDown(self):
- # llfuse.close is buggy, so use fusermount instead.
- #llfuse.close(unmount=True)
- subprocess.call(["fusermount", "-u", self.mounttmp])
+class FuseTagsTest(MountTestBase):
+ def setUp(self):
+ super(FuseTagsTest, self).setUp()
- os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
+ cw = arvados.CollectionWriter()
+
+ cw.start_new_file('foo')
+ cw.write("foo")
+
+ self.testcollection = cw.finish()
-class FuseTagsTest(unittest.TestCase):
- def setUp(self):
run_test_server.run()
def runTest(self):
run_test_server.authorize_with("admin")
+ api = arvados.api('v1')
+
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(d1, ['foo_tag'])
+
+ d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
+ d2.sort()
+ self.assertEqual(d2, ['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'])
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
+ d3.sort()
+ self.assertEqual(d3, ['foo'])
+
+ files = {}
+ files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
+
+ for k, v in files.items():
+ with open(os.path.join(self.mounttmp, k)) as f:
+ self.assertEqual(f.read(), v)
+
def tearDown(self):
run_test_server.stop()
+
+ super(FuseTagsTest, self).tearDown()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list