[ARVADOS] updated: ef47dfeb871015d9da6d5e2e0a04100c2e9e8eee
git at public.curoverse.com
git at public.curoverse.com
Thu May 8 16:54:21 EDT 2014
Summary of changes:
sdk/python/MANIFEST.in | 1 +
sdk/python/run_test_server.py | 173 +++++++++++++++++++++++++++------
sdk/python/setup_header.py | 12 ++-
sdk/python/test_keep_client.py | 23 +++--
sdk/python/test_mount.py | 10 +-
sdk/python/test_pipeline_template.py | 8 ++
sdk/python/test_websockets.py | 2 +-
7 files changed, 180 insertions(+), 49 deletions(-)
create mode 100644 sdk/python/MANIFEST.in
via ef47dfeb871015d9da6d5e2e0a04100c2e9e8eee (commit)
via 383aa9f906c32876a71599024084283627070adb (commit)
from f990e6880050eafaf3b6bfd063e36aa016e8682a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ef47dfeb871015d9da6d5e2e0a04100c2e9e8eee
Merge: 383aa9f f990e68
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu May 8 16:54:18 2014 -0400
Merge branch 'master' of git.clinicalfuture.com:arvados
# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.
commit 383aa9f906c32876a71599024084283627070adb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu May 8 16:52:45 2014 -0400
- Added MANIFEST.in to include setup_header.py in the source package. Now gets
the version number from PKG-INFO metadata if available, or git if not, which
should handle all the cases of using sdist, pip install, or install from git
checkout.
- Improved run_test_server.py and fixed Python SDK tests to use it.
diff --git a/sdk/python/MANIFEST.in b/sdk/python/MANIFEST.in
new file mode 100644
index 0000000..e03678f
--- /dev/null
+++ b/sdk/python/MANIFEST.in
@@ -0,0 +1 @@
+include setup_header.py
\ No newline at end of file
diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index b4886f0..6aceb05 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -6,67 +6,165 @@ import yaml
import sys
import argparse
import arvados.config
+import shutil
+import tempfile
ARV_API_SERVER_DIR = '../../services/api'
+KEEP_SERVER_DIR = '../../services/keep'
SERVER_PID_PATH = 'tmp/pids/server.pid'
WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger.3001.pid'
-def find_server_pid(PID_PATH):
- timeout = time.time() + 10
+def find_server_pid(PID_PATH, wait=10):
+ now = time.time()
+ timeout = now + wait
good_pid = False
- while (not good_pid) and (time.time() < timeout):
+ while (not good_pid) and (now <= timeout):
time.sleep(0.2)
try:
with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
good_pid = (os.kill(server_pid, 0) == None)
- except Exception:
+ except IOError:
good_pid = False
+ except OSError:
+ good_pid = False
+ now = time.time()
if not good_pid:
- raise Exception("could not find API server Rails pid")
+ return None
return server_pid
-def run(websockets=False):
+def kill_server_pid(PID_PATH, wait=10):
+ try:
+ now = time.time()
+ timeout = now + wait
+ while now <= timeout:
+ with open(PID_PATH, 'r') as f:
+ server_pid = int(f.read())
+ os.kill(server_pid, signal.SIGTERM) == None
+ now = time.time()
+ except IOError:
+ good_pid = False
+ except OSError:
+ good_pid = False
+
+def run(websockets=False, reuse_server=False):
cwd = os.getcwd()
- os.chdir(ARV_API_SERVER_DIR)
- os.environ["RAILS_ENV"] = "test"
- subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
- subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
+ os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+
if websockets:
- os.environ["ARVADOS_WEBSOCKETS"] = "true"
- subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
- '-out', './self-signed.pem',
- '-keyout', './self-signed.key',
- '-days', '3650',
- '-subj', '/CN=localhost'])
- subprocess.call(['passenger', 'start', '-d', '-p3001', '--ssl',
- '--ssl-certificate', 'self-signed.pem',
- '--ssl-certificate-key', 'self-signed.key'])
- find_server_pid(WEBSOCKETS_SERVER_PID_PATH)
+ pid_file = WEBSOCKETS_SERVER_PID_PATH
else:
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
- find_server_pid(SERVER_PID_PATH)
+ pid_file = SERVER_PID_PATH
+
+ test_pid = find_server_pid(pid_file, 0)
+
+ if test_pid == None or not reuse_server:
+ if test_pid != None:
+ stop()
+
+ # delete cached discovery document
+ shutil.rmtree(os.path.join("~", ".cache", "arvados", "discovery"), True)
+
+ # Setup database
+ os.environ["RAILS_ENV"] = "test"
+ subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
+ subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
+
+ if websockets:
+ os.environ["ARVADOS_WEBSOCKETS"] = "true"
+ subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
+ '-out', './self-signed.pem',
+ '-keyout', './self-signed.key',
+ '-days', '3650',
+ '-subj', '/CN=localhost'])
+ subprocess.call(['passenger', 'start', '-d', '-p3001', '--ssl',
+ '--ssl-certificate', 'self-signed.pem',
+ '--ssl-certificate-key', 'self-signed.key'])
+ else:
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d', '-p3001'])
+
+ pid = find_server_pid(SERVER_PID_PATH)
+
#os.environ["ARVADOS_API_HOST"] = "localhost:3001"
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
+ os.environ["ARVADOS_API_TOKEN"] = ""
os.chdir(cwd)
-def stop(websockets=False):
+def stop():
cwd = os.getcwd()
- os.chdir(ARV_API_SERVER_DIR)
- if websockets:
- os.kill(find_server_pid(WEBSOCKETS_SERVER_PID_PATH), signal.SIGTERM)
+ os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+
+ kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
+ kill_server_pid(SERVER_PID_PATH, 0)
+
+ try:
os.unlink('self-signed.pem')
+ except:
+ pass
+
+ try:
os.unlink('self-signed.key')
- else:
- os.kill(find_server_pid(SERVER_PID_PATH), signal.SIGTERM)
+ except:
+ pass
+
+ os.chdir(cwd)
+
+def _start_keep(n):
+ keep0 = tempfile.mkdtemp()
+ kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])
+ with open("tmp/keep{}.pid".format(n), 'w') as f:
+ f.write(str(kp0.pid))
+ with open("tmp/keep{}.volume".format(n), 'w') as f:
+ f.write(keep0)
+
+def run_keep():
+ stop_keep()
+
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ os.environ["GOPATH"] = os.getcwd()
+
+ shutil.rmtree("tmp", True)
+ os.mkdir("tmp")
+
+ keep1 = tempfile.mkdtemp()
+ subprocess.call(["go", "install", "keep"])
+ _start_keep(0)
+ _start_keep(1)
+
+ authorize_with("admin")
+ api = arvados.api('v1')
+ a = api.keep_disks().list().execute()
+ for d in api.keep_disks().list().execute()['items']:
+ api.keep_disks().delete(uuid=d['uuid']).execute()
+
+ api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25107} }).execute()
+ api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25108} }).execute()
+
+ os.chdir(cwd)
+
+def stop_keep():
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+
+ kill_server_pid("keep0.pid", 0)
+ kill_server_pid("keep1.pid", 0)
+
+ with open("keep0.volume", 'r') as r:
+ shutil.rmtree(r.read(), True)
+ with open("keep1.volume", 'r') as r:
+ shutil.rmtree(r.read(), True)
+
+ shutil.rmtree("tmp", True)
+
os.chdir(cwd)
def fixture(fix):
'''load a fixture yaml file'''
- with open(os.path.join(ARV_API_SERVER_DIR, "test", "fixtures",
+ with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
fix + ".yml")) as f:
return yaml.load(f.read())
@@ -76,11 +174,22 @@ def authorize_with(token):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
- parser.add_argument('action', type=str, help='''one of "start" or "stop"''')
+ parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
parser.add_argument('--websockets', action='store_true', default=False)
+ parser.add_argument('--reuse', action='store_true', default=False)
+ parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
args = parser.parse_args()
if args.action == 'start':
- run(args.websockets)
+ run(args.websockets, args.reuse)
+ if args.auth != None:
+ authorize_with(args.auth)
+ print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
+ print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
+ print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
elif args.action == 'stop':
- stop(args.websockets)
+ stop()
+ elif args.action == 'start_keep':
+ run_keep()
+ elif args.action == 'stop_keep':
+ stop_keep()
diff --git a/sdk/python/setup_header.py b/sdk/python/setup_header.py
index c97f94d..21b9500 100644
--- a/sdk/python/setup_header.py
+++ b/sdk/python/setup_header.py
@@ -1,9 +1,17 @@
import shutil
import os
import sys
+import re
-with os.popen("git log --format=format:%ct.%h -n1 .") as m:
- minor_version=m.read()
+if os.path.exists("PKG-INFO"):
+ with open("PKG-INFO", "r") as p:
+ for l in p:
+ g = re.match(r'Version: (.*)', l)
+ if g != None:
+ minor_version = g.group(1)
+else:
+ with os.popen("git log --format=format:%ct.%h -n1 .") as m:
+ minor_version = m.read()
# setup.py and setup_fuse.py both share the build/ directory (argh!) so
# make sure to delete it to avoid scooping up the wrong files.
diff --git a/sdk/python/test_keep_client.py b/sdk/python/test_keep_client.py
index 23fd582..aa79b0d 100644
--- a/sdk/python/test_keep_client.py
+++ b/sdk/python/test_keep_client.py
@@ -5,16 +5,24 @@
import unittest
import arvados
import os
+import run_test_server
class KeepTestCase(unittest.TestCase):
- def setUp(self):
+ @classmethod
+ def setUpClass(cls):
try:
del os.environ['KEEP_LOCAL_STORE']
except KeyError:
pass
+ run_test_server.run()
+ run_test_server.run_keep()
-class KeepBasicRWTest(KeepTestCase):
- def runTest(self):
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop()
+ run_test_server.stop_keep()
+
+ def test_KeepBasicRWTest(self):
foo_locator = arvados.Keep.put('foo')
self.assertEqual(foo_locator,
'acbd18db4cc2f85cedef654fccc4a4d8+3',
@@ -23,8 +31,7 @@ class KeepBasicRWTest(KeepTestCase):
'foo',
'wrong content from Keep.get(md5("foo"))')
-class KeepBinaryRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepBinaryRWTest(self):
blob_str = '\xff\xfe\xf7\x00\x01\x02'
blob_locator = arvados.Keep.put(blob_str)
self.assertEqual(blob_locator,
@@ -35,8 +42,7 @@ class KeepBinaryRWTest(KeepTestCase):
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
-class KeepLongBinaryRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepLongBinaryRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
for i in range(0,23):
blob_str = blob_str + blob_str
@@ -49,8 +55,7 @@ class KeepLongBinaryRWTest(KeepTestCase):
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
-class KeepSingleCopyRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepSingleCopyRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
blob_locator = arvados.Keep.put(blob_str, copies=1)
self.assertEqual(blob_locator,
diff --git a/sdk/python/test_mount.py b/sdk/python/test_mount.py
index a462799..f2a84cc 100644
--- a/sdk/python/test_mount.py
+++ b/sdk/python/test_mount.py
@@ -218,7 +218,7 @@ class FuseTagsUpdateTestBase(MountTestBase):
d1 = os.listdir(self.mounttmp)
d1.sort()
- self.assertEqual(d1, ['foo_tag'])
+ self.assertEqual(['foo_tag'], d1)
api.links().create(body={'link': {
'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
@@ -266,7 +266,7 @@ class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
self.runRealTest()
def tearDown(self):
- run_test_server.stop(True)
+ run_test_server.stop()
super(FuseTagsUpdateTestWebsockets, self).tearDown()
@@ -279,7 +279,7 @@ class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
self.runRealTest()
def tearDown(self):
- run_test_server.stop(False)
+ run_test_server.stop()
super(FuseTagsUpdateTestPoll, self).tearDown()
@@ -312,14 +312,14 @@ class FuseGroupsTest(MountTestBase):
"I'm a template in a folder",
"zzzzz-j58dm-5gid26432uujf79",
"zzzzz-j58dm-7r18rnd5nzhg5yk",
+ "zzzzz-j58dm-ypsjlol9dofwijz",
"zzzzz-j7d0g-axqo7eu9pwvna1x"
], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
d3.sort()
self.assertEqual(["I'm in a subfolder, too",
- "zzzzz-j58dm-c40lddwcqqr1ffs",
- "zzzzz-o0j2j-ryhm1bn83ni03sn"
+ "zzzzz-j58dm-c40lddwcqqr1ffs"
], d3)
with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a folder")) as f:
diff --git a/sdk/python/test_pipeline_template.py b/sdk/python/test_pipeline_template.py
index d7cb105..bc82271 100644
--- a/sdk/python/test_pipeline_template.py
+++ b/sdk/python/test_pipeline_template.py
@@ -5,9 +5,14 @@
import unittest
import arvados
import apiclient
+import run_test_server
class PipelineTemplateTest(unittest.TestCase):
+ def setUp(self):
+ run_test_server.run()
+
def runTest(self):
+ run_test_server.authorize_with("admin")
pt_uuid = arvados.api('v1').pipeline_templates().create(
body={'name':__file__}
).execute()['uuid']
@@ -50,3 +55,6 @@ class PipelineTemplateTest(unittest.TestCase):
geterror_response = arvados.api('v1').pipeline_templates().get(
uuid=pt_uuid
).execute()
+
+ def tearDown(self):
+ run_test_server.stop()
diff --git a/sdk/python/test_websockets.py b/sdk/python/test_websockets.py
index b9f6502..92974ca 100644
--- a/sdk/python/test_websockets.py
+++ b/sdk/python/test_websockets.py
@@ -29,4 +29,4 @@ class WebsocketTest(unittest.TestCase):
time.sleep(1)
def tearDown(self):
- run_test_server.stop(True)
+ run_test_server.stop()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list