[ARVADOS] created: 4d716aad11806b2f0f027e8c6f7ac0ae4f5bda6a
git at public.curoverse.com
git at public.curoverse.com
Wed Aug 20 09:53:18 EDT 2014
at 4d716aad11806b2f0f027e8c6f7ac0ae4f5bda6a (commit)
commit 4d716aad11806b2f0f027e8c6f7ac0ae4f5bda6a
Author: Brett Smith <brett at curoverse.com>
Date: Wed Aug 20 09:24:22 2014 -0400
2800: Use local KeepClients in Python SDK tests.
This improves test isolation, and demonstrates the preferred way to
use the API.
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 68c9698..1300c6c 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -29,9 +29,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
def setUpClass(cls):
super(ArvadosCollectionsTest, cls).setUpClass()
run_test_server.authorize_with('active')
+ cls.api_client = arvados.api('v1')
+ cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
+ local_store=cls.local_store)
def write_foo_bar_baz(self):
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
self.assertEqual(cw.current_stream_name(), '.',
'current_stream_name() should be "." now')
cw.set_current_file_name('foo.txt')
@@ -46,8 +49,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
return cw.finish()
def test_keep_local_store(self):
- self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
- self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
+ self.assertEqual(self.keep_client.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
+ self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
def test_local_collection_writer(self):
self.assertEqual(self.write_foo_bar_baz(),
@@ -56,7 +59,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
def test_local_collection_reader(self):
self.write_foo_bar_baz()
- cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle')
+ cr = arvados.CollectionReader(
+ 'd6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle', self.api_client)
got = []
for s in cr.all_streams():
for f in s.all_files():
@@ -78,7 +82,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
'reading zero bytes should have returned empty string')
def _test_subset(self, collection, expected):
- cr = arvados.CollectionReader(collection)
+ cr = arvados.CollectionReader(collection, self.api_client)
for s in cr.all_streams():
for ex in expected:
if ex[0] == s:
@@ -95,29 +99,29 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
[3, '.', 'foo.txt', 'foo'],
[3, './baz', 'baz.txt', 'baz']])
self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
- (arvados.Keep.put("foo"),
- arvados.Keep.put("bar"))),
+ (self.keep_client.put("foo"),
+ self.keep_client.put("bar"))),
[[3, '.', 'bar.txt', 'bar'],
[3, '.', 'foo.txt', 'foo']])
self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
- (arvados.Keep.put("foo"),
- arvados.Keep.put("bar"))),
+ (self.keep_client.put("foo"),
+ self.keep_client.put("bar"))),
[[2, '.', 'fo.txt', 'fo'],
[4, '.', 'obar.txt', 'obar']])
self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
- (arvados.Keep.put("foo"),
- arvados.Keep.put("bar"))),
+ (self.keep_client.put("foo"),
+ self.keep_client.put("bar"))),
[[2, '.', 'ar.txt', 'ar'],
[2, '.', 'fo.txt', 'fo'],
[2, '.', 'ob.txt', 'ob'],
[0, '.', 'zero.txt', '']])
def _test_readline(self, what_in, what_out):
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('test.txt')
cw.write(what_in)
test1 = cw.finish()
- cr = arvados.CollectionReader(test1)
+ cr = arvados.CollectionReader(test1, self.api_client)
got = []
for x in list(cr.all_files())[0].readlines():
got += [x]
@@ -132,13 +136,13 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
["ab\n", "cd\n"])
def test_collection_empty_file(self):
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('zero.txt')
cw.write('')
self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n")
self.check_manifest_file_sizes(cw.manifest_text(), [0])
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('zero.txt')
cw.write('')
cw.start_new_file('one.txt')
@@ -149,7 +153,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
self.check_manifest_file_sizes(cw.manifest_text(), [1,0,0])
def check_manifest_file_sizes(self, manifest_text, expect_sizes):
- cr = arvados.CollectionReader(manifest_text)
+ cr = arvados.CollectionReader(manifest_text, self.api_client)
got_sizes = []
for f in cr.all_files():
got_sizes += [f.size()]
@@ -161,12 +165,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
for x in xrange(0, 18):
data_in += data_in
compressed_data_in = bz2.compress(data_in)
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('test.bz2')
cw.write(compressed_data_in)
bz2_manifest = cw.manifest_text()
- cr = arvados.CollectionReader(bz2_manifest)
+ cr = arvados.CollectionReader(bz2_manifest, self.api_client)
got = 0
for x in list(cr.all_files())[0].readlines():
@@ -188,12 +192,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
shell=False, close_fds=True)
compressed_data_in, stderrdata = p.communicate(data_in)
- cw = arvados.CollectionWriter()
+ cw = arvados.CollectionWriter(self.api_client)
cw.start_new_file('test.gz')
cw.write(compressed_data_in)
gzip_manifest = cw.manifest_text()
- cr = arvados.CollectionReader(gzip_manifest)
+ cr = arvados.CollectionReader(gzip_manifest, self.api_client)
got = 0
for x in list(cr.all_files())[0].readlines():
self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
@@ -206,25 +210,25 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
- self.assertEqual(arvados.CollectionReader(m1).manifest_text(),
+ self.assertEqual(arvados.CollectionReader(m1, self.api_client).manifest_text(),
""". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt
""")
m2 = """. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
"""
- self.assertEqual(arvados.CollectionReader(m2).manifest_text(), m2)
+ self.assertEqual(arvados.CollectionReader(m2, self.api_client).manifest_text(), m2)
m3 = """. 5348b82a029fd9e971a811ce1f71360b+43 3:40:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
- self.assertEqual(arvados.CollectionReader(m3).manifest_text(),
+ self.assertEqual(arvados.CollectionReader(m3, self.api_client).manifest_text(),
""". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 3:124:md5sum.txt
""")
m4 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
./foo 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar"""
- self.assertEqual(arvados.CollectionReader(m4).manifest_text(),
+ self.assertEqual(arvados.CollectionReader(m4, self.api_client).manifest_text(),
"""./foo 204e43b8a1185621ca55a94839582e6f+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar 67108864:3:bar
./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
""")
@@ -232,22 +236,22 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
m5 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
./foo 204e43b8a1185621ca55a94839582e6f+67108864 3:3:bar"""
- self.assertEqual(arvados.CollectionReader(m5).manifest_text(),
+ self.assertEqual(arvados.CollectionReader(m5, self.api_client).manifest_text(),
"""./foo 204e43b8a1185621ca55a94839582e6f+67108864 0:6:bar
./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
""")
with self.data_file('1000G_ref_manifest') as f6:
m6 = f6.read()
- self.assertEqual(arvados.CollectionReader(m6).manifest_text(), m6)
+ self.assertEqual(arvados.CollectionReader(m6, self.api_client).manifest_text(), m6)
with self.data_file('jlake_manifest') as f7:
m7 = f7.read()
- self.assertEqual(arvados.CollectionReader(m7).manifest_text(), m7)
+ self.assertEqual(arvados.CollectionReader(m7, self.api_client).manifest_text(), m7)
m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
"""
- self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8)
+ self.assertEqual(arvados.CollectionReader(m8, self.api_client).manifest_text(), m8)
def test_locators_and_ranges(self):
blocks2 = [['a', 10, 0],
@@ -472,22 +476,22 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt"""
- m2 = arvados.CollectionReader(m1)
+ m2 = arvados.CollectionReader(m1, self.api_client)
self.assertEqual(m2.manifest_text(),
". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
- self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md5sum.txt'].as_manifest(),
+ self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md5sum.txt'].as_manifest(),
". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
- self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md6sum.txt'].as_manifest(),
+ self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md6sum.txt'].as_manifest(),
". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
- self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md7sum.txt'].as_manifest(),
+ self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md7sum.txt'].as_manifest(),
". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
- self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md9sum.txt'].as_manifest(),
+ self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md9sum.txt'].as_manifest(),
". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
def test_write_directory_tree(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
cwriter.write_directory_tree(self.build_directory_tree(
['basefile', 'subdir/subfile']))
self.assertEqual(cwriter.manifest_text(),
@@ -495,7 +499,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
./subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
def test_write_named_directory_tree(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
cwriter.write_directory_tree(self.build_directory_tree(
['basefile', 'subdir/subfile']), 'root')
self.assertEqual(
@@ -504,7 +508,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
./root/subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
def test_write_directory_tree_in_one_stream(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
cwriter.write_directory_tree(self.build_directory_tree(
['basefile', 'subdir/subfile']), max_manifest_depth=0)
self.assertEqual(cwriter.manifest_text(),
@@ -512,7 +516,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
./subdir 4ace875ffdc6824a04950f06858f4465+22 8:14:subfile\n""")
def test_write_directory_tree_with_limited_recursion(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
cwriter.write_directory_tree(
self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
max_manifest_depth=1)
@@ -522,7 +526,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
./d1/d2 50170217e5b04312024aa5cd42934494+13 0:8:f3\n""")
def test_write_one_file(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
with self.make_test_file() as testfile:
cwriter.write_file(testfile.name)
self.assertEqual(
@@ -531,14 +535,14 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
os.path.basename(testfile.name)))
def test_write_named_file(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
with self.make_test_file() as testfile:
cwriter.write_file(testfile.name, 'foo')
self.assertEqual(cwriter.manifest_text(),
". 098f6bcd4621d373cade4e832627b4f6+4 0:4:foo\n")
def test_write_multiple_files(self):
- cwriter = arvados.CollectionWriter()
+ cwriter = arvados.CollectionWriter(self.api_client)
for letter in 'ABC':
with self.make_test_file(letter) as testfile:
cwriter.write_file(testfile.name, letter)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index fb52ab4..6198919 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -2,51 +2,42 @@
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
-import contextlib
import os
import unittest
import arvados
import run_test_server
- at contextlib.contextmanager
-def unauthenticated_client(keep_client=None):
- if keep_client is None:
- keep_client = arvados.keep.global_client_object
- if not hasattr(keep_client, 'api_token'):
- yield keep_client
- else:
- orig_token = keep_client.api_token
- keep_client.api_token = ''
- yield keep_client
- keep_client.api_token = orig_token
-
class KeepTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
KEEP_SERVER = {}
- def setUp(self):
- arvados.keep.global_client_object = None
+ @classmethod
+ def setUpClass(cls):
+ super(KeepTestCase, cls).setUpClass()
run_test_server.authorize_with("admin")
+ cls.api_client = arvados.api('v1')
+ cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
+ proxy='', local_store='')
def test_KeepBasicRWTest(self):
- foo_locator = arvados.Keep.put('foo')
+ foo_locator = self.keep_client.put('foo')
self.assertRegexpMatches(
foo_locator,
'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
- self.assertEqual(arvados.Keep.get(foo_locator),
+ self.assertEqual(self.keep_client.get(foo_locator),
'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepBinaryRWTest(self):
blob_str = '\xff\xfe\xf7\x00\x01\x02'
- blob_locator = arvados.Keep.put(blob_str)
+ blob_locator = self.keep_client.put(blob_str)
self.assertRegexpMatches(
blob_locator,
'^7fc7c53b45e53926ba52821140fef396\+6',
('wrong locator from Keep.put(<binarydata>):' + blob_locator))
- self.assertEqual(arvados.Keep.get(blob_locator),
+ self.assertEqual(self.keep_client.get(blob_locator),
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
@@ -54,23 +45,23 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
for i in range(0,23):
blob_str = blob_str + blob_str
- blob_locator = arvados.Keep.put(blob_str)
+ blob_locator = self.keep_client.put(blob_str)
self.assertRegexpMatches(
blob_locator,
'^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
- self.assertEqual(arvados.Keep.get(blob_locator),
+ self.assertEqual(self.keep_client.get(blob_locator),
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
def test_KeepSingleCopyRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
- blob_locator = arvados.Keep.put(blob_str, copies=1)
+ blob_locator = self.keep_client.put(blob_str, copies=1)
self.assertRegexpMatches(
blob_locator,
'^c902006bc98a3eb4a3663b65ab4a6fab\+8',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
- self.assertEqual(arvados.Keep.get(blob_locator),
+ self.assertEqual(self.keep_client.get(blob_locator),
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
@@ -82,24 +73,25 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
- foo_locator = arvados.Keep.put('foo')
+ keep_client = arvados.KeepClient()
+ foo_locator = keep_client.put('foo')
self.assertRegexpMatches(
foo_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
- self.assertEqual(arvados.Keep.get(foo_locator),
+ self.assertEqual(keep_client.get(foo_locator),
'foo',
'wrong content from Keep.get(md5("foo"))')
# GET with an unsigned locator => NotFound
- bar_locator = arvados.Keep.put('bar')
+ bar_locator = keep_client.put('bar')
unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
self.assertRegexpMatches(
bar_locator,
r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("bar"): ' + bar_locator)
self.assertRaises(arvados.errors.NotFoundError,
- arvados.Keep.get,
+ keep_client.get,
unsigned_bar_locator)
# GET from a different user => NotFound
@@ -110,13 +102,13 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
# Unauthenticated GET for a signed locator => NotFound
# Unauthenticated GET for an unsigned locator => NotFound
- with unauthenticated_client():
- self.assertRaises(arvados.errors.NotFoundError,
- arvados.Keep.get,
- bar_locator)
- self.assertRaises(arvados.errors.NotFoundError,
- arvados.Keep.get,
- unsigned_bar_locator)
+ keep_client.api_token = ''
+ self.assertRaises(arvados.errors.NotFoundError,
+ keep_client.get,
+ bar_locator)
+ self.assertRaises(arvados.errors.NotFoundError,
+ keep_client.get,
+ unsigned_bar_locator)
# KeepOptionalPermission: starts Keep with --permission-key-file
@@ -133,57 +125,54 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
'enforce_permissions': False}
- def test_KeepAuthenticatedSignedTest(self):
- run_test_server.authorize_with('active')
- signed_locator = arvados.Keep.put('foo')
+ @classmethod
+ def setUpClass(cls):
+ super(KeepOptionalPermission, cls).setUpClass()
+ run_test_server.authorize_with("admin")
+ cls.api_client = arvados.api('v1')
+
+ def setUp(self):
+ super(KeepOptionalPermission, self).setUp()
+ self.keep_client = arvados.KeepClient(api_client=self.api_client,
+ proxy='', local_store='')
+
+ def _put_foo_and_check(self):
+ signed_locator = self.keep_client.put('foo')
self.assertRegexpMatches(
signed_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
- self.assertEqual(arvados.Keep.get(signed_locator),
+ return signed_locator
+
+ def test_KeepAuthenticatedSignedTest(self):
+ signed_locator = self._put_foo_and_check()
+ self.assertEqual(self.keep_client.get(signed_locator),
'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepAuthenticatedUnsignedTest(self):
- run_test_server.authorize_with('active')
- signed_locator = arvados.Keep.put('foo')
- self.assertRegexpMatches(
- signed_locator,
- r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
- 'invalid locator from Keep.put("foo"): ' + signed_locator)
- self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+ signed_locator = self._put_foo_and_check()
+ self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
'foo',
'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedSignedTest(self):
- # Since --enforce-permissions is not in effect, GET requests
- # need not be authenticated.
- run_test_server.authorize_with('active')
- signed_locator = arvados.Keep.put('foo')
- self.assertRegexpMatches(
- signed_locator,
- r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
- 'invalid locator from Keep.put("foo"): ' + signed_locator)
-
- with unauthenticated_client():
- self.assertEqual(arvados.Keep.get(signed_locator),
- 'foo',
- 'wrong content from Keep.get(md5("foo"))')
+ # Check that signed GET requests work even when permissions
+ # enforcement is off.
+ signed_locator = self._put_foo_and_check()
+ self.keep_client.api_token = ''
+ self.assertEqual(self.keep_client.get(signed_locator),
+ 'foo',
+ 'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedUnsignedTest(self):
# Since --enforce-permissions is not in effect, GET requests
# need not be authenticated.
- run_test_server.authorize_with('active')
- signed_locator = arvados.Keep.put('foo')
- self.assertRegexpMatches(
- signed_locator,
- r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
- 'invalid locator from Keep.put("foo"): ' + signed_locator)
-
- with unauthenticated_client():
- self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- 'foo',
- 'wrong content from Keep.get(md5("foo"))')
+ signed_locator = self._put_foo_and_check()
+ self.keep_client.api_token = ''
+ self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+ 'foo',
+ 'wrong content from Keep.get(md5("foo"))')
class KeepProxyTestCase(run_test_server.TestCaseWithServers):
@@ -194,43 +183,39 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
@classmethod
def setUpClass(cls):
super(KeepProxyTestCase, cls).setUpClass()
- cls.proxy_addr = os.environ['ARVADOS_KEEP_PROXY']
+ cls.api_client = arvados.api('v1')
- def setUp(self):
- arvados.keep.global_client_object = None
- os.environ['ARVADOS_KEEP_PROXY'] = self.proxy_addr
- os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
+ def tearDown(self):
+ arvados.config.settings().pop('ARVADOS_EXTERNAL_CLIENT', None)
+ super(KeepProxyTestCase, self).tearDown()
def test_KeepProxyTest1(self):
# Will use ARVADOS_KEEP_PROXY environment variable that is set by
# setUpClass().
- baz_locator = arvados.Keep.put('baz')
+ keep_client = arvados.KeepClient(api_client=self.api_client,
+ local_store='')
+ baz_locator = keep_client.put('baz')
self.assertRegexpMatches(
baz_locator,
'^73feffa4b7f6bb68e44cf984c85f6e88\+3',
'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
- self.assertEqual(arvados.Keep.get(baz_locator),
+ self.assertEqual(keep_client.get(baz_locator),
'baz',
'wrong content from Keep.get(md5("baz"))')
-
- self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
+ self.assertTrue(keep_client.using_proxy)
def test_KeepProxyTest2(self):
- # We don't want to use ARVADOS_KEEP_PROXY from run_keep_proxy() in
- # setUpClass(), so clear it and set ARVADOS_EXTERNAL_CLIENT which will
- # contact the API server.
- os.environ["ARVADOS_KEEP_PROXY"] = ""
- os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
-
- # Will send X-External-Client to server and get back the proxy from
- # keep_services/accessible
- baz_locator = arvados.Keep.put('baz2')
+ # Don't instantiate the proxy directly, but set the X-External-Client
+ # header. The API server should direct us to the proxy.
+ arvados.config.settings()['ARVADOS_EXTERNAL_CLIENT'] = 'true'
+ keep_client = arvados.KeepClient(api_client=self.api_client,
+ proxy='', local_store='')
+ baz_locator = keep_client.put('baz2')
self.assertRegexpMatches(
baz_locator,
'^91f372a266fe2bf2823cb8ec7fda31ce\+4',
'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
- self.assertEqual(arvados.Keep.get(baz_locator),
+ self.assertEqual(keep_client.get(baz_locator),
'baz2',
'wrong content from Keep.get(md5("baz2"))')
-
- self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
+ self.assertTrue(keep_client.using_proxy)
commit c84954c16aa7ed2dad2525de97365006b9711a31
Author: Brett Smith <brett at curoverse.com>
Date: Wed Aug 20 09:46:49 2014 -0400
2800: Introduce TestCaseWithServers to Python SDK.
This is a subclass of unittest.TestCase. It looks for specific class
variables to launch supporting Arvados servers for subclass test
cases, taking care to adjust the environment and local Arvados
configuration to match. Put another away, this refactors a number of
similar setUpClass/tearDownClass methods throughout the SDK tests.
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index cd86d80..369d561 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -27,20 +27,6 @@ class ArvadosBaseTestCase(unittest.TestCase):
basedir = '.'
return open(os.path.join(basedir, 'data', filename))
-
-class ArvadosKeepLocalStoreTestCase(ArvadosBaseTestCase):
- def setUp(self):
- super(ArvadosKeepLocalStoreTestCase, self).setUp()
- self._orig_keep_local_store = os.environ.get('KEEP_LOCAL_STORE')
- os.environ['KEEP_LOCAL_STORE'] = self.make_tmpdir()
-
- def tearDown(self):
- if self._orig_keep_local_store is None:
- del os.environ['KEEP_LOCAL_STORE']
- else:
- os.environ['KEEP_LOCAL_STORE'] = self._orig_keep_local_store
- super(ArvadosKeepLocalStoreTestCase, self).tearDown()
-
def build_directory_tree(self, tree):
tree_root = self.make_tmpdir()
for leaf in tree:
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index a225b74..0173718 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -8,6 +8,7 @@ import subprocess
import sys
import tempfile
import time
+import unittest
import yaml
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
@@ -216,7 +217,7 @@ def run_keep_proxy(auth):
api = arvados.api('v1', cache=False)
api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute()
- arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+ os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
def stop_keep_proxy():
kill_server_pid("tests/tmp/keepproxy.pid", 0)
@@ -233,6 +234,65 @@ def authorize_with(token):
arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
+class TestCaseWithServers(unittest.TestCase):
+ """TestCase to start and stop supporting Arvados servers.
+
+ Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
+ class variables as a dictionary of keyword arguments. If you do,
+ setUpClass will start the corresponding servers by passing these
+ keyword arguments to the run, run_keep, and/or run_keep_server
+ functions, respectively. It will also set Arvados environment
+ variables to point to these servers appropriately. If you don't
+ run a Keep or Keep proxy server, setUpClass will set up a
+ temporary directory for Keep local storage, and set it as
+ KEEP_LOCAL_STORE.
+
+ tearDownClass will stop any servers started, and restore the
+ original environment.
+ """
+ MAIN_SERVER = None
+ KEEP_SERVER = None
+ KEEP_PROXY_SERVER = None
+
+ @staticmethod
+ def _restore_dict(src, dest):
+ for key in dest.keys():
+ if key not in src:
+ del dest[key]
+ dest.update(src)
+
+ @classmethod
+ def setUpClass(cls):
+ cls._orig_environ = os.environ.copy()
+ cls._orig_config = arvados.config.settings().copy()
+ cls._cleanup_funcs = []
+ for server_kwargs, start_func, stop_func in (
+ (cls.MAIN_SERVER, run, stop),
+ (cls.KEEP_SERVER, run_keep, stop_keep),
+ (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
+ if server_kwargs is not None:
+ start_func(**server_kwargs)
+ cls._cleanup_funcs.append(stop_func)
+ os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
+ if cls.KEEP_PROXY_SERVER is None:
+ os.environ.pop('ARVADOS_KEEP_PROXY', None)
+ if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
+ cls.local_store = tempfile.mkdtemp()
+ os.environ['KEEP_LOCAL_STORE'] = cls.local_store
+ cls._cleanup_funcs.append(
+ lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
+ else:
+ os.environ.pop('KEEP_LOCAL_STORE', None)
+ arvados.config.initialize()
+
+ @classmethod
+ def tearDownClass(cls):
+ for clean_func in cls._cleanup_funcs:
+ clean_func()
+ cls._restore_dict(cls._orig_environ, os.environ)
+ cls._restore_dict(cls._orig_config, arvados.config.settings())
+
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 5ce475e..de4fdd2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -18,7 +18,7 @@ from cStringIO import StringIO
import arvados
import arvados.commands.put as arv_put
-from arvados_testutil import ArvadosBaseTestCase, ArvadosKeepLocalStoreTestCase
+from arvados_testutil import ArvadosBaseTestCase
import run_test_server
class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
@@ -196,9 +196,11 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
arv_put.ResumeCache, path)
-class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
+ ArvadosBaseTestCase):
def setUp(self):
super(ArvadosPutCollectionWriterTest, self).setUp()
+ run_test_server.authorize_with('active')
with tempfile.NamedTemporaryFile(delete=False) as cachefile:
self.cache = arv_put.ResumeCache(cachefile.name)
self.cache_filename = cachefile.name
@@ -398,7 +400,9 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
['--project-uuid', self.Z_UUID, '--stream'])
-class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
+ MAIN_SERVER = {}
+
def call_main_with_args(self, args):
self.main_stdout = StringIO()
self.main_stderr = StringIO()
@@ -415,6 +419,7 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
def setUp(self):
super(ArvadosPutTest, self).setUp()
+ run_test_server.authorize_with('active')
arv_put.api_client = None
def tearDown(self):
@@ -454,41 +459,32 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
['--name', 'test without Collection',
'--stream', '/dev/null'])
-class ArvPutIntegrationTest(unittest.TestCase):
- PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
- ENVIRON = os.environ
- ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
-
- @classmethod
- def setUpClass(cls):
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
-
- # Use the blob_signing_key from the Rails "test" configuration
- # to provision the Keep server.
- config_blob_signing_key = None
+class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
+ ArvadosBaseTestCase):
+ def _getKeepServerConfig():
for config_file in ['application.yml', 'application.default.yml']:
with open(os.path.join(run_test_server.SERVICES_SRC_DIR,
"api", "config", config_file)) as f:
rails_config = yaml.load(f.read())
for config_section in ['test', 'common']:
try:
- config_blob_signing_key = rails_config[config_section]["blob_signing_key"]
- break
- except KeyError:
+ key = rails_config[config_section]["blob_signing_key"]
+ except (KeyError, TypeError):
pass
- if config_blob_signing_key is not None:
- break
- run_test_server.run()
- run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
- enforce_permissions=(config_blob_signing_key != None))
+ else:
+ return {'blob_signing_key': key,
+ 'enforce_permissions': True}
+ return {'blog_signing_key': None, 'enforce_permissions': False}
+
+ MAIN_SERVER = {}
+ KEEP_SERVER = _getKeepServerConfig()
+ PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
@classmethod
- def tearDownClass(cls):
- run_test_server.stop()
- run_test_server.stop_keep()
+ def setUpClass(cls):
+ super(ArvPutIntegrationTest, cls).setUpClass()
+ cls.ENVIRON = os.environ.copy()
+ cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
def setUp(self):
super(ArvPutIntegrationTest, self).setUp()
@@ -499,7 +495,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
+ self.ENVIRON[v] = arvados.config.settings()[v]
arv_put.api_client = arvados.api('v1', cache=False)
def current_user(self):
@@ -556,7 +552,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
notfound = arv_put.api_client.collections().get(
uuid=manifest_uuid).execute()
- datadir = tempfile.mkdtemp()
+ datadir = self.make_tmpdir()
with open(os.path.join(datadir, "foo"), "w") as f:
f.write("The quick brown fox jumped over the lazy dog")
p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
@@ -573,9 +569,6 @@ class ArvPutIntegrationTest(unittest.TestCase):
c['manifest_text'],
r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
- os.remove(os.path.join(datadir, "foo"))
- os.rmdir(datadir)
-
def run_and_find_link(self, text, extra_args=[]):
self.authorize_with('active')
pipe = subprocess.Popen(
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index b1af723..68c9698 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -11,7 +11,8 @@ import subprocess
import tempfile
import unittest
-from arvados_testutil import ArvadosKeepLocalStoreTestCase
+import run_test_server
+from arvados_testutil import ArvadosBaseTestCase
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
@@ -20,7 +21,15 @@ class TestResumableWriter(arvados.ResumableCollectionWriter):
return self.dump_state(copy.deepcopy)
-class ArvadosCollectionsTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
+ ArvadosBaseTestCase):
+ MAIN_SERVER = {}
+
+ @classmethod
+ def setUpClass(cls):
+ super(ArvadosCollectionsTest, cls).setUpClass()
+ run_test_server.authorize_with('active')
+
def write_foo_bar_baz(self):
cw = arvados.CollectionWriter()
self.assertEqual(cw.current_stream_name(), '.',
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 9d3cecd..fb52ab4 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -2,38 +2,33 @@
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
+import contextlib
import os
import unittest
import arvados
import run_test_server
-class KeepTestCase(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- super(KeepTestCase, cls).setUpClass()
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
-
- # Make sure these are clear, we want to talk to the Keep servers
- # directly.
- os.environ["ARVADOS_KEEP_PROXY"] = ""
- os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
-
- run_test_server.run()
- run_test_server.run_keep()
+ at contextlib.contextmanager
+def unauthenticated_client(keep_client=None):
+ if keep_client is None:
+ keep_client = arvados.keep.global_client_object
+ if not hasattr(keep_client, 'api_token'):
+ yield keep_client
+ else:
+ orig_token = keep_client.api_token
+ keep_client.api_token = ''
+ yield keep_client
+ keep_client.api_token = orig_token
+
+class KeepTestCase(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {}
+
+ def setUp(self):
arvados.keep.global_client_object = None
- arvados.config._settings = None
run_test_server.authorize_with("admin")
- @classmethod
- def tearDownClass(cls):
- super(KeepTestCase, cls).tearDownClass()
- run_test_server.stop()
- run_test_server.stop_keep()
-
def test_KeepBasicRWTest(self):
foo_locator = arvados.Keep.put('foo')
self.assertRegexpMatches(
@@ -79,22 +74,11 @@ class KeepTestCase(unittest.TestCase):
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
-class KeepPermissionTestCase(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
- run_test_server.run()
- run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
- enforce_permissions=True)
-
- @classmethod
- def tearDownClass(cls):
- run_test_server.stop()
- run_test_server.stop_keep()
+class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
+ 'enforce_permissions': True}
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
@@ -126,13 +110,14 @@ class KeepPermissionTestCase(unittest.TestCase):
# Unauthenticated GET for a signed locator => NotFound
# Unauthenticated GET for an unsigned locator => NotFound
- arvados.keep.global_client_object.api_token = ''
- self.assertRaises(arvados.errors.NotFoundError,
- arvados.Keep.get,
- bar_locator)
- self.assertRaises(arvados.errors.NotFoundError,
- arvados.Keep.get,
- unsigned_bar_locator)
+ with unauthenticated_client():
+ self.assertRaises(arvados.errors.NotFoundError,
+ arvados.Keep.get,
+ bar_locator)
+ self.assertRaises(arvados.errors.NotFoundError,
+ arvados.Keep.get,
+ unsigned_bar_locator)
+
# KeepOptionalPermission: starts Keep with --permission-key-file
# but not --enforce-permissions (i.e. generate signatures on PUT
@@ -143,22 +128,10 @@ class KeepPermissionTestCase(unittest.TestCase):
# * authenticated request, unsigned locator
# * unauthenticated request, signed locator
# * unauthenticated request, unsigned locator
-
-class KeepOptionalPermission(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
- run_test_server.run()
- run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
- enforce_permissions=False)
-
- @classmethod
- def tearDownClass(cls):
- run_test_server.stop()
- run_test_server.stop_keep()
+class KeepOptionalPermission(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
+ 'enforce_permissions': False}
def test_KeepAuthenticatedSignedTest(self):
run_test_server.authorize_with('active')
@@ -192,10 +165,10 @@ class KeepOptionalPermission(unittest.TestCase):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
- arvados.keep.global_client_object.api_token = ''
- self.assertEqual(arvados.Keep.get(signed_locator),
- 'foo',
- 'wrong content from Keep.get(md5("foo"))')
+ with unauthenticated_client():
+ self.assertEqual(arvados.Keep.get(signed_locator),
+ 'foo',
+ 'wrong content from Keep.get(md5("foo"))')
def test_KeepUnauthenticatedUnsignedTest(self):
# Since --enforce-permissions is not in effect, GET requests
@@ -207,48 +180,30 @@ class KeepOptionalPermission(unittest.TestCase):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
- arvados.keep.global_client_object.api_token = ''
- self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- 'foo',
- 'wrong content from Keep.get(md5("foo"))')
+ with unauthenticated_client():
+ self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+ 'foo',
+ 'wrong content from Keep.get(md5("foo"))')
-class KeepProxyTestCase(unittest.TestCase):
+class KeepProxyTestCase(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {}
+ KEEP_PROXY_SERVER = {'auth': 'admin'}
+
@classmethod
def setUpClass(cls):
super(KeepProxyTestCase, cls).setUpClass()
+ cls.proxy_addr = os.environ['ARVADOS_KEEP_PROXY']
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
-
- os.environ["ARVADOS_KEEP_PROXY"] = ""
- os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
-
- run_test_server.run()
- run_test_server.run_keep()
+ def setUp(self):
arvados.keep.global_client_object = None
- arvados.config._settings = None
- run_test_server.run_keep_proxy("admin")
- KeepProxyTestCase.arvados_keep_proxy = arvados.config.get("ARVADOS_KEEP_PROXY")
-
- @classmethod
- def tearDownClass(cls):
- super(KeepProxyTestCase, cls).tearDownClass()
- run_test_server.stop()
- run_test_server.stop_keep()
- run_test_server.stop_keep_proxy()
+ os.environ['ARVADOS_KEEP_PROXY'] = self.proxy_addr
+ os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
def test_KeepProxyTest1(self):
# Will use ARVADOS_KEEP_PROXY environment variable that is set by
- # run_keep_proxy() in setUpClass()
-
- os.environ["ARVADOS_KEEP_PROXY"] = KeepProxyTestCase.arvados_keep_proxy
- os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
- arvados.keep.global_client_object = None
- arvados.config._settings = None
-
+ # setUpClass().
baz_locator = arvados.Keep.put('baz')
self.assertRegexpMatches(
baz_locator,
@@ -266,12 +221,9 @@ class KeepProxyTestCase(unittest.TestCase):
# contact the API server.
os.environ["ARVADOS_KEEP_PROXY"] = ""
os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
- arvados.keep.global_client_object = None
- arvados.config._settings = None
# Will send X-External-Client to server and get back the proxy from
# keep_services/accessible
-
baz_locator = arvados.Keep.put('baz2')
self.assertRegexpMatches(
baz_locator,
diff --git a/sdk/python/tests/test_websockets.py b/sdk/python/tests/test_websockets.py
index 6b57fe3..1dae978 100644
--- a/sdk/python/tests/test_websockets.py
+++ b/sdk/python/tests/test_websockets.py
@@ -4,9 +4,8 @@ import arvados
import arvados.events
import time
-class WebsocketTest(unittest.TestCase):
- def setUp(self):
- run_test_server.run(websockets=True)
+class WebsocketTest(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {'websockets': True}
def on_event(self, ev):
if self.state == 1:
@@ -27,6 +26,3 @@ class WebsocketTest(unittest.TestCase):
time.sleep(1)
self.h = api.humans().create(body={}).execute()
time.sleep(1)
-
- def tearDown(self):
- run_test_server.stop()
commit 13fe1f607b5f8c589e32a3616a11dd7c6db9e8f4
Author: Brett Smith <brett at curoverse.com>
Date: Tue Aug 19 16:41:33 2014 -0400
2800: Make run_test_server put Keep tempfiles in tests/tmp/.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 8ac34b8..a225b74 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -141,29 +141,29 @@ def _start_keep(n, keep_args):
keep_cmd = ["keepstore",
"-volumes={}".format(keep0),
"-listen=:{}".format(25107+n),
- "-pid={}".format("tmp/keep{}.pid".format(n))]
+ "-pid={}".format("tests/tmp/keep{}.pid".format(n))]
for arg, val in keep_args.iteritems():
keep_cmd.append("{}={}".format(arg, val))
kp0 = subprocess.Popen(keep_cmd)
- with open("tmp/keep{}.pid".format(n), 'w') as f:
+ with open("tests/tmp/keep{}.pid".format(n), 'w') as f:
f.write(str(kp0.pid))
- with open("tmp/keep{}.volume".format(n), 'w') as f:
+ with open("tests/tmp/keep{}.volume".format(n), 'w') as f:
f.write(keep0)
def run_keep(blob_signing_key=None, enforce_permissions=False):
stop_keep()
- if not os.path.exists("tmp"):
- os.mkdir("tmp")
+ if not os.path.exists("tests/tmp"):
+ os.mkdir("tests/tmp")
keep_args = {}
if blob_signing_key:
- with open("tmp/keep.blob_signing_key", "w") as f:
+ with open("tests/tmp/keep.blob_signing_key", "w") as f:
f.write(blob_signing_key)
- keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
+ keep_args['--permission-key-file'] = 'tests/tmp/keep.blob_signing_key'
if enforce_permissions:
keep_args['--enforce-permissions'] = 'true'
@@ -186,13 +186,13 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
def _stop_keep(n):
- kill_server_pid("tmp/keep{}.pid".format(n), 0)
- if os.path.exists("tmp/keep{}.volume".format(n)):
- with open("tmp/keep{}.volume".format(n), 'r') as r:
+ kill_server_pid("tests/tmp/keep{}.pid".format(n), 0)
+ if os.path.exists("tests/tmp/keep{}.volume".format(n)):
+ with open("tests/tmp/keep{}.volume".format(n), 'r') as r:
shutil.rmtree(r.read(), True)
- os.unlink("tmp/keep{}.volume".format(n))
- if os.path.exists("tmp/keep.blob_signing_key"):
- os.remove("tmp/keep.blob_signing_key")
+ os.unlink("tests/tmp/keep{}.volume".format(n))
+ if os.path.exists("tests/tmp/keep.blob_signing_key"):
+ os.remove("tests/tmp/keep.blob_signing_key")
def stop_keep():
_stop_keep(0)
@@ -201,15 +201,16 @@ def stop_keep():
def run_keep_proxy(auth):
stop_keep_proxy()
- if not os.path.exists("tmp"):
- os.mkdir("tmp")
+ if not os.path.exists("tests/tmp"):
+ os.mkdir("tests/tmp")
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
kp0 = subprocess.Popen(["keepproxy",
- "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
+ "-pid=tests/tmp/keepproxy.pid",
+ "-listen=:{}".format(25101)])
authorize_with("admin")
api = arvados.api('v1', cache=False)
@@ -218,7 +219,7 @@ def run_keep_proxy(auth):
arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
def stop_keep_proxy():
- kill_server_pid("tmp/keepproxy.pid", 0)
+ kill_server_pid("tests/tmp/keepproxy.pid", 0)
def fixture(fix):
'''load a fixture yaml file'''
commit 5e303407f6ac27d0828f2ec7a860f62e8f537094
Author: Brett Smith <brett at curoverse.com>
Date: Tue Aug 19 16:37:42 2014 -0400
2800: Use dedicated logger in Python SDK api module.
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 1eb8f51..cb716f1 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -99,7 +99,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
if not version:
version = 'v1'
- logging.info("Using default API version. " +
+ _logger.info("Using default API version. " +
"Call arvados.api('%s') instead." %
version)
if 'discoveryServiceUrl' in kwargs:
commit 4a2f470dcd71fb18f29ed3948892cef01334bf07
Author: Brett Smith <brett at curoverse.com>
Date: Wed Aug 20 09:46:33 2014 -0400
2800: Migrate from Keep to KeepClient in the Python SDK.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 7aed681..40b5889 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -92,7 +92,9 @@ def normalize(collection):
class CollectionReader(object):
- def __init__(self, manifest_locator_or_text):
+ def __init__(self, manifest_locator_or_text, api_client=None):
+ self._api_client = api_client
+ self._keep_client = None
if re.match(r'[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
self._manifest_locator = manifest_locator_or_text
self._manifest_text = None
@@ -111,17 +113,21 @@ class CollectionReader(object):
pass
def _populate(self):
- if self._streams != None:
+ if self._streams is not None:
return
if not self._manifest_text:
+ if self._api_client is None:
+ self._api_client = arvados.api('v1')
+ if self._keep_client is None:
+ self._keep_client = KeepClient(api_client=self._api_client)
try:
- c = arvados.api('v1').collections().get(
+ c = self._api_client.collections().get(
uuid=self._manifest_locator).execute()
self._manifest_text = c['manifest_text']
except Exception as e:
_logger.warning("API lookup failed for collection %s (%s: %s)",
self._manifest_locator, type(e), str(e))
- self._manifest_text = Keep.get(self._manifest_locator)
+ self._manifest_text = self._keep_client.get(self._manifest_locator)
self._streams = []
for stream_line in self._manifest_text.split("\n"):
if stream_line != '':
@@ -159,7 +165,9 @@ class CollectionReader(object):
class CollectionWriter(object):
KEEP_BLOCK_SIZE = 2**26
- def __init__(self):
+ def __init__(self, api_client=None):
+ self._api_client = api_client
+ self._keep_client = None
self._data_buffer = []
self._data_buffer_len = 0
self._current_stream_files = []
@@ -180,6 +188,10 @@ class CollectionWriter(object):
def __exit__(self):
self.finish()
+ def _prep_keep_client(self):
+ if self._keep_client is None:
+ self._keep_client = KeepClient(api_client=self._api_client)
+
def do_queued_work(self):
# The work queue consists of three pieces:
# * _queued_file: The file object we're currently writing to the
@@ -285,7 +297,9 @@ class CollectionWriter(object):
def flush_data(self):
data_buffer = ''.join(self._data_buffer)
if data_buffer != '':
- self._current_stream_locators += [Keep.put(data_buffer[0:self.KEEP_BLOCK_SIZE])]
+ self._prep_keep_client()
+ self._current_stream_locators.append(
+ self._keep_client.put(data_buffer[0:self.KEEP_BLOCK_SIZE]))
self._data_buffer = [data_buffer[self.KEEP_BLOCK_SIZE:]]
self._data_buffer_len = len(self._data_buffer[0])
@@ -355,7 +369,8 @@ class CollectionWriter(object):
def finish(self):
# Store the manifest in Keep and return its locator.
- return Keep.put(self.manifest_text())
+ self._prep_keep_client()
+ return self._keep_client.put(self.manifest_text())
def stripped_manifest(self):
"""
@@ -403,9 +418,9 @@ class ResumableCollectionWriter(CollectionWriter):
'_data_buffer', '_dependencies', '_finished_streams',
'_queued_dirents', '_queued_trees']
- def __init__(self):
+ def __init__(self, api_client=None):
self._dependencies = {}
- super(ResumableCollectionWriter, self).__init__()
+ super(ResumableCollectionWriter, self).__init__(api_client)
@classmethod
def from_state(cls, state, *init_args, **init_kwargs):
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 335ef17..e4b763a 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -22,6 +22,7 @@ import tempfile
import arvados.commands._util as arv_cmd
CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
+api_client = None
upload_opts = argparse.ArgumentParser(add_help=False)
@@ -238,13 +239,14 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
STATE_PROPS = (arvados.ResumableCollectionWriter.STATE_PROPS +
['bytes_written', '_seen_inputs'])
- def __init__(self, cache=None, reporter=None, bytes_expected=None):
+ def __init__(self, cache=None, reporter=None, bytes_expected=None,
+ api_client=None):
self.bytes_written = 0
self._seen_inputs = []
self.cache = cache
self.reporter = reporter
self.bytes_expected = bytes_expected
- super(ArvPutCollectionWriter, self).__init__()
+ super(ArvPutCollectionWriter, self).__init__(api_client)
@classmethod
def from_cache(cls, cache, reporter=None, bytes_expected=None):
@@ -342,7 +344,7 @@ def exit_signal_handler(sigcode, frame):
def check_project_exists(project_uuid):
try:
- arvados.api('v1').groups().get(uuid=project_uuid).execute()
+ api_client.groups().get(uuid=project_uuid).execute()
except (apiclient.errors.Error, arvados.errors.NotFoundError) as error:
raise ValueError("Project {} not found ({})".format(project_uuid,
error))
@@ -362,7 +364,7 @@ def prep_project_link(args, stderr, project_exists=check_project_exists):
'link_class': 'name',
'name': args.name}
if not link['tail_uuid']:
- link['tail_uuid'] = arvados.api('v1').users().current().execute()['uuid']
+ link['tail_uuid'] = api_client.users().current().execute()['uuid']
elif not project_exists(link['tail_uuid']):
raise ValueError("Project {} not found".format(args.project_uuid))
if not link['name']:
@@ -377,9 +379,12 @@ def prep_project_link(args, stderr, project_exists=check_project_exists):
def create_project_link(locator, link):
link['head_uuid'] = locator
- return arvados.api('v1').links().create(body=link).execute()
+ return api_client.links().create(body=link).execute()
def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
+ global api_client
+ if api_client is None:
+ api_client = arvados.api('v1')
status = 0
args = parse_arguments(arguments)
@@ -445,7 +450,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
output = ','.join(writer.data_locators())
else:
# Register the resulting collection in Arvados.
- collection = arvados.api().collections().create(
+ collection = api_client.collections().create(
body={
'manifest_text': writer.manifest_text(),
},
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 7a29100..064a4f2 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -204,10 +204,9 @@ class StreamReader(object):
self._data_locators = []
self._files = collections.OrderedDict()
- if keep != None:
- self._keep = keep
- else:
- self._keep = Keep.global_client_object()
+ if keep is None:
+ keep = KeepClient()
+ self._keep = keep
streamoffset = 0L
diff --git a/sdk/python/bin/arv-get b/sdk/python/bin/arv-get
index 7f02cf7..0d403d1 100755
--- a/sdk/python/bin/arv-get
+++ b/sdk/python/bin/arv-get
@@ -126,6 +126,7 @@ if args.r and not get_prefix:
todo = []
todo_bytes = 0
+api_client = arvados.api('v1')
if not get_prefix:
try:
if not args.n:
@@ -133,15 +134,15 @@ if not get_prefix:
abort('Local file %s already exists.' % (args.destination,))
with open(args.destination, 'wb') as f:
try:
- c = arvados.api('v1').collections().get(
- uuid=collection).execute()
+ c = api_client.collections().get(uuid=collection).execute()
manifest = c['manifest_text']
except Exception as e:
logger.warning(
"Collection %s not found. " +
"Trying to fetch directly from Keep (deprecated).",
collection)
- manifest = arvados.Keep.get(collection)
+ manifest = arvados.KeepClient(
+ api_client=api_client).get(collection)
f.write(manifest)
sys.exit(0)
except arvados.errors.NotFoundError as e:
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 9bc385d..5ce475e 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -413,6 +413,10 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
'098f6bcd4621d373cade4e832627b4f6')),
"did not find file stream in Keep store")
+ def setUp(self):
+ super(ArvadosPutTest, self).setUp()
+ arv_put.api_client = None
+
def tearDown(self):
for outbuf in ['main_stdout', 'main_stderr']:
if hasattr(self, outbuf):
@@ -473,9 +477,9 @@ class ArvPutIntegrationTest(unittest.TestCase):
try:
config_blob_signing_key = rails_config[config_section]["blob_signing_key"]
break
- except KeyError, AttributeError:
+ except KeyError:
pass
- if config_blob_signing_key != None:
+ if config_blob_signing_key is not None:
break
run_test_server.run()
run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
@@ -486,22 +490,29 @@ class ArvPutIntegrationTest(unittest.TestCase):
run_test_server.stop()
run_test_server.stop_keep()
+ def setUp(self):
+ super(ArvPutIntegrationTest, self).setUp()
+ arv_put.api_client = None
+
def authorize_with(self, token_name):
run_test_server.authorize_with(token_name)
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
+ arv_put.api_client = arvados.api('v1', cache=False)
def current_user(self):
- return arvados.api('v1').users().current().execute()
+ return arv_put.api_client.users().current().execute()
def test_check_real_project_found(self):
+ self.authorize_with('active')
self.assertTrue(arv_put.check_project_exists(self.PROJECT_UUID),
"did not correctly find test fixture project")
def test_check_error_finding_nonexistent_project(self):
BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+ self.authorize_with('active')
try:
result = arv_put.check_project_exists(BAD_UUID)
except ValueError as error:
@@ -540,10 +551,10 @@ class ArvPutIntegrationTest(unittest.TestCase):
# Before doing anything, demonstrate that the collection
# we're about to create is not present in our test fixture.
- api = arvados.api('v1', cache=False)
manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
with self.assertRaises(apiclient.errors.HttpError):
- notfound = api.collections().get(uuid=manifest_uuid).execute()
+ notfound = arv_put.api_client.collections().get(
+ uuid=manifest_uuid).execute()
datadir = tempfile.mkdtemp()
with open(os.path.join(datadir, "foo"), "w") as f:
@@ -557,7 +568,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
# The manifest text stored in the API server under the same
# manifest UUID must use signed locators.
- c = api.collections().get(uuid=manifest_uuid).execute()
+ c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
self.assertRegexpMatches(
c['manifest_text'],
r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
@@ -572,7 +583,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=self.ENVIRON)
stdout, stderr = pipe.communicate(text)
- link_list = arvados.api('v1', cache=False).links().list(
+ link_list = arv_put.api_client.links().list(
filters=[['head_uuid', '=', stdout.strip()],
['link_class', '=', 'name']]).execute().get('items', [])
self.assertEqual(1, len(link_list))
commit 7283c47292ebd73feea85f55381b8a350d7b4b14
Author: Brett Smith <brett at curoverse.com>
Date: Tue Aug 19 10:17:57 2014 -0400
2800: Remove global state from KeepClient.
This commit makes it possible to build and use a KeepClient that isn't
influenced by changes in outside state. Changing the KeepClient based
on global state has been pushed up to the simple Keep class.
This commit changes the tests as little as possible to demonstrate
backward compatibility. Refactoring the tests to build KeepClients
directly will come in the future.
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index e7348a1..1eb8f51 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -154,6 +154,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
kwargs['http'] = credentials.authorize(kwargs['http'])
svc = apiclient.discovery.build('arvados', version, **kwargs)
+ svc.api_token = token
kwargs['http'].cache = None
if cache:
conncache[connprofile] = svc
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 3667829..561d34c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -24,8 +24,8 @@ import ssl
_logger = logging.getLogger('arvados.keep')
global_client_object = None
-from api import *
-import config
+import arvados
+import arvados.config as config
import arvados.errors
import arvados.util
@@ -106,12 +106,31 @@ class KeepLocator(object):
return self.perm_expiry <= as_of_dt
-class Keep:
- @staticmethod
- def global_client_object():
+class Keep(object):
+ """Simple interface to a global KeepClient object.
+
+ THIS CLASS IS DEPRECATED. Please instantiate your own KeepClient with your
+ own API client. The global KeepClient will build an API client from the
+ current Arvados configuration, which may not match the one you built.
+ """
+ _last_key = None
+
+ @classmethod
+ def global_client_object(cls):
global global_client_object
- if global_client_object == None:
+ # Previously, KeepClient would change its behavior at runtime based
+ # on these configuration settings. We simulate that behavior here
+ # by checking the values and returning a new KeepClient if any of
+ # them have changed.
+ key = (config.get('ARVADOS_API_HOST'),
+ config.get('ARVADOS_API_TOKEN'),
+ config.flag_is_true('ARVADOS_API_HOST_INSECURE'),
+ config.get('ARVADOS_KEEP_PROXY'),
+ config.get('ARVADOS_EXTERNAL_CLIENT') == 'true',
+ os.environ.get('KEEP_LOCAL_STORE'))
+ if (global_client_object is None) or (cls._last_key != key):
global_client_object = KeepClient()
+ cls._last_key = key
return global_client_object
@staticmethod
@@ -122,8 +141,8 @@ class Keep:
def put(data, **kwargs):
return Keep.global_client_object().put(data, **kwargs)
-class KeepClient(object):
+class KeepClient(object):
class ThreadLimiter(object):
"""
Limit the number of threads running at a given time to
@@ -180,14 +199,16 @@ class KeepClient(object):
with self._done_lock:
return self._done
+
class KeepWriterThread(threading.Thread):
"""
Write a blob of data to the given Keep server. On success, call
save_response() of the given ThreadLimiter to save the returned
locator.
"""
- def __init__(self, **kwargs):
+ def __init__(self, api_token, **kwargs):
super(KeepClient.KeepWriterThread, self).__init__()
+ self._api_token = api_token
self.args = kwargs
self._success = False
@@ -209,8 +230,7 @@ class KeepClient(object):
self.args['service_root'])
h = httplib2.Http(timeout=self.args.get('timeout', None))
url = self.args['service_root'] + self.args['data_hash']
- api_token = config.get('ARVADOS_API_TOKEN')
- headers = {'Authorization': "OAuth2 %s" % api_token}
+ headers = {'Authorization': "OAuth2 %s" % (self._api_token,)}
if self.args['using_proxy']:
# We're using a proxy, so tell the proxy how many copies we
@@ -259,51 +279,91 @@ class KeepClient(object):
_logger.debug("Request fail: PUT %s => %s: %s",
url, type(e), str(e))
- def __init__(self, **kwargs):
+
+ def __init__(self, api_client=None, proxy=None, timeout=60,
+ api_token=None, local_store=None):
+ """Initialize a new KeepClient.
+
+ Arguments:
+ * api_client: The API client to use to find Keep services. If not
+ provided, KeepClient will build one from available Arvados
+ configuration.
+ * proxy: If specified, this KeepClient will send requests to this
+ Keep proxy. Otherwise, KeepClient will fall back to the setting
+ of the ARVADOS_KEEP_PROXY configuration setting. If you want to
+ ensure KeepClient does not use a proxy, pass in an empty string.
+ * timeout: The timeout for all HTTP requests, in seconds. Default
+ 60.
+ * api_token: If you're not using an API client, but only talking
+ directly to a Keep proxy, this parameter specifies an API token
+ to authenticate Keep requests. It is an error to specify both
+ api_client and api_token. If you specify neither, KeepClient
+ will use one available from the Arvados configuration.
+ * local_store: If specified, this KeepClient will bypass Keep
+ services, and save data to the named directory. If unspecified,
+ KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
+ environment variable. If you want to ensure KeepClient does not
+ use local storage, pass in an empty string. This is primarily
+ intended to mock a server for testing.
+ """
self.lock = threading.Lock()
- self.service_roots = None
- self._cache_lock = threading.Lock()
- self._cache = []
- # default 256 megabyte cache
- self.cache_max = 256 * 1024 * 1024
- self.using_proxy = False
- self.timeout = kwargs.get('timeout', 60)
+ if proxy is None:
+ proxy = config.get('ARVADOS_KEEP_PROXY')
+ if api_token is None:
+ api_token = config.get('ARVADOS_API_TOKEN')
+ elif api_client is not None:
+ raise ValueError(
+ "can't build KeepClient with both API client and token")
+ if local_store is None:
+ local_store = os.environ.get('KEEP_LOCAL_STORE')
+
+ if local_store:
+ self.local_store = local_store
+ self.get = self.local_store_get
+ self.put = self.local_store_put
+ else:
+ self.timeout = timeout
+ self.cache_max = 256 * 1024 * 1024 # Cache is 256MiB
+ self._cache = []
+ self._cache_lock = threading.Lock()
+ if proxy:
+ if not proxy.endswith('/'):
+ proxy += '/'
+ self.api_token = api_token
+ self.service_roots = [proxy]
+ self.using_proxy = True
+ else:
+ # It's important to avoid instantiating an API client
+ # unless we actually need one, for testing's sake.
+ if api_client is None:
+ api_client = arvados.api('v1')
+ self.api_client = api_client
+ self.api_token = api_client.api_token
+ self.service_roots = None
+ self.using_proxy = None
def shuffled_service_roots(self, hash):
- if self.service_roots == None:
- self.lock.acquire()
+ if self.service_roots is None:
+ with self.lock:
+ try:
+ keep_services = self.api_client.keep_services().accessible()
+ except Exception: # API server predates Keep services.
+ keep_services = self.api_client.keep_disks().list()
- # Override normal keep disk lookup with an explict proxy
- # configuration.
- keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
- if keep_proxy_env != None and len(keep_proxy_env) > 0:
+ keep_services = keep_services.execute().get('items')
+ if not keep_services:
+ raise arvados.errors.NoKeepServersError()
- if keep_proxy_env[-1:] != '/':
- keep_proxy_env += "/"
- self.service_roots = [keep_proxy_env]
- self.using_proxy = True
- else:
- try:
- try:
- keep_services = arvados.api().keep_services().accessible().execute()['items']
- except Exception:
- keep_services = arvados.api().keep_disks().list().execute()['items']
-
- if len(keep_services) == 0:
- raise arvados.errors.NoKeepServersError()
-
- if 'service_type' in keep_services[0] and keep_services[0]['service_type'] == 'proxy':
- self.using_proxy = True
-
- roots = (("http%s://%s:%d/" %
- ('s' if f['service_ssl_flag'] else '',
- f['service_host'],
- f['service_port']))
- for f in keep_services)
- self.service_roots = sorted(set(roots))
- _logger.debug(str(self.service_roots))
- finally:
- self.lock.release()
+ self.using_proxy = (keep_services[0].get('service_type') ==
+ 'proxy')
+
+ roots = (("http%s://%s:%d/" %
+ ('s' if f['service_ssl_flag'] else '',
+ f['service_host'],
+ f['service_port']))
+ for f in keep_services)
+ self.service_roots = sorted(set(roots))
+ _logger.debug(str(self.service_roots))
# Build an ordering with which to query the Keep servers based on the
# contents of the hash.
@@ -403,12 +463,11 @@ class KeepClient(object):
finally:
self._cache_lock.release()
- def get(self, locator):
- if re.search(r',', locator):
- return ''.join(self.get(x) for x in locator.split(','))
- if 'KEEP_LOCAL_STORE' in os.environ:
- return KeepClient.local_store_get(locator)
- expect_hash = re.sub(r'\+.*', '', locator)
+ def get(self, loc_s):
+ if ',' in loc_s:
+ return ''.join(self.get(x) for x in loc_s.split(','))
+ locator = KeepLocator(loc_s)
+ expect_hash = locator.md5sum
slot, first = self.reserve_cache(expect_hash)
@@ -418,9 +477,8 @@ class KeepClient(object):
try:
for service_root in self.shuffled_service_roots(expect_hash):
- url = service_root + locator
- api_token = config.get('ARVADOS_API_TOKEN')
- headers = {'Authorization': "OAuth2 %s" % api_token,
+ url = service_root + loc_s
+ headers = {'Authorization': "OAuth2 %s" % (self.api_token,),
'Accept': 'application/octet-stream'}
blob = self.get_url(url, headers, expect_hash)
if blob:
@@ -428,9 +486,10 @@ class KeepClient(object):
self.cap_cache()
return blob
- for location_hint in re.finditer(r'\+K@([a-z0-9]+)', locator):
- instance = location_hint.group(1)
- url = 'http://keep.' + instance + '.arvadosapi.com/' + locator
+ for hint in locator.hints:
+ if not hint.startswith('K@'):
+ continue
+ url = 'http://keep.' + hint[2:] + '.arvadosapi.com/' + loc_s
blob = self.get_url(url, {}, expect_hash)
if blob:
slot.set(blob)
@@ -456,9 +515,7 @@ class KeepClient(object):
len(content), t.msecs,
(len(content)/(1024*1024))/t.secs)
if re.match(r'^2\d\d$', resp['status']):
- m = hashlib.new('md5')
- m.update(content)
- md5 = m.hexdigest()
+ md5 = hashlib.md5(content).hexdigest()
if md5 == expect_hash:
return content
_logger.warning("Checksum fail: md5(%s) = %s", url, md5)
@@ -467,20 +524,17 @@ class KeepClient(object):
url, type(e), str(e))
return None
- def put(self, data, **kwargs):
- if 'KEEP_LOCAL_STORE' in os.environ:
- return KeepClient.local_store_put(data)
- m = hashlib.new('md5')
- m.update(data)
- data_hash = m.hexdigest()
+ def put(self, data, copies=2):
+ data_hash = hashlib.md5(data).hexdigest()
have_copies = 0
- want_copies = kwargs.get('copies', 2)
+ want_copies = copies
if not (want_copies > 0):
return data_hash
threads = []
thread_limiter = KeepClient.ThreadLimiter(want_copies)
for service_root in self.shuffled_service_roots(data_hash):
t = KeepClient.KeepWriterThread(
+ self.api_token,
data=data,
data_hash=data_hash,
service_root=service_root,
@@ -502,7 +556,8 @@ class KeepClient(object):
t.args['service_root'],
t.args['data_hash'])
retry_with_args = t.args.copy()
- t_retry = KeepClient.KeepWriterThread(**retry_with_args)
+ t_retry = KeepClient.KeepWriterThread(self.api_token,
+ **retry_with_args)
t_retry.start()
threads_retry += [t_retry]
for t in threads_retry:
@@ -519,26 +574,22 @@ class KeepClient(object):
def sign_for_old_server(data_hash, data):
return (("-----BEGIN PGP SIGNED MESSAGE-----\n\n\n%d %s\n-----BEGIN PGP SIGNATURE-----\n\n-----END PGP SIGNATURE-----\n" % (int(time.time()), data_hash)) + data)
-
- @staticmethod
- def local_store_put(data):
- m = hashlib.new('md5')
- m.update(data)
- md5 = m.hexdigest()
+ def local_store_put(self, data):
+ md5 = hashlib.md5(data).hexdigest()
locator = '%s+%d' % (md5, len(data))
- with open(os.path.join(os.environ['KEEP_LOCAL_STORE'], md5 + '.tmp'), 'w') as f:
+ with open(os.path.join(self.local_store, md5 + '.tmp'), 'w') as f:
f.write(data)
- os.rename(os.path.join(os.environ['KEEP_LOCAL_STORE'], md5 + '.tmp'),
- os.path.join(os.environ['KEEP_LOCAL_STORE'], md5))
+ os.rename(os.path.join(self.local_store, md5 + '.tmp'),
+ os.path.join(self.local_store, md5))
return locator
- @staticmethod
- def local_store_get(locator):
- r = re.search('^([0-9a-f]{32,})', locator)
- if not r:
+ def local_store_get(self, loc_s):
+ try:
+ locator = KeepLocator(loc_s)
+ except ValueError:
raise arvados.errors.NotFoundError(
- "Invalid data locator: '%s'" % locator)
- if r.group(0) == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
+ "Invalid data locator: '%s'" % loc_s)
+ if locator.md5sum == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
return ''
- with open(os.path.join(os.environ['KEEP_LOCAL_STORE'], r.group(0)), 'r') as f:
+ with open(os.path.join(self.local_store, locator.md5sum), 'r') as f:
return f.read()
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 8706e21..9d3cecd 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -126,7 +126,7 @@ class KeepPermissionTestCase(unittest.TestCase):
# Unauthenticated GET for a signed locator => NotFound
# Unauthenticated GET for an unsigned locator => NotFound
- del arvados.config.settings()["ARVADOS_API_TOKEN"]
+ arvados.keep.global_client_object.api_token = ''
self.assertRaises(arvados.errors.NotFoundError,
arvados.Keep.get,
bar_locator)
@@ -192,7 +192,7 @@ class KeepOptionalPermission(unittest.TestCase):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
- del arvados.config.settings()["ARVADOS_API_TOKEN"]
+ arvados.keep.global_client_object.api_token = ''
self.assertEqual(arvados.Keep.get(signed_locator),
'foo',
'wrong content from Keep.get(md5("foo"))')
@@ -207,7 +207,7 @@ class KeepOptionalPermission(unittest.TestCase):
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
- del arvados.config.settings()["ARVADOS_API_TOKEN"]
+ arvados.keep.global_client_object.api_token = ''
self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
'foo',
'wrong content from Keep.get(md5("foo"))')
commit b56f5fa969694403bf96b743e8873da10eb7af4b
Author: Brett Smith <brett at curoverse.com>
Date: Tue Aug 19 10:01:36 2014 -0400
2800: Improve spec conformance of Python SDK KeepLocator.
* Require size to immediately follow digest.
* Accept all valid hints.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index bffb02a..3667829 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -31,28 +31,30 @@ import arvados.util
class KeepLocator(object):
EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0)
+ HINT_RE = re.compile(r'^[A-Z][A-Za-z0-9 at _-]+$')
def __init__(self, locator_str):
- self.size = None
- self.loc_hint = None
+ self.hints = []
self._perm_sig = None
self._perm_expiry = None
pieces = iter(locator_str.split('+'))
self.md5sum = next(pieces)
+ try:
+ self.size = int(next(pieces))
+ except StopIteration:
+ self.size = None
for hint in pieces:
- if hint.startswith('A'):
+ if self.HINT_RE.match(hint) is None:
+ raise ValueError("unrecognized hint data {}".format(hint))
+ elif hint.startswith('A'):
self.parse_permission_hint(hint)
- elif hint.startswith('K'):
- self.loc_hint = hint # FIXME
- elif hint.isdigit():
- self.size = int(hint)
else:
- raise ValueError("unrecognized hint data {}".format(hint))
+ self.hints.append(hint)
def __str__(self):
return '+'.join(
- str(s) for s in [self.md5sum, self.size, self.loc_hint,
- self.permission_hint()]
+ str(s) for s in [self.md5sum, self.size,
+ self.permission_hint()] + self.hints
if s is not None)
def _make_hex_prop(name, length):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 5354f3a..b1af723 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -586,7 +586,7 @@ class ArvadosCollectionsTest(ArvadosKeepLocalStoreTestCase):
state = cwriter.current_state()
# Add an expired locator to the state.
state['_current_stream_locators'].append(''.join([
- 'a' * 32, '+A', 'b' * 40, '@', '10000000']))
+ 'a' * 32, '+1+A', 'b' * 40, '@', '10000000']))
self.assertRaises(arvados.errors.StaleWriterStateError,
TestResumableWriter.from_state, state)
diff --git a/sdk/python/tests/test_keep_locator.py b/sdk/python/tests/test_keep_locator.py
index e9d6356..a7e9cb1 100644
--- a/sdk/python/tests/test_keep_locator.py
+++ b/sdk/python/tests/test_keep_locator.py
@@ -8,7 +8,7 @@ import unittest
from arvados.keep import KeepLocator
-class ArvadosPutResumeCacheTest(unittest.TestCase):
+class ArvadosKeepLocatorTest(unittest.TestCase):
DEFAULT_TEST_COUNT = 10
def numstrs(fmtstr, base, exponent):
@@ -22,13 +22,17 @@ class ArvadosPutResumeCacheTest(unittest.TestCase):
signatures = numstrs('{:040x}', 16, 40)
timestamps = numstrs('{:08x}', 16, 8)
+ def base_locators(self, count=DEFAULT_TEST_COUNT):
+ return ('+'.join(pair) for pair in
+ itertools.izip(self.checksums(count), self.sizes(count)))
+
def perm_hints(self, count=DEFAULT_TEST_COUNT):
for sig, ts in itertools.izip(self.signatures(count),
self.timestamps(count)):
yield 'A{}@{}'.format(sig, ts)
def test_good_locators_returned(self):
- for hint_gens in [(), (self.sizes(),), (self.perm_hints(),),
+ for hint_gens in [(), (self.sizes(),),
(self.sizes(), self.perm_hints())]:
for loc_data in itertools.izip(self.checksums(), *hint_gens):
locator = '+'.join(loc_data)
@@ -40,24 +44,36 @@ class ArvadosPutResumeCacheTest(unittest.TestCase):
'3+8f9e68d957b504a29ba76c526c3145d9']:
self.assertRaises(ValueError, KeepLocator, badstr)
+ def test_unknown_hints_accepted(self):
+ base = next(self.base_locators(1))
+ for weirdhint in ['Zfoo', 'Ybar234', 'Xa at b_c-372', 'W99']:
+ locator = '+'.join([base, weirdhint])
+ self.assertEquals(locator, str(KeepLocator(locator)))
+
def test_bad_hints_rejected(self):
- checksum = next(self.checksums(1))
- for badhint in ['', 'nonsense', '+32', checksum]:
+ base = next(self.base_locators(1))
+ for badhint in ['', 'A', 'lowercase', '+32']:
self.assertRaises(ValueError, KeepLocator,
- '+'.join([checksum, badhint]))
+ '+'.join([base, badhint]))
+
+ def test_multiple_locator_hints_accepted(self):
+ base = next(self.base_locators(1))
+ for loc_hints in itertools.permutations(['Kab1cd', 'Kef2gh', 'Kij3kl']):
+ locator = '+'.join((base,) + loc_hints)
+ self.assertEquals(locator, str(KeepLocator(locator)))
def test_expiry_passed(self):
- checksum = next(self.checksums(1))
+ base = next(self.base_locators(1))
signature = next(self.signatures(1))
dt1980 = datetime.datetime(1980, 1, 1)
dt2000 = datetime.datetime(2000, 2, 2)
dt2080 = datetime.datetime(2080, 3, 3)
- locator = KeepLocator(checksum)
+ locator = KeepLocator(base)
self.assertFalse(locator.permission_expired())
self.assertFalse(locator.permission_expired(dt1980))
self.assertFalse(locator.permission_expired(dt2080))
# Timestamped to 1987-01-05 18:48:32.
- locator = KeepLocator('{}+A{}@20000000'.format(checksum, signature))
+ locator = KeepLocator('{}+A{}@20000000'.format(base, signature))
self.assertTrue(locator.permission_expired())
self.assertTrue(locator.permission_expired(dt2000))
self.assertFalse(locator.permission_expired(dt1980))
commit 1e243a79a37e2bbf73a8f3f6a6a4b50e09c24689
Author: Brett Smith <brett at curoverse.com>
Date: Tue Aug 19 09:59:53 2014 -0400
2800: Introduce config.flag_is_true() to Python SDK.
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 3df2430..e7348a1 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -119,8 +119,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
raise ValueError("%s is not set. Aborting." % x)
host = config.get('ARVADOS_API_HOST')
token = config.get('ARVADOS_API_TOKEN')
- insecure = (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in
- ('yes', 'true', '1'))
+ insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
else:
# Caller provided one but not the other
if not host:
diff --git a/sdk/python/arvados/config.py b/sdk/python/arvados/config.py
index e205e92..2b8374a 100644
--- a/sdk/python/arvados/config.py
+++ b/sdk/python/arvados/config.py
@@ -25,6 +25,9 @@ def initialize(config_file=default_config_file):
if var.startswith('ARVADOS_'):
_settings[var] = os.environ[var]
+def flag_is_true(key):
+ return get(key, '').lower() in set(['1', 't', 'true', 'y', 'yes'])
+
def get(key, default_val=None):
return settings().get(key, default_val)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list