[ARVADOS] updated: 05bea2c50474edeb9d0e3fb8daaf838b58ea9a54

git at public.curoverse.com git at public.curoverse.com
Wed Aug 20 14:17:09 EDT 2014


Summary of changes:
 sdk/python/arvados/api.py             |   6 +-
 sdk/python/arvados/collection.py      |  33 +++--
 sdk/python/arvados/commands/put.py    |  17 ++-
 sdk/python/arvados/config.py          |   3 +
 sdk/python/arvados/keep.py            | 251 ++++++++++++++++++++--------------
 sdk/python/arvados/stream.py          |   7 +-
 sdk/python/bin/arv-get                |   7 +-
 sdk/python/tests/arvados_testutil.py  |  14 --
 sdk/python/tests/run_test_server.py   |  97 ++++++++++---
 sdk/python/tests/test_arv_put.py      |  82 +++++------
 sdk/python/tests/test_collections.py  |  99 ++++++++------
 sdk/python/tests/test_keep_client.py  | 221 +++++++++++-------------------
 sdk/python/tests/test_keep_locator.py |  32 +++--
 sdk/python/tests/test_websockets.py   |   8 +-
 14 files changed, 483 insertions(+), 394 deletions(-)

       via  05bea2c50474edeb9d0e3fb8daaf838b58ea9a54 (commit)
       via  40d92dda0dd3e31cbfd5b887ea8465f9940b7540 (commit)
       via  3b9b5c14f6b1cd0cbed75e0d4adf36e10323dbe7 (commit)
       via  da01d7e185a6ce5e96d542c180856ce84e94ed63 (commit)
       via  e17db6aa8bae683f31d06500bbd78b043dce8d9a (commit)
       via  327e1768a130476a411baf4cd4fb602f7a73c5ed (commit)
       via  f05bc0963563ab50341ff042439462b631894de6 (commit)
       via  1afe2c7bbb571004736daf347f5178a27128704c (commit)
       via  f6e6e01802e09e1a108c067519d361caf0fc606f (commit)
      from  35a3223ee35279c04552d5d3d372998711a956ab (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 05bea2c50474edeb9d0e3fb8daaf838b58ea9a54
Merge: 35a3223 40d92dd
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Aug 20 14:18:24 2014 -0400

    Merge branch '2800-pysdk-no-global-keep-client-wip'
    
    Closes #2800, #3639.


commit 40d92dda0dd3e31cbfd5b887ea8465f9940b7540
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Aug 20 09:24:22 2014 -0400

    2800: Use local KeepClients in Python SDK tests.
    
    This improves test isolation, and demonstrates the preferred way to
    use the API.

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 68c9698..1300c6c 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -29,9 +29,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
     def setUpClass(cls):
         super(ArvadosCollectionsTest, cls).setUpClass()
         run_test_server.authorize_with('active')
+        cls.api_client = arvados.api('v1')
+        cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
+                                             local_store=cls.local_store)
 
     def write_foo_bar_baz(self):
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         self.assertEqual(cw.current_stream_name(), '.',
                          'current_stream_name() should be "." now')
         cw.set_current_file_name('foo.txt')
@@ -46,8 +49,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         return cw.finish()
 
     def test_keep_local_store(self):
-        self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
-        self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
+        self.assertEqual(self.keep_client.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
+        self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
 
     def test_local_collection_writer(self):
         self.assertEqual(self.write_foo_bar_baz(),
@@ -56,7 +59,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 
     def test_local_collection_reader(self):
         self.write_foo_bar_baz()
-        cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle')
+        cr = arvados.CollectionReader(
+            'd6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle', self.api_client)
         got = []
         for s in cr.all_streams():
             for f in s.all_files():
@@ -78,7 +82,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                          'reading zero bytes should have returned empty string')
 
     def _test_subset(self, collection, expected):
-        cr = arvados.CollectionReader(collection)
+        cr = arvados.CollectionReader(collection, self.api_client)
         for s in cr.all_streams():
             for ex in expected:
                 if ex[0] == s:
@@ -95,29 +99,29 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                            [3, '.',     'foo.txt', 'foo'],
                            [3, './baz', 'baz.txt', 'baz']])
         self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
-                           (arvados.Keep.put("foo"),
-                            arvados.Keep.put("bar"))),
+                           (self.keep_client.put("foo"),
+                            self.keep_client.put("bar"))),
                           [[3, '.', 'bar.txt', 'bar'],
                            [3, '.', 'foo.txt', 'foo']])
         self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
-                           (arvados.Keep.put("foo"),
-                            arvados.Keep.put("bar"))),
+                           (self.keep_client.put("foo"),
+                            self.keep_client.put("bar"))),
                           [[2, '.', 'fo.txt', 'fo'],
                            [4, '.', 'obar.txt', 'obar']])
         self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
-                           (arvados.Keep.put("foo"),
-                            arvados.Keep.put("bar"))),
+                           (self.keep_client.put("foo"),
+                            self.keep_client.put("bar"))),
                           [[2, '.', 'ar.txt', 'ar'],
                            [2, '.', 'fo.txt', 'fo'],
                            [2, '.', 'ob.txt', 'ob'],
                            [0, '.', 'zero.txt', '']])
 
     def _test_readline(self, what_in, what_out):
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         cw.start_new_file('test.txt')
         cw.write(what_in)
         test1 = cw.finish()
-        cr = arvados.CollectionReader(test1)
+        cr = arvados.CollectionReader(test1, self.api_client)
         got = []
         for x in list(cr.all_files())[0].readlines():
             got += [x]
@@ -132,13 +136,13 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                             ["ab\n", "cd\n"])
 
     def test_collection_empty_file(self):
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         cw.start_new_file('zero.txt')
         cw.write('')
 
         self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n")
         self.check_manifest_file_sizes(cw.manifest_text(), [0])
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         cw.start_new_file('zero.txt')
         cw.write('')
         cw.start_new_file('one.txt')
@@ -149,7 +153,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         self.check_manifest_file_sizes(cw.manifest_text(), [1,0,0])
 
     def check_manifest_file_sizes(self, manifest_text, expect_sizes):
-        cr = arvados.CollectionReader(manifest_text)
+        cr = arvados.CollectionReader(manifest_text, self.api_client)
         got_sizes = []
         for f in cr.all_files():
             got_sizes += [f.size()]
@@ -161,12 +165,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         for x in xrange(0, 18):
             data_in += data_in
         compressed_data_in = bz2.compress(data_in)
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         cw.start_new_file('test.bz2')
         cw.write(compressed_data_in)
         bz2_manifest = cw.manifest_text()
 
-        cr = arvados.CollectionReader(bz2_manifest)
+        cr = arvados.CollectionReader(bz2_manifest, self.api_client)
 
         got = 0
         for x in list(cr.all_files())[0].readlines():
@@ -188,12 +192,12 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                              shell=False, close_fds=True)
         compressed_data_in, stderrdata = p.communicate(data_in)
 
-        cw = arvados.CollectionWriter()
+        cw = arvados.CollectionWriter(self.api_client)
         cw.start_new_file('test.gz')
         cw.write(compressed_data_in)
         gzip_manifest = cw.manifest_text()
 
-        cr = arvados.CollectionReader(gzip_manifest)
+        cr = arvados.CollectionReader(gzip_manifest, self.api_client)
         got = 0
         for x in list(cr.all_files())[0].readlines():
             self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
@@ -206,25 +210,25 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
-        self.assertEqual(arvados.CollectionReader(m1).manifest_text(),
+        self.assertEqual(arvados.CollectionReader(m1, self.api_client).manifest_text(),
                          """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt
 """)
 
         m2 = """. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
 """
-        self.assertEqual(arvados.CollectionReader(m2).manifest_text(), m2)
+        self.assertEqual(arvados.CollectionReader(m2, self.api_client).manifest_text(), m2)
 
         m3 = """. 5348b82a029fd9e971a811ce1f71360b+43 3:40:md5sum.txt
 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
-        self.assertEqual(arvados.CollectionReader(m3).manifest_text(),
+        self.assertEqual(arvados.CollectionReader(m3, self.api_client).manifest_text(),
                          """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 3:124:md5sum.txt
 """)
 
         m4 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
 ./foo 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar"""
-        self.assertEqual(arvados.CollectionReader(m4).manifest_text(),
+        self.assertEqual(arvados.CollectionReader(m4, self.api_client).manifest_text(),
                          """./foo 204e43b8a1185621ca55a94839582e6f+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar 67108864:3:bar
 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
 """)
@@ -232,22 +236,22 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         m5 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
 ./foo 204e43b8a1185621ca55a94839582e6f+67108864 3:3:bar"""
-        self.assertEqual(arvados.CollectionReader(m5).manifest_text(),
+        self.assertEqual(arvados.CollectionReader(m5, self.api_client).manifest_text(),
                          """./foo 204e43b8a1185621ca55a94839582e6f+67108864 0:6:bar
 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
 """)
 
         with self.data_file('1000G_ref_manifest') as f6:
             m6 = f6.read()
-            self.assertEqual(arvados.CollectionReader(m6).manifest_text(), m6)
+            self.assertEqual(arvados.CollectionReader(m6, self.api_client).manifest_text(), m6)
 
         with self.data_file('jlake_manifest') as f7:
             m7 = f7.read()
-            self.assertEqual(arvados.CollectionReader(m7).manifest_text(), m7)
+            self.assertEqual(arvados.CollectionReader(m7, self.api_client).manifest_text(), m7)
 
         m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
 """
-        self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8)
+        self.assertEqual(arvados.CollectionReader(m8, self.api_client).manifest_text(), m8)
 
     def test_locators_and_ranges(self):
         blocks2 = [['a', 10, 0],
@@ -472,22 +476,22 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt"""
 
-        m2 = arvados.CollectionReader(m1)
+        m2 = arvados.CollectionReader(m1, self.api_client)
 
         self.assertEqual(m2.manifest_text(),
                          ". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
 
-        self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md5sum.txt'].as_manifest(),
+        self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md5sum.txt'].as_manifest(),
                          ". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
-        self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md6sum.txt'].as_manifest(),
+        self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md6sum.txt'].as_manifest(),
                          ". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
-        self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md7sum.txt'].as_manifest(),
+        self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md7sum.txt'].as_manifest(),
                          ". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
-        self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md9sum.txt'].as_manifest(),
+        self.assertEqual(arvados.CollectionReader(m1, self.api_client).all_streams()[0].files()['md9sum.txt'].as_manifest(),
                          ". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
 
     def test_write_directory_tree(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         cwriter.write_directory_tree(self.build_directory_tree(
                 ['basefile', 'subdir/subfile']))
         self.assertEqual(cwriter.manifest_text(),
@@ -495,7 +499,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 ./subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
 
     def test_write_named_directory_tree(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         cwriter.write_directory_tree(self.build_directory_tree(
                 ['basefile', 'subdir/subfile']), 'root')
         self.assertEqual(
@@ -504,7 +508,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 ./root/subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
 
     def test_write_directory_tree_in_one_stream(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         cwriter.write_directory_tree(self.build_directory_tree(
                 ['basefile', 'subdir/subfile']), max_manifest_depth=0)
         self.assertEqual(cwriter.manifest_text(),
@@ -512,7 +516,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 ./subdir 4ace875ffdc6824a04950f06858f4465+22 8:14:subfile\n""")
 
     def test_write_directory_tree_with_limited_recursion(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         cwriter.write_directory_tree(
             self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
             max_manifest_depth=1)
@@ -522,7 +526,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
 ./d1/d2 50170217e5b04312024aa5cd42934494+13 0:8:f3\n""")
 
     def test_write_one_file(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         with self.make_test_file() as testfile:
             cwriter.write_file(testfile.name)
             self.assertEqual(
@@ -531,14 +535,14 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                     os.path.basename(testfile.name)))
 
     def test_write_named_file(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         with self.make_test_file() as testfile:
             cwriter.write_file(testfile.name, 'foo')
             self.assertEqual(cwriter.manifest_text(),
                              ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:foo\n")
 
     def test_write_multiple_files(self):
-        cwriter = arvados.CollectionWriter()
+        cwriter = arvados.CollectionWriter(self.api_client)
         for letter in 'ABC':
             with self.make_test_file(letter) as testfile:
                 cwriter.write_file(testfile.name, letter)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index fb52ab4..6198919 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -2,51 +2,42 @@
 #
 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
 
-import contextlib
 import os
 import unittest
 
 import arvados
 import run_test_server
 
- at contextlib.contextmanager
-def unauthenticated_client(keep_client=None):
-    if keep_client is None:
-        keep_client = arvados.keep.global_client_object
-    if not hasattr(keep_client, 'api_token'):
-        yield keep_client
-    else:
-        orig_token = keep_client.api_token
-        keep_client.api_token = ''
-        yield keep_client
-        keep_client.api_token = orig_token
-
 class KeepTestCase(run_test_server.TestCaseWithServers):
     MAIN_SERVER = {}
     KEEP_SERVER = {}
 
-    def setUp(self):
-        arvados.keep.global_client_object = None
+    @classmethod
+    def setUpClass(cls):
+        super(KeepTestCase, cls).setUpClass()
         run_test_server.authorize_with("admin")
+        cls.api_client = arvados.api('v1')
+        cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
+                                             proxy='', local_store='')
 
     def test_KeepBasicRWTest(self):
-        foo_locator = arvados.Keep.put('foo')
+        foo_locator = self.keep_client.put('foo')
         self.assertRegexpMatches(
             foo_locator,
             '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
             'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
-        self.assertEqual(arvados.Keep.get(foo_locator),
+        self.assertEqual(self.keep_client.get(foo_locator),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
 
     def test_KeepBinaryRWTest(self):
         blob_str = '\xff\xfe\xf7\x00\x01\x02'
-        blob_locator = arvados.Keep.put(blob_str)
+        blob_locator = self.keep_client.put(blob_str)
         self.assertRegexpMatches(
             blob_locator,
             '^7fc7c53b45e53926ba52821140fef396\+6',
             ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
-        self.assertEqual(arvados.Keep.get(blob_locator),
+        self.assertEqual(self.keep_client.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
 
@@ -54,23 +45,23 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
         for i in range(0,23):
             blob_str = blob_str + blob_str
-        blob_locator = arvados.Keep.put(blob_str)
+        blob_locator = self.keep_client.put(blob_str)
         self.assertRegexpMatches(
             blob_locator,
             '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
-        self.assertEqual(arvados.Keep.get(blob_locator),
+        self.assertEqual(self.keep_client.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
 
     def test_KeepSingleCopyRWTest(self):
         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
-        blob_locator = arvados.Keep.put(blob_str, copies=1)
+        blob_locator = self.keep_client.put(blob_str, copies=1)
         self.assertRegexpMatches(
             blob_locator,
             '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
-        self.assertEqual(arvados.Keep.get(blob_locator),
+        self.assertEqual(self.keep_client.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
 
@@ -82,24 +73,25 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
 
     def test_KeepBasicRWTest(self):
         run_test_server.authorize_with('active')
-        foo_locator = arvados.Keep.put('foo')
+        keep_client = arvados.KeepClient()
+        foo_locator = keep_client.put('foo')
         self.assertRegexpMatches(
             foo_locator,
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + foo_locator)
-        self.assertEqual(arvados.Keep.get(foo_locator),
+        self.assertEqual(keep_client.get(foo_locator),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
 
         # GET with an unsigned locator => NotFound
-        bar_locator = arvados.Keep.put('bar')
+        bar_locator = keep_client.put('bar')
         unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
         self.assertRegexpMatches(
             bar_locator,
             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("bar"): ' + bar_locator)
         self.assertRaises(arvados.errors.NotFoundError,
-                          arvados.Keep.get,
+                          keep_client.get,
                           unsigned_bar_locator)
 
         # GET from a different user => NotFound
@@ -110,13 +102,13 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
 
         # Unauthenticated GET for a signed locator => NotFound
         # Unauthenticated GET for an unsigned locator => NotFound
-        with unauthenticated_client():
-            self.assertRaises(arvados.errors.NotFoundError,
-                              arvados.Keep.get,
-                              bar_locator)
-            self.assertRaises(arvados.errors.NotFoundError,
-                              arvados.Keep.get,
-                              unsigned_bar_locator)
+        keep_client.api_token = ''
+        self.assertRaises(arvados.errors.NotFoundError,
+                          keep_client.get,
+                          bar_locator)
+        self.assertRaises(arvados.errors.NotFoundError,
+                          keep_client.get,
+                          unsigned_bar_locator)
 
 
 # KeepOptionalPermission: starts Keep with --permission-key-file
@@ -133,57 +125,54 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers):
     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
                    'enforce_permissions': False}
 
-    def test_KeepAuthenticatedSignedTest(self):
-        run_test_server.authorize_with('active')
-        signed_locator = arvados.Keep.put('foo')
+    @classmethod
+    def setUpClass(cls):
+        super(KeepOptionalPermission, cls).setUpClass()
+        run_test_server.authorize_with("admin")
+        cls.api_client = arvados.api('v1')
+
+    def setUp(self):
+        super(KeepOptionalPermission, self).setUp()
+        self.keep_client = arvados.KeepClient(api_client=self.api_client,
+                                              proxy='', local_store='')
+
+    def _put_foo_and_check(self):
+        signed_locator = self.keep_client.put('foo')
         self.assertRegexpMatches(
             signed_locator,
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + signed_locator)
-        self.assertEqual(arvados.Keep.get(signed_locator),
+        return signed_locator
+
+    def test_KeepAuthenticatedSignedTest(self):
+        signed_locator = self._put_foo_and_check()
+        self.assertEqual(self.keep_client.get(signed_locator),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
 
     def test_KeepAuthenticatedUnsignedTest(self):
-        run_test_server.authorize_with('active')
-        signed_locator = arvados.Keep.put('foo')
-        self.assertRegexpMatches(
-            signed_locator,
-            r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
-            'invalid locator from Keep.put("foo"): ' + signed_locator)
-        self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+        signed_locator = self._put_foo_and_check()
+        self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
 
     def test_KeepUnauthenticatedSignedTest(self):
-        # Since --enforce-permissions is not in effect, GET requests
-        # need not be authenticated.
-        run_test_server.authorize_with('active')
-        signed_locator = arvados.Keep.put('foo')
-        self.assertRegexpMatches(
-            signed_locator,
-            r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
-            'invalid locator from Keep.put("foo"): ' + signed_locator)
-
-        with unauthenticated_client():
-            self.assertEqual(arvados.Keep.get(signed_locator),
-                             'foo',
-                             'wrong content from Keep.get(md5("foo"))')
+        # Check that signed GET requests work even when permissions
+        # enforcement is off.
+        signed_locator = self._put_foo_and_check()
+        self.keep_client.api_token = ''
+        self.assertEqual(self.keep_client.get(signed_locator),
+                         'foo',
+                         'wrong content from Keep.get(md5("foo"))')
 
     def test_KeepUnauthenticatedUnsignedTest(self):
         # Since --enforce-permissions is not in effect, GET requests
         # need not be authenticated.
-        run_test_server.authorize_with('active')
-        signed_locator = arvados.Keep.put('foo')
-        self.assertRegexpMatches(
-            signed_locator,
-            r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
-            'invalid locator from Keep.put("foo"): ' + signed_locator)
-
-        with unauthenticated_client():
-            self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
-                             'foo',
-                             'wrong content from Keep.get(md5("foo"))')
+        signed_locator = self._put_foo_and_check()
+        self.keep_client.api_token = ''
+        self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+                         'foo',
+                         'wrong content from Keep.get(md5("foo"))')
 
 
 class KeepProxyTestCase(run_test_server.TestCaseWithServers):
@@ -194,43 +183,39 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
     @classmethod
     def setUpClass(cls):
         super(KeepProxyTestCase, cls).setUpClass()
-        cls.proxy_addr = os.environ['ARVADOS_KEEP_PROXY']
+        cls.api_client = arvados.api('v1')
 
-    def setUp(self):
-        arvados.keep.global_client_object = None
-        os.environ['ARVADOS_KEEP_PROXY'] = self.proxy_addr
-        os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
+    def tearDown(self):
+        arvados.config.settings().pop('ARVADOS_EXTERNAL_CLIENT', None)
+        super(KeepProxyTestCase, self).tearDown()
 
     def test_KeepProxyTest1(self):
         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
         # setUpClass().
-        baz_locator = arvados.Keep.put('baz')
+        keep_client = arvados.KeepClient(api_client=self.api_client,
+                                         local_store='')
+        baz_locator = keep_client.put('baz')
         self.assertRegexpMatches(
             baz_locator,
             '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
             'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
-        self.assertEqual(arvados.Keep.get(baz_locator),
+        self.assertEqual(keep_client.get(baz_locator),
                          'baz',
                          'wrong content from Keep.get(md5("baz"))')
-
-        self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
+        self.assertTrue(keep_client.using_proxy)
 
     def test_KeepProxyTest2(self):
-        # We don't want to use ARVADOS_KEEP_PROXY from run_keep_proxy() in
-        # setUpClass(), so clear it and set ARVADOS_EXTERNAL_CLIENT which will
-        # contact the API server.
-        os.environ["ARVADOS_KEEP_PROXY"] = ""
-        os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
-
-        # Will send X-External-Client to server and get back the proxy from
-        # keep_services/accessible
-        baz_locator = arvados.Keep.put('baz2')
+        # Don't instantiate the proxy directly, but set the X-External-Client
+        # header.  The API server should direct us to the proxy.
+        arvados.config.settings()['ARVADOS_EXTERNAL_CLIENT'] = 'true'
+        keep_client = arvados.KeepClient(api_client=self.api_client,
+                                         proxy='', local_store='')
+        baz_locator = keep_client.put('baz2')
         self.assertRegexpMatches(
             baz_locator,
             '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
             'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
-        self.assertEqual(arvados.Keep.get(baz_locator),
+        self.assertEqual(keep_client.get(baz_locator),
                          'baz2',
                          'wrong content from Keep.get(md5("baz2"))')
-
-        self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
+        self.assertTrue(keep_client.using_proxy)

commit 3b9b5c14f6b1cd0cbed75e0d4adf36e10323dbe7
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Aug 20 09:46:49 2014 -0400

    2800: Introduce TestCaseWithServers to Python SDK.
    
    This is a subclass of unittest.TestCase.  It looks for specific class
    variables to launch supporting Arvados servers for subclass test
    cases, taking care to adjust the environment and local Arvados
    configuration to match.  Put another away, this refactors a number of
    similar setUpClass/tearDownClass methods throughout the SDK tests.

diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index cd86d80..369d561 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -27,20 +27,6 @@ class ArvadosBaseTestCase(unittest.TestCase):
             basedir = '.'
         return open(os.path.join(basedir, 'data', filename))
 
-
-class ArvadosKeepLocalStoreTestCase(ArvadosBaseTestCase):
-    def setUp(self):
-        super(ArvadosKeepLocalStoreTestCase, self).setUp()
-        self._orig_keep_local_store = os.environ.get('KEEP_LOCAL_STORE')
-        os.environ['KEEP_LOCAL_STORE'] = self.make_tmpdir()
-
-    def tearDown(self):
-        if self._orig_keep_local_store is None:
-            del os.environ['KEEP_LOCAL_STORE']
-        else:
-            os.environ['KEEP_LOCAL_STORE'] = self._orig_keep_local_store
-        super(ArvadosKeepLocalStoreTestCase, self).tearDown()
-
     def build_directory_tree(self, tree):
         tree_root = self.make_tmpdir()
         for leaf in tree:
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index a225b74..0173718 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -8,6 +8,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import unittest
 import yaml
 
 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
@@ -216,7 +217,7 @@ def run_keep_proxy(auth):
     api = arvados.api('v1', cache=False)
     api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
 
-    arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+    os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 
 def stop_keep_proxy():
     kill_server_pid("tests/tmp/keepproxy.pid", 0)
@@ -233,6 +234,65 @@ def authorize_with(token):
     arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
     arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
 
+class TestCaseWithServers(unittest.TestCase):
+    """TestCase to start and stop supporting Arvados servers.
+
+    Define any of MAIN_SERVER, KEEP_SERVER, and/or KEEP_PROXY_SERVER
+    class variables as a dictionary of keyword arguments.  If you do,
+    setUpClass will start the corresponding servers by passing these
+    keyword arguments to the run, run_keep, and/or run_keep_server
+    functions, respectively.  It will also set Arvados environment
+    variables to point to these servers appropriately.  If you don't
+    run a Keep or Keep proxy server, setUpClass will set up a
+    temporary directory for Keep local storage, and set it as
+    KEEP_LOCAL_STORE.
+
+    tearDownClass will stop any servers started, and restore the
+    original environment.
+    """
+    MAIN_SERVER = None
+    KEEP_SERVER = None
+    KEEP_PROXY_SERVER = None
+
+    @staticmethod
+    def _restore_dict(src, dest):
+        for key in dest.keys():
+            if key not in src:
+                del dest[key]
+        dest.update(src)
+
+    @classmethod
+    def setUpClass(cls):
+        cls._orig_environ = os.environ.copy()
+        cls._orig_config = arvados.config.settings().copy()
+        cls._cleanup_funcs = []
+        for server_kwargs, start_func, stop_func in (
+              (cls.MAIN_SERVER, run, stop),
+              (cls.KEEP_SERVER, run_keep, stop_keep),
+              (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
+            if server_kwargs is not None:
+                start_func(**server_kwargs)
+                cls._cleanup_funcs.append(stop_func)
+        os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
+        if cls.KEEP_PROXY_SERVER is None:
+            os.environ.pop('ARVADOS_KEEP_PROXY', None)
+        if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
+            cls.local_store = tempfile.mkdtemp()
+            os.environ['KEEP_LOCAL_STORE'] = cls.local_store
+            cls._cleanup_funcs.append(
+                lambda: shutil.rmtree(cls.local_store, ignore_errors=True))
+        else:
+            os.environ.pop('KEEP_LOCAL_STORE', None)
+        arvados.config.initialize()
+
+    @classmethod
+    def tearDownClass(cls):
+        for clean_func in cls._cleanup_funcs:
+            clean_func()
+        cls._restore_dict(cls._orig_environ, os.environ)
+        cls._restore_dict(cls._orig_config, arvados.config.settings())
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 5ce475e..de4fdd2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -18,7 +18,7 @@ from cStringIO import StringIO
 import arvados
 import arvados.commands.put as arv_put
 
-from arvados_testutil import ArvadosBaseTestCase, ArvadosKeepLocalStoreTestCase
+from arvados_testutil import ArvadosBaseTestCase
 import run_test_server
 
 class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
@@ -196,9 +196,11 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
                           arv_put.ResumeCache, path)
 
 
-class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers,
+                                     ArvadosBaseTestCase):
     def setUp(self):
         super(ArvadosPutCollectionWriterTest, self).setUp()
+        run_test_server.authorize_with('active')
         with tempfile.NamedTemporaryFile(delete=False) as cachefile:
             self.cache = arv_put.ResumeCache(cachefile.name)
             self.cache_filename = cachefile.name
@@ -398,7 +400,9 @@ class ArvadosPutProjectLinkTest(ArvadosBaseTestCase):
                           ['--project-uuid', self.Z_UUID, '--stream'])
 
 
-class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
+    MAIN_SERVER = {}
+
     def call_main_with_args(self, args):
         self.main_stdout = StringIO()
         self.main_stderr = StringIO()
@@ -415,6 +419,7 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
 
     def setUp(self):
         super(ArvadosPutTest, self).setUp()
+        run_test_server.authorize_with('active')
         arv_put.api_client = None
 
     def tearDown(self):
@@ -454,41 +459,32 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
                           ['--name', 'test without Collection',
                            '--stream', '/dev/null'])
 
-class ArvPutIntegrationTest(unittest.TestCase):
-    PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
-    ENVIRON = os.environ
-    ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
-
-    @classmethod
-    def setUpClass(cls):
-        try:
-            del os.environ['KEEP_LOCAL_STORE']
-        except KeyError:
-            pass
-
-        # Use the blob_signing_key from the Rails "test" configuration
-        # to provision the Keep server.
-        config_blob_signing_key = None
+class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
+                            ArvadosBaseTestCase):
+    def _getKeepServerConfig():
         for config_file in ['application.yml', 'application.default.yml']:
             with open(os.path.join(run_test_server.SERVICES_SRC_DIR,
                                    "api", "config", config_file)) as f:
                 rails_config = yaml.load(f.read())
                 for config_section in ['test', 'common']:
                     try:
-                        config_blob_signing_key = rails_config[config_section]["blob_signing_key"]
-                        break
-                    except KeyError:
+                        key = rails_config[config_section]["blob_signing_key"]
+                    except (KeyError, TypeError):
                         pass
-            if config_blob_signing_key is not None:
-                break
-        run_test_server.run()
-        run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
-                                 enforce_permissions=(config_blob_signing_key != None))
+                    else:
+                        return {'blob_signing_key': key,
+                                'enforce_permissions': True}
+        return {'blog_signing_key': None, 'enforce_permissions': False}
+
+    MAIN_SERVER = {}
+    KEEP_SERVER = _getKeepServerConfig()
+    PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
 
     @classmethod
-    def tearDownClass(cls):
-        run_test_server.stop()
-        run_test_server.stop_keep()
+    def setUpClass(cls):
+        super(ArvPutIntegrationTest, cls).setUpClass()
+        cls.ENVIRON = os.environ.copy()
+        cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
 
     def setUp(self):
         super(ArvPutIntegrationTest, self).setUp()
@@ -499,7 +495,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
         for v in ["ARVADOS_API_HOST",
                   "ARVADOS_API_HOST_INSECURE",
                   "ARVADOS_API_TOKEN"]:
-            os.environ[v] = arvados.config.settings()[v]
+            self.ENVIRON[v] = arvados.config.settings()[v]
         arv_put.api_client = arvados.api('v1', cache=False)
 
     def current_user(self):
@@ -556,7 +552,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
             notfound = arv_put.api_client.collections().get(
                 uuid=manifest_uuid).execute()
 
-        datadir = tempfile.mkdtemp()
+        datadir = self.make_tmpdir()
         with open(os.path.join(datadir, "foo"), "w") as f:
             f.write("The quick brown fox jumped over the lazy dog")
         p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
@@ -573,9 +569,6 @@ class ArvPutIntegrationTest(unittest.TestCase):
             c['manifest_text'],
             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
 
-        os.remove(os.path.join(datadir, "foo"))
-        os.rmdir(datadir)
-
     def run_and_find_link(self, text, extra_args=[]):
         self.authorize_with('active')
         pipe = subprocess.Popen(
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index b1af723..68c9698 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -11,7 +11,8 @@ import subprocess
 import tempfile
 import unittest
 
-from arvados_testutil import ArvadosKeepLocalStoreTestCase
+import run_test_server
+from arvados_testutil import ArvadosBaseTestCase
 
 class TestResumableWriter(arvados.ResumableCollectionWriter):
     KEEP_BLOCK_SIZE = 1024  # PUT to Keep every 1K.
@@ -20,7 +21,15 @@ class TestResumableWriter(arvados.ResumableCollectionWriter):
         return self.dump_state(copy.deepcopy)
 
 
-class ArvadosCollectionsTest(ArvadosKeepLocalStoreTestCase):
+class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
+                             ArvadosBaseTestCase):
+    MAIN_SERVER = {}
+
+    @classmethod
+    def setUpClass(cls):
+        super(ArvadosCollectionsTest, cls).setUpClass()
+        run_test_server.authorize_with('active')
+
     def write_foo_bar_baz(self):
         cw = arvados.CollectionWriter()
         self.assertEqual(cw.current_stream_name(), '.',
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 9d3cecd..fb52ab4 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -2,38 +2,33 @@
 #
 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
 
+import contextlib
 import os
 import unittest
 
 import arvados
 import run_test_server
 
-class KeepTestCase(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        super(KeepTestCase, cls).setUpClass()
-        try:
-            del os.environ['KEEP_LOCAL_STORE']
-        except KeyError:
-            pass
-
-        # Make sure these are clear, we want to talk to the Keep servers
-        # directly.
-        os.environ["ARVADOS_KEEP_PROXY"] = ""
-        os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
-
-        run_test_server.run()
-        run_test_server.run_keep()
+ at contextlib.contextmanager
+def unauthenticated_client(keep_client=None):
+    if keep_client is None:
+        keep_client = arvados.keep.global_client_object
+    if not hasattr(keep_client, 'api_token'):
+        yield keep_client
+    else:
+        orig_token = keep_client.api_token
+        keep_client.api_token = ''
+        yield keep_client
+        keep_client.api_token = orig_token
+
+class KeepTestCase(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {}
+    KEEP_SERVER = {}
+
+    def setUp(self):
         arvados.keep.global_client_object = None
-        arvados.config._settings = None
         run_test_server.authorize_with("admin")
 
-    @classmethod
-    def tearDownClass(cls):
-        super(KeepTestCase, cls).tearDownClass()
-        run_test_server.stop()
-        run_test_server.stop_keep()
-
     def test_KeepBasicRWTest(self):
         foo_locator = arvados.Keep.put('foo')
         self.assertRegexpMatches(
@@ -79,22 +74,11 @@ class KeepTestCase(unittest.TestCase):
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
 
-class KeepPermissionTestCase(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        try:
-            del os.environ['KEEP_LOCAL_STORE']
-        except KeyError:
-            pass
 
-        run_test_server.run()
-        run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
-                                 enforce_permissions=True)
-
-    @classmethod
-    def tearDownClass(cls):
-        run_test_server.stop()
-        run_test_server.stop_keep()
+class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {}
+    KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
+                   'enforce_permissions': True}
 
     def test_KeepBasicRWTest(self):
         run_test_server.authorize_with('active')
@@ -126,13 +110,14 @@ class KeepPermissionTestCase(unittest.TestCase):
 
         # Unauthenticated GET for a signed locator => NotFound
         # Unauthenticated GET for an unsigned locator => NotFound
-        arvados.keep.global_client_object.api_token = ''
-        self.assertRaises(arvados.errors.NotFoundError,
-                          arvados.Keep.get,
-                          bar_locator)
-        self.assertRaises(arvados.errors.NotFoundError,
-                          arvados.Keep.get,
-                          unsigned_bar_locator)
+        with unauthenticated_client():
+            self.assertRaises(arvados.errors.NotFoundError,
+                              arvados.Keep.get,
+                              bar_locator)
+            self.assertRaises(arvados.errors.NotFoundError,
+                              arvados.Keep.get,
+                              unsigned_bar_locator)
+
 
 # KeepOptionalPermission: starts Keep with --permission-key-file
 # but not --enforce-permissions (i.e. generate signatures on PUT
@@ -143,22 +128,10 @@ class KeepPermissionTestCase(unittest.TestCase):
 # * authenticated request, unsigned locator
 # * unauthenticated request, signed locator
 # * unauthenticated request, unsigned locator
-
-class KeepOptionalPermission(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        try:
-            del os.environ['KEEP_LOCAL_STORE']
-        except KeyError:
-            pass
-        run_test_server.run()
-        run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
-                                 enforce_permissions=False)
-
-    @classmethod
-    def tearDownClass(cls):
-        run_test_server.stop()
-        run_test_server.stop_keep()
+class KeepOptionalPermission(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {}
+    KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
+                   'enforce_permissions': False}
 
     def test_KeepAuthenticatedSignedTest(self):
         run_test_server.authorize_with('active')
@@ -192,10 +165,10 @@ class KeepOptionalPermission(unittest.TestCase):
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + signed_locator)
 
-        arvados.keep.global_client_object.api_token = ''
-        self.assertEqual(arvados.Keep.get(signed_locator),
-                         'foo',
-                         'wrong content from Keep.get(md5("foo"))')
+        with unauthenticated_client():
+            self.assertEqual(arvados.Keep.get(signed_locator),
+                             'foo',
+                             'wrong content from Keep.get(md5("foo"))')
 
     def test_KeepUnauthenticatedUnsignedTest(self):
         # Since --enforce-permissions is not in effect, GET requests
@@ -207,48 +180,30 @@ class KeepOptionalPermission(unittest.TestCase):
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + signed_locator)
 
-        arvados.keep.global_client_object.api_token = ''
-        self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
-                         'foo',
-                         'wrong content from Keep.get(md5("foo"))')
+        with unauthenticated_client():
+            self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
+                             'foo',
+                             'wrong content from Keep.get(md5("foo"))')
 
 
-class KeepProxyTestCase(unittest.TestCase):
+class KeepProxyTestCase(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {}
+    KEEP_SERVER = {}
+    KEEP_PROXY_SERVER = {'auth': 'admin'}
+
     @classmethod
     def setUpClass(cls):
         super(KeepProxyTestCase, cls).setUpClass()
+        cls.proxy_addr = os.environ['ARVADOS_KEEP_PROXY']
 
-        try:
-            del os.environ['KEEP_LOCAL_STORE']
-        except KeyError:
-            pass
-
-        os.environ["ARVADOS_KEEP_PROXY"] = ""
-        os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
-
-        run_test_server.run()
-        run_test_server.run_keep()
+    def setUp(self):
         arvados.keep.global_client_object = None
-        arvados.config._settings = None
-        run_test_server.run_keep_proxy("admin")
-        KeepProxyTestCase.arvados_keep_proxy = arvados.config.get("ARVADOS_KEEP_PROXY")
-
-    @classmethod
-    def tearDownClass(cls):
-        super(KeepProxyTestCase, cls).tearDownClass()
-        run_test_server.stop()
-        run_test_server.stop_keep()
-        run_test_server.stop_keep_proxy()
+        os.environ['ARVADOS_KEEP_PROXY'] = self.proxy_addr
+        os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
 
     def test_KeepProxyTest1(self):
         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
-        # run_keep_proxy() in setUpClass()
-
-        os.environ["ARVADOS_KEEP_PROXY"] = KeepProxyTestCase.arvados_keep_proxy
-        os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
-        arvados.keep.global_client_object = None
-        arvados.config._settings = None
-
+        # setUpClass().
         baz_locator = arvados.Keep.put('baz')
         self.assertRegexpMatches(
             baz_locator,
@@ -266,12 +221,9 @@ class KeepProxyTestCase(unittest.TestCase):
         # contact the API server.
         os.environ["ARVADOS_KEEP_PROXY"] = ""
         os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
-        arvados.keep.global_client_object = None
-        arvados.config._settings = None
 
         # Will send X-External-Client to server and get back the proxy from
         # keep_services/accessible
-
         baz_locator = arvados.Keep.put('baz2')
         self.assertRegexpMatches(
             baz_locator,
diff --git a/sdk/python/tests/test_websockets.py b/sdk/python/tests/test_websockets.py
index 6b57fe3..1dae978 100644
--- a/sdk/python/tests/test_websockets.py
+++ b/sdk/python/tests/test_websockets.py
@@ -4,9 +4,8 @@ import arvados
 import arvados.events
 import time
 
-class WebsocketTest(unittest.TestCase):
-    def setUp(self):
-        run_test_server.run(websockets=True)
+class WebsocketTest(run_test_server.TestCaseWithServers):
+    MAIN_SERVER = {'websockets': True}
 
     def on_event(self, ev):
         if self.state == 1:
@@ -27,6 +26,3 @@ class WebsocketTest(unittest.TestCase):
         time.sleep(1)
         self.h = api.humans().create(body={}).execute()
         time.sleep(1)
-
-    def tearDown(self):
-        run_test_server.stop()

commit da01d7e185a6ce5e96d542c180856ce84e94ed63
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Aug 19 16:41:33 2014 -0400

    2800: Make run_test_server put Keep tempfiles in tests/tmp/.

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 8ac34b8..a225b74 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -141,29 +141,29 @@ def _start_keep(n, keep_args):
     keep_cmd = ["keepstore",
                 "-volumes={}".format(keep0),
                 "-listen=:{}".format(25107+n),
-                "-pid={}".format("tmp/keep{}.pid".format(n))]
+                "-pid={}".format("tests/tmp/keep{}.pid".format(n))]
 
     for arg, val in keep_args.iteritems():
         keep_cmd.append("{}={}".format(arg, val))
 
     kp0 = subprocess.Popen(keep_cmd)
-    with open("tmp/keep{}.pid".format(n), 'w') as f:
+    with open("tests/tmp/keep{}.pid".format(n), 'w') as f:
         f.write(str(kp0.pid))
 
-    with open("tmp/keep{}.volume".format(n), 'w') as f:
+    with open("tests/tmp/keep{}.volume".format(n), 'w') as f:
         f.write(keep0)
 
 def run_keep(blob_signing_key=None, enforce_permissions=False):
     stop_keep()
 
-    if not os.path.exists("tmp"):
-        os.mkdir("tmp")
+    if not os.path.exists("tests/tmp"):
+        os.mkdir("tests/tmp")
 
     keep_args = {}
     if blob_signing_key:
-        with open("tmp/keep.blob_signing_key", "w") as f:
+        with open("tests/tmp/keep.blob_signing_key", "w") as f:
             f.write(blob_signing_key)
-        keep_args['--permission-key-file'] = 'tmp/keep.blob_signing_key'
+        keep_args['--permission-key-file'] = 'tests/tmp/keep.blob_signing_key'
     if enforce_permissions:
         keep_args['--enforce-permissions'] = 'true'
 
@@ -186,13 +186,13 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
     api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
 
 def _stop_keep(n):
-    kill_server_pid("tmp/keep{}.pid".format(n), 0)
-    if os.path.exists("tmp/keep{}.volume".format(n)):
-        with open("tmp/keep{}.volume".format(n), 'r') as r:
+    kill_server_pid("tests/tmp/keep{}.pid".format(n), 0)
+    if os.path.exists("tests/tmp/keep{}.volume".format(n)):
+        with open("tests/tmp/keep{}.volume".format(n), 'r') as r:
             shutil.rmtree(r.read(), True)
-        os.unlink("tmp/keep{}.volume".format(n))
-    if os.path.exists("tmp/keep.blob_signing_key"):
-        os.remove("tmp/keep.blob_signing_key")
+        os.unlink("tests/tmp/keep{}.volume".format(n))
+    if os.path.exists("tests/tmp/keep.blob_signing_key"):
+        os.remove("tests/tmp/keep.blob_signing_key")
 
 def stop_keep():
     _stop_keep(0)
@@ -201,15 +201,16 @@ def stop_keep():
 def run_keep_proxy(auth):
     stop_keep_proxy()
 
-    if not os.path.exists("tmp"):
-        os.mkdir("tmp")
+    if not os.path.exists("tests/tmp"):
+        os.mkdir("tests/tmp")
 
     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
     os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 
     kp0 = subprocess.Popen(["keepproxy",
-                            "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
+                            "-pid=tests/tmp/keepproxy.pid",
+                            "-listen=:{}".format(25101)])
 
     authorize_with("admin")
     api = arvados.api('v1', cache=False)
@@ -218,7 +219,7 @@ def run_keep_proxy(auth):
     arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
 
 def stop_keep_proxy():
-    kill_server_pid("tmp/keepproxy.pid", 0)
+    kill_server_pid("tests/tmp/keepproxy.pid", 0)
 
 def fixture(fix):
     '''load a fixture yaml file'''

commit e17db6aa8bae683f31d06500bbd78b043dce8d9a
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Aug 19 16:37:42 2014 -0400

    2800: Use dedicated logger in Python SDK api module.

diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 1eb8f51..cb716f1 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -99,7 +99,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
 
     if not version:
         version = 'v1'
-        logging.info("Using default API version. " +
+        _logger.info("Using default API version. " +
                      "Call arvados.api('%s') instead." %
                      version)
     if 'discoveryServiceUrl' in kwargs:

commit 327e1768a130476a411baf4cd4fb602f7a73c5ed
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Aug 20 09:46:33 2014 -0400

    2800: Migrate from Keep to KeepClient in the Python SDK.

diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 7aed681..40b5889 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -92,7 +92,9 @@ def normalize(collection):
 
 
 class CollectionReader(object):
-    def __init__(self, manifest_locator_or_text):
+    def __init__(self, manifest_locator_or_text, api_client=None):
+        self._api_client = api_client
+        self._keep_client = None
         if re.match(r'[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
             self._manifest_locator = manifest_locator_or_text
             self._manifest_text = None
@@ -111,17 +113,21 @@ class CollectionReader(object):
         pass
 
     def _populate(self):
-        if self._streams != None:
+        if self._streams is not None:
             return
         if not self._manifest_text:
+            if self._api_client is None:
+                self._api_client = arvados.api('v1')
+            if self._keep_client is None:
+                self._keep_client = KeepClient(api_client=self._api_client)
             try:
-                c = arvados.api('v1').collections().get(
+                c = self._api_client.collections().get(
                     uuid=self._manifest_locator).execute()
                 self._manifest_text = c['manifest_text']
             except Exception as e:
                 _logger.warning("API lookup failed for collection %s (%s: %s)",
                                 self._manifest_locator, type(e), str(e))
-                self._manifest_text = Keep.get(self._manifest_locator)
+                self._manifest_text = self._keep_client.get(self._manifest_locator)
         self._streams = []
         for stream_line in self._manifest_text.split("\n"):
             if stream_line != '':
@@ -159,7 +165,9 @@ class CollectionReader(object):
 class CollectionWriter(object):
     KEEP_BLOCK_SIZE = 2**26
 
-    def __init__(self):
+    def __init__(self, api_client=None):
+        self._api_client = api_client
+        self._keep_client = None
         self._data_buffer = []
         self._data_buffer_len = 0
         self._current_stream_files = []
@@ -180,6 +188,10 @@ class CollectionWriter(object):
     def __exit__(self):
         self.finish()
 
+    def _prep_keep_client(self):
+        if self._keep_client is None:
+            self._keep_client = KeepClient(api_client=self._api_client)
+
     def do_queued_work(self):
         # The work queue consists of three pieces:
         # * _queued_file: The file object we're currently writing to the
@@ -285,7 +297,9 @@ class CollectionWriter(object):
     def flush_data(self):
         data_buffer = ''.join(self._data_buffer)
         if data_buffer != '':
-            self._current_stream_locators += [Keep.put(data_buffer[0:self.KEEP_BLOCK_SIZE])]
+            self._prep_keep_client()
+            self._current_stream_locators.append(
+                self._keep_client.put(data_buffer[0:self.KEEP_BLOCK_SIZE]))
             self._data_buffer = [data_buffer[self.KEEP_BLOCK_SIZE:]]
             self._data_buffer_len = len(self._data_buffer[0])
 
@@ -355,7 +369,8 @@ class CollectionWriter(object):
 
     def finish(self):
         # Store the manifest in Keep and return its locator.
-        return Keep.put(self.manifest_text())
+        self._prep_keep_client()
+        return self._keep_client.put(self.manifest_text())
 
     def stripped_manifest(self):
         """
@@ -403,9 +418,9 @@ class ResumableCollectionWriter(CollectionWriter):
                    '_data_buffer', '_dependencies', '_finished_streams',
                    '_queued_dirents', '_queued_trees']
 
-    def __init__(self):
+    def __init__(self, api_client=None):
         self._dependencies = {}
-        super(ResumableCollectionWriter, self).__init__()
+        super(ResumableCollectionWriter, self).__init__(api_client)
 
     @classmethod
     def from_state(cls, state, *init_args, **init_kwargs):
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 335ef17..e4b763a 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -22,6 +22,7 @@ import tempfile
 import arvados.commands._util as arv_cmd
 
 CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
+api_client = None
 
 upload_opts = argparse.ArgumentParser(add_help=False)
 
@@ -238,13 +239,14 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
     STATE_PROPS = (arvados.ResumableCollectionWriter.STATE_PROPS +
                    ['bytes_written', '_seen_inputs'])
 
-    def __init__(self, cache=None, reporter=None, bytes_expected=None):
+    def __init__(self, cache=None, reporter=None, bytes_expected=None,
+                 api_client=None):
         self.bytes_written = 0
         self._seen_inputs = []
         self.cache = cache
         self.reporter = reporter
         self.bytes_expected = bytes_expected
-        super(ArvPutCollectionWriter, self).__init__()
+        super(ArvPutCollectionWriter, self).__init__(api_client)
 
     @classmethod
     def from_cache(cls, cache, reporter=None, bytes_expected=None):
@@ -342,7 +344,7 @@ def exit_signal_handler(sigcode, frame):
 
 def check_project_exists(project_uuid):
     try:
-        arvados.api('v1').groups().get(uuid=project_uuid).execute()
+        api_client.groups().get(uuid=project_uuid).execute()
     except (apiclient.errors.Error, arvados.errors.NotFoundError) as error:
         raise ValueError("Project {} not found ({})".format(project_uuid,
                                                             error))
@@ -362,7 +364,7 @@ def prep_project_link(args, stderr, project_exists=check_project_exists):
             'link_class': 'name',
             'name': args.name}
     if not link['tail_uuid']:
-        link['tail_uuid'] = arvados.api('v1').users().current().execute()['uuid']
+        link['tail_uuid'] = api_client.users().current().execute()['uuid']
     elif not project_exists(link['tail_uuid']):
         raise ValueError("Project {} not found".format(args.project_uuid))
     if not link['name']:
@@ -377,9 +379,12 @@ def prep_project_link(args, stderr, project_exists=check_project_exists):
 
 def create_project_link(locator, link):
     link['head_uuid'] = locator
-    return arvados.api('v1').links().create(body=link).execute()
+    return api_client.links().create(body=link).execute()
 
 def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
+    global api_client
+    if api_client is None:
+        api_client = arvados.api('v1')
     status = 0
 
     args = parse_arguments(arguments)
@@ -445,7 +450,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
         output = ','.join(writer.data_locators())
     else:
         # Register the resulting collection in Arvados.
-        collection = arvados.api().collections().create(
+        collection = api_client.collections().create(
             body={
                 'manifest_text': writer.manifest_text(),
                 },
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 7a29100..064a4f2 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -204,10 +204,9 @@ class StreamReader(object):
         self._data_locators = []
         self._files = collections.OrderedDict()
 
-        if keep != None:
-            self._keep = keep
-        else:
-            self._keep = Keep.global_client_object()
+        if keep is None:
+            keep = KeepClient()
+        self._keep = keep
 
         streamoffset = 0L
 
diff --git a/sdk/python/bin/arv-get b/sdk/python/bin/arv-get
index 7f02cf7..0d403d1 100755
--- a/sdk/python/bin/arv-get
+++ b/sdk/python/bin/arv-get
@@ -126,6 +126,7 @@ if args.r and not get_prefix:
 
 todo = []
 todo_bytes = 0
+api_client = arvados.api('v1')
 if not get_prefix:
     try:
         if not args.n:
@@ -133,15 +134,15 @@ if not get_prefix:
                 abort('Local file %s already exists.' % (args.destination,))
             with open(args.destination, 'wb') as f:
                 try:
-                    c = arvados.api('v1').collections().get(
-                        uuid=collection).execute()
+                    c = api_client.collections().get(uuid=collection).execute()
                     manifest = c['manifest_text']
                 except Exception as e:
                     logger.warning(
                         "Collection %s not found. " +
                         "Trying to fetch directly from Keep (deprecated).",
                         collection)
-                    manifest = arvados.Keep.get(collection)
+                    manifest = arvados.KeepClient(
+                        api_client=api_client).get(collection)
                 f.write(manifest)
         sys.exit(0)
     except arvados.errors.NotFoundError as e:
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 9bc385d..5ce475e 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -413,6 +413,10 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
                                         '098f6bcd4621d373cade4e832627b4f6')),
             "did not find file stream in Keep store")
 
+    def setUp(self):
+        super(ArvadosPutTest, self).setUp()
+        arv_put.api_client = None
+
     def tearDown(self):
         for outbuf in ['main_stdout', 'main_stderr']:
             if hasattr(self, outbuf):
@@ -473,9 +477,9 @@ class ArvPutIntegrationTest(unittest.TestCase):
                     try:
                         config_blob_signing_key = rails_config[config_section]["blob_signing_key"]
                         break
-                    except KeyError, AttributeError:
+                    except KeyError:
                         pass
-            if config_blob_signing_key != None:
+            if config_blob_signing_key is not None:
                 break
         run_test_server.run()
         run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
@@ -486,22 +490,29 @@ class ArvPutIntegrationTest(unittest.TestCase):
         run_test_server.stop()
         run_test_server.stop_keep()
 
+    def setUp(self):
+        super(ArvPutIntegrationTest, self).setUp()
+        arv_put.api_client = None
+
     def authorize_with(self, token_name):
         run_test_server.authorize_with(token_name)
         for v in ["ARVADOS_API_HOST",
                   "ARVADOS_API_HOST_INSECURE",
                   "ARVADOS_API_TOKEN"]:
             os.environ[v] = arvados.config.settings()[v]
+        arv_put.api_client = arvados.api('v1', cache=False)
 
     def current_user(self):
-        return arvados.api('v1').users().current().execute()
+        return arv_put.api_client.users().current().execute()
 
     def test_check_real_project_found(self):
+        self.authorize_with('active')
         self.assertTrue(arv_put.check_project_exists(self.PROJECT_UUID),
                         "did not correctly find test fixture project")
 
     def test_check_error_finding_nonexistent_project(self):
         BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
+        self.authorize_with('active')
         try:
             result = arv_put.check_project_exists(BAD_UUID)
         except ValueError as error:
@@ -540,10 +551,10 @@ class ArvPutIntegrationTest(unittest.TestCase):
 
         # Before doing anything, demonstrate that the collection
         # we're about to create is not present in our test fixture.
-        api = arvados.api('v1', cache=False)
         manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
         with self.assertRaises(apiclient.errors.HttpError):
-            notfound = api.collections().get(uuid=manifest_uuid).execute()
+            notfound = arv_put.api_client.collections().get(
+                uuid=manifest_uuid).execute()
 
         datadir = tempfile.mkdtemp()
         with open(os.path.join(datadir, "foo"), "w") as f:
@@ -557,7 +568,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
 
         # The manifest text stored in the API server under the same
         # manifest UUID must use signed locators.
-        c = api.collections().get(uuid=manifest_uuid).execute()
+        c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
         self.assertRegexpMatches(
             c['manifest_text'],
             r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
@@ -572,7 +583,7 @@ class ArvPutIntegrationTest(unittest.TestCase):
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE, env=self.ENVIRON)
         stdout, stderr = pipe.communicate(text)
-        link_list = arvados.api('v1', cache=False).links().list(
+        link_list = arv_put.api_client.links().list(
             filters=[['head_uuid', '=', stdout.strip()],
                      ['link_class', '=', 'name']]).execute().get('items', [])
         self.assertEqual(1, len(link_list))

commit f05bc0963563ab50341ff042439462b631894de6
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Aug 19 10:17:57 2014 -0400

    2800: Remove global state from KeepClient.
    
    This commit makes it possible to build and use a KeepClient that isn't
    influenced by changes in outside state.  Changing the KeepClient based
    on global state has been pushed up to the simple Keep class.
    
    This commit changes the tests as little as possible to demonstrate
    backward compatibility.  Refactoring the tests to build KeepClients
    directly will come in the future.

diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index e7348a1..1eb8f51 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -154,6 +154,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
     kwargs['http'] = credentials.authorize(kwargs['http'])
 
     svc = apiclient.discovery.build('arvados', version, **kwargs)
+    svc.api_token = token
     kwargs['http'].cache = None
     if cache:
         conncache[connprofile] = svc
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 3667829..561d34c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -24,8 +24,8 @@ import ssl
 _logger = logging.getLogger('arvados.keep')
 global_client_object = None
 
-from api import *
-import config
+import arvados
+import arvados.config as config
 import arvados.errors
 import arvados.util
 
@@ -106,12 +106,31 @@ class KeepLocator(object):
         return self.perm_expiry <= as_of_dt
 
 
-class Keep:
-    @staticmethod
-    def global_client_object():
+class Keep(object):
+    """Simple interface to a global KeepClient object.
+
+    THIS CLASS IS DEPRECATED.  Please instantiate your own KeepClient with your
+    own API client.  The global KeepClient will build an API client from the
+    current Arvados configuration, which may not match the one you built.
+    """
+    _last_key = None
+
+    @classmethod
+    def global_client_object(cls):
         global global_client_object
-        if global_client_object == None:
+        # Previously, KeepClient would change its behavior at runtime based
+        # on these configuration settings.  We simulate that behavior here
+        # by checking the values and returning a new KeepClient if any of
+        # them have changed.
+        key = (config.get('ARVADOS_API_HOST'),
+               config.get('ARVADOS_API_TOKEN'),
+               config.flag_is_true('ARVADOS_API_HOST_INSECURE'),
+               config.get('ARVADOS_KEEP_PROXY'),
+               config.get('ARVADOS_EXTERNAL_CLIENT') == 'true',
+               os.environ.get('KEEP_LOCAL_STORE'))
+        if (global_client_object is None) or (cls._last_key != key):
             global_client_object = KeepClient()
+            cls._last_key = key
         return global_client_object
 
     @staticmethod
@@ -122,8 +141,8 @@ class Keep:
     def put(data, **kwargs):
         return Keep.global_client_object().put(data, **kwargs)
 
-class KeepClient(object):
 
+class KeepClient(object):
     class ThreadLimiter(object):
         """
         Limit the number of threads running at a given time to
@@ -180,14 +199,16 @@ class KeepClient(object):
             with self._done_lock:
                 return self._done
 
+
     class KeepWriterThread(threading.Thread):
         """
         Write a blob of data to the given Keep server. On success, call
         save_response() of the given ThreadLimiter to save the returned
         locator.
         """
-        def __init__(self, **kwargs):
+        def __init__(self, api_token, **kwargs):
             super(KeepClient.KeepWriterThread, self).__init__()
+            self._api_token = api_token
             self.args = kwargs
             self._success = False
 
@@ -209,8 +230,7 @@ class KeepClient(object):
                           self.args['service_root'])
             h = httplib2.Http(timeout=self.args.get('timeout', None))
             url = self.args['service_root'] + self.args['data_hash']
-            api_token = config.get('ARVADOS_API_TOKEN')
-            headers = {'Authorization': "OAuth2 %s" % api_token}
+            headers = {'Authorization': "OAuth2 %s" % (self._api_token,)}
 
             if self.args['using_proxy']:
                 # We're using a proxy, so tell the proxy how many copies we
@@ -259,51 +279,91 @@ class KeepClient(object):
                 _logger.debug("Request fail: PUT %s => %s: %s",
                                 url, type(e), str(e))
 
-    def __init__(self, **kwargs):
+
+    def __init__(self, api_client=None, proxy=None, timeout=60,
+                 api_token=None, local_store=None):
+        """Initialize a new KeepClient.
+
+        Arguments:
+        * api_client: The API client to use to find Keep services.  If not
+          provided, KeepClient will build one from available Arvados
+          configuration.
+        * proxy: If specified, this KeepClient will send requests to this
+          Keep proxy.  Otherwise, KeepClient will fall back to the setting
+          of the ARVADOS_KEEP_PROXY configuration setting.  If you want to
+          ensure KeepClient does not use a proxy, pass in an empty string.
+        * timeout: The timeout for all HTTP requests, in seconds.  Default
+          60.
+        * api_token: If you're not using an API client, but only talking
+          directly to a Keep proxy, this parameter specifies an API token
+          to authenticate Keep requests.  It is an error to specify both
+          api_client and api_token.  If you specify neither, KeepClient
+          will use one available from the Arvados configuration.
+        * local_store: If specified, this KeepClient will bypass Keep
+          services, and save data to the named directory.  If unspecified,
+          KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
+          environment variable.  If you want to ensure KeepClient does not
+          use local storage, pass in an empty string.  This is primarily
+          intended to mock a server for testing.
+        """
         self.lock = threading.Lock()
-        self.service_roots = None
-        self._cache_lock = threading.Lock()
-        self._cache = []
-        # default 256 megabyte cache
-        self.cache_max = 256 * 1024 * 1024
-        self.using_proxy = False
-        self.timeout = kwargs.get('timeout', 60)
+        if proxy is None:
+            proxy = config.get('ARVADOS_KEEP_PROXY')
+        if api_token is None:
+            api_token = config.get('ARVADOS_API_TOKEN')
+        elif api_client is not None:
+            raise ValueError(
+                "can't build KeepClient with both API client and token")
+        if local_store is None:
+            local_store = os.environ.get('KEEP_LOCAL_STORE')
+
+        if local_store:
+            self.local_store = local_store
+            self.get = self.local_store_get
+            self.put = self.local_store_put
+        else:
+            self.timeout = timeout
+            self.cache_max = 256 * 1024 * 1024  # Cache is 256MiB
+            self._cache = []
+            self._cache_lock = threading.Lock()
+            if proxy:
+                if not proxy.endswith('/'):
+                    proxy += '/'
+                self.api_token = api_token
+                self.service_roots = [proxy]
+                self.using_proxy = True
+            else:
+                # It's important to avoid instantiating an API client
+                # unless we actually need one, for testing's sake.
+                if api_client is None:
+                    api_client = arvados.api('v1')
+                self.api_client = api_client
+                self.api_token = api_client.api_token
+                self.service_roots = None
+                self.using_proxy = None
 
     def shuffled_service_roots(self, hash):
-        if self.service_roots == None:
-            self.lock.acquire()
+        if self.service_roots is None:
+            with self.lock:
+                try:
+                    keep_services = self.api_client.keep_services().accessible()
+                except Exception:  # API server predates Keep services.
+                    keep_services = self.api_client.keep_disks().list()
 
-            # Override normal keep disk lookup with an explict proxy
-            # configuration.
-            keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
-            if keep_proxy_env != None and len(keep_proxy_env) > 0:
+                keep_services = keep_services.execute().get('items')
+                if not keep_services:
+                    raise arvados.errors.NoKeepServersError()
 
-                if keep_proxy_env[-1:] != '/':
-                    keep_proxy_env += "/"
-                self.service_roots = [keep_proxy_env]
-                self.using_proxy = True
-            else:
-                try:
-                    try:
-                        keep_services = arvados.api().keep_services().accessible().execute()['items']
-                    except Exception:
-                        keep_services = arvados.api().keep_disks().list().execute()['items']
-
-                    if len(keep_services) == 0:
-                        raise arvados.errors.NoKeepServersError()
-
-                    if 'service_type' in keep_services[0] and keep_services[0]['service_type'] == 'proxy':
-                        self.using_proxy = True
-
-                    roots = (("http%s://%s:%d/" %
-                              ('s' if f['service_ssl_flag'] else '',
-                               f['service_host'],
-                               f['service_port']))
-                             for f in keep_services)
-                    self.service_roots = sorted(set(roots))
-                    _logger.debug(str(self.service_roots))
-                finally:
-                    self.lock.release()
+                self.using_proxy = (keep_services[0].get('service_type') ==
+                                    'proxy')
+
+                roots = (("http%s://%s:%d/" %
+                          ('s' if f['service_ssl_flag'] else '',
+                           f['service_host'],
+                           f['service_port']))
+                         for f in keep_services)
+                self.service_roots = sorted(set(roots))
+                _logger.debug(str(self.service_roots))
 
         # Build an ordering with which to query the Keep servers based on the
         # contents of the hash.
@@ -403,12 +463,11 @@ class KeepClient(object):
         finally:
             self._cache_lock.release()
 
-    def get(self, locator):
-        if re.search(r',', locator):
-            return ''.join(self.get(x) for x in locator.split(','))
-        if 'KEEP_LOCAL_STORE' in os.environ:
-            return KeepClient.local_store_get(locator)
-        expect_hash = re.sub(r'\+.*', '', locator)
+    def get(self, loc_s):
+        if ',' in loc_s:
+            return ''.join(self.get(x) for x in loc_s.split(','))
+        locator = KeepLocator(loc_s)
+        expect_hash = locator.md5sum
 
         slot, first = self.reserve_cache(expect_hash)
 
@@ -418,9 +477,8 @@ class KeepClient(object):
 
         try:
             for service_root in self.shuffled_service_roots(expect_hash):
-                url = service_root + locator
-                api_token = config.get('ARVADOS_API_TOKEN')
-                headers = {'Authorization': "OAuth2 %s" % api_token,
+                url = service_root + loc_s
+                headers = {'Authorization': "OAuth2 %s" % (self.api_token,),
                            'Accept': 'application/octet-stream'}
                 blob = self.get_url(url, headers, expect_hash)
                 if blob:
@@ -428,9 +486,10 @@ class KeepClient(object):
                     self.cap_cache()
                     return blob
 
-            for location_hint in re.finditer(r'\+K@([a-z0-9]+)', locator):
-                instance = location_hint.group(1)
-                url = 'http://keep.' + instance + '.arvadosapi.com/' + locator
+            for hint in locator.hints:
+                if not hint.startswith('K@'):
+                    continue
+                url = 'http://keep.' + hint[2:] + '.arvadosapi.com/' + loc_s
                 blob = self.get_url(url, {}, expect_hash)
                 if blob:
                     slot.set(blob)
@@ -456,9 +515,7 @@ class KeepClient(object):
                          len(content), t.msecs,
                          (len(content)/(1024*1024))/t.secs)
             if re.match(r'^2\d\d$', resp['status']):
-                m = hashlib.new('md5')
-                m.update(content)
-                md5 = m.hexdigest()
+                md5 = hashlib.md5(content).hexdigest()
                 if md5 == expect_hash:
                     return content
                 _logger.warning("Checksum fail: md5(%s) = %s", url, md5)
@@ -467,20 +524,17 @@ class KeepClient(object):
                          url, type(e), str(e))
         return None
 
-    def put(self, data, **kwargs):
-        if 'KEEP_LOCAL_STORE' in os.environ:
-            return KeepClient.local_store_put(data)
-        m = hashlib.new('md5')
-        m.update(data)
-        data_hash = m.hexdigest()
+    def put(self, data, copies=2):
+        data_hash = hashlib.md5(data).hexdigest()
         have_copies = 0
-        want_copies = kwargs.get('copies', 2)
+        want_copies = copies
         if not (want_copies > 0):
             return data_hash
         threads = []
         thread_limiter = KeepClient.ThreadLimiter(want_copies)
         for service_root in self.shuffled_service_roots(data_hash):
             t = KeepClient.KeepWriterThread(
+                self.api_token,
                 data=data,
                 data_hash=data_hash,
                 service_root=service_root,
@@ -502,7 +556,8 @@ class KeepClient(object):
                                     t.args['service_root'],
                                     t.args['data_hash'])
                     retry_with_args = t.args.copy()
-                    t_retry = KeepClient.KeepWriterThread(**retry_with_args)
+                    t_retry = KeepClient.KeepWriterThread(self.api_token,
+                                                          **retry_with_args)
                     t_retry.start()
                     threads_retry += [t_retry]
             for t in threads_retry:
@@ -519,26 +574,22 @@ class KeepClient(object):
     def sign_for_old_server(data_hash, data):
         return (("-----BEGIN PGP SIGNED MESSAGE-----\n\n\n%d %s\n-----BEGIN PGP SIGNATURE-----\n\n-----END PGP SIGNATURE-----\n" % (int(time.time()), data_hash)) + data)
 
-
-    @staticmethod
-    def local_store_put(data):
-        m = hashlib.new('md5')
-        m.update(data)
-        md5 = m.hexdigest()
+    def local_store_put(self, data):
+        md5 = hashlib.md5(data).hexdigest()
         locator = '%s+%d' % (md5, len(data))
-        with open(os.path.join(os.environ['KEEP_LOCAL_STORE'], md5 + '.tmp'), 'w') as f:
+        with open(os.path.join(self.local_store, md5 + '.tmp'), 'w') as f:
             f.write(data)
-        os.rename(os.path.join(os.environ['KEEP_LOCAL_STORE'], md5 + '.tmp'),
-                  os.path.join(os.environ['KEEP_LOCAL_STORE'], md5))
+        os.rename(os.path.join(self.local_store, md5 + '.tmp'),
+                  os.path.join(self.local_store, md5))
         return locator
 
-    @staticmethod
-    def local_store_get(locator):
-        r = re.search('^([0-9a-f]{32,})', locator)
-        if not r:
+    def local_store_get(self, loc_s):
+        try:
+            locator = KeepLocator(loc_s)
+        except ValueError:
             raise arvados.errors.NotFoundError(
-                "Invalid data locator: '%s'" % locator)
-        if r.group(0) == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
+                "Invalid data locator: '%s'" % loc_s)
+        if locator.md5sum == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
             return ''
-        with open(os.path.join(os.environ['KEEP_LOCAL_STORE'], r.group(0)), 'r') as f:
+        with open(os.path.join(self.local_store, locator.md5sum), 'r') as f:
             return f.read()
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 8706e21..9d3cecd 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -126,7 +126,7 @@ class KeepPermissionTestCase(unittest.TestCase):
 
         # Unauthenticated GET for a signed locator => NotFound
         # Unauthenticated GET for an unsigned locator => NotFound
-        del arvados.config.settings()["ARVADOS_API_TOKEN"]
+        arvados.keep.global_client_object.api_token = ''
         self.assertRaises(arvados.errors.NotFoundError,
                           arvados.Keep.get,
                           bar_locator)
@@ -192,7 +192,7 @@ class KeepOptionalPermission(unittest.TestCase):
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + signed_locator)
 
-        del arvados.config.settings()["ARVADOS_API_TOKEN"]
+        arvados.keep.global_client_object.api_token = ''
         self.assertEqual(arvados.Keep.get(signed_locator),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')
@@ -207,7 +207,7 @@ class KeepOptionalPermission(unittest.TestCase):
             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
             'invalid locator from Keep.put("foo"): ' + signed_locator)
 
-        del arvados.config.settings()["ARVADOS_API_TOKEN"]
+        arvados.keep.global_client_object.api_token = ''
         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
                          'foo',
                          'wrong content from Keep.get(md5("foo"))')

commit 1afe2c7bbb571004736daf347f5178a27128704c
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Aug 19 10:01:36 2014 -0400

    2800: Improve spec conformance of Python SDK KeepLocator.
    
    * Require size to immediately follow digest.
    * Accept all valid hints.

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index bffb02a..3667829 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -31,28 +31,30 @@ import arvados.util
 
 class KeepLocator(object):
     EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0)
+    HINT_RE = re.compile(r'^[A-Z][A-Za-z0-9 at _-]+$')
 
     def __init__(self, locator_str):
-        self.size = None
-        self.loc_hint = None
+        self.hints = []
         self._perm_sig = None
         self._perm_expiry = None
         pieces = iter(locator_str.split('+'))
         self.md5sum = next(pieces)
+        try:
+            self.size = int(next(pieces))
+        except StopIteration:
+            self.size = None
         for hint in pieces:
-            if hint.startswith('A'):
+            if self.HINT_RE.match(hint) is None:
+                raise ValueError("unrecognized hint data {}".format(hint))
+            elif hint.startswith('A'):
                 self.parse_permission_hint(hint)
-            elif hint.startswith('K'):
-                self.loc_hint = hint  # FIXME
-            elif hint.isdigit():
-                self.size = int(hint)
             else:
-                raise ValueError("unrecognized hint data {}".format(hint))
+                self.hints.append(hint)
 
     def __str__(self):
         return '+'.join(
-            str(s) for s in [self.md5sum, self.size, self.loc_hint,
-                             self.permission_hint()]
+            str(s) for s in [self.md5sum, self.size,
+                             self.permission_hint()] + self.hints
             if s is not None)
 
     def _make_hex_prop(name, length):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 5354f3a..b1af723 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -586,7 +586,7 @@ class ArvadosCollectionsTest(ArvadosKeepLocalStoreTestCase):
         state = cwriter.current_state()
         # Add an expired locator to the state.
         state['_current_stream_locators'].append(''.join([
-                    'a' * 32, '+A', 'b' * 40, '@', '10000000']))
+                    'a' * 32, '+1+A', 'b' * 40, '@', '10000000']))
         self.assertRaises(arvados.errors.StaleWriterStateError,
                           TestResumableWriter.from_state, state)
 
diff --git a/sdk/python/tests/test_keep_locator.py b/sdk/python/tests/test_keep_locator.py
index e9d6356..a7e9cb1 100644
--- a/sdk/python/tests/test_keep_locator.py
+++ b/sdk/python/tests/test_keep_locator.py
@@ -8,7 +8,7 @@ import unittest
 
 from arvados.keep import KeepLocator
 
-class ArvadosPutResumeCacheTest(unittest.TestCase):
+class ArvadosKeepLocatorTest(unittest.TestCase):
     DEFAULT_TEST_COUNT = 10
 
     def numstrs(fmtstr, base, exponent):
@@ -22,13 +22,17 @@ class ArvadosPutResumeCacheTest(unittest.TestCase):
     signatures = numstrs('{:040x}', 16, 40)
     timestamps = numstrs('{:08x}', 16, 8)
 
+    def base_locators(self, count=DEFAULT_TEST_COUNT):
+        return ('+'.join(pair) for pair in
+                itertools.izip(self.checksums(count), self.sizes(count)))
+
     def perm_hints(self, count=DEFAULT_TEST_COUNT):
         for sig, ts in itertools.izip(self.signatures(count),
                                       self.timestamps(count)):
             yield 'A{}@{}'.format(sig, ts)
 
     def test_good_locators_returned(self):
-        for hint_gens in [(), (self.sizes(),), (self.perm_hints(),),
+        for hint_gens in [(), (self.sizes(),),
                           (self.sizes(), self.perm_hints())]:
             for loc_data in itertools.izip(self.checksums(), *hint_gens):
                 locator = '+'.join(loc_data)
@@ -40,24 +44,36 @@ class ArvadosPutResumeCacheTest(unittest.TestCase):
                        '3+8f9e68d957b504a29ba76c526c3145d9']:
             self.assertRaises(ValueError, KeepLocator, badstr)
 
+    def test_unknown_hints_accepted(self):
+        base = next(self.base_locators(1))
+        for weirdhint in ['Zfoo', 'Ybar234', 'Xa at b_c-372', 'W99']:
+            locator = '+'.join([base, weirdhint])
+            self.assertEquals(locator, str(KeepLocator(locator)))
+
     def test_bad_hints_rejected(self):
-        checksum = next(self.checksums(1))
-        for badhint in ['', 'nonsense', '+32', checksum]:
+        base = next(self.base_locators(1))
+        for badhint in ['', 'A', 'lowercase', '+32']:
             self.assertRaises(ValueError, KeepLocator,
-                              '+'.join([checksum, badhint]))
+                              '+'.join([base, badhint]))
+
+    def test_multiple_locator_hints_accepted(self):
+        base = next(self.base_locators(1))
+        for loc_hints in itertools.permutations(['Kab1cd', 'Kef2gh', 'Kij3kl']):
+            locator = '+'.join((base,) + loc_hints)
+            self.assertEquals(locator, str(KeepLocator(locator)))
 
     def test_expiry_passed(self):
-        checksum = next(self.checksums(1))
+        base = next(self.base_locators(1))
         signature = next(self.signatures(1))
         dt1980 = datetime.datetime(1980, 1, 1)
         dt2000 = datetime.datetime(2000, 2, 2)
         dt2080 = datetime.datetime(2080, 3, 3)
-        locator = KeepLocator(checksum)
+        locator = KeepLocator(base)
         self.assertFalse(locator.permission_expired())
         self.assertFalse(locator.permission_expired(dt1980))
         self.assertFalse(locator.permission_expired(dt2080))
         # Timestamped to 1987-01-05 18:48:32.
-        locator = KeepLocator('{}+A{}@20000000'.format(checksum, signature))
+        locator = KeepLocator('{}+A{}@20000000'.format(base, signature))
         self.assertTrue(locator.permission_expired())
         self.assertTrue(locator.permission_expired(dt2000))
         self.assertFalse(locator.permission_expired(dt1980))

commit f6e6e01802e09e1a108c067519d361caf0fc606f
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Aug 19 09:59:53 2014 -0400

    2800: Introduce config.flag_is_true() to Python SDK.

diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 3df2430..e7348a1 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -119,8 +119,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
                 raise ValueError("%s is not set. Aborting." % x)
         host = config.get('ARVADOS_API_HOST')
         token = config.get('ARVADOS_API_TOKEN')
-        insecure = (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in
-                       ('yes', 'true', '1'))
+        insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
     else:
         # Caller provided one but not the other
         if not host:
diff --git a/sdk/python/arvados/config.py b/sdk/python/arvados/config.py
index e205e92..2b8374a 100644
--- a/sdk/python/arvados/config.py
+++ b/sdk/python/arvados/config.py
@@ -25,6 +25,9 @@ def initialize(config_file=default_config_file):
         if var.startswith('ARVADOS_'):
             _settings[var] = os.environ[var]
 
+def flag_is_true(key):
+    return get(key, '').lower() in set(['1', 't', 'true', 'y', 'yes'])
+
 def get(key, default_val=None):
     return settings().get(key, default_val)
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list