[ARVADOS] created: 0bd1c28bed9a0756c61037947d5a9dccd5066f00
git at public.curoverse.com
git at public.curoverse.com
Sat Aug 16 02:06:55 EDT 2014
at 0bd1c28bed9a0756c61037947d5a9dccd5066f00 (commit)
commit 0bd1c28bed9a0756c61037947d5a9dccd5066f00
Merge: b5a2787 c6db15d
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Aug 16 02:06:20 2014 -0400
Merge branch '2800-python-global-state' into 2800-pgs
Conflicts:
sdk/python/arvados/api.py
diff --cc sdk/python/arvados/api.py
index 7c60f51,64c2125..1db694f
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@@ -12,10 -12,12 +13,13 @@@ import confi
import errors
import util
-services = {}
+_logger = logging.getLogger('arvados.api')
- services = {}
++conncache = {}
+
+ class CredentialsFromToken(object):
+ def __init__(self, api_token):
+ self.api_token = api_token
- class CredentialsFromEnv(object):
@staticmethod
def http_request(self, uri, **kwargs):
from httplib import BadStatusLine
@@@ -70,54 -64,56 +75,85 @@@ def http_cache(data_type)
path = None
return path
- def api(version=None, cache=True, **kwargs):
-def api(version=None, cache=True, host=None, token=None, insecure=False):
- global services
-
- if 'ARVADOS_DEBUG' in config.settings():
- logging.basicConfig(level=logging.DEBUG)
++def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+ """Return an apiclient Resources object for an Arvados instance.
+
+ Arguments:
+ * version: A string naming the version of the Arvados API to use (for
+ example, 'v1').
- * cache: If True (default), return an existing resources object, or use
- a cached discovery document to build one.
++ * cache: If True (default), return an existing Resources object if
++ one already exists with the same endpoint and credentials. If
++ False, create a new one, and do not keep it in the cache (i.e.,
++ do not return it from subsequent api(cache=True) calls with
++ matching endpoint and credentials).
++ * host: The Arvados API server host (and optional :port) to connect to.
++ * token: The authentication token to send with each API call.
++ * insecure: If True, ignore SSL certificate validation errors.
+
+ Additional keyword arguments will be passed directly to
- `apiclient.discovery.build`. If the `discoveryServiceUrl` or `http`
- keyword arguments are missing, this function will set default values for
- them, based on the current Arvados configuration settings."""
- if not cache or not services.get(version):
- if not version:
- version = 'v1'
- _logger.info("Using default API version. " +
- "Call arvados.api('%s') instead.",
- version)
-
- if 'discoveryServiceUrl' not in kwargs:
- api_host = config.get('ARVADOS_API_HOST')
- if not api_host:
- raise ValueError(
- "No discoveryServiceUrl or ARVADOS_API_HOST set.")
- kwargs['discoveryServiceUrl'] = (
- 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' %
- (api_host,))
-
- if 'http' not in kwargs:
- http_kwargs = {}
- # Prefer system's CA certificates (if available) over httplib2's.
- certs_path = '/etc/ssl/certs/ca-certificates.crt'
- if os.path.exists(certs_path):
- http_kwargs['ca_certs'] = certs_path
- if cache:
- http_kwargs['cache'] = http_cache('discovery')
- if (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in
- ('yes', 'true', '1')):
- http_kwargs['disable_ssl_certificate_validation'] = True
- kwargs['http'] = httplib2.Http(**http_kwargs)
-
- kwargs['http'] = CredentialsFromEnv().authorize(kwargs['http'])
- services[version] = apiclient.discovery.build('arvados', version,
- **kwargs)
- kwargs['http'].cache = None
- return services[version]
-
- def uncache_api(version):
- if version in services:
- del services[version]
++ `apiclient.discovery.build` if a new Resource object is created.
++ If the `discoveryServiceUrl` or `http` keyword arguments are
++ missing, this function will set default values for them, based on
++ the current Arvados configuration settings.
++
++ """
+
+ if not version:
+ version = 'v1'
+ logging.info("Using default API version. " +
+ "Call arvados.api('%s') instead." %
+ version)
+ if host and token:
- # Provided by caller
- pass
++ apiinsecure = insecure
+ elif not host and not token:
+ # Load from user configuration or environment
+ for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
+ if x not in config.settings():
+ raise Exception("%s is not set. Aborting." % x)
+ host = config.get('ARVADOS_API_HOST')
+ token = config.get('ARVADOS_API_TOKEN')
- apiinsecure = re.match(r'(?i)^(true|1|yes)$',
- config.get('ARVADOS_API_HOST_INSECURE', 'no'))
++ apiinsecure = (config.get('ARVADOS_API_HOST_INSECURE', '').lower() in
++ ('yes', 'true', '1'))
+ else:
+ # Caller provided one but not the other
+ if not host:
+ raise Exception("token argument provided, but host missing.")
+ else:
+ raise Exception("host argument provided, but token missing.")
+
- connprofile = hashlib.sha1(' '.join([
- version, host, token, ('y' if apiinsecure else 'n')
- ])).hexdigest()
++ if cache:
++ connprofile = hashlib.sha1(' '.join([
++ version, host, token, ('y' if apiinsecure else 'n')
++ ])).hexdigest()
++ svc = conncache.get(connprofile)
++ if svc:
++ return svc
++
++ if 'http' not in kwargs:
++ http_kwargs = {}
++ # Prefer system's CA certificates (if available) over httplib2's.
++ certs_path = '/etc/ssl/certs/ca-certificates.crt'
++ if os.path.exists(certs_path):
++ http_kwargs['ca_certs'] = certs_path
++ if cache:
++ http_kwargs['cache'] = http_cache('discovery')
++ if apiinsecure:
++ http_kwargs['disable_ssl_certificate_validation'] = True
++ kwargs['http'] = httplib2.Http(**http_kwargs)
+
- if not cache or not services.get(connprofile):
- url = 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % host
- credentials = CredentialsFromToken(api_token=token)
++ credentials = CredentialsFromToken(api_token=token)
++ kwargs['http'] = credentials.authorize(kwargs['http'])
+
- # Use system's CA certificates (if we find them) instead of httplib2's
- ca_certs = '/etc/ssl/certs/ca-certificates.crt'
- if not os.path.exists(ca_certs):
- ca_certs = None # use httplib2 default
++ if 'discoveryServiceUrl' not in kwargs:
++ kwargs['discoveryServiceUrl'] = (
++ 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
+
- http = httplib2.Http(ca_certs=ca_certs,
- cache=(http_cache('discovery') if cache else None))
- http = credentials.authorize(http)
- if apiinsecure:
- http.disable_ssl_certificate_validation = True
- services[connprofile] = apiclient.discovery.build(
- 'arvados', version, http=http, discoveryServiceUrl=url)
- http.cache = None
++ svc = apiclient.discovery.build('arvados', version, **kwargs)
++ kwargs['http'].cache = None
++ if cache:
++ conncache[connprofile] = svc
++ return svc
+
- return services[connprofile]
++def unload_connection_cache():
++ for connprofile in conncache:
++ del conncache[connprofile]
diff --cc sdk/python/tests/test_api.py
index 4c48571,0000000..4917d00
mode 100644,000000..100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@@ -1,76 -1,0 +1,72 @@@
+#!/usr/bin/env python
+
+import apiclient.errors
+import arvados
+import httplib2
+import json
+import mimetypes
+import unittest
++import os
++import run_test_server
+
+from apiclient.http import RequestMockBuilder
+from httplib import responses as HTTP_RESPONSES
+
+if not mimetypes.inited:
+ mimetypes.init()
+
+class ArvadosApiClientTest(unittest.TestCase):
+ @classmethod
+ def response_from_code(cls, code):
+ return httplib2.Response(
+ {'status': code,
+ 'reason': HTTP_RESPONSES.get(code, "Unknown Response"),
+ 'Content-Type': mimetypes.types_map['.json']})
+
+ @classmethod
+ def api_error_response(cls, code, *errors):
+ return (cls.response_from_code(code),
+ json.dumps({'errors': errors,
+ 'error_token': '1234567890+12345678'}))
+
+ @classmethod
+ def setUpClass(cls):
- # The apiclient library has support for mocking requests for
- # testing, but it doesn't extend to the discovery document
- # itself. Point it at a known stable discovery document for now.
- # FIXME: Figure out a better way to stub this out.
- cls.orig_api_host = arvados.config.get('ARVADOS_API_HOST')
- arvados.config.settings()['ARVADOS_API_HOST'] = 'qr1hi.arvadosapi.com'
++ run_test_server.run()
+ mock_responses = {
+ 'arvados.humans.delete': (cls.response_from_code(500), ""),
+ 'arvados.humans.get': cls.api_error_response(
+ 422, "Bad UUID format", "Bad output format"),
+ 'arvados.humans.list': (None, json.dumps(
+ {'items_available': 0, 'items': []})),
+ }
+ req_builder = RequestMockBuilder(mock_responses)
- cls.api = arvados.api('v1', False, requestBuilder=req_builder)
++ cls.api = arvados.api('v1', cache=False,
++ host=os.environ['ARVADOS_API_HOST'],
++ token='discovery-doc-only-no-token-needed',
++ insecure=True,
++ requestBuilder=req_builder)
+
+ @classmethod
+ def tearDownClass(cls):
- if cls.orig_api_host is None:
- del arvados.config.settings()['ARVADOS_API_HOST']
- else:
- arvados.config.settings()['ARVADOS_API_HOST'] = cls.orig_api_host
- # Prevent other tests from using our mocked API client.
- arvados.uncache_api('v1')
++ run_test_server.stop()
+
+ def test_basic_list(self):
+ answer = self.api.humans().list(
+ filters=[['uuid', 'is', None]]).execute()
+ self.assertEqual(answer['items_available'], len(answer['items']))
+
+ def test_exceptions_include_errors(self):
+ with self.assertRaises(apiclient.errors.HttpError) as err_ctx:
+ self.api.humans().get(uuid='xyz-xyz-abcdef').execute()
+ err_s = str(err_ctx.exception)
+ for msg in ["Bad UUID format", "Bad output format"]:
+ self.assertIn(msg, err_s)
+
+ def test_exceptions_without_errors_have_basic_info(self):
+ with self.assertRaises(apiclient.errors.HttpError) as err_ctx:
+ self.api.humans().delete(uuid='xyz-xyz-abcdef').execute()
+ self.assertIn("500", str(err_ctx.exception))
+
+
+if __name__ == '__main__':
+ unittest.main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list