[ARVADOS] updated: fdb1bc1446407e8589ced3ea32f267b38e954921
git at public.curoverse.com
git at public.curoverse.com
Fri Oct 17 10:23:24 EDT 2014
Summary of changes:
.../tutorials/tutorial-keep-mount.html.textile.liquid | 14 +++++++++-----
sdk/python/arvados/api.py | 18 +++++++++---------
sdk/python/arvados/commands/put.py | 6 +++---
sdk/python/arvados/errors.py | 4 ++--
sdk/python/setup.py | 2 +-
sdk/python/tests/test_api.py | 10 +++++-----
services/nodemanager/arvnodeman/config.py | 2 +-
7 files changed, 30 insertions(+), 26 deletions(-)
via fdb1bc1446407e8589ced3ea32f267b38e954921 (commit)
via 95f99e05352e2a87ad627a0b4c775f7551ef9265 (commit)
from 0b3873cf0f05ec1e08c1cee81aed68c2421f158f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fdb1bc1446407e8589ced3ea32f267b38e954921
Author: Nancy Ouyang <nancy.ouyang at gmail.com>
Date: Fri Oct 17 09:52:17 2014 -0400
added by_id subdirectory notes and arvados#collection
No issue #.
diff --git a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
index e07be5d..a41fede 100644
--- a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
@@ -16,17 +16,21 @@ h2. Arv-mount
* It is easy for existing tools to access files in Keep.
* Data is downloaded on demand. It is not necessary to download an entire file or collection to start processing.
-The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by their keep locator.
+The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by their Keep locator. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
<notextile>
<pre><code>~$ <span class="userinput">mkdir -p keep</span>
~$ <span class="userinput">arv-mount keep</span>
-~$ <span class="userinput">cd keep/c1bad4b39ca5a924e481008009d94e32+210</span>
-~/keep/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">ls</span>
+~$ <span class="userinput">cd keep/by_id/c1bad4b39ca5a924e481008009d94e32+210</span>
+~/keep/by_id/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">ls</span>
var-GS000016015-ASM.tsv.bz2
-~/keep/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">md5sum var-GS000016015-ASM.tsv.bz2</span>
+~/keep/by_id/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">md5sum var-GS000016015-ASM.tsv.bz2</span>
44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
-~/keep/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">cd ../..</span>
+~/keep/by_id/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">cd ../..</span>
~$ <span class="userinput">fusermount -u keep</span>
</code></pre>
</notextile>
+
+The last line unmounts Keep. Subdirectories will no longer be accessible.
+
+Within each directory on Keep, there is a @.arvados#collection@ file that does not show up with @ls at . Its contents include, for instance, the @portable_data_hash@, which is the same as the Keep locator.
commit 95f99e05352e2a87ad627a0b4c775f7551ef9265
Author: Brett Smith <brett at curoverse.com>
Date: Fri Oct 17 10:22:01 2014 -0400
4239: Update Python SDK for google-api-python-client 1.3.
This package renames the module from apiclient to googleapiclient. It
provides a shim apiclient module for compatibility, but it only
exports the appropriate names, not real submodules. Using it requires
changing imports like
import apiclient.errors
to
from apiclient import errors
This approach works (has been tested) with both versions 1.2 and 1.3.
Closes #4239.
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index cb716f1..c618fc3 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -6,8 +6,8 @@ import re
import types
import apiclient
-import apiclient.discovery
-import apiclient.errors
+from apiclient import discovery as apiclient_discovery
+from apiclient import errors as apiclient_errors
import config
import errors
import util
@@ -47,7 +47,7 @@ class CredentialsFromToken(object):
# Monkey patch discovery._cast() so objects and arrays get serialized
# with json.dumps() instead of str().
-_cast_orig = apiclient.discovery._cast
+_cast_orig = apiclient_discovery._cast
def _cast_objects_too(value, schema_type):
global _cast_orig
if (type(value) != type('') and
@@ -55,16 +55,16 @@ def _cast_objects_too(value, schema_type):
return json.dumps(value)
else:
return _cast_orig(value, schema_type)
-apiclient.discovery._cast = _cast_objects_too
+apiclient_discovery._cast = _cast_objects_too
# Convert apiclient's HttpErrors into our own API error subclass for better
# error reporting.
-# Reassigning apiclient.errors.HttpError is not sufficient because most of the
+# Reassigning apiclient_errors.HttpError is not sufficient because most of the
# apiclient submodules import the class into their own namespace.
def _new_http_error(cls, *args, **kwargs):
- return super(apiclient.errors.HttpError, cls).__new__(
+ return super(apiclient_errors.HttpError, cls).__new__(
errors.ApiError, *args, **kwargs)
-apiclient.errors.HttpError.__new__ = staticmethod(_new_http_error)
+apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
def http_cache(data_type):
path = os.environ['HOME'] + '/.cache/arvados/' + data_type
@@ -90,7 +90,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
* insecure: If True, ignore SSL certificate validation errors.
Additional keyword arguments will be passed directly to
- `apiclient.discovery.build` if a new Resource object is created.
+ `apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
missing, this function will set default values for them, based on
the current Arvados configuration settings.
@@ -153,7 +153,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
credentials = CredentialsFromToken(api_token=token)
kwargs['http'] = credentials.authorize(kwargs['http'])
- svc = apiclient.discovery.build('arvados', version, **kwargs)
+ svc = apiclient_discovery.build('arvados', version, **kwargs)
svc.api_token = token
kwargs['http'].cache = None
if cache:
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 53d6f56..4a926c7 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -3,7 +3,6 @@
# TODO:
# --md5sum - display md5 of each file as read from disk
-import apiclient.errors
import argparse
import arvados
import base64
@@ -18,6 +17,7 @@ import signal
import socket
import sys
import tempfile
+from apiclient import errors as apiclient_errors
import arvados.commands._util as arv_cmd
@@ -390,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
try:
project_uuid = desired_project_uuid(api_client, args.project_uuid,
args.retries)
- except (apiclient.errors.Error, ValueError) as error:
+ except (apiclient_errors.Error, ValueError) as error:
print >>stderr, error
sys.exit(1)
@@ -468,7 +468,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
else:
output = collection['uuid']
- except apiclient.errors.Error as error:
+ except apiclient_errors.Error as error:
print >>stderr, (
"arv-put: Error creating Collection on project: {}.".format(
error))
diff --git a/sdk/python/arvados/errors.py b/sdk/python/arvados/errors.py
index 89910aa..4740a2d 100644
--- a/sdk/python/arvados/errors.py
+++ b/sdk/python/arvados/errors.py
@@ -1,9 +1,9 @@
# errors.py - Arvados-specific exceptions.
-import apiclient.errors
import json
+from apiclient import errors as apiclient_errors
-class ApiError(apiclient.errors.HttpError):
+class ApiError(apiclient_errors.HttpError):
def _get_reason(self):
try:
return '; '.join(json.loads(self.content)['errors'])
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index ee5b15d..03637cb 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -42,7 +42,7 @@ setup(name='arvados-python-client',
],
install_requires=[
'python-gflags',
- 'google-api-python-client<1.3',
+ 'google-api-python-client',
'httplib2',
'urllib3',
'ws4py'
diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py
index e9cb838..0d81fdf 100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python
-import apiclient.errors
import arvados
import httplib2
import json
@@ -8,8 +7,9 @@ import mimetypes
import os
import run_test_server
import unittest
+from apiclient import errors as apiclient_errors
+from apiclient import http as apiclient_http
-from apiclient.http import RequestMockBuilder
from arvados_testutil import fake_httplib2_response
if not mimetypes.inited:
@@ -41,7 +41,7 @@ class ArvadosApiClientTest(unittest.TestCase):
'arvados.humans.list': (None, json.dumps(
{'items_available': 0, 'items': []})),
}
- req_builder = RequestMockBuilder(mock_responses)
+ req_builder = apiclient_http.RequestMockBuilder(mock_responses)
cls.api = arvados.api('v1', cache=False,
host=os.environ['ARVADOS_API_HOST'],
token='discovery-doc-only-no-token-needed',
@@ -58,14 +58,14 @@ class ArvadosApiClientTest(unittest.TestCase):
self.assertEqual(answer['items_available'], len(answer['items']))
def test_exceptions_include_errors(self):
- with self.assertRaises(apiclient.errors.HttpError) as err_ctx:
+ with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
self.api.humans().get(uuid='xyz-xyz-abcdef').execute()
err_s = str(err_ctx.exception)
for msg in ["Bad UUID format", "Bad output format"]:
self.assertIn(msg, err_s)
def test_exceptions_without_errors_have_basic_info(self):
- with self.assertRaises(apiclient.errors.HttpError) as err_ctx:
+ with self.assertRaises(apiclient_errors.HttpError) as err_ctx:
self.api.humans().delete(uuid='xyz-xyz-abcdef').execute()
self.assertIn("500", str(err_ctx.exception))
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index 07504e2..1699584 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -7,11 +7,11 @@ import importlib
import logging
import ssl
-import apiclient.errors as apierror
import arvados
import httplib2
import libcloud.common.types as cloud_types
import pykka
+from apiclient import errors as apierror
# IOError is the base class for socket.error and friends.
# It seems like it hits the sweet spot for operations we want to retry:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list