[ARVADOS] created: 1.1.1-162-gb7eae78
Git user
git at public.curoverse.com
Fri Dec 8 11:10:07 EST 2017
at b7eae78616dd72ddda597f414f3c208b099c0a27 (commit)
commit b7eae78616dd72ddda597f414f3c208b099c0a27
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 8 11:09:13 2017 -0500
12167: Use one X-Request-Id per arv-put; display it if not --silent.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index ec4ae8f..8bfe520 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -398,7 +398,7 @@ class ArvPutUploadJob(object):
}
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
- name=None, owner_uuid=None,
+ name=None, owner_uuid=None, api_client=None,
ensure_unique_name=False, num_retries=None,
put_threads=None, replication_desired=None,
filename=None, update_time=60.0, update_collection=None,
@@ -421,6 +421,7 @@ class ArvPutUploadJob(object):
self.replication_desired = replication_desired
self.put_threads = put_threads
self.filename = filename
+ self._api_client = api_client
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
self._current_files = [] # Current run file list
@@ -775,7 +776,8 @@ class ArvPutUploadJob(object):
if update_collection and re.match(arvados.util.collection_uuid_pattern,
update_collection):
try:
- self._remote_collection = arvados.collection.Collection(update_collection)
+ self._remote_collection = arvados.collection.Collection(
+ update_collection, api_client=self._api_client)
except arvados.errors.ApiError as error:
raise CollectionUpdateError("Cannot read collection {} ({})".format(update_collection, error))
else:
@@ -822,7 +824,11 @@ class ArvPutUploadJob(object):
# No cache file, set empty state
self._state = copy.deepcopy(self.EMPTY_STATE)
# Load the previous manifest so we can check if files were modified remotely.
- self._local_collection = arvados.collection.Collection(self._state['manifest'], replication_desired=self.replication_desired, put_threads=self.put_threads)
+ self._local_collection = arvados.collection.Collection(
+ self._state['manifest'],
+ replication_desired=self.replication_desired,
+ put_threads=self.put_threads,
+ api_client=self._api_client)
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
@@ -977,8 +983,12 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
else:
logger.setLevel(logging.INFO)
status = 0
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
# Determine the name to use
if args.name:
@@ -1063,6 +1073,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
use_cache = args.use_cache,
filename = args.filename,
reporter = reporter,
+ api_client = api_client,
num_retries = args.retries,
replication_desired = args.replication,
put_threads = args.threads,
commit 1588323a9c02c652877d53e12b0b5d85b18c175e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 8 10:16:08 2017 -0500
12167: Use one X-Request-Id per arv-get process; display it if -v.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py
index 881fdd6..1e52714 100755
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@ -81,6 +81,10 @@ Overwrite existing files while writing. The default behavior is to
refuse to write *anything* if any of the output files already
exist. As a special case, -f is not needed to write to stdout.
""")
+group.add_argument('-v', action='count', default=0,
+ help="""
+Once for verbose mode, twice for debug mode.
+""")
group.add_argument('--skip-existing', action='store_true',
help="""
Skip files that already exist. The default behavior is to refuse to
@@ -140,8 +144,13 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
stdout = stdout.buffer
args = parse_arguments(arguments, stdout, stderr)
+ logger.setLevel(logging.WARNING - 10 * args.v)
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
r = re.search(r'^(.*?)(/.*)?$', args.locator)
col_loc = r.group(1)
@@ -157,14 +166,15 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
open_flags |= os.O_EXCL
try:
if args.destination == "-":
- write_block_or_manifest(dest=stdout, src=col_loc,
- api_client=api_client, args=args)
+ write_block_or_manifest(
+ dest=stdout, src=col_loc,
+ api_client=api_client, args=args)
else:
out_fd = os.open(args.destination, open_flags)
with os.fdopen(out_fd, 'wb') as out_file:
- write_block_or_manifest(dest=out_file,
- src=col_loc, api_client=api_client,
- args=args)
+ write_block_or_manifest(
+ dest=out_file, src=col_loc,
+ api_client=api_client, args=args)
except (IOError, OSError) as error:
logger.error("can't write to '{}': {}".format(args.destination, error))
return 1
@@ -180,7 +190,8 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
return 0
try:
- reader = arvados.CollectionReader(col_loc, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ col_loc, api_client=api_client, num_retries=args.retries)
except Exception as error:
logger.error("failed to read collection: {}".format(error))
return 1
@@ -305,5 +316,6 @@ def write_block_or_manifest(dest, src, api_client, args):
dest.write(kc.get(src, num_retries=args.retries))
else:
# collection UUID or portable data hash
- reader = arvados.CollectionReader(src, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ src, api_client=api_client, num_retries=args.retries)
dest.write(reader.manifest_text(strip=args.strip_manifest).encode())
commit 1e9d8203ad92df6d5d0353da9d23fa9b2b3a8c9e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 8 10:13:27 2017 -0500
12167: Send caller-provided or random X-Request-Id in Keep requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index e6e93f0..351f7f5 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -291,7 +291,8 @@ class KeepClient(object):
def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
- download_counter=None, **headers):
+ download_counter=None,
+ headers={}):
self.root = root
self._user_agent_pool = user_agent_pool
self._result = {'error': None}
@@ -920,7 +921,7 @@ class KeepClient(object):
_logger.debug("{}: {}".format(locator, sorted_roots))
return sorted_roots
- def map_new_services(self, roots_map, locator, force_rebuild, need_writable, **headers):
+ def map_new_services(self, roots_map, locator, force_rebuild, need_writable, headers):
# roots_map is a dictionary, mapping Keep service root strings
# to KeepService objects. Poll for Keep services, and add any
# new ones to roots_map. Return the current list of local
@@ -933,7 +934,7 @@ class KeepClient(object):
root, self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter,
- **headers)
+ headers=headers)
return local_roots
@staticmethod
@@ -963,14 +964,14 @@ class KeepClient(object):
return None
@retry.retry_method
- def head(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="HEAD", num_retries=num_retries)
+ def head(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="HEAD", **kwargs)
@retry.retry_method
- def get(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="GET", num_retries=num_retries)
+ def get(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="GET", **kwargs)
- def _get_or_head(self, loc_s, method="GET", num_retries=None):
+ def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None):
"""Get data from Keep.
This method fetches one or more blocks of data from Keep. It
@@ -1005,6 +1006,12 @@ class KeepClient(object):
self.misses_counter.add(1)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ }
+
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
@@ -1021,7 +1028,8 @@ class KeepClient(object):
roots_map = {
root: self.KeepService(root, self._user_agent_pool,
upload_counter=self.upload_counter,
- download_counter=self.download_counter)
+ download_counter=self.download_counter,
+ headers=headers)
for root in hint_roots
}
@@ -1040,7 +1048,8 @@ class KeepClient(object):
sorted_roots = self.map_new_services(
roots_map, locator,
force_rebuild=(tries_left < num_retries),
- need_writable=False)
+ need_writable=False,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
@@ -1084,7 +1093,7 @@ class KeepClient(object):
"failed to read {}".format(loc_s), service_errors, label="service")
@retry.retry_method
- def put(self, data, copies=2, num_retries=None):
+ def put(self, data, copies=2, num_retries=None, request_id=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
@@ -1114,9 +1123,12 @@ class KeepClient(object):
return loc_s
locator = KeepLocator(loc_s)
- headers = {}
- # Tell the proxy how many copies we want it to store
- headers['X-Keep-Desired-Replicas'] = str(copies)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ 'X-Keep-Desired-Replicas': str(copies),
+ }
roots_map = {}
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
@@ -1125,7 +1137,9 @@ class KeepClient(object):
try:
sorted_roots = self.map_new_services(
roots_map, locator,
- force_rebuild=(tries_left < num_retries), need_writable=True, **headers)
+ force_rebuild=(tries_left < num_retries),
+ need_writable=True,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
commit e7d8544fae381f4b529c5bb63be909980e31727d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Dec 8 10:11:21 2017 -0500
12167: Send caller-provided or random X-Request-Id in API requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index e69f1a1..e14d0a9 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -52,19 +52,17 @@ class OrderedJsonModel(apiclient.model.JsonModel):
return body
-def _intercept_http_request(self, uri, method="GET", **kwargs):
+def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- if 'headers' not in kwargs:
- kwargs['headers'] = {}
-
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
- kwargs['headers']['X-External-Client'] = '1'
+ headers['X-External-Client'] = '1'
- kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers.setdefault('X-Request-Id', self._request_id or util.new_request_id())
retryable = method in [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
@@ -83,7 +81,7 @@ def _intercept_http_request(self, uri, method="GET", **kwargs):
for _ in range(retry_count):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
@@ -101,9 +99,9 @@ def _intercept_http_request(self, uri, method="GET", **kwargs):
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
-def _patch_http_request(http, api_token):
+def _patch_http_request(http, api_token, request_id):
http.arvados_api_token = api_token
http.max_request_size = 0
http.orig_http_request = http.request
@@ -113,6 +111,7 @@ def _patch_http_request(http, api_token):
http._retry_delay_initial = RETRY_DELAY_INITIAL
http._retry_delay_backoff = RETRY_DELAY_BACKOFF
http._retry_count = RETRY_COUNT
+ http._request_id = request_id
return http
# Monkey patch discovery._cast() so objects and arrays get serialized
@@ -148,7 +147,8 @@ def http_cache(data_type):
return None
return cache.SafeHTTPCache(path, max_age=60*60*24*2)
-def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False,
+ request_id=None, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
:version:
@@ -168,6 +168,12 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
:insecure:
If True, ignore SSL certificate validation errors.
+ :request_id:
+ Default X-Request-Id header value for outgoing requests that
+ don't already provide one. If None or omitted, generate a random
+ ID. When retrying failed requests, the same ID is used on all
+ attempts.
+
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
@@ -192,7 +198,8 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
elif host and token:
pass
elif not host and not token:
- return api_from_config(version=version, cache=cache, **kwargs)
+ return api_from_config(
+ version=version, cache=cache, request_id=request_id, **kwargs)
else:
# Caller provided one but not the other
if not host:
@@ -213,11 +220,12 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
http_kwargs['disable_ssl_certificate_validation'] = True
kwargs['http'] = httplib2.Http(**http_kwargs)
- kwargs['http'] = _patch_http_request(kwargs['http'], token)
+ kwargs['http'] = _patch_http_request(kwargs['http'], token, request_id)
svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
svc.api_token = token
svc.insecure = insecure
+ svc.request_id = request_id
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
return svc
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 1a97358..8076337 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -2,10 +2,14 @@
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import division
+from builtins import range
+
import fcntl
import hashlib
import httplib2
import os
+import random
import re
import subprocess
import errno
@@ -399,3 +403,16 @@ def ca_certs_path(fallback=httplib2.CA_CERTS):
if os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
+
+def new_request_id():
+ id = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ id += chr(c+ord('0'))
+ else:
+ id += chr(c+ord('a')-10)
+ n = n // 36
+ return id
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list