[ARVADOS] updated: e31d1ec3c19d88042b47516fad9ed952d6561fd6
git at public.curoverse.com
git at public.curoverse.com
Fri Sep 25 13:00:43 EDT 2015
Summary of changes:
apps/workbench/config/application.default.yml | 47 ++++++++++++++++++++++++---
sdk/cli/bin/crunch-job | 10 +++---
sdk/go/keepclient/keepclient.go | 17 ++++++----
sdk/go/keepclient/support.go | 44 ++++++++++++++-----------
services/api/config/application.default.yml | 43 ++++++++++++++++++++++--
services/datamanager/keep/keep.go | 19 ++++++++---
6 files changed, 138 insertions(+), 42 deletions(-)
via e31d1ec3c19d88042b47516fad9ed952d6561fd6 (commit)
via c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd (commit)
via 72748d17a195be1b65f847b078d1924f4648d45a (commit)
via fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c (commit)
via 57694f580feff10b274d6b13e9b4b013bfdad937 (commit)
via b8ad249763a75f3eb7bea758b88cdc87389639f1 (commit)
via 0ba145115071473ba46a3fac30c2dc98b74f49ad (commit)
via 0fdfc23c640c831cb2c0ee1e04fb639eb032de6f (commit)
via 73ce0cf7675e060d33e75488edfa4f533c177f82 (commit)
via 32a29ee7d171dccdb424990ff9c73e4b893dc3e4 (commit)
via 5a55113805af906145449be18ec86b1a95a5017b (commit)
via 285c0092f8ae4d757e31754c91314baa4db5af44 (commit)
via 559dc0ac07ea3d6820f5e220b9f6faf850ae63bf (commit)
via 5a974b0568f7645a7e4377394da04940fa1ec137 (commit)
via 2b20a0ec9a32abfd85c09474e044cfd4c09f4963 (commit)
via c21c6330cc8a3c8cc96d83ff9a315fe10dff7c8f (commit)
via 9db403a87830f7afec86dae5bb279cf87df8fd0e (commit)
via 67f730557e062608f842fa3abc612fb717fbce2f (commit)
via 7fdfbc3eae8de4fe7f6912c160e408ef57a89487 (commit)
via 0333c360cdd24b8fdd28a88d6d99bebe7e7a9bed (commit)
via c7b4e4802762eada81652224878a74b538a1d704 (commit)
via 0219fab6b0eb8b4710af2ae03c5231bf4d932a28 (commit)
via 6cca3e129b6de3fc4b34a1664f494fcbbbd940d1 (commit)
via bb663993cc851971d9444d834acfc416363931b2 (commit)
from ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e31d1ec3c19d88042b47516fad9ed952d6561fd6
Merge: c2a9349 ee9c164
Author: radhika <radhika at curoverse.com>
Date: Fri Sep 25 12:44:15 2015 -0400
Merge branch '7161-support-all-service-types' of git.curoverse.com:arvados into 7161-support-all-service-types
commit c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd
Author: radhika <radhika at curoverse.com>
Date: Fri Sep 25 10:13:49 2015 -0400
7161: Use default value of 1 for max_replicas_per_service and set it to None when writable non-disk type services are found.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 50b7e01..0754744 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -651,7 +651,7 @@ class KeepClient(object):
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
- self.max_replicas_per_service = None
+ self.max_replicas_per_service = 1
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
@@ -664,7 +664,7 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
- self.max_replicas_per_service = None
+ self.max_replicas_per_service = 1
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -722,11 +722,10 @@ class KeepClient(object):
self.using_proxy = any(ks.get('service_type') == 'proxy'
for ks in self._keep_services)
- # Set max_replicas_per_service to 1 for disk typed services.
- # In case of, non-disk typed services, we will use None to mean unknown.
- self.max_replicas_per_service = 1
+ # For disk type services, max_replicas_per_service is 1
+ # It is unknown or unlimited for non-disk typed services.
for ks in accessible:
- if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+ if ('disk' != ks.get('service_type')) and (not ks.get('read_only')):
self.max_replicas_per_service = None
def _service_weight(self, data_hash, service_uuid):
commit 72748d17a195be1b65f847b078d1924f4648d45a
Author: radhika <radhika at curoverse.com>
Date: Thu Sep 24 14:57:40 2015 -0400
7161: white space
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 1b076b6..50b7e01 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -752,8 +752,8 @@ class KeepClient(object):
for hint in locator.hints:
if hint.startswith('K@'):
if len(hint) == 7:
- sorted_roots.append(
- "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+ sorted_roots.append(
+ "https://keep.{}.arvadosapi.com/".format(hint[2:]))
elif len(hint) == 29:
svc = self._gateway_services.get(hint[2:])
if svc:
commit fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c
Author: radhika <radhika at curoverse.com>
Date: Thu Sep 24 14:52:39 2015 -0400
7161: clarify max_replicas_per_service
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index ee3419c..1b076b6 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,7 +22,6 @@ import time
import timer
import types
import UserDict
-import util
import zlib
import arvados
@@ -652,7 +651,7 @@ class KeepClient(object):
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
- self.thread_count = None
+ self.max_replicas_per_service = None
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
@@ -665,7 +664,7 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
- self.thread_count = None
+ self.max_replicas_per_service = None
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -723,10 +722,12 @@ class KeepClient(object):
self.using_proxy = any(ks.get('service_type') == 'proxy'
for ks in self._keep_services)
- # Use a thread_count of 1 if the service is not a disk
+ # Set max_replicas_per_service to 1 for disk typed services.
+ # In case of, non-disk typed services, we will use None to mean unknown.
+ self.max_replicas_per_service = 1
for ks in accessible:
- if ('disk' != ks.get('service_type')) and (True != ks.get('read_only')):
- self.thread_count = 1
+ if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+ self.max_replicas_per_service = None
def _service_weight(self, data_hash, service_uuid):
"""Compute the weight of a Keep service endpoint for a data
@@ -745,18 +746,15 @@ class KeepClient(object):
self.build_services_list(force_rebuild)
sorted_roots = []
-
- # Use the services indicated by the given hints that are
- # not size or authorization hints.
- # If it is a K@ hint of size 7, it is a keepproxy
- # Otherwise, expect the hint to be of len 29 and a uuid
- # of a remote service that can be resolved to a URI.
+ # Use the services indicated by the given +K at ... remote
+ # service hints, if any are present and can be resolved to a
+ # URI.
for hint in locator.hints:
- if not hint.startswith('A') and not hint[0].isdigit():
- if len(hint) == 7 and hint.startswith('K@'):
- sorted_roots.append(
- "https://keep.{}.arvadosapi.com/".format(hint[2:]))
- elif len(hint) == 29 and re.match(util.uuid_pattern, hint[2:]):
+ if hint.startswith('K@'):
+ if len(hint) == 7:
+ sorted_roots.append(
+ "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+ elif len(hint) == 29:
svc = self._gateway_services.get(hint[2:])
if svc:
sorted_roots.append(svc['_service_root'])
@@ -950,7 +948,7 @@ class KeepClient(object):
# Tell the proxy how many copies we want it to store
headers['X-Keep-Desired-Replication'] = str(copies)
roots_map = {}
- thread_limiter = KeepClient.ThreadLimiter(1 if 1 == self.thread_count else copies)
+ thread_limiter = KeepClient.ThreadLimiter(1 if self.max_replicas_per_service is None else copies)
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
for tries_left in loop:
commit 57694f580feff10b274d6b13e9b4b013bfdad937
Author: radhika <radhika at curoverse.com>
Date: Wed Sep 23 15:32:44 2015 -0400
7161: Look for all service hints, not just keep proxy. Also, use thread count of 1 when it not a disk.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 63b99da..ee3419c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,6 +22,7 @@ import time
import timer
import types
import UserDict
+import util
import zlib
import arvados
@@ -651,6 +652,7 @@ class KeepClient(object):
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
+ self.thread_count = None
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
@@ -663,6 +665,7 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
+ self.thread_count = None
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -720,6 +723,10 @@ class KeepClient(object):
self.using_proxy = any(ks.get('service_type') == 'proxy'
for ks in self._keep_services)
+ # Use a thread_count of 1 if the service is not a disk
+ for ks in accessible:
+ if ('disk' != ks.get('service_type')) and (True != ks.get('read_only')):
+ self.thread_count = 1
def _service_weight(self, data_hash, service_uuid):
"""Compute the weight of a Keep service endpoint for a data
@@ -739,15 +746,17 @@ class KeepClient(object):
sorted_roots = []
- # Use the services indicated by the given +K at ... remote
- # service hints, if any are present and can be resolved to a
- # URI.
+ # Use the services indicated by the given hints that are
+ # not size or authorization hints.
+ # If it is a K@ hint of size 7, it is a keepproxy
+ # Otherwise, expect the hint to be of len 29 and a uuid
+ # of a remote service that can be resolved to a URI.
for hint in locator.hints:
- if hint.startswith('K@'):
- if len(hint) == 7:
+ if not hint.startswith('A') and not hint[0].isdigit():
+ if len(hint) == 7 and hint.startswith('K@'):
sorted_roots.append(
"https://keep.{}.arvadosapi.com/".format(hint[2:]))
- elif len(hint) == 29:
+ elif len(hint) == 29 and re.match(util.uuid_pattern, hint[2:]):
svc = self._gateway_services.get(hint[2:])
if svc:
sorted_roots.append(svc['_service_root'])
@@ -938,11 +947,10 @@ class KeepClient(object):
locator = KeepLocator(loc_s)
headers = {}
- if self.using_proxy:
- # Tell the proxy how many copies we want it to store
- headers['X-Keep-Desired-Replication'] = str(copies)
+ # Tell the proxy how many copies we want it to store
+ headers['X-Keep-Desired-Replication'] = str(copies)
roots_map = {}
- thread_limiter = KeepClient.ThreadLimiter(copies)
+ thread_limiter = KeepClient.ThreadLimiter(1 if 1 == self.thread_count else copies)
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
for tries_left in loop:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list