[ARVADOS] updated: ee9c164a23be2aabc22da20737bce362fad8ed7e
git at public.curoverse.com
git at public.curoverse.com
Fri Sep 25 10:15:23 EDT 2015
Summary of changes:
docker/jobs/Dockerfile | 2 +-
sdk/cli/bin/crunch-job | 7 +-
sdk/python/arvados/keep.py | 29 +++--
services/api/script/crunch-dispatch.rb | 5 +
services/keepstore/collision.go | 2 +-
.../keepstore/handlers_with_generic_volume_test.go | 119 +++++++++++++++++++++
services/keepstore/keepstore.go | 6 +-
services/keepstore/keepstore_test.go | 4 +
services/keepstore/mock_mutex_for_test.go | 6 +-
services/keepstore/volume_generic_test.go | 81 ++++++++------
services/keepstore/volume_unix_test.go | 16 +++
11 files changed, 215 insertions(+), 62 deletions(-)
create mode 100644 services/keepstore/handlers_with_generic_volume_test.go
via ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)
via 743022a885b563e47bc5d31e144b5165ab4688c9 (commit)
via 99349abd0ee7347b5bac3d4a9638853c6d4b97ab (commit)
via acd1241cec9260d54c2dca55785e309644334c41 (commit)
via e45aa811c8489cae3ac83836aceca7f2ce6ba398 (commit)
via 20ad8adb02808a85ac38f865fd870b00fb400b74 (commit)
via af901a9a0434c9b5ed8b9349835a1749f45f95d3 (commit)
via 33596fce13c3bd5c368f5efeb89d625f684a622d (commit)
via e668f9e2424cb1224b540865374b12fa9e1e3a8f (commit)
via 127c47f949fb71738186b2ae7b98adfd4bdef25e (commit)
via fdbf2ead3452bf7c8ba8f3f275cf486339b5406e (commit)
via 501325d396c1ac30637b5a48ecda17f77cd73206 (commit)
via a3e32a2f4a702c076d46a5b19305dd20a1ee3012 (commit)
via 6d352c4bfe2d0e9a405a52863c28b00de47f4be5 (commit)
via 43748be08c5d67171cb2f5565670dd0eef889b07 (commit)
via 87892d98eac028934480c378748cfb65a780d1cc (commit)
via 0fc6e0671e822cd4c5671519c7e7c58cff06c91f (commit)
via 67c6f66d2b954f1f0f887011b7b6d5153b2d7bf1 (commit)
via 9bd1f604e8f5ad0dd33b4501c535d9915924e8bd (commit)
via c447d4a79f653bbf2c172a0a715d30db896a4a32 (commit)
from b2ed333572a4db15e82ba23a162b0f065d24c2b7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ee9c164a23be2aabc22da20737bce362fad8ed7e
Author: radhika <radhika at curoverse.com>
Date: Fri Sep 25 10:13:49 2015 -0400
7161: working
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 50b7e01..0754744 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -651,7 +651,7 @@ class KeepClient(object):
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
- self.max_replicas_per_service = None
+ self.max_replicas_per_service = 1
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
@@ -664,7 +664,7 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
- self.max_replicas_per_service = None
+ self.max_replicas_per_service = 1
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -722,11 +722,10 @@ class KeepClient(object):
self.using_proxy = any(ks.get('service_type') == 'proxy'
for ks in self._keep_services)
- # Set max_replicas_per_service to 1 for disk typed services.
- # In case of, non-disk typed services, we will use None to mean unknown.
- self.max_replicas_per_service = 1
+ # For disk type services, max_replicas_per_service is 1
+ # It is unknown or unlimited for non-disk typed services.
for ks in accessible:
- if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+ if ('disk' != ks.get('service_type')) and (not ks.get('read_only')):
self.max_replicas_per_service = None
def _service_weight(self, data_hash, service_uuid):
commit 743022a885b563e47bc5d31e144b5165ab4688c9
Merge: 20ad8ad 99349ab
Author: radhika <radhika at curoverse.com>
Date: Fri Sep 25 08:32:46 2015 -0400
Merge branch 'master' into 7161-support-all-service-types
commit 20ad8adb02808a85ac38f865fd870b00fb400b74
Author: radhika <radhika at curoverse.com>
Date: Thu Sep 24 14:57:40 2015 -0400
7161: white space
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 1b076b6..50b7e01 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -752,8 +752,8 @@ class KeepClient(object):
for hint in locator.hints:
if hint.startswith('K@'):
if len(hint) == 7:
- sorted_roots.append(
- "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+ sorted_roots.append(
+ "https://keep.{}.arvadosapi.com/".format(hint[2:]))
elif len(hint) == 29:
svc = self._gateway_services.get(hint[2:])
if svc:
commit af901a9a0434c9b5ed8b9349835a1749f45f95d3
Author: radhika <radhika at curoverse.com>
Date: Thu Sep 24 14:52:39 2015 -0400
7161: clarify max_replicas_per_service
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index ee3419c..1b076b6 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,7 +22,6 @@ import time
import timer
import types
import UserDict
-import util
import zlib
import arvados
@@ -652,7 +651,7 @@ class KeepClient(object):
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
- self.thread_count = None
+ self.max_replicas_per_service = None
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
@@ -665,7 +664,7 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
- self.thread_count = None
+ self.max_replicas_per_service = None
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -723,10 +722,12 @@ class KeepClient(object):
self.using_proxy = any(ks.get('service_type') == 'proxy'
for ks in self._keep_services)
- # Use a thread_count of 1 if the service is not a disk
+ # Set max_replicas_per_service to 1 for disk typed services.
+ # In case of, non-disk typed services, we will use None to mean unknown.
+ self.max_replicas_per_service = 1
for ks in accessible:
- if ('disk' != ks.get('service_type')) and (True != ks.get('read_only')):
- self.thread_count = 1
+ if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+ self.max_replicas_per_service = None
def _service_weight(self, data_hash, service_uuid):
"""Compute the weight of a Keep service endpoint for a data
@@ -745,18 +746,15 @@ class KeepClient(object):
self.build_services_list(force_rebuild)
sorted_roots = []
-
- # Use the services indicated by the given hints that are
- # not size or authorization hints.
- # If it is a K@ hint of size 7, it is a keepproxy
- # Otherwise, expect the hint to be of len 29 and a uuid
- # of a remote service that can be resolved to a URI.
+ # Use the services indicated by the given +K at ... remote
+ # service hints, if any are present and can be resolved to a
+ # URI.
for hint in locator.hints:
- if not hint.startswith('A') and not hint[0].isdigit():
- if len(hint) == 7 and hint.startswith('K@'):
- sorted_roots.append(
- "https://keep.{}.arvadosapi.com/".format(hint[2:]))
- elif len(hint) == 29 and re.match(util.uuid_pattern, hint[2:]):
+ if hint.startswith('K@'):
+ if len(hint) == 7:
+ sorted_roots.append(
+ "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+ elif len(hint) == 29:
svc = self._gateway_services.get(hint[2:])
if svc:
sorted_roots.append(svc['_service_root'])
@@ -950,7 +948,7 @@ class KeepClient(object):
# Tell the proxy how many copies we want it to store
headers['X-Keep-Desired-Replication'] = str(copies)
roots_map = {}
- thread_limiter = KeepClient.ThreadLimiter(1 if 1 == self.thread_count else copies)
+ thread_limiter = KeepClient.ThreadLimiter(1 if self.max_replicas_per_service is None else copies)
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
for tries_left in loop:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list