[ARVADOS] updated: e31d1ec3c19d88042b47516fad9ed952d6561fd6

git at public.curoverse.com git at public.curoverse.com
Fri Sep 25 13:00:43 EDT 2015


Summary of changes:
 apps/workbench/config/application.default.yml | 47 ++++++++++++++++++++++++---
 sdk/cli/bin/crunch-job                        | 10 +++---
 sdk/go/keepclient/keepclient.go               | 17 ++++++----
 sdk/go/keepclient/support.go                  | 44 ++++++++++++++-----------
 services/api/config/application.default.yml   | 43 ++++++++++++++++++++++--
 services/datamanager/keep/keep.go             | 19 ++++++++---
 6 files changed, 138 insertions(+), 42 deletions(-)

       via  e31d1ec3c19d88042b47516fad9ed952d6561fd6 (commit)
       via  c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd (commit)
       via  72748d17a195be1b65f847b078d1924f4648d45a (commit)
       via  fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c (commit)
       via  57694f580feff10b274d6b13e9b4b013bfdad937 (commit)
       via  b8ad249763a75f3eb7bea758b88cdc87389639f1 (commit)
       via  0ba145115071473ba46a3fac30c2dc98b74f49ad (commit)
       via  0fdfc23c640c831cb2c0ee1e04fb639eb032de6f (commit)
       via  73ce0cf7675e060d33e75488edfa4f533c177f82 (commit)
       via  32a29ee7d171dccdb424990ff9c73e4b893dc3e4 (commit)
       via  5a55113805af906145449be18ec86b1a95a5017b (commit)
       via  285c0092f8ae4d757e31754c91314baa4db5af44 (commit)
       via  559dc0ac07ea3d6820f5e220b9f6faf850ae63bf (commit)
       via  5a974b0568f7645a7e4377394da04940fa1ec137 (commit)
       via  2b20a0ec9a32abfd85c09474e044cfd4c09f4963 (commit)
       via  c21c6330cc8a3c8cc96d83ff9a315fe10dff7c8f (commit)
       via  9db403a87830f7afec86dae5bb279cf87df8fd0e (commit)
       via  67f730557e062608f842fa3abc612fb717fbce2f (commit)
       via  7fdfbc3eae8de4fe7f6912c160e408ef57a89487 (commit)
       via  0333c360cdd24b8fdd28a88d6d99bebe7e7a9bed (commit)
       via  c7b4e4802762eada81652224878a74b538a1d704 (commit)
       via  0219fab6b0eb8b4710af2ae03c5231bf4d932a28 (commit)
       via  6cca3e129b6de3fc4b34a1664f494fcbbbd940d1 (commit)
       via  bb663993cc851971d9444d834acfc416363931b2 (commit)
      from  ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit e31d1ec3c19d88042b47516fad9ed952d6561fd6
Merge: c2a9349 ee9c164
Author: radhika <radhika at curoverse.com>
Date:   Fri Sep 25 12:44:15 2015 -0400

    Merge branch '7161-support-all-service-types' of git.curoverse.com:arvados into 7161-support-all-service-types


commit c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd
Author: radhika <radhika at curoverse.com>
Date:   Fri Sep 25 10:13:49 2015 -0400

    7161: Use default value of 1 for max_replicas_per_service and set it to None when writable non-disk type services are found.

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 50b7e01..0754744 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -651,7 +651,7 @@ class KeepClient(object):
                 self._writable_services = self._keep_services
                 self.using_proxy = True
                 self._static_services_list = True
-                self.max_replicas_per_service = None
+                self.max_replicas_per_service = 1
             else:
                 # It's important to avoid instantiating an API client
                 # unless we actually need one, for testing's sake.
@@ -664,7 +664,7 @@ class KeepClient(object):
                 self._writable_services = None
                 self.using_proxy = None
                 self._static_services_list = False
-                self.max_replicas_per_service = None
+                self.max_replicas_per_service = 1
 
     def current_timeout(self, attempt_number):
         """Return the appropriate timeout to use for this client.
@@ -722,11 +722,10 @@ class KeepClient(object):
 
             self.using_proxy = any(ks.get('service_type') == 'proxy'
                                    for ks in self._keep_services)
-            # Set max_replicas_per_service to 1 for disk typed services.
-            # In case of, non-disk typed services, we will use None to mean unknown.
-            self.max_replicas_per_service = 1
+            # For disk type services, max_replicas_per_service is 1
+            # It is unknown or unlimited for non-disk typed services.
             for ks in accessible:
-                if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+                if ('disk' != ks.get('service_type')) and (not ks.get('read_only')):
                     self.max_replicas_per_service = None
 
     def _service_weight(self, data_hash, service_uuid):

commit 72748d17a195be1b65f847b078d1924f4648d45a
Author: radhika <radhika at curoverse.com>
Date:   Thu Sep 24 14:57:40 2015 -0400

    7161: white space

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 1b076b6..50b7e01 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -752,8 +752,8 @@ class KeepClient(object):
         for hint in locator.hints:
             if hint.startswith('K@'):
                 if len(hint) == 7:
-                     sorted_roots.append(
-                         "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+                    sorted_roots.append(
+                        "https://keep.{}.arvadosapi.com/".format(hint[2:]))
                 elif len(hint) == 29:
                     svc = self._gateway_services.get(hint[2:])
                     if svc:

commit fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c
Author: radhika <radhika at curoverse.com>
Date:   Thu Sep 24 14:52:39 2015 -0400

    7161: clarify max_replicas_per_service

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index ee3419c..1b076b6 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,7 +22,6 @@ import time
 import timer
 import types
 import UserDict
-import util
 import zlib
 
 import arvados
@@ -652,7 +651,7 @@ class KeepClient(object):
                 self._writable_services = self._keep_services
                 self.using_proxy = True
                 self._static_services_list = True
-                self.thread_count = None
+                self.max_replicas_per_service = None
             else:
                 # It's important to avoid instantiating an API client
                 # unless we actually need one, for testing's sake.
@@ -665,7 +664,7 @@ class KeepClient(object):
                 self._writable_services = None
                 self.using_proxy = None
                 self._static_services_list = False
-                self.thread_count = None
+                self.max_replicas_per_service = None
 
     def current_timeout(self, attempt_number):
         """Return the appropriate timeout to use for this client.
@@ -723,10 +722,12 @@ class KeepClient(object):
 
             self.using_proxy = any(ks.get('service_type') == 'proxy'
                                    for ks in self._keep_services)
-            # Use a thread_count of 1 if the service is not a disk
+            # Set max_replicas_per_service to 1 for disk typed services.
+            # In case of, non-disk typed services, we will use None to mean unknown.
+            self.max_replicas_per_service = 1
             for ks in accessible:
-                if ('disk' != ks.get('service_type')) and (True != ks.get('read_only')):
-                    self.thread_count = 1
+                if ('disk' != ks.get('service_type')) and not ks.get('read_only'):
+                    self.max_replicas_per_service = None
 
     def _service_weight(self, data_hash, service_uuid):
         """Compute the weight of a Keep service endpoint for a data
@@ -745,18 +746,15 @@ class KeepClient(object):
         self.build_services_list(force_rebuild)
 
         sorted_roots = []
-
-        # Use the services indicated by the given hints that are
-        # not size or authorization hints.
-        # If it is a K@ hint of size 7, it is a keepproxy
-        # Otherwise, expect the hint to be of len 29 and a uuid
-        # of a remote service that can be resolved to a URI.
+        # Use the services indicated by the given +K at ... remote
+        # service hints, if any are present and can be resolved to a
+        # URI.
         for hint in locator.hints:
-            if not hint.startswith('A') and not hint[0].isdigit():
-                if len(hint) == 7 and hint.startswith('K@'):
-                    sorted_roots.append(
-                        "https://keep.{}.arvadosapi.com/".format(hint[2:]))
-                elif len(hint) == 29 and re.match(util.uuid_pattern, hint[2:]):
+            if hint.startswith('K@'):
+                if len(hint) == 7:
+                     sorted_roots.append(
+                         "https://keep.{}.arvadosapi.com/".format(hint[2:]))
+                elif len(hint) == 29:
                     svc = self._gateway_services.get(hint[2:])
                     if svc:
                         sorted_roots.append(svc['_service_root'])
@@ -950,7 +948,7 @@ class KeepClient(object):
         # Tell the proxy how many copies we want it to store
         headers['X-Keep-Desired-Replication'] = str(copies)
         roots_map = {}
-        thread_limiter = KeepClient.ThreadLimiter(1 if 1 == self.thread_count else copies)
+        thread_limiter = KeepClient.ThreadLimiter(1 if self.max_replicas_per_service is None else copies)
         loop = retry.RetryLoop(num_retries, self._check_loop_result,
                                backoff_start=2)
         for tries_left in loop:

commit 57694f580feff10b274d6b13e9b4b013bfdad937
Author: radhika <radhika at curoverse.com>
Date:   Wed Sep 23 15:32:44 2015 -0400

    7161: Look for all service hints, not just keep proxy. Also, use thread count of 1 when it not a disk.

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 63b99da..ee3419c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -22,6 +22,7 @@ import time
 import timer
 import types
 import UserDict
+import util
 import zlib
 
 import arvados
@@ -651,6 +652,7 @@ class KeepClient(object):
                 self._writable_services = self._keep_services
                 self.using_proxy = True
                 self._static_services_list = True
+                self.thread_count = None
             else:
                 # It's important to avoid instantiating an API client
                 # unless we actually need one, for testing's sake.
@@ -663,6 +665,7 @@ class KeepClient(object):
                 self._writable_services = None
                 self.using_proxy = None
                 self._static_services_list = False
+                self.thread_count = None
 
     def current_timeout(self, attempt_number):
         """Return the appropriate timeout to use for this client.
@@ -720,6 +723,10 @@ class KeepClient(object):
 
             self.using_proxy = any(ks.get('service_type') == 'proxy'
                                    for ks in self._keep_services)
+            # Use a thread_count of 1 if the service is not a disk
+            for ks in accessible:
+                if ('disk' != ks.get('service_type')) and (True != ks.get('read_only')):
+                    self.thread_count = 1
 
     def _service_weight(self, data_hash, service_uuid):
         """Compute the weight of a Keep service endpoint for a data
@@ -739,15 +746,17 @@ class KeepClient(object):
 
         sorted_roots = []
 
-        # Use the services indicated by the given +K at ... remote
-        # service hints, if any are present and can be resolved to a
-        # URI.
+        # Use the services indicated by the given hints that are
+        # not size or authorization hints.
+        # If it is a K@ hint of size 7, it is a keepproxy
+        # Otherwise, expect the hint to be of len 29 and a uuid
+        # of a remote service that can be resolved to a URI.
         for hint in locator.hints:
-            if hint.startswith('K@'):
-                if len(hint) == 7:
+            if not hint.startswith('A') and not hint[0].isdigit():
+                if len(hint) == 7 and hint.startswith('K@'):
                     sorted_roots.append(
                         "https://keep.{}.arvadosapi.com/".format(hint[2:]))
-                elif len(hint) == 29:
+                elif len(hint) == 29 and re.match(util.uuid_pattern, hint[2:]):
                     svc = self._gateway_services.get(hint[2:])
                     if svc:
                         sorted_roots.append(svc['_service_root'])
@@ -938,11 +947,10 @@ class KeepClient(object):
         locator = KeepLocator(loc_s)
 
         headers = {}
-        if self.using_proxy:
-            # Tell the proxy how many copies we want it to store
-            headers['X-Keep-Desired-Replication'] = str(copies)
+        # Tell the proxy how many copies we want it to store
+        headers['X-Keep-Desired-Replication'] = str(copies)
         roots_map = {}
-        thread_limiter = KeepClient.ThreadLimiter(copies)
+        thread_limiter = KeepClient.ThreadLimiter(1 if 1 == self.thread_count else copies)
         loop = retry.RetryLoop(num_retries, self._check_loop_result,
                                backoff_start=2)
         for tries_left in loop:

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list