[ARVADOS] created: 7232f7451793ef7f07e30e46d951ade91c878f5b

[git at public.curoverse.com]

        at  7232f7451793ef7f07e30e46d951ade91c878f5b (commit)


commit 7232f7451793ef7f07e30e46d951ade91c878f5b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Feb 8 21:50:52 2016 -0500

    Add KEEP_CACHE and KEEP_PREFETCH env vars.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 71af644..a9aba94 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -400,7 +400,7 @@ class _BlockManager(object):
     DEFAULT_PUT_THREADS = 2
     DEFAULT_GET_THREADS = 2
 
-    def __init__(self, keep):
+    def __init__(self, keep, prefetch_size=None):
         """keep: KeepClient object to use"""
         self._keep = keep
         self._bufferblocks = {}
@@ -412,6 +412,7 @@ class _BlockManager(object):
         self.prefetch_enabled = True
         self.num_put_threads = _BlockManager.DEFAULT_PUT_THREADS
         self.num_get_threads = _BlockManager.DEFAULT_GET_THREADS
+        self.prefetch_size = prefetch_size if prefetch_size is not None else config.KEEP_BLOCK_SIZE
 
     @synchronized
     def alloc_bufferblock(self, blockid=None, starting_capacity=2**14, owner=None):
@@ -820,7 +821,9 @@ class ArvadosFile(object):
             if size == 0 or offset >= self.size():
                 return ''
             readsegs = locators_and_ranges(self._segments, offset, size)
-            prefetch = locators_and_ranges(self._segments, offset + size, config.KEEP_BLOCK_SIZE, limit=32)
+            prefetch = locators_and_ranges(self._segments, offset + size,
+                                           self.parent._my_block_manager().prefetch_size,
+                                           limit=32)
 
         locs = set()
         data = []
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 8450bd1..5b40ecd 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1231,7 +1231,8 @@ class Collection(RichCollectionBase):
     @synchronized
     def _my_api(self):
         if self._api_client is None:
-            self._api_client = ThreadSafeApiCache(self._config)
+            block_cache = keep.KeepBlockCache(int(self._config["KEEP_CACHE"]) if "KEEP_CACHE" in self._config else None)
+            self._api_client = ThreadSafeApiCache(self._config, keep_params={"block_cache": block_cache})
             self._keep_client = self._api_client.keep
         return self._api_client
 
@@ -1241,13 +1242,15 @@ class Collection(RichCollectionBase):
             if self._api_client is None:
                 self._my_api()
             else:
-                self._keep_client = KeepClient(api_client=self._api_client)
+                block_cache = keep.KeepBlockCache(int(self._config["KEEP_CACHE"]) if "KEEP_CACHE" in self._config else None)
+                self._keep_client = KeepClient(api_client=self._api_client, block_cache=block_cache)
         return self._keep_client
 
     @synchronized
     def _my_block_manager(self):
         if self._block_manager is None:
-            self._block_manager = _BlockManager(self._my_keep())
+            prefetch_size = int(self._config["KEEP_PREFETCH"]) if "KEEP_PREFETCH" in self._config else None
+            self._block_manager = _BlockManager(self._my_keep(), prefetch_size=prefetch_size)
         return self._block_manager
 
     def _remember_api_response(self, response):

-----------------------------------------------------------------------


hooks/post-receive
--