[ARVADOS] updated: 1.1.0-99-ga707277

Git user git at public.curoverse.com
Tue Nov 7 13:53:03 EST 2017


Summary of changes:
 sdk/cwl/arvados_cwl/fsaccess.py |  8 ++++---
 sdk/cwl/tests/test_fsaccess.py  | 47 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 3 deletions(-)

       via  a70727762dafee667b022331307f6c0f949fd7e7 (commit)
      from  d59645f3e566e691ba757f74bba503c13773dbe8 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit a70727762dafee667b022331307f6c0f949fd7e7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Nov 7 13:52:28 2017 -0500

    12568: Add test for collection cache management behavior
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 935fe65..0e96684 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -27,14 +27,16 @@ from schema_salad.ref_resolver import DefaultFetcher
 logger = logging.getLogger('arvados.cwl-runner')
 
 class CollectionCache(object):
-    def __init__(self, api_client, keep_client, num_retries):
+    def __init__(self, api_client, keep_client, num_retries,
+                 cap=256*1024*1024,
+                 min_entries=4):
         self.api_client = api_client
         self.keep_client = keep_client
         self.collections = OrderedDict()
         self.lock = threading.Lock()
         self.total = 0
-        self.cap = 256*1024*1024
-        self.min_entries = 4
+        self.cap = cap
+        self.min_entries = min_entries
 
     def cap_cache(self):
         if self.total > self.cap:
diff --git a/sdk/cwl/tests/test_fsaccess.py b/sdk/cwl/tests/test_fsaccess.py
index dd53595..43a508f 100644
--- a/sdk/cwl/tests/test_fsaccess.py
+++ b/sdk/cwl/tests/test_fsaccess.py
@@ -30,3 +30,50 @@ class TestFsAccess(unittest.TestCase):
         self.assertEqual(1, cr.call_count)
         c3 = cache.get("99999999999999999999999999999992+99")
         self.assertEqual(2, cr.call_count)
+
+    @mock.patch("arvados.collection.CollectionReader")
+    def test_collection_cache_limit(self, cr):
+        cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+        cr().manifest_text.return_value = 'x' * 524289
+        self.assertEqual(0, cache.total)
+        c1 = cache.get("99999999999999999999999999999991+99")
+        self.assertEqual((524289*128)*1, cache.total)
+
+        c2 = cache.get("99999999999999999999999999999992+99")
+        self.assertEqual((524289*128)*2, cache.total)
+
+        c1 = cache.get("99999999999999999999999999999991+99")
+        self.assertEqual((524289*128)*2, cache.total)
+
+        c3 = cache.get("99999999999999999999999999999993+99")
+        self.assertEqual((524289*128)*3, cache.total)
+        c4 = cache.get("99999999999999999999999999999994+99")
+        self.assertEqual((524289*128)*3, cache.total)
+        c5 = cache.get("99999999999999999999999999999995+99")
+        self.assertEqual((524289*128)*3, cache.total)
+        c6 = cache.get("99999999999999999999999999999996+99")
+        self.assertEqual((524289*128)*3, cache.total)
+
+
+    @mock.patch("arvados.collection.CollectionReader")
+    def test_collection_cache_limit2(self, cr):
+        cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+        cr().manifest_text.return_value = 'x' * 524287
+        self.assertEqual(0, cache.total)
+        c1 = cache.get("99999999999999999999999999999991+99")
+        self.assertEqual((524287*128)*1, cache.total)
+
+        c2 = cache.get("99999999999999999999999999999992+99")
+        self.assertEqual((524287*128)*2, cache.total)
+
+        c1 = cache.get("99999999999999999999999999999991+99")
+        self.assertEqual((524287*128)*2, cache.total)
+
+        c3 = cache.get("99999999999999999999999999999993+99")
+        self.assertEqual((524287*128)*3, cache.total)
+        c4 = cache.get("99999999999999999999999999999994+99")
+        self.assertEqual((524287*128)*4, cache.total)
+        c5 = cache.get("99999999999999999999999999999995+99")
+        self.assertEqual((524287*128)*4, cache.total)
+        c6 = cache.get("99999999999999999999999999999996+99")
+        self.assertEqual((524287*128)*4, cache.total)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list