[ARVADOS] updated: 1.1.0-99-ga707277
Git user
git at public.curoverse.com
Tue Nov 7 13:53:03 EST 2017
Summary of changes:
sdk/cwl/arvados_cwl/fsaccess.py | 8 ++++---
sdk/cwl/tests/test_fsaccess.py | 47 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 52 insertions(+), 3 deletions(-)
via a70727762dafee667b022331307f6c0f949fd7e7 (commit)
from d59645f3e566e691ba757f74bba503c13773dbe8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a70727762dafee667b022331307f6c0f949fd7e7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 7 13:52:28 2017 -0500
12568: Add test for collection cache management behavior
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 935fe65..0e96684 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -27,14 +27,16 @@ from schema_salad.ref_resolver import DefaultFetcher
logger = logging.getLogger('arvados.cwl-runner')
class CollectionCache(object):
- def __init__(self, api_client, keep_client, num_retries):
+ def __init__(self, api_client, keep_client, num_retries,
+ cap=256*1024*1024,
+ min_entries=4):
self.api_client = api_client
self.keep_client = keep_client
self.collections = OrderedDict()
self.lock = threading.Lock()
self.total = 0
- self.cap = 256*1024*1024
- self.min_entries = 4
+ self.cap = cap
+ self.min_entries = min_entries
def cap_cache(self):
if self.total > self.cap:
diff --git a/sdk/cwl/tests/test_fsaccess.py b/sdk/cwl/tests/test_fsaccess.py
index dd53595..43a508f 100644
--- a/sdk/cwl/tests/test_fsaccess.py
+++ b/sdk/cwl/tests/test_fsaccess.py
@@ -30,3 +30,50 @@ class TestFsAccess(unittest.TestCase):
self.assertEqual(1, cr.call_count)
c3 = cache.get("99999999999999999999999999999992+99")
self.assertEqual(2, cr.call_count)
+
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_collection_cache_limit(self, cr):
+ cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+ cr().manifest_text.return_value = 'x' * 524289
+ self.assertEqual(0, cache.total)
+ c1 = cache.get("99999999999999999999999999999991+99")
+ self.assertEqual((524289*128)*1, cache.total)
+
+ c2 = cache.get("99999999999999999999999999999992+99")
+ self.assertEqual((524289*128)*2, cache.total)
+
+ c1 = cache.get("99999999999999999999999999999991+99")
+ self.assertEqual((524289*128)*2, cache.total)
+
+ c3 = cache.get("99999999999999999999999999999993+99")
+ self.assertEqual((524289*128)*3, cache.total)
+ c4 = cache.get("99999999999999999999999999999994+99")
+ self.assertEqual((524289*128)*3, cache.total)
+ c5 = cache.get("99999999999999999999999999999995+99")
+ self.assertEqual((524289*128)*3, cache.total)
+ c6 = cache.get("99999999999999999999999999999996+99")
+ self.assertEqual((524289*128)*3, cache.total)
+
+
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_collection_cache_limit2(self, cr):
+ cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+ cr().manifest_text.return_value = 'x' * 524287
+ self.assertEqual(0, cache.total)
+ c1 = cache.get("99999999999999999999999999999991+99")
+ self.assertEqual((524287*128)*1, cache.total)
+
+ c2 = cache.get("99999999999999999999999999999992+99")
+ self.assertEqual((524287*128)*2, cache.total)
+
+ c1 = cache.get("99999999999999999999999999999991+99")
+ self.assertEqual((524287*128)*2, cache.total)
+
+ c3 = cache.get("99999999999999999999999999999993+99")
+ self.assertEqual((524287*128)*3, cache.total)
+ c4 = cache.get("99999999999999999999999999999994+99")
+ self.assertEqual((524287*128)*4, cache.total)
+ c5 = cache.get("99999999999999999999999999999995+99")
+ self.assertEqual((524287*128)*4, cache.total)
+ c6 = cache.get("99999999999999999999999999999996+99")
+ self.assertEqual((524287*128)*4, cache.total)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list