[ARVADOS] updated: 91f1e7009305109db0bfb6405adbd11357745bfc

[Git user]

Summary of changes:
 sdk/cwl/arvados_cwl/__init__.py |  5 ++++-
 sdk/cwl/arvados_cwl/fsaccess.py | 13 ++++++++-----
 sdk/cwl/tests/test_fsaccess.py  | 28 ++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 6 deletions(-)
 create mode 100644 sdk/cwl/tests/test_fsaccess.py

       via  91f1e7009305109db0bfb6405adbd11357745bfc (commit)
      from  3a88a5d8753593be89373c5b124fc5a8398457a0 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 91f1e7009305109db0bfb6405adbd11357745bfc
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Apr 14 11:08:01 2017 -0400

    11462: Log ArgumentError in make_output_collection.  Add test for CollectionCache.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 1d2fa29..ed784f5 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -247,10 +247,13 @@ class ArvCwlRunner(object):
                 raise Exception("Output source is not in keep or a literal")
             sp = k.split("/")
             srccollection = sp[0][5:]
-            reader = self.collection_cache.get(srccollection)
             try:
+                reader = self.collection_cache.get(srccollection)
                 srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
                 final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
+            except arvados.errors.ArgumentError as e:
+                logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
+                raise
             except IOError as e:
                 logger.warn("While preparing output collection: %s", e)
 
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 34d9cea..534a675 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -4,6 +4,7 @@ import errno
 import urlparse
 import re
 import logging
+import threading
 
 import ruamel.yaml as yaml
 
@@ -25,13 +26,15 @@ class CollectionCache(object):
         self.api_client = api_client
         self.keep_client = keep_client
         self.collections = {}
+        self.lock = threading.Lock()
 
     def get(self, pdh):
-        if pdh not in self.collections:
-            logger.debug("Creating collection reader for %s", pdh)
-            self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
-                                                                        keep_client=self.keep_client)
-        return self.collections[pdh]
+        with self.lock:
+            if pdh not in self.collections:
+                logger.debug("Creating collection reader for %s", pdh)
+                self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
+                                                                            keep_client=self.keep_client)
+            return self.collections[pdh]
 
 
 class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
diff --git a/sdk/cwl/tests/test_fsaccess.py b/sdk/cwl/tests/test_fsaccess.py
new file mode 100644
index 0000000..4395661
--- /dev/null
+++ b/sdk/cwl/tests/test_fsaccess.py
@@ -0,0 +1,28 @@
+import functools
+import mock
+import sys
+import unittest
+import json
+import logging
+import os
+
+import arvados
+import arvados.keep
+import arvados.collection
+import arvados_cwl
+
+from cwltool.pathmapper import MapperEnt
+from .mock_discovery import get_rootDesc
+
+from arvados_cwl.fsaccess import CollectionCache
+
+class TestFsAccess(unittest.TestCase):
+    @mock.patch("arvados.collection.CollectionReader")
+    def test_collection_cache(self, cr):
+        cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+        c1 = cache.get("99999999999999999999999999999991+99")
+        c2 = cache.get("99999999999999999999999999999991+99")
+        self.assertIs(c1, c2)
+        self.assertEqual(1, cr.call_count)
+        c3 = cache.get("99999999999999999999999999999992+99")
+        self.assertEqual(2, cr.call_count)

-----------------------------------------------------------------------


hooks/post-receive
--