[ARVADOS] updated: 1.2.0-455-geff370384
Git user
git at public.curoverse.com
Fri Nov 30 10:52:42 EST 2018
Summary of changes:
sdk/cwl/arvados_cwl/executor.py | 3 +-
sdk/cwl/arvados_cwl/runner.py | 12 +++----
sdk/cwl/tests/test_fsaccess.py | 78 ++++++++++++++++++++---------------------
sdk/cwl/tests/test_submit.py | 3 +-
4 files changed, 48 insertions(+), 48 deletions(-)
via eff370384f2301c5673820690ed0ae039182f4d6 (commit)
from be64ccd6498eb08960310e19a6ef2d840a33dffc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit eff370384f2301c5673820690ed0ae039182f4d6
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 30 10:50:20 2018 -0500
14510: Bugfix order of precedence for setting collection cache size
Fix a couple other buggy tests.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 27e6ad41b..9595b5591 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -631,7 +631,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
merged_map=merged_map,
priority=runtimeContext.priority,
secret_store=self.secret_store,
- collection_cache_size=runtimeContext.collection_cache_size)
+ collection_cache_size=runtimeContext.collection_cache_size,
+ collection_cache_is_default=self.should_estimate_cache_size)
elif self.work_api == "jobs":
runnerjob = RunnerJob(self, tool, job_order, runtimeContext.enable_reuse,
self.output_name,
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index c1a98e745..6094cfe24 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -364,7 +364,9 @@ class Runner(object):
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
intermediate_output_ttl=0, merged_map=None,
- priority=None, secret_store=None, collection_cache_size=None):
+ priority=None, secret_store=None,
+ collection_cache_size=256,
+ collection_cache_is_default=True):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -389,7 +391,7 @@ class Runner(object):
self.submit_runner_cores = 1
self.submit_runner_ram = 1024 # defaut 1 GiB
- self.collection_cache_size = 256
+ self.collection_cache_size = collection_cache_size
runner_resource_req, _ = self.tool.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
if runner_resource_req:
@@ -397,17 +399,13 @@ class Runner(object):
self.submit_runner_cores = runner_resource_req["coresMin"]
if runner_resource_req.get("ramMin"):
self.submit_runner_ram = runner_resource_req["ramMin"]
- if runner_resource_req.get("keep_cache"):
+ if runner_resource_req.get("keep_cache") and collection_cache_is_default:
self.collection_cache_size = runner_resource_req["keep_cache"]
if submit_runner_ram:
# Command line / initializer overrides default and/or spec from workflow
self.submit_runner_ram = submit_runner_ram
- if collection_cache_size:
- # Command line / initializer overrides default and/or spec from workflow
- self.collection_cache_size = collection_cache_size
-
if self.submit_runner_ram <= 0:
raise Exception("Value of submit-runner-ram must be greater than zero")
diff --git a/sdk/cwl/tests/test_fsaccess.py b/sdk/cwl/tests/test_fsaccess.py
index d52e94871..f83612a8b 100644
--- a/sdk/cwl/tests/test_fsaccess.py
+++ b/sdk/cwl/tests/test_fsaccess.py
@@ -36,34 +36,34 @@ class TestFsAccess(unittest.TestCase):
cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
cr().manifest_text.return_value = 'x' * 524289
self.assertEqual(0, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*1, cache.total)
- c2 = cache.get("99999999999999999999999999999992+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c2 = cache.get("99999999999999999999999999999992+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*2, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*2, cache.total)
- c3 = cache.get("99999999999999999999999999999993+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c3 = cache.get("99999999999999999999999999999993+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
- c4 = cache.get("99999999999999999999999999999994+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c4 = cache.get("99999999999999999999999999999994+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
- c5 = cache.get("99999999999999999999999999999995+99")
- self.assertNotIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c5 = cache.get("99999999999999999999999999999995+524289")
+ self.assertNotIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
@@ -72,37 +72,37 @@ class TestFsAccess(unittest.TestCase):
cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
cr().manifest_text.return_value = 'x' * 524287
self.assertEqual(0, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*1, cache.total)
- c2 = cache.get("99999999999999999999999999999992+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c2 = cache.get("99999999999999999999999999999992+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*2, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*2, cache.total)
- c3 = cache.get("99999999999999999999999999999993+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c3 = cache.get("99999999999999999999999999999993+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*3, cache.total)
- c4 = cache.get("99999999999999999999999999999994+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c4 = cache.get("99999999999999999999999999999994+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
- c5 = cache.get("99999999999999999999999999999995+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c5 = cache.get("99999999999999999999999999999995+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
- c6 = cache.get("99999999999999999999999999999996+99")
- self.assertNotIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c6 = cache.get("99999999999999999999999999999996+524287")
+ self.assertNotIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index a7a21e709..8a40fbaf0 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1115,10 +1115,11 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=60.0', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=500",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list