[arvados] updated: 2.1.0-3158-g0531cc9f3
git repository hosting
git at public.arvados.org
Tue Dec 6 22:13:30 UTC 2022
Summary of changes:
lib/crunchrun/crunchrun_test.go | 4 ++--
sdk/cwl/arvados_cwl/arvcontainer.py | 8 +++-----
sdk/cwl/tests/test_container.py | 14 ++++++++++++++
3 files changed, 19 insertions(+), 7 deletions(-)
via 0531cc9f35e78a7be1eec7eb96fb6cc668ebcefa (commit)
via e68b04895316643762c38e3d52324744442ab095 (commit)
from 7d79a88a17d851fab8e4a7235448682fb269b374 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0531cc9f35e78a7be1eec7eb96fb6cc668ebcefa
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Dec 6 17:13:19 2022 -0500
19847: Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 76289b951..8da933a73 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -1299,7 +1299,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
- "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
+ "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
"--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{
"/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
@@ -1382,7 +1382,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
bindmounts, err := cr.SetupMounts()
c.Check(err, IsNil)
c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
- "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
+ "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
"--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
c.Check(bindmounts, DeepEquals, map[string]bindmount{
"/tmp": {realTemp + "/tmp2", false},
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 75371e2b7..b8bc70782 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -135,6 +135,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -299,6 +300,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -430,6 +432,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -663,6 +666,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -787,6 +791,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -885,6 +890,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -930,6 +936,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -1005,6 +1012,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -1100,6 +1108,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -1204,6 +1213,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
@@ -1295,6 +1305,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -1387,6 +1398,7 @@ class TestContainer(unittest.TestCase):
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": rev}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -1474,6 +1486,7 @@ class TestWorkflow(unittest.TestCase):
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
+ api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
@@ -1606,6 +1619,7 @@ class TestWorkflow(unittest.TestCase):
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
+ api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
commit e68b04895316643762c38e3d52324744442ab095
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Dec 6 16:41:15 2022 -0500
19847: Change disk cache size heuristic
Now sets disk cache to (2 GB <= the size of the RAM request <= 32 GiB)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4be2e2bd7..c6030d723 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -291,11 +291,9 @@ class ArvadosContainer(JobBase):
}
if use_disk_cache and "keep_cache_disk" not in runtime_constraints:
- # Cache size wasn't explicitly set so calculate a default
- # based on 2x RAM request or 1 GB per core, whichever is
- # smaller. This is to avoid requesting 100s of GB of disk
- # cache when requesting a node with a huge amount of RAM.
- runtime_constraints["keep_cache_disk"] = min(runtime_constraints["ram"] * 2, runtime_constraints["vcpus"] * (1024*1024*1024))
+ # Cache size wasn't explicitly set so set the default to
+ # 2 GB <= the size of the RAM request <= 32 GiB
+ runtime_constraints["keep_cache_disk"] = min(max(2*1024*1024*1024, runtime_constraints["ram"]), 32*1024*1024*1024)
partition_req, _ = self.get_requirement("http://arvados.org/cwl#PartitionRequirement")
if partition_req:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list