[arvados] created: 2.1.0-2967-gc453e7667
git repository hosting
git at public.arvados.org
Tue Oct 25 15:28:27 UTC 2022
at c453e7667a87053ebc34d70c44995e116601cb93 (commit)
commit c453e7667a87053ebc34d70c44995e116601cb93
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Oct 25 11:27:53 2022 -0400
18842: Add KeepCacheDisk to EstimateScratchSpace used by dispatchers
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index 7c7643bfc..5b85d57ea 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -56,7 +56,7 @@ func estimateDockerImageSize(collectionPDH string) int64 {
// EstimateScratchSpace estimates how much available disk space (in
// bytes) is needed to run the container by summing the capacity
// requested by 'tmp' mounts plus disk space required to load the
-// Docker image.
+// Docker image plus arv-mount block cache.
func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
for _, m := range ctr.Mounts {
if m.Kind == "tmp" {
@@ -80,6 +80,9 @@ func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
// Now reserve space for the extracted image on disk.
needScratch += dockerImageSize
+ // Now reserve space the arv-mount disk cache
+ needScratch += ctr.RuntimeConstraints.KeepCacheDisk
+
return
}
commit 2bb2e562faaa5ae3160469cf3ef5b36f1b884504
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Oct 25 11:21:47 2022 -0400
18842: Initial work adding disk cache option to configuration
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 9b5547248..88154f05e 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -953,8 +953,15 @@ Clusters:
# troubleshooting purposes.
LogReuseDecisions: false
- # Default value for keep_cache_ram of a container's runtime_constraints.
- DefaultKeepCacheRAM: 268435456
+ # Default value for keep_cache_ram of a container's
+ # runtime_constraints. Note: this gets added to the RAM request
+ # used to allocate a VM or submit an HPC job
+ DefaultKeepCacheRAM: 0
+
+ # Default value for keep_cache_disk of a container's
+ # runtime_constraints. Note: this gets added to the disk
+ # request used to allocate a VM or submit an HPC job
+ DefaultKeepCacheDisk: 8589934592
# Number of times a container can be unlocked before being
# automatically cancelled.
diff --git a/lib/config/export.go b/lib/config/export.go
index e7cf094eb..9877b85a3 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -121,6 +121,7 @@ var whitelist = map[string]bool{
"Containers.CrunchRunArgumentsList": false,
"Containers.CrunchRunCommand": false,
"Containers.DefaultKeepCacheRAM": true,
+ "Containers.DefaultKeepCacheDisk": true,
"Containers.DispatchPrivateKey": false,
"Containers.JobsAPI": true,
"Containers.JobsAPI.Enable": true,
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index ee9115d8d..67454cd0f 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -427,7 +427,11 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
arvMountCmd = append(arvMountCmd, "--allow-other")
}
- if runner.Container.RuntimeConstraints.KeepCacheRAM > 0 {
+ if runner.Container.RuntimeConstraints.KeepCacheDisk > 0 {
+ keepcachedir, err = runner.MkTempDir(runner.parentTemp, "keepcache")
+ arvMountCmd = append(arvMountCmd, "--disk-cache", "--disk-cache-dir", keepcachedir, "--file-cache", fmt.Sprintf("%d", runner.Container.RuntimeConstraints.KeepCacheDisk))
+ }
+ else if runner.Container.RuntimeConstraints.KeepCacheRAM > 0 {
arvMountCmd = append(arvMountCmd, "--file-cache", fmt.Sprintf("%d", runner.Container.RuntimeConstraints.KeepCacheRAM))
}
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index e9b58bc83..9b5f32233 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -265,7 +265,11 @@ class ArvadosContainer(JobBase):
runtime_req, _ = self.get_requirement("http://arvados.org/cwl#RuntimeConstraints")
if runtime_req:
if "keep_cache" in runtime_req:
- runtime_constraints["keep_cache_ram"] = math.ceil(runtime_req["keep_cache"] * 2**20)
+ if self.arvrunner.api.config()["Containers"].get("DefaultKeepCacheDisk", 0) > 0:
+ # If DefaultKeepCacheDisk is non-zero it means we should use disk cache.
+ runtime_constraints["keep_cache_disk"] = math.ceil(runtime_req["keep_cache"] * 2**20)
+ else:
+ runtime_constraints["keep_cache_ram"] = math.ceil(runtime_req["keep_cache"] * 2**20)
if "outputDirType" in runtime_req:
if runtime_req["outputDirType"] == "local_output_dir":
# Currently the default behavior.
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 45f92017c..165c8112e 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -107,11 +107,12 @@ type CUDARuntimeConstraints struct {
// RuntimeConstraints specify a container's compute resources (RAM,
// CPU) and network connectivity.
type RuntimeConstraints struct {
- API bool `json:"API"`
- RAM int64 `json:"ram"`
- VCPUs int `json:"vcpus"`
- KeepCacheRAM int64 `json:"keep_cache_ram"`
- CUDA CUDARuntimeConstraints `json:"cuda"`
+ API bool `json:"API"`
+ RAM int64 `json:"ram"`
+ VCPUs int `json:"vcpus"`
+ KeepCacheRAM int64 `json:"keep_cache_ram"`
+ KeepCacheDisk int64 `json:"keep_cache_disk"`
+ CUDA CUDARuntimeConstraints `json:"cuda"`
}
// SchedulingParameters specify a container's scheduling parameters
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index c2725506c..7143378d2 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -959,6 +959,7 @@ class ArvadosModel < ApplicationRecord
'hardware_capability' => '',
},
'keep_cache_ram' => 0,
+ 'keep_cache_disk' => 0,
'ram' => 0,
'vcpus' => 0,
}.merge(attributes['runtime_constraints'] || {})
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 3e3f73b83..3334fdcac 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -227,6 +227,10 @@ class Container < ArvadosModel
if rc['keep_cache_ram'] == 0
rc['keep_cache_ram'] = Rails.configuration.Containers.DefaultKeepCacheRAM
end
+ if rc['keep_cache_disk'] == 0 and rc['keep_cache_ram'] == 0
+ # Only set if keep_cache_ram isn't set.
+ rc['keep_cache_disk'] = Rails.configuration.Containers.DefaultKeepCacheDisk
+ end
rc
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list