[ARVADOS] updated: 2.1.0-1726-gcb2d52217
Git user
git at public.arvados.org
Fri Dec 17 15:56:22 UTC 2021
Summary of changes:
.../_container_runtime_constraints.liquid | 11 +-
lib/crunchrun/crunchrun.go | 8 +-
lib/dispatchcloud/node_size.go | 12 +-
lib/dispatchcloud/node_size_test.go | 139 ++++++++++++---------
sdk/go/arvados/container.go | 18 +--
5 files changed, 111 insertions(+), 77 deletions(-)
via cb2d522176c17f2d388098b70fdbaa90fb30e682 (commit)
from 9908d25991d607687c7691548a862d1fb73788d5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit cb2d522176c17f2d388098b70fdbaa90fb30e682
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Dec 17 10:55:00 2021 -0500
18321: Add CUDARuntimeConstraints matching CUDAFeatures
Adjust CUDA version comparison. Refactor tests.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/_includes/_container_runtime_constraints.liquid b/doc/_includes/_container_runtime_constraints.liquid
index dcdc29cf3..6926b9d3d 100644
--- a/doc/_includes/_container_runtime_constraints.liquid
+++ b/doc/_includes/_container_runtime_constraints.liquid
@@ -14,6 +14,11 @@ table(table table-bordered table-condensed).
|vcpus|integer|Number of cores to be used to run this process.|Optional. However, a ContainerRequest that is in "Committed" state must provide this.|
|keep_cache_ram|integer|Number of keep cache bytes to be used to run this process.|Optional.|
|API|boolean|When set, ARVADOS_API_HOST and ARVADOS_API_TOKEN will be set, and container will have networking enabled to access the Arvados API server.|Optional.|
-|cuda_driver_version|string|Minimum CUDA driver version.|Optional.|
-|cuda_hardware_capability|string|Minimum CUDA hardware capability.|Optional.|
-|cuda_device_count|int|Number of GPUs to request.|Optional.|
+|cuda|object|Request CUDA GPU support, see below|Optional.|
+
+h3. CUDA GPU support
+
+table(table table-bordered table-condensed).
+|device_count|int|Number of GPUs to request.|Required to request a GPU node.|
+|driver_version|string|Minimum CUDA driver version.|Optional.|
+|hardware_capability|string|Minimum CUDA hardware capability.|Optional.|
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 7e68dcd33..52d9c4b0f 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -988,11 +988,11 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st
runner.executorStderr = stderr
cudaDeviceCount := 0
- if runner.Container.RuntimeConstraints.CUDADriverVersion != "" ||
- runner.Container.RuntimeConstraints.CUDAHardwareCapability != "" ||
- runner.Container.RuntimeConstraints.CUDADeviceCount != 0 {
+ if runner.Container.RuntimeConstraints.CUDA.DriverVersion != "" ||
+ runner.Container.RuntimeConstraints.CUDA.HardwareCapability != "" ||
+ runner.Container.RuntimeConstraints.CUDA.DeviceCount != 0 {
// if any of these are set, enable CUDA GPU support
- cudaDeviceCount = runner.Container.RuntimeConstraints.CUDADeviceCount
+ cudaDeviceCount = runner.Container.RuntimeConstraints.CUDA.DeviceCount
if cudaDeviceCount == 0 {
cudaDeviceCount = 1
}
diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index aa2cd7d56..6831d8d8e 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -83,8 +83,8 @@ func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
return
}
-// compareVersion returns true if vs1 >= vs2, otherwise false
-func compareVersion(vs1 string, vs2 string) bool {
+// compareVersion returns true if vs1 < vs2, otherwise false
+func versionLess(vs1 string, vs2 string) bool {
v1, err := strconv.ParseFloat(vs1, 64)
if err != nil {
return false
@@ -93,7 +93,7 @@ func compareVersion(vs1 string, vs2 string) bool {
if err != nil {
return false
}
- return v1 >= v2
+ return v1 < v2
}
// ChooseInstanceType returns the cheapest available
@@ -123,9 +123,9 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
case it.VCPUs < needVCPUs: // insufficient VCPUs
case it.Preemptible != ctr.SchedulingParameters.Preemptible: // wrong preemptable setting
case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs): // same price, worse specs
- case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDADeviceCount: // insufficient CUDA devices
- case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDADriverVersion): // insufficient driver version
- case it.CUDA.DeviceCount > 0 && !compareVersion(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDAHardwareCapability): // insufficient hardware capability
+ case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDA.DeviceCount: // insufficient CUDA devices
+ case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && versionLess(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDA.DriverVersion): // insufficient driver version
+ case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && versionLess(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDA.HardwareCapability): // insufficient hardware capability
// Don't select this node
default:
// Didn't reject the node, so select it
diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index cdcf4033f..06255462b 100644
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -154,69 +154,94 @@ func (*NodeSizeSuite) TestChooseGPU(c *check.C) {
"low_capability": {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_capability", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "8.0", DriverVersion: "11.0"}},
"best": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "11.0"}},
"low_driver": {Price: 2.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "low_driver", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "9.0", DriverVersion: "10.0"}},
- "small": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+ "cheap_gpu": {Price: 2.0, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "cheap_gpu", CUDA: arvados.CUDAFeatures{DeviceCount: 1, HardwareCapability: "8.0", DriverVersion: "10.0"}},
+ "non_gpu": {Price: 1.1, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "non_gpu"},
}
- best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
- Mounts: map[string]arvados.Mount{
- "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+
+ type GPUTestCase struct {
+ CUDA arvados.CUDARuntimeConstraints
+ SelectedInstance string
+ }
+ cases := []GPUTestCase{
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "9.0",
+ DriverVersion: "11.0",
+ },
+ SelectedInstance: "best",
},
- RuntimeConstraints: arvados.RuntimeConstraints{
- VCPUs: 2,
- RAM: 987654321,
- KeepCacheRAM: 123456789,
- CUDADeviceCount: 1,
- CUDAHardwareCapability: "9.0",
- CUDADriverVersion: "11.0",
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 2,
+ HardwareCapability: "9.0",
+ DriverVersion: "11.0",
+ },
+ SelectedInstance: "costly",
},
- })
- c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "best")
- c.Check(best.RAM >= 1234567890, check.Equals, true)
- c.Check(best.VCPUs >= 2, check.Equals, true)
- c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
- c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
- c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
-
- best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
- Mounts: map[string]arvados.Mount{
- "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "8.0",
+ DriverVersion: "11.0",
+ },
+ SelectedInstance: "low_capability",
},
- RuntimeConstraints: arvados.RuntimeConstraints{
- VCPUs: 2,
- RAM: 987654321,
- KeepCacheRAM: 123456789,
- CUDADeviceCount: 2,
- CUDAHardwareCapability: "9.0",
- CUDADriverVersion: "11.0",
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "9.0",
+ DriverVersion: "10.0",
+ },
+ SelectedInstance: "low_driver",
},
- })
- c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "costly")
- c.Check(best.RAM >= 1234567890, check.Equals, true)
- c.Check(best.VCPUs >= 2, check.Equals, true)
- c.Check(best.CUDA.DeviceCount >= 2, check.Equals, true)
- c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
- c.Check(best.CUDA.HardwareCapability, check.Equals, "9.0")
-
- best, err = ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
- Mounts: map[string]arvados.Mount{
- "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "",
+ DriverVersion: "10.0",
+ },
+ SelectedInstance: "cheap_gpu",
},
- RuntimeConstraints: arvados.RuntimeConstraints{
- VCPUs: 2,
- RAM: 987654321,
- KeepCacheRAM: 123456789,
- CUDADeviceCount: 1,
- CUDAHardwareCapability: "8.0",
- CUDADriverVersion: "11.0",
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "8.0",
+ DriverVersion: "",
+ },
+ SelectedInstance: "cheap_gpu",
},
- })
- c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "low_capability")
- c.Check(best.RAM >= 1234567890, check.Equals, true)
- c.Check(best.VCPUs >= 2, check.Equals, true)
- c.Check(best.CUDA.DeviceCount >= 1, check.Equals, true)
- c.Check(best.CUDA.DriverVersion, check.Equals, "11.0")
- c.Check(best.CUDA.HardwareCapability, check.Equals, "8.0")
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 1,
+ HardwareCapability: "",
+ DriverVersion: "",
+ },
+ SelectedInstance: "cheap_gpu",
+ },
+ GPUTestCase{
+ CUDA: arvados.CUDARuntimeConstraints{
+ DeviceCount: 0,
+ HardwareCapability: "9.0",
+ DriverVersion: "11.0",
+ },
+ SelectedInstance: "non_gpu",
+ },
+ }
+ for _, tc := range cases {
+ best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
+ Mounts: map[string]arvados.Mount{
+ "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 2,
+ RAM: 987654321,
+ KeepCacheRAM: 123456789,
+ CUDA: tc.CUDA,
+ },
+ })
+ c.Check(err, check.IsNil)
+ c.Check(best.Name, check.Equals, tc.SelectedInstance)
+ }
}
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 27afc1a3a..d0b127333 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -93,16 +93,20 @@ type Mount struct {
GitURL string `json:"git_url"` // only if kind=="git_tree"
}
+type CUDARuntimeConstraints struct {
+ DriverVersion string `json:"driver_version,omitempty"`
+ HardwareCapability string `json:"hardware_capability,omitempty"`
+ DeviceCount int `json:"device_count,omitempty"`
+}
+
// RuntimeConstraints specify a container's compute resources (RAM,
// CPU) and network connectivity.
type RuntimeConstraints struct {
- API bool `json:"API"`
- RAM int64 `json:"ram"`
- VCPUs int `json:"vcpus"`
- KeepCacheRAM int64 `json:"keep_cache_ram"`
- CUDADriverVersion string `json:"cuda_driver_version,omitempty"`
- CUDAHardwareCapability string `json:"cuda_hardware_capability,omitempty"`
- CUDADeviceCount int `json:"cuda_device_count,omitempty"`
+ API bool `json:"API"`
+ RAM int64 `json:"ram"`
+ VCPUs int `json:"vcpus"`
+ KeepCacheRAM int64 `json:"keep_cache_ram"`
+ CUDA CUDARuntimeConstraints `json:"cuda,omitempty"`
}
// SchedulingParameters specify a container's scheduling parameters
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list