[ARVADOS] created: 1.3.0-8-g934d880aa

[Git user]

        at  934d880aa5d10ed3382f9924a9a9f5694b41f266 (commit)


commit 934d880aa5d10ed3382f9924a9a9f5694b41f266
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Dec 10 11:12:28 2018 -0500

    14495: Add dispatchcloud.EstimateScratchSpace()
    
    Estimate size of Docker image based on collection size, and
    incorporate that into disk space requirement.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index 1c36d6cf5..7706e411b 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -8,7 +8,9 @@ import (
 	"errors"
 	"log"
 	"os/exec"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -27,6 +29,65 @@ type ConstraintsNotSatisfiableError struct {
 	AvailableTypes []arvados.InstanceType
 }
 
+var pdhRegexp = regexp.MustCompile(`^[0-9a-f]{32}\+(\d+)$`)
+
+// estimateDockerImageSize estimates how much disk space will be used
+// by a Docker image, given the PDH of a collection containing a
+// Docker image that was created by "arv-keepdocker".  Returns
+// estimated number of bytes of disk space that should be reserved.
+func estimateDockerImageSize(collectionPDH string) int64 {
+	m := pdhRegexp.FindStringSubmatch(collectionPDH)
+	if m == nil {
+		return 0
+	}
+	n, err := strconv.ParseInt(m[1], 10, 64)
+	if err != nil || n < 122 {
+		return 0
+	}
+	// To avoid having to fetch the collection, take advantage of
+	// the fact that the manifest storing a container image
+	// uploaded by arv-keepdocker has a predictable format, which
+	// allows us to estimate the size of the image based on just
+	// the size of the manifest.
+	//
+	// Use the following heuristic:
+	// - Start with the length of the mainfest (n)
+	// - Subtract 80 characters for the filename and file segment
+	// - Divide by 42 to get the number of block identifiers ('hash\+size\ ' is 32+1+8+1)
+	// - Assume each block is full, multiply by 64 MiB
+	return ((n - 80) / 42) * (64 * 1024 * 1024)
+}
+
+// EstimateScratchSpace estimates how much available disk space (in
+// bytes) is needed to run the container by summing the capacity
+// requested by 'tmp' mounts plus disk space required to load the
+// Docker image.
+func EstimateScratchSpace(ctr *arvados.Container) (needScratch int64) {
+	for _, m := range ctr.Mounts {
+		if m.Kind == "tmp" {
+			needScratch += m.Capacity
+		}
+	}
+
+	// Account for disk space usage by Docker, assumes the following behavior:
+	// - Layer tarballs are buffered to disk during "docker load".
+	// - Individual layer tarballs are extracted from buffered
+	// copy to the filesystem
+	dockerImageSize := estimateDockerImageSize(ctr.ContainerImage)
+
+	// The buffer is only needed during image load, so make sure
+	// the baseline scratch space at least covers dockerImageSize,
+	// and assume it will be released to the job afterwards.
+	if needScratch < dockerImageSize {
+		needScratch = dockerImageSize
+	}
+
+	// Now reserve space for the extracted image on disk.
+	needScratch += dockerImageSize
+
+	return
+}
+
 // ChooseInstanceType returns the cheapest available
 // arvados.InstanceType big enough to run ctr.
 func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
@@ -35,12 +96,7 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
 		return
 	}
 
-	needScratch := int64(0)
-	for _, m := range ctr.Mounts {
-		if m.Kind == "tmp" {
-			needScratch += m.Capacity
-		}
-	}
+	needScratch := EstimateScratchSpace(ctr)
 
 	needVCPUs := ctr.RuntimeConstraints.VCPUs
 
diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index 91c6bb104..e94853816 100644
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -119,3 +119,13 @@ func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
 	c.Check(best.Scratch >= 2*GiB, check.Equals, true)
 	c.Check(best.Preemptible, check.Equals, true)
 }
+
+func (*NodeSizeSuite) TestScratchForDockerImage(c *check.C) {
+	n := EstimateScratchSpace(&arvados.Container{
+		ContainerImage: "d5025c0f29f6eef304a7358afa82a822+342",
+	})
+	// Actual image is 371.1 MiB (according to workbench)
+	// Estimated size is 384 MiB (402653184 bytes)
+	// Want to reserve 2x the estimated size, so 805306368 bytes
+	c.Check(n, check.Equals, int64(805306368))
+}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 084700d39..c4d535c99 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -229,12 +229,7 @@ func (disp *Dispatcher) checkSqueueForOrphans() {
 func (disp *Dispatcher) slurmConstraintArgs(container arvados.Container) []string {
 	mem := int64(math.Ceil(float64(container.RuntimeConstraints.RAM+container.RuntimeConstraints.KeepCacheRAM+disp.ReserveExtraRAM) / float64(1048576)))
 
-	var disk int64
-	for _, m := range container.Mounts {
-		if m.Kind == "tmp" {
-			disk += m.Capacity
-		}
-	}
+	disk := dispatchcloud.EstimateScratchSpace(&container)
 	disk = int64(math.Ceil(float64(disk) / float64(1048576)))
 	return []string{
 		fmt.Sprintf("--mem=%d", mem),

-----------------------------------------------------------------------


hooks/post-receive
--