[ARVADOS] updated: 1.1.4-489-g43fce79bb

Git user git at public.curoverse.com
Wed Jun 20 15:37:55 EDT 2018


Summary of changes:
 lib/dispatchcloud/node_size.go | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

  discards  ac9841cbdd577987c93db32c6ab7d23356796f83 (commit)
       via  43fce79bb4793c2d184d6ff8f9a6cc86d7e3e8b9 (commit)
       via  bab20d5535f413f63b6ceefdfa21dd54b941da4b (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (ac9841cbdd577987c93db32c6ab7d23356796f83)
            \
             N -- N -- N (43fce79bb4793c2d184d6ff8f9a6cc86d7e3e8b9)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 43fce79bb4793c2d184d6ff8f9a6cc86d7e3e8b9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Jun 20 15:30:02 2018 -0400

    13611: Don't build heavy error unless needed.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index f148c2c04..e59c550ed 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -47,20 +47,10 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
 	needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
 	needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
 
-	availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
-	for _, t := range cc.InstanceTypes {
-		availableTypes = append(availableTypes, t)
-	}
-	sort.Slice(availableTypes, func(a, b int) bool {
-		return availableTypes[a].Price < availableTypes[b].Price
-	})
-	err = ConstraintsNotSatisfiableError{
-		errors.New("constraints not satisfiable by any configured instance type"),
-		availableTypes,
-	}
+	ok := false
 	for _, it := range cc.InstanceTypes {
 		switch {
-		case err == nil && it.Price > best.Price:
+		case ok && it.Price > best.Price:
 		case it.Scratch < needScratch:
 		case it.RAM < needRAM:
 		case it.VCPUs < needVCPUs:
@@ -70,8 +60,22 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
 		default:
 			// Lower price || (same price && better specs)
 			best = it
-			err = nil
+			ok = true
+		}
+	}
+	if !ok {
+		availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
+		for _, t := range cc.InstanceTypes {
+			availableTypes = append(availableTypes, t)
+		}
+		sort.Slice(availableTypes, func(a, b int) bool {
+			return availableTypes[a].Price < availableTypes[b].Price
+		})
+		err = ConstraintsNotSatisfiableError{
+			errors.New("constraints not satisfiable by any configured instance type"),
+			availableTypes,
 		}
+		return
 	}
 	return
 }

commit bab20d5535f413f63b6ceefdfa21dd54b941da4b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Jun 20 15:25:29 2018 -0400

    13611: Change InstanceTypes config from array to hash.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index b5fd0262a..f148c2c04 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -47,8 +47,10 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
 	needRAM := ctr.RuntimeConstraints.RAM + ctr.RuntimeConstraints.KeepCacheRAM
 	needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
 
-	availableTypes := make([]arvados.InstanceType, len(cc.InstanceTypes))
-	copy(availableTypes, cc.InstanceTypes)
+	availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
+	for _, t := range cc.InstanceTypes {
+		availableTypes = append(availableTypes, t)
+	}
 	sort.Slice(availableTypes, func(a, b int) bool {
 		return availableTypes[a].Price < availableTypes[b].Price
 	})
diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index d6b7c6bf9..0e704cc41 100644
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -27,10 +27,10 @@ func (*NodeSizeSuite) TestChooseNotConfigured(c *check.C) {
 
 func (*NodeSizeSuite) TestChooseUnsatisfiable(c *check.C) {
 	checkUnsatisfiable := func(ctr *arvados.Container) {
-		_, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: []arvados.InstanceType{
-			{Price: 1.1, RAM: 1000000000, VCPUs: 2, Name: "small1"},
-			{Price: 2.2, RAM: 2000000000, VCPUs: 4, Name: "small2"},
-			{Price: 4.4, RAM: 4000000000, VCPUs: 8, Name: "small4", Scratch: GiB},
+		_, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: map[string]arvados.InstanceType{
+			"small1": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Name: "small1"},
+			"small2": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Name: "small2"},
+			"small4": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Name: "small4", Scratch: GiB},
 		}}, ctr)
 		c.Check(err, check.FitsTypeOf, ConstraintsNotSatisfiableError{})
 	}
@@ -49,29 +49,29 @@ func (*NodeSizeSuite) TestChooseUnsatisfiable(c *check.C) {
 }
 
 func (*NodeSizeSuite) TestChoose(c *check.C) {
-	for _, menu := range [][]arvados.InstanceType{
+	for _, menu := range []map[string]arvados.InstanceType{
 		{
-			{Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
-			{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
-			{Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+			"costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+			"best":   {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+			"small":  {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
 		},
 		{
-			{Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
-			{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
-			{Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
-			{Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+			"costly":     {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+			"goodenough": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
+			"best":       {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+			"small":      {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
 		},
 		{
-			{Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
-			{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
-			{Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
-			{Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+			"small":      {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Name: "small"},
+			"goodenough": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "goodenough"},
+			"best":       {Price: 2.2, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+			"costly":     {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
 		},
 		{
-			{Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: GiB, Name: "small"},
-			{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: GiB, Name: "nearly"},
-			{Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
-			{Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
+			"small":  {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: GiB, Name: "small"},
+			"nearly": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: GiB, Name: "nearly"},
+			"best":   {Price: 3.3, RAM: 4000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+			"costly": {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Name: "costly"},
 		},
 	} {
 		best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
@@ -93,11 +93,11 @@ func (*NodeSizeSuite) TestChoose(c *check.C) {
 }
 
 func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
-	menu := []arvados.InstanceType{
-		{Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Preemptable: true, Name: "costly"},
-		{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "almost best"},
-		{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Preemptable: true, Name: "best"},
-		{Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Preemptable: true, Name: "small"},
+	menu := map[string]arvados.InstanceType{
+		"costly":      {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Preemptable: true, Name: "costly"},
+		"almost best": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "almost best"},
+		"best":        {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Preemptable: true, Name: "best"},
+		"small":       {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Preemptable: true, Name: "small"},
 	}
 	best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
 		Mounts: map[string]arvados.Mount{
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 841f95281..ed067d45b 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -5,6 +5,8 @@
 package arvados
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 
@@ -52,7 +54,7 @@ type Cluster struct {
 	ClusterID          string `json:"-"`
 	ManagementToken    string
 	NodeProfiles       map[string]NodeProfile
-	InstanceTypes      []InstanceType
+	InstanceTypes      InstanceTypeMap
 	HTTPRequestTimeout Duration
 }
 
@@ -66,6 +68,46 @@ type InstanceType struct {
 	Preemptable  bool
 }
 
+type InstanceTypeMap map[string]InstanceType
+
+var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
+
+// UnmarshalJSON handles old config files that provide an array of
+// instance types instead of a hash.
+func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
+	if len(data) > 0 && data[0] == '[' {
+		var arr []InstanceType
+		err := json.Unmarshal(data, &arr)
+		if err != nil {
+			return err
+		}
+		if len(arr) == 0 {
+			*it = nil
+			return nil
+		}
+		*it = make(map[string]InstanceType, len(arr))
+		for _, t := range arr {
+			if _, ok := (*it)[t.Name]; ok {
+				return errDuplicateInstanceTypeName
+			}
+			(*it)[t.Name] = t
+		}
+		return nil
+	}
+	var hash map[string]InstanceType
+	err := json.Unmarshal(data, &hash)
+	if err != nil {
+		return err
+	}
+	// Fill in Name field using hash key.
+	*it = InstanceTypeMap(hash)
+	for name, t := range *it {
+		t.Name = name
+		(*it)[name] = t
+	}
+	return nil
+}
+
 // GetNodeProfile returns a NodeProfile for the given hostname. An
 // error is returned if the appropriate configuration can't be
 // determined (e.g., this does not appear to be a system node). If
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index b4033e78b..23a8a0ca0 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -367,17 +367,17 @@ func (s *StubbedSuite) TestSbatchInstanceTypeConstraint(c *C) {
 	}
 
 	for _, trial := range []struct {
-		types      []arvados.InstanceType
+		types      map[string]arvados.InstanceType
 		sbatchArgs []string
 		err        error
 	}{
 		// Choose node type => use --constraint arg
 		{
-			types: []arvados.InstanceType{
-				{Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
-				{Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
-				{Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
-				{Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
+			types: map[string]arvados.InstanceType{
+				"a1.tiny":   {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+				"a1.small":  {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
+				"a1.medium": {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
+				"a1.large":  {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
 			},
 			sbatchArgs: []string{"--constraint=instancetype=a1.medium"},
 		},
@@ -388,8 +388,8 @@ func (s *StubbedSuite) TestSbatchInstanceTypeConstraint(c *C) {
 		},
 		// No node type is big enough => error
 		{
-			types: []arvados.InstanceType{
-				{Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+			types: map[string]arvados.InstanceType{
+				"a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
 			},
 			err: dispatchcloud.ConstraintsNotSatisfiableError{},
 		},

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list