[arvados] updated: 2.6.0-196-g84c80680c

[git repository hosting]

Summary of changes:
 lib/dispatchcloud/scheduler/run_queue.go      |  8 +--
 lib/dispatchcloud/scheduler/run_queue_test.go | 96 +++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 5 deletions(-)

       via  84c80680c3ac6b0b1cd44de40a05f77eb82ca1c9 (commit)
      from  f3b5289610db5f0b54c73ff5ec0cd80e51f1ae94 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 84c80680c3ac6b0b1cd44de40a05f77eb82ca1c9
Author: Tom Clegg <tom at curii.com>
Date:   Thu May 25 13:43:34 2023 -0400

    20511: Don't shutdown/unlock based on dynamic maxConcurrency.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 8f4c2e083..db6e97b59 100644
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -120,8 +120,7 @@ tryrun:
 		case arvados.ContainerStateQueued:
 			if sch.maxConcurrency > 0 && trying >= sch.maxConcurrency {
 				logger.Tracef("not locking: already at maxConcurrency %d", sch.maxConcurrency)
-				overquota = sorted[i:]
-				break tryrun
+				continue
 			}
 			trying++
 			if unalloc[it] < 1 && sch.pool.AtQuota() {
@@ -137,9 +136,8 @@ tryrun:
 			unalloc[it]--
 		case arvados.ContainerStateLocked:
 			if sch.maxConcurrency > 0 && trying >= sch.maxConcurrency {
-				logger.Debugf("not starting: already at maxConcurrency %d", sch.maxConcurrency)
-				overquota = sorted[i:]
-				break tryrun
+				logger.Tracef("not starting: already at maxConcurrency %d", sch.maxConcurrency)
+				continue
 			}
 			trying++
 			if unalloc[it] > 0 {
diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go
index 3278c7de6..2deff69e9 100644
--- a/lib/dispatchcloud/scheduler/run_queue_test.go
+++ b/lib/dispatchcloud/scheduler/run_queue_test.go
@@ -278,6 +278,102 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) {
 	}
 }
 
+// Don't unlock containers or shutdown unalloc (booting/idle) nodes
+// just because some 503 errors caused us to reduce maxConcurrency
+// below the current load level.
+//
+// We expect to raise maxConcurrency soon when we stop seeing 503s. If
+// that doesn't happen soon, the idle timeout will take care of the
+// excess nodes.
+func (*SchedulerSuite) TestIdleIn503QuietPeriod(c *check.C) {
+	ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c))
+	queue := test.Queue{
+		ChooseType: chooseType,
+		Containers: []arvados.Container{
+			// scheduled on an instance (but not Running yet)
+			{
+				UUID:     test.ContainerUUID(1),
+				Priority: 1000,
+				State:    arvados.ContainerStateLocked,
+				RuntimeConstraints: arvados.RuntimeConstraints{
+					VCPUs: 2,
+					RAM:   2 << 30,
+				},
+			},
+			// not yet scheduled
+			{
+				UUID:     test.ContainerUUID(2),
+				Priority: 1000,
+				State:    arvados.ContainerStateLocked,
+				RuntimeConstraints: arvados.RuntimeConstraints{
+					VCPUs: 2,
+					RAM:   2 << 30,
+				},
+			},
+			// scheduled on an instance (but not Running yet)
+			{
+				UUID:     test.ContainerUUID(3),
+				Priority: 1000,
+				State:    arvados.ContainerStateLocked,
+				RuntimeConstraints: arvados.RuntimeConstraints{
+					VCPUs: 3,
+					RAM:   3 << 30,
+				},
+			},
+			// not yet scheduled
+			{
+				UUID:     test.ContainerUUID(4),
+				Priority: 1000,
+				State:    arvados.ContainerStateLocked,
+				RuntimeConstraints: arvados.RuntimeConstraints{
+					VCPUs: 3,
+					RAM:   3 << 30,
+				},
+			},
+			// not yet locked
+			{
+				UUID:     test.ContainerUUID(5),
+				Priority: 1000,
+				State:    arvados.ContainerStateQueued,
+				RuntimeConstraints: arvados.RuntimeConstraints{
+					VCPUs: 3,
+					RAM:   3 << 30,
+				},
+			},
+		},
+	}
+	queue.Update()
+	pool := stubPool{
+		quota: 16,
+		unalloc: map[arvados.InstanceType]int{
+			test.InstanceType(2): 2,
+			test.InstanceType(3): 2,
+		},
+		idle: map[arvados.InstanceType]int{
+			test.InstanceType(2): 1,
+			test.InstanceType(3): 1,
+		},
+		running: map[string]time.Time{
+			test.ContainerUUID(1): {},
+			test.ContainerUUID(3): {},
+		},
+		creates:   []arvados.InstanceType{},
+		starts:    []string{},
+		canCreate: 0,
+	}
+	sch := New(ctx, arvados.NewClientFromEnv(), &queue, &pool, nil, time.Millisecond, time.Millisecond, 0)
+	sch.last503time = time.Now()
+	sch.maxConcurrency = 3
+	sch.sync()
+	sch.runQueue()
+	sch.sync()
+
+	c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(2)})
+	c.Check(pool.shutdowns, check.Equals, 0)
+	c.Check(pool.creates, check.HasLen, 0)
+	c.Check(queue.StateChanges(), check.HasLen, 0)
+}
+
 // Don't flap lock/unlock when equal-priority containers compete for
 // limited workers.
 //

-----------------------------------------------------------------------


hooks/post-receive
--