[ARVADOS] updated: 1.2.0-14-g70e5c7a3c
Git user
git at public.curoverse.com
Thu Aug 16 17:42:03 EDT 2018
Summary of changes:
sdk/go/dispatch/dispatch.go | 23 +++++++++++++++++++++-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 4 ++++
.../crunch-dispatch-slurm_test.go | 2 +-
services/crunch-dispatch-slurm/usage.go | 1 +
4 files changed, 28 insertions(+), 2 deletions(-)
via 70e5c7a3c6a5860d702d5e5c219dc0f3a3696d35 (commit)
via a96cca47cc60c482316d3b1a25cfe1cb4b838f41 (commit)
via fd4d26e448b1f9f45f84c0ff9ec10db54471d880 (commit)
via 420a8e7fb7b159452da834062cc3e040dd1b411b (commit)
from 34d989929d8ecf3620640b5b7a5e89b89da70a89 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 70e5c7a3c6a5860d702d5e5c219dc0f3a3696d35
Merge: 34d989929 a96cca47c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Aug 16 17:25:11 2018 -0400
Merge branch '13933-dispatch-batch-size'
closes #13933
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit a96cca47cc60c482316d3b1a25cfe1cb4b838f41
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Aug 16 17:23:59 2018 -0400
13933: Update error text check.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 719ec98d2..c0f2945a0 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -246,7 +246,7 @@ func (s *StubbedSuite) TestAPIErrorGettingContainers(c *C) {
apiStubResponses["/arvados/v1/api_client_authorizations/current"] = arvadostest.StubResponse{200, `{"uuid":"` + arvadostest.Dispatch1AuthUUID + `"}`}
apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)}
- s.testWithServerStub(c, apiStubResponses, "echo", "Error getting list of containers")
+ s.testWithServerStub(c, apiStubResponses, "echo", "error getting count of containers")
}
func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) {
commit fd4d26e448b1f9f45f84c0ff9ec10db54471d880
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Aug 16 17:23:50 2018 -0400
13933: Fix formatting.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/dispatch/dispatch.go b/sdk/go/dispatch/dispatch.go
index e9fb6b6d9..ba2cf289c 100644
--- a/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@ -169,19 +169,19 @@ func (d *Dispatcher) checkForUpdates(filters [][]interface{}, todo map[string]*r
var countList arvados.ContainerList
params := arvadosclient.Dict{
"filters": filters,
- "count": "exact",
- "limit": 0,
+ "count": "exact",
+ "limit": 0,
"order": []string{"priority desc"}}
err := d.Arv.List("containers", params, &countList)
if err != nil {
- log.Printf("Error getting count of containers: %q", err)
+ log.Printf("error getting count of containers: %q", err)
return false
}
- itemsAvailable := countList.ItemsAvailable
+ itemsAvailable := countList.ItemsAvailable
params = arvadosclient.Dict{
"filters": filters,
- "count": "none",
- "limit": d.BatchSize,
+ "count": "none",
+ "limit": d.BatchSize,
"order": []string{"priority desc"}}
offset := 0
for {
diff --git a/services/crunch-dispatch-slurm/usage.go b/services/crunch-dispatch-slurm/usage.go
index db2395cdf..bcfa5b8a3 100644
--- a/services/crunch-dispatch-slurm/usage.go
+++ b/services/crunch-dispatch-slurm/usage.go
@@ -22,7 +22,7 @@ var exampleConfigFile = []byte(`
"PollPeriod": "10s",
"SbatchArguments": ["--partition=foo", "--exclude=node13"],
"ReserveExtraRAM": 268435456,
- "BatchSize": 10000,
+ "BatchSize": 10000
}`)
func usage(fs *flag.FlagSet) {
commit 420a8e7fb7b159452da834062cc3e040dd1b411b
Author: Joshua C. Randall <jcrandall at alum.mit.edu>
Date: Mon Aug 13 12:01:10 2018 +0000
Make dispatcher Run loop use count=none and BatchSize limit
Arvados-DCO-1.1-Signed-off-by: Joshua C. Randall <jcrandall at alum.mit.edu>
diff --git a/sdk/go/dispatch/dispatch.go b/sdk/go/dispatch/dispatch.go
index c3d603099..e9fb6b6d9 100644
--- a/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@ -29,6 +29,9 @@ const (
type Dispatcher struct {
Arv *arvadosclient.ArvadosClient
+ // Batch size for container queries
+ BatchSize int64
+
// Queue polling frequency
PollPeriod time.Duration
@@ -72,6 +75,10 @@ func (d *Dispatcher) Run(ctx context.Context) error {
poll := time.NewTicker(d.PollPeriod)
defer poll.Stop()
+ if d.BatchSize == 0 {
+ d.BatchSize = 100
+ }
+
for {
select {
case <-poll.C:
@@ -159,8 +166,22 @@ func (d *Dispatcher) start(c arvados.Container) *runTracker {
}
func (d *Dispatcher) checkForUpdates(filters [][]interface{}, todo map[string]*runTracker) bool {
+ var countList arvados.ContainerList
params := arvadosclient.Dict{
"filters": filters,
+ "count": "exact",
+ "limit": 0,
+ "order": []string{"priority desc"}}
+ err := d.Arv.List("containers", params, &countList)
+ if err != nil {
+ log.Printf("Error getting count of containers: %q", err)
+ return false
+ }
+ itemsAvailable := countList.ItemsAvailable
+ params = arvadosclient.Dict{
+ "filters": filters,
+ "count": "none",
+ "limit": d.BatchSize,
"order": []string{"priority desc"}}
offset := 0
for {
@@ -173,7 +194,7 @@ func (d *Dispatcher) checkForUpdates(filters [][]interface{}, todo map[string]*r
}
d.checkListForUpdates(list.Items, todo)
offset += len(list.Items)
- if len(list.Items) == 0 || list.ItemsAvailable <= offset {
+ if len(list.Items) == 0 || itemsAvailable <= offset {
return true
}
}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index d1f19dd7b..c1009a5d8 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -57,6 +57,9 @@ type Dispatcher struct {
// Minimum time between two attempts to run the same container
MinRetryPeriod arvados.Duration
+
+ // Batch size for container queries
+ BatchSize int64
}
func main() {
@@ -164,6 +167,7 @@ func (disp *Dispatcher) setup() {
}
disp.Dispatcher = &dispatch.Dispatcher{
Arv: arv,
+ BatchSize: disp.BatchSize,
RunContainer: disp.runContainer,
PollPeriod: time.Duration(disp.PollPeriod),
MinRetryPeriod: time.Duration(disp.MinRetryPeriod),
diff --git a/services/crunch-dispatch-slurm/usage.go b/services/crunch-dispatch-slurm/usage.go
index 032d86284..db2395cdf 100644
--- a/services/crunch-dispatch-slurm/usage.go
+++ b/services/crunch-dispatch-slurm/usage.go
@@ -22,6 +22,7 @@ var exampleConfigFile = []byte(`
"PollPeriod": "10s",
"SbatchArguments": ["--partition=foo", "--exclude=node13"],
"ReserveExtraRAM": 268435456,
+ "BatchSize": 10000,
}`)
func usage(fs *flag.FlagSet) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list