[ARVADOS] created: 2.1.0-915-g08c8b9cc4

Git user git at public.arvados.org
Tue Jun 8 15:29:18 UTC 2021


        at  08c8b9cc496627bc3fd3d87ae333fadce4797eaa (commit)


commit 08c8b9cc496627bc3fd3d87ae333fadce4797eaa
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 8 02:08:43 2021 -0400

    17609: Tweak messages.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 1c7b98baf..b0241b3ae 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -72,15 +72,15 @@ type diagnoser struct {
 }
 
 func (diag *diagnoser) debugf(f string, args ...interface{}) {
-	diag.logger.Debugf(f, args...)
+	diag.logger.Debugf("  ... "+f, args...)
 }
 
 func (diag *diagnoser) infof(f string, args ...interface{}) {
-	diag.logger.Infof(f, args...)
+	diag.logger.Infof("  ... "+f, args...)
 }
 
 func (diag *diagnoser) warnf(f string, args ...interface{}) {
-	diag.logger.Warnf(f, args...)
+	diag.logger.Warnf("  ... "+f, args...)
 }
 
 func (diag *diagnoser) errorf(f string, args ...interface{}) {
@@ -101,14 +101,14 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
 	}
 	diag.done[id] = true
 
-	diag.infof("%4d: %s", id, title)
+	diag.logger.Infof("%4d: %s", id, title)
 	t0 := time.Now()
 	err := fn()
-	elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
+	elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
 	if err != nil {
 		diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
 	} else {
-		diag.debugf("%4d: %s (%s): ok", id, title, elapsed)
+		diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
 	}
 }
 
@@ -533,7 +533,7 @@ func (diag *diagnoser) runtests() {
 
 	diag.dotest(160, "running a container", func() error {
 		if diag.priority < 1 {
-			diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
+			diag.infof("skipping (use priority > 0 if you want to run a container)")
 			return nil
 		}
 		if project.UUID == "" {

commit 3975b33706d85857e8d7a8ec2677edbb35984a14
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 8 01:53:12 2021 -0400

    17609: Behave better when project creation fails.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 2f43263e5..1c7b98baf 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -31,7 +31,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
 	f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
 	f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
 	f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
-	f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000)")
+	f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
 	f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
 	err := f.Parse(args)
 	if err == flag.ErrHelp {
@@ -334,13 +334,17 @@ func (diag *diagnoser) runtests() {
 
 	var collection arvados.Collection
 	diag.dotest(90, "creating temporary collection", func() error {
+		if project.UUID == "" {
+			return fmt.Errorf("skipping, no project to work in")
+		}
 		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 		defer cancel()
 		err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 			"ensure_unique_name": true,
 			"collection": map[string]interface{}{
-				"name":     "test collection",
-				"trash_at": time.Now().Add(time.Hour)}})
+				"owner_uuid": project.UUID,
+				"name":       "test collection",
+				"trash_at":   time.Now().Add(time.Hour)}})
 		if err != nil {
 			return err
 		}
@@ -532,6 +536,9 @@ func (diag *diagnoser) runtests() {
 			diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
 			return nil
 		}
+		if project.UUID == "" {
+			return fmt.Errorf("skipping, no project to work in")
+		}
 
 		var cr arvados.ContainerRequest
 		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))

commit 93cf55af0be9ef96c79d17323d82ba85eb1b9968
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 8 01:45:55 2021 -0400

    17609: Improve output formatting.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/go.mod b/go.mod
index aa289761b..9062c667d 100644
--- a/go.mod
+++ b/go.mod
@@ -55,13 +55,13 @@ require (
 	github.com/prometheus/common v0.7.0
 	github.com/satori/go.uuid v1.2.1-0.20180103174451-36e9d2ebbde5 // indirect
 	github.com/sergi/go-diff v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.4.2
+	github.com/sirupsen/logrus v1.8.1
 	github.com/src-d/gcfg v1.3.0 // indirect
 	github.com/xanzy/ssh-agent v0.1.0 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/net v0.0.0-20201021035429-f5854403a974
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/sys v0.0.0-20210603125802-9665404d3644
 	golang.org/x/tools v0.1.0 // indirect
 	google.golang.org/api v0.13.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
diff --git a/go.sum b/go.sum
index 006118ad9..23731f69a 100644
--- a/go.sum
+++ b/go.sum
@@ -234,6 +234,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
 github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg=
@@ -301,9 +303,12 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 4f3ad8b0a..2f43263e5 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -41,7 +41,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
 		return 2
 	}
 	diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
-	diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+	diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
 	diag.runtests()
 	if len(diag.errors) == 0 {
 		diag.logger.Info("--- no errors ---")
@@ -101,14 +101,15 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
 	}
 	diag.done[id] = true
 
-	diag.infof("%4d %s", id, title)
+	diag.infof("%4d: %s", id, title)
 	t0 := time.Now()
 	err := fn()
 	elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
 	if err != nil {
-		diag.errorf("%s (%s): %s", title, elapsed, err)
+		diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
+	} else {
+		diag.debugf("%4d: %s (%s): ok", id, title, elapsed)
 	}
-	diag.debugf("%4d %s (%s): ok", id, title, elapsed)
 }
 
 func (diag *diagnoser) runtests() {

commit 56d721569988522e5cca008a303b19d008b937dd
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 8 01:45:40 2021 -0400

    17609: Add "run container" test.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 54c78d07c..4f3ad8b0a 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -31,6 +31,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
 	f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
 	f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
 	f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+	f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000)")
 	f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
 	err := f.Parse(args)
 	if err == flag.ErrHelp {
@@ -60,6 +61,7 @@ type diagnoser struct {
 	stdout        io.Writer
 	stderr        io.Writer
 	logLevel      string
+	priority      int
 	projectName   string
 	checkInternal bool
 	checkExternal bool
@@ -523,4 +525,80 @@ func (diag *diagnoser) runtests() {
 		}
 		return nil
 	})
+
+	diag.dotest(160, "running a container", func() error {
+		if diag.priority < 1 {
+			diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
+			return nil
+		}
+
+		var cr arvados.ContainerRequest
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+
+		timestamp := time.Now().Format(time.RFC3339)
+		err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+			"owner_uuid":      project.UUID,
+			"name":            fmt.Sprintf("diagnostics container request %s", timestamp),
+			"container_image": "arvados/jobs",
+			"command":         []string{"echo", timestamp},
+			"use_existing":    false,
+			"output_path":     "/mnt/output",
+			"output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
+			"priority":        diag.priority,
+			"state":           arvados.ContainerRequestStateCommitted,
+			"mounts": map[string]map[string]interface{}{
+				"/mnt/output": {
+					"kind":     "collection",
+					"writable": true,
+				},
+			},
+			"runtime_constraints": arvados.RuntimeConstraints{
+				VCPUs:        1,
+				RAM:          1 << 26,
+				KeepCacheRAM: 1 << 26,
+			},
+		}})
+		if err != nil {
+			return err
+		}
+		diag.debugf("container request uuid = %s", cr.UUID)
+		diag.debugf("container uuid = %s", cr.ContainerUUID)
+
+		timeout := 10 * time.Minute
+		diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
+		ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
+		defer cancel()
+
+		var c arvados.Container
+		for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
+			ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
+			defer cancel()
+
+			crStateWas := cr.State
+			err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
+			if err != nil {
+				return err
+			}
+			if cr.State != crStateWas {
+				diag.debugf("container request state = %s", cr.State)
+			}
+
+			cStateWas := c.State
+			err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
+			if err != nil {
+				return err
+			}
+			if c.State != cStateWas {
+				diag.debugf("container state = %s", c.State)
+			}
+		}
+
+		if c.State != arvados.ContainerStateComplete {
+			return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
+		} else if c.ExitCode != 0 {
+			return fmt.Errorf("container exited %d", c.ExitCode)
+		}
+		return nil
+	})
 }

commit dd7268b61d18f37036f40479c67b4b9f446eb2a0
Author: Tom Clegg <tom at curii.com>
Date:   Wed Jun 2 15:23:56 2021 -0400

    17609: Improve log text alignment.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 3b3ed400b..54c78d07c 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -99,14 +99,14 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
 	}
 	diag.done[id] = true
 
-	diag.infof("%d %s", id, title)
+	diag.infof("%4d %s", id, title)
 	t0 := time.Now()
 	err := fn()
 	elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
 	if err != nil {
 		diag.errorf("%s (%s): %s", title, elapsed, err)
 	}
-	diag.debugf("%d %s (%s): ok", id, title, elapsed)
+	diag.debugf("%4d %s (%s): ok", id, title, elapsed)
 }
 
 func (diag *diagnoser) runtests() {

commit 6c517a23f8af60ecbb87099be26f8df7a7652f4f
Author: Tom Clegg <tom at curii.com>
Date:   Wed Jun 2 15:17:13 2021 -0400

    17609: Use timeout on all http requests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 7021bb0ee..3b3ed400b 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -120,7 +120,9 @@ func (diag *diagnoser) runtests() {
 	var dd arvados.DiscoveryDocument
 	ddpath := "discovery/v1/apis/arvados/v1/rest"
 	diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
-		err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+		err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
 		if err != nil {
 			return err
 		}
@@ -131,7 +133,9 @@ func (diag *diagnoser) runtests() {
 	var cluster arvados.Cluster
 	cfgpath := "arvados/v1/config"
 	diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
-		err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+		err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
 		if err != nil {
 			return err
 		}
@@ -141,7 +145,9 @@ func (diag *diagnoser) runtests() {
 
 	var user arvados.User
 	diag.dotest(30, "getting current user record", func() error {
-		err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+		err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
 		if err != nil {
 			return err
 		}
@@ -163,6 +169,8 @@ func (diag *diagnoser) runtests() {
 		&cluster.Services.Workbench2,
 	} {
 		diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
+			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+			defer cancel()
 			u := svc.ExternalURL
 			if strings.HasPrefix(u.Scheme, "ws") {
 				// We can do a real websocket test elsewhere,
@@ -173,7 +181,7 @@ func (diag *diagnoser) runtests() {
 			if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
 				u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
 			}
-			req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 			if err != nil {
 				return err
 			}
@@ -192,7 +200,9 @@ func (diag *diagnoser) runtests() {
 		cluster.Services.WebDAVDownload.ExternalURL.String(),
 	} {
 		diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
-			req, err := http.NewRequest("GET", url, nil)
+			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+			defer cancel()
+			req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 			if err != nil {
 				return err
 			}
@@ -210,7 +220,9 @@ func (diag *diagnoser) runtests() {
 
 	var keeplist arvados.KeepServiceList
 	diag.dotest(60, "checking internal/external client detection", func() error {
-		err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+		err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 		if err != nil {
 			return fmt.Errorf("error getting keep services list: %s", err)
 		} else if len(keeplist.Items) == 0 {
@@ -288,8 +300,10 @@ func (diag *diagnoser) runtests() {
 
 	var project arvados.Group
 	diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
 		var grplist arvados.GroupList
-		err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+		err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
 			Filters: []arvados.Filter{
 				{"name", "=", diag.projectName},
 				{"group_class", "=", "project"},
@@ -304,7 +318,7 @@ func (diag *diagnoser) runtests() {
 			return nil
 		}
 		diag.debugf("list groups: ok, no results")
-		err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
+		err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 			"name":        diag.projectName,
 			"group_class": "project",
 		}})
@@ -317,7 +331,9 @@ func (diag *diagnoser) runtests() {
 
 	var collection arvados.Collection
 	diag.dotest(90, "creating temporary collection", func() error {
-		err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
+		err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 			"ensure_unique_name": true,
 			"collection": map[string]interface{}{
 				"name":     "test collection",
@@ -332,16 +348,20 @@ func (diag *diagnoser) runtests() {
 	if collection.UUID != "" {
 		defer func() {
 			diag.dotest(9990, "deleting temporary collection", func() error {
-				return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+				ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+				defer cancel()
+				return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
 			})
 		}()
 	}
 
 	diag.dotest(100, "uploading file via webdav", func() error {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
 		if collection.UUID == "" {
 			return fmt.Errorf("skipping, no test collection")
 		}
-		req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+		req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 		if err != nil {
 			return fmt.Errorf("BUG? http.NewRequest: %s", err)
 		}
@@ -355,7 +375,7 @@ func (diag *diagnoser) runtests() {
 			return fmt.Errorf("status %s", resp.Status)
 		}
 		diag.debugf("ok, status %s", resp.Status)
-		err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
+		err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 		if err != nil {
 			return fmt.Errorf("get updated collection: %s", err)
 		}
@@ -387,10 +407,12 @@ func (diag *diagnoser) runtests() {
 		{true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 	} {
 		diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+			defer cancel()
 			if trial.needcoll && collection.UUID == "" {
 				return fmt.Errorf("skipping, no test collection")
 			}
-			req, err := http.NewRequest("GET", trial.fileurl, nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
 			if err != nil {
 				return err
 			}
@@ -416,8 +438,10 @@ func (diag *diagnoser) runtests() {
 
 	var vm arvados.VirtualMachine
 	diag.dotest(130, "getting list of virtual machines", func() error {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
 		var vmlist arvados.VirtualMachineList
-		err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+		err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
 		if err != nil {
 			return err
 		}
@@ -429,12 +453,14 @@ func (diag *diagnoser) runtests() {
 	})
 
 	diag.dotest(140, "getting workbench1 webshell page", func() error {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+		defer cancel()
 		if vm.UUID == "" {
 			return fmt.Errorf("skipping, no vm available")
 		}
 		webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
 		diag.debugf("url %s", webshelltermurl)
-		req, err := http.NewRequest("GET", webshelltermurl, nil)
+		req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
 		if err != nil {
 			return err
 		}

commit 6560ade0a1fca34182814e0881b8b543216ad328
Author: Tom Clegg <tom at curii.com>
Date:   Wed Jun 2 15:12:58 2021 -0400

    17609: Fix limit args to work with older API.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 60860e257..7021bb0ee 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -210,7 +210,7 @@ func (diag *diagnoser) runtests() {
 
 	var keeplist arvados.KeepServiceList
 	diag.dotest(60, "checking internal/external client detection", func() error {
-		err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+		err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
 		if err != nil {
 			return fmt.Errorf("error getting keep services list: %s", err)
 		} else if len(keeplist.Items) == 0 {
@@ -294,7 +294,7 @@ func (diag *diagnoser) runtests() {
 				{"name", "=", diag.projectName},
 				{"group_class", "=", "project"},
 				{"owner_uuid", "=", user.UUID}},
-			Limit: -1})
+			Limit: 999999})
 		if err != nil {
 			return fmt.Errorf("list groups: %s", err)
 		}

commit a98105b60cb4bd19ed0723bc14b7a5e4514e7a83
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 1 16:23:08 2021 -0400

    17609: Refactor tests to add some structure.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 66b678a8e..60860e257 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
 	"strings"
@@ -21,17 +22,16 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type Command struct {
-	projectName string
-}
+type Command struct{}
 
-func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	var diag diagnoser
 	f := flag.NewFlagSet(prog, flag.ContinueOnError)
 	f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
-	loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
-	checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
-	checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
-	timeout := f.Duration("timeout", 10*time.Second, "timeout for http requests")
+	f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+	f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
+	f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+	f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
 	err := f.Parse(args)
 	if err == flag.ErrHelp {
 		return 0
@@ -39,72 +39,122 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 		fmt.Fprintln(stderr, err)
 		return 2
 	}
+	diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
+	diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+	diag.runtests()
+	if len(diag.errors) == 0 {
+		diag.logger.Info("--- no errors ---")
+		return 0
+	} else {
+		if diag.logger.Level > logrus.ErrorLevel {
+			fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
+			for _, e := range diag.errors {
+				diag.logger.Error(e)
+			}
+		}
+		return 1
+	}
+}
 
-	ctx := context.Background()
+type diagnoser struct {
+	stdout        io.Writer
+	stderr        io.Writer
+	logLevel      string
+	projectName   string
+	checkInternal bool
+	checkExternal bool
+	timeout       time.Duration
+	logger        *logrus.Logger
+	errors        []string
+	done          map[int]bool
+}
+
+func (diag *diagnoser) debugf(f string, args ...interface{}) {
+	diag.logger.Debugf(f, args...)
+}
+
+func (diag *diagnoser) infof(f string, args ...interface{}) {
+	diag.logger.Infof(f, args...)
+}
+
+func (diag *diagnoser) warnf(f string, args ...interface{}) {
+	diag.logger.Warnf(f, args...)
+}
 
-	logger := ctxlog.New(stdout, "text", *loglevel)
-	logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+func (diag *diagnoser) errorf(f string, args ...interface{}) {
+	diag.logger.Errorf(f, args...)
+	diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
+}
 
-	infof := logger.Infof
-	warnf := logger.Warnf
-	debugf := logger.Debugf
-	var errors []string
-	errorf := func(f string, args ...interface{}) {
-		logger.Errorf(f, args...)
-		errors = append(errors, fmt.Sprintf(f, args...))
+// Run the given func, logging appropriate messages before and after,
+// adding timing info, etc.
+//
+// The id argument should be unique among tests, and shouldn't change
+// when other tests are added/removed.
+func (diag *diagnoser) dotest(id int, title string, fn func() error) {
+	if diag.done == nil {
+		diag.done = map[int]bool{}
+	} else if diag.done[id] {
+		diag.errorf("(bug) reused test id %d", id)
 	}
-	defer func() {
-		if len(errors) == 0 {
-			logger.Info("--- no errors ---")
-		} else {
-			fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
-			for _, e := range errors {
-				logger.Error(e)
-			}
-		}
-	}()
+	diag.done[id] = true
+
+	diag.infof("%d %s", id, title)
+	t0 := time.Now()
+	err := fn()
+	elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
+	if err != nil {
+		diag.errorf("%s (%s): %s", title, elapsed, err)
+	}
+	diag.debugf("%d %s (%s): ok", id, title, elapsed)
+}
 
+func (diag *diagnoser) runtests() {
 	client := arvados.NewClientFromEnv()
 
+	if client.APIHost == "" || client.AuthToken == "" {
+		diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
+		return
+	}
+
 	var dd arvados.DiscoveryDocument
 	ddpath := "discovery/v1/apis/arvados/v1/rest"
-	testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
-	logger.Info(testname)
-	err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
-	if err != nil {
-		errorf("%s: %s", testname, err)
-	} else {
-		infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
-	}
+	diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
+		err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+		if err != nil {
+			return err
+		}
+		diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
+		return nil
+	})
 
 	var cluster arvados.Cluster
 	cfgpath := "arvados/v1/config"
-	testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
-	logger.Info(testname)
-	err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
-	if err != nil {
-		errorf("%s: %s", testname, err)
-	} else {
-		infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
-	}
+	diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
+		err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+		if err != nil {
+			return err
+		}
+		diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
+		return nil
+	})
 
 	var user arvados.User
-	testname = "getting current user record"
-	logger.Info(testname)
-	err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
-	if err != nil {
-		errorf("%s: %s", testname, err)
-		return 2
-	} else {
-		infof("%s: ok, uuid = %s", testname, user.UUID)
-	}
+	diag.dotest(30, "getting current user record", func() error {
+		err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+		if err != nil {
+			return err
+		}
+		diag.debugf("user uuid = %s", user.UUID)
+		return nil
+	})
 
 	// uncomment to create some spurious errors
 	// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
 
 	// TODO: detect routing errors here, like finding wb2 at the
 	// wb1 address.
-	for _, svc := range []*arvados.Service{
+	for i, svc := range []*arvados.Service{
 		&cluster.Services.Keepproxy,
 		&cluster.Services.WebDAV,
 		&cluster.Services.WebDAVDownload,
@@ -112,278 +162,303 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 		&cluster.Services.Workbench1,
 		&cluster.Services.Workbench2,
 	} {
-		testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
-		logger.Info(testname)
-		u := svc.ExternalURL
-		if strings.HasPrefix(u.Scheme, "ws") {
-			// We can do a real websocket test elsewhere,
-			// but for now we'll just check the https
-			// connection.
-			u.Scheme = "http" + u.Scheme[2:]
-		}
-		if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
-			u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
-		}
-		req, err := http.NewRequest(http.MethodGet, u.String(), nil)
-		if err != nil {
-			errorf("%s: %s", testname, err)
-			continue
-		}
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			errorf("%s: %s", testname, err)
-			continue
-		}
-		resp.Body.Close()
-		infof("%s: ok", testname)
+		diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
+			u := svc.ExternalURL
+			if strings.HasPrefix(u.Scheme, "ws") {
+				// We can do a real websocket test elsewhere,
+				// but for now we'll just check the https
+				// connection.
+				u.Scheme = "http" + u.Scheme[2:]
+			}
+			if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
+				u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
+			}
+			req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+			if err != nil {
+				return err
+			}
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			resp.Body.Close()
+			return nil
+		})
 	}
 
-	for _, url := range []string{
+	for i, url := range []string{
 		cluster.Services.Controller.ExternalURL.String(),
 		cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
 		cluster.Services.WebDAVDownload.ExternalURL.String(),
 	} {
-		testname = fmt.Sprintf("checking CORS headers at %s", url)
-		logger.Info(testname)
-		req, err := http.NewRequest("GET", url, nil)
-		if err != nil {
-			errorf("%s: %s", testname, err)
-			continue
-		}
-		req.Header.Set("Origin", "https://example.com")
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			errorf("%s: %s", testname, err)
-			continue
-		}
-		if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
-			warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
-		} else {
-			infof("%s: ok", testname)
-		}
+		diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
+			req, err := http.NewRequest("GET", url, nil)
+			if err != nil {
+				return err
+			}
+			req.Header.Set("Origin", "https://example.com")
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
+				return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
+			}
+			return nil
+		})
 	}
 
 	var keeplist arvados.KeepServiceList
-	testname = "checking internal/external client detection"
-	logger.Info(testname)
-	err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
-	if err != nil {
-		errorf("%s: error getting keep services list: %s", testname, err)
-	} else if len(keeplist.Items) == 0 {
-		errorf("%s: controller did not return any keep services", testname)
-	} else {
+	diag.dotest(60, "checking internal/external client detection", func() error {
+		err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+		if err != nil {
+			return fmt.Errorf("error getting keep services list: %s", err)
+		} else if len(keeplist.Items) == 0 {
+			return fmt.Errorf("controller did not return any keep services")
+		}
 		found := map[string]int{}
 		for _, ks := range keeplist.Items {
 			found[ks.ServiceType]++
 		}
-		infof := infof
 		isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
 		isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
-		if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
-			infof = errorf
-		}
 		if isExternal {
-			infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
+			diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
 		} else if isInternal {
-			infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
-		} else {
-			errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
+			diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
+		}
+		if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
+			return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
+		}
+		return nil
+	})
+
+	for i, ks := range keeplist.Items {
+		u := url.URL{
+			Scheme: "http",
+			Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
+			Path:   "/",
+		}
+		if ks.ServiceSSLFlag {
+			u.Scheme = "https"
 		}
+		diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
+			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+			defer cancel()
+			req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
+			if err != nil {
+				return err
+			}
+			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return fmt.Errorf("reading response body: %s", err)
+			}
+			loc := strings.TrimSpace(string(body))
+			if !strings.HasPrefix(loc, "d41d8") {
+				return fmt.Errorf("unexpected response from write: %q", body)
+			}
+
+			req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
+			if err != nil {
+				return err
+			}
+			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+			resp, err = http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			body, err = ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return fmt.Errorf("reading response body: %s", err)
+			}
+			if len(body) != 0 {
+				return fmt.Errorf("unexpected response from read: %q", body)
+			}
+
+			return nil
+		})
 	}
 
 	var project arvados.Group
-	var grplist arvados.GroupList
-	testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
-	logger.Info(testname)
-	err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
-		Filters: []arvados.Filter{
-			{"name", "=", diag.projectName},
-			{"group_class", "=", "project"},
-			{"owner_uuid", "=", user.UUID}},
-		Limit: -1})
-	if err != nil {
-		errorf("%s: list groups: %s", testname, err)
-	} else if len(grplist.Items) < 1 {
-		infof("%s: list groups: ok, no results", testname)
+	diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
+		var grplist arvados.GroupList
+		err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+			Filters: []arvados.Filter{
+				{"name", "=", diag.projectName},
+				{"group_class", "=", "project"},
+				{"owner_uuid", "=", user.UUID}},
+			Limit: -1})
+		if err != nil {
+			return fmt.Errorf("list groups: %s", err)
+		}
+		if len(grplist.Items) > 0 {
+			project = grplist.Items[0]
+			diag.debugf("using existing project, uuid = %s", project.UUID)
+			return nil
+		}
+		diag.debugf("list groups: ok, no results")
 		err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
 			"name":        diag.projectName,
 			"group_class": "project",
 		}})
 		if err != nil {
-			errorf("%s: create project: %s", testname, err)
-		} else {
-			infof("%s: created project, uuid = %s", testname, project.UUID)
+			return fmt.Errorf("create project: %s", err)
 		}
-	} else {
-		project = grplist.Items[0]
-		infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
-	}
+		diag.debugf("created project, uuid = %s", project.UUID)
+		return nil
+	})
 
-	testname = "creating temporary collection"
-	logger.Info(testname)
 	var collection arvados.Collection
-	err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
-		"ensure_unique_name": true,
-		"collection": map[string]interface{}{
-			"name":     "test collection",
-			"trash_at": time.Now().Add(time.Hour)}})
-	if err != nil {
-		errorf("%s: %s", testname, err)
-	} else {
-		infof("%s: ok, uuid = %s", testname, collection.UUID)
+	diag.dotest(90, "creating temporary collection", func() error {
+		err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+			"ensure_unique_name": true,
+			"collection": map[string]interface{}{
+				"name":     "test collection",
+				"trash_at": time.Now().Add(time.Hour)}})
+		if err != nil {
+			return err
+		}
+		diag.debugf("ok, uuid = %s", collection.UUID)
+		return nil
+	})
+
+	if collection.UUID != "" {
 		defer func() {
-			testname := "deleting temporary collection"
-			logger.Info(testname)
-			err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
-			if err != nil {
-				errorf("%s: %s", testname, err)
-			} else {
-				infof("%s: ok", testname)
-			}
+			diag.dotest(9990, "deleting temporary collection", func() error {
+				return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+			})
 		}()
 	}
 
-	testname = "uploading file via webdav"
-	logger.Info(testname)
-	func() {
+	diag.dotest(100, "uploading file via webdav", func() error {
 		if collection.UUID == "" {
-			infof("%s: skipping, no test collection", testname)
-			return
+			return fmt.Errorf("skipping, no test collection")
 		}
 		req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
 		if err != nil {
-			errorf("%s: BUG? http.NewRequest: %s", testname, err)
-			return
+			return fmt.Errorf("BUG? http.NewRequest: %s", err)
 		}
 		req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
-			errorf("%s: error performing http request: %s", testname, err)
-			return
+			return fmt.Errorf("error performing http request: %s", err)
 		}
 		resp.Body.Close()
 		if resp.StatusCode != http.StatusCreated {
-			errorf("%s: status %s", testname, resp.Status)
-			return
+			return fmt.Errorf("status %s", resp.Status)
 		}
-		infof("%s: ok, status %s", testname, resp.Status)
+		diag.debugf("ok, status %s", resp.Status)
 		err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
 		if err != nil {
-			errorf("%s: get updated collection: %s", testname, err)
-			return
+			return fmt.Errorf("get updated collection: %s", err)
 		}
-		infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
-	}()
+		diag.debugf("ok, pdh %s", collection.PortableDataHash)
+		return nil
+	})
 
 	davurl := cluster.Services.WebDAV.ExternalURL
-	testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
-	logger.Info(testname)
-	if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
-		infof("%s: looks ok", testname)
-	} else if davurl.Host == "" {
-		warnf("%s: host missing - content previews will not work", testname)
-	} else {
-		warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
-	}
+	diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
+		if davurl.Host == "" {
+			return fmt.Errorf("host missing - content previews will not work")
+		}
+		if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
+			diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
+		}
+		return nil
+	})
 
-	for _, trial := range []struct {
-		status  int
-		fileurl string
+	for i, trial := range []struct {
+		needcoll bool
+		status   int
+		fileurl  string
 	}{
-		{http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
-		{http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
-		{http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
-		{http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
-		{http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
-		{http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+		{false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+		{false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+		{false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+		{false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+		{true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+		{true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
 	} {
-		func() {
-			testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
-			logger.Info(testname)
-			if collection.UUID == "" {
-				errorf("%s: skipping, no test collection", testname)
-				return
+		diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+			if trial.needcoll && collection.UUID == "" {
+				return fmt.Errorf("skipping, no test collection")
 			}
 			req, err := http.NewRequest("GET", trial.fileurl, nil)
 			if err != nil {
-				errorf("%s: %s", testname, err)
-				return
+				return err
 			}
 			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 			resp, err := http.DefaultClient.Do(req)
 			if err != nil {
-				errorf("%s: %s", testname, err)
-				return
+				return err
 			}
 			defer resp.Body.Close()
 			body, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
-				errorf("%s: error reading response: %s", testname, err)
+				return fmt.Errorf("reading response: %s", err)
 			}
 			if resp.StatusCode != trial.status {
-				errorf("%s: unexpected response status: %s", testname, resp.Status)
-			} else if trial.status == http.StatusOK && string(body) != "testfiledata" {
-				errorf("%s: unexpected response content: %q", testname, body)
-			} else {
-				infof("%s: ok", testname)
+				return fmt.Errorf("unexpected response status: %s", resp.Status)
 			}
-		}()
+			if trial.status == http.StatusOK && string(body) != "testfiledata" {
+				return fmt.Errorf("unexpected response content: %q", body)
+			}
+			return nil
+		})
 	}
 
 	var vm arvados.VirtualMachine
-	var vmlist arvados.VirtualMachineList
-	testname = "getting list of virtual machines"
-	logger.Info(testname)
-	err = client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
-	if err != nil {
-		errorf("%s: %s", testname, err)
-	} else if len(vmlist.Items) < 1 {
-		errorf("%s: none found", testname)
-	} else {
+	diag.dotest(130, "getting list of virtual machines", func() error {
+		var vmlist arvados.VirtualMachineList
+		err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+		if err != nil {
+			return err
+		}
+		if len(vmlist.Items) < 1 {
+			return fmt.Errorf("no VMs found")
+		}
 		vm = vmlist.Items[0]
-		infof("%s: ok", testname)
-	}
+		return nil
+	})
 
-	testname = "getting workbench1 webshell page"
-	logger.Info(testname)
-	func() {
+	diag.dotest(140, "getting workbench1 webshell page", func() error {
 		if vm.UUID == "" {
-			errorf("%s: skipping, no vm available", testname)
-			return
+			return fmt.Errorf("skipping, no vm available")
 		}
 		webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
-		debugf("%s: url %s", testname, webshelltermurl)
+		diag.debugf("url %s", webshelltermurl)
 		req, err := http.NewRequest("GET", webshelltermurl, nil)
 		if err != nil {
-			errorf("%s: %s", testname, err)
-			return
+			return err
 		}
 		req.Header.Set("Authorization", "Bearer "+client.AuthToken)
 		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
-			errorf("%s: %s", testname, err)
-			return
+			return err
 		}
 		defer resp.Body.Close()
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			errorf("%s: error reading response: %s", testname, err)
+			return fmt.Errorf("reading response: %s", err)
 		}
 		if resp.StatusCode != http.StatusOK {
-			errorf("%s: unexpected response status: %s %q", testname, resp.Status, body)
-			return
+			return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
 		}
-		infof("%s: ok", testname)
-	}()
+		return nil
+	})
 
-	testname = "connecting to webshell service"
-	logger.Info(testname)
-	func() {
-		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*timeout))
+	diag.dotest(150, "connecting to webshell service", func() error {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
 		defer cancel()
 		if vm.UUID == "" {
-			errorf("%s: skipping, no vm available", testname)
-			return
+			return fmt.Errorf("skipping, no vm available")
 		}
 		u := cluster.Services.WebShell.ExternalURL
 		webshellurl := u.String() + vm.Hostname + "?"
@@ -391,7 +466,7 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 			u.Host = vm.Hostname + u.Host[1:]
 			webshellurl = u.String() + "?"
 		}
-		debugf("%s: url %s", testname, webshellurl)
+		diag.debugf("url %s", webshellurl)
 		req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
 			"width":   {"80"},
 			"height":  {"25"},
@@ -399,31 +474,27 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 			"rooturl": {webshellurl},
 		}.Encode()))
 		if err != nil {
-			errorf("%s: %s", testname, err)
-			return
+			return err
 		}
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
-			errorf("%s: %s", testname, err)
-			return
+			return err
 		}
 		defer resp.Body.Close()
-		debugf("%s: response status %s", testname, resp.Status)
+		diag.debugf("response status %s", resp.Status)
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			errorf("%s: error reading response: %s", testname, err)
+			return fmt.Errorf("reading response: %s", err)
 		}
-		debugf("%s: response body %q", testname, body)
+		diag.debugf("response body %q", body)
 		// We don't speak the protocol, so we get a 400 error
 		// from the webshell server even if everything is
 		// OK. Anything else (404, 502, ???) indicates a
 		// problem.
 		if resp.StatusCode != http.StatusBadRequest {
-			errorf("%s: unexpected response status: %s, %q", testname, resp.Status, body)
-			return
+			return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
 		}
-		infof("%s: ok", testname)
-	}()
-	return 0
+		return nil
+	})
 }

commit 5b3f3eee0352885c8daaca67f9d42f4480c761dd
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jun 1 11:28:54 2021 -0400

    17609: Add webshell test.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 26bbdfe1d..66b678a8e 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -6,11 +6,13 @@ package diagnostics
 
 import (
 	"bytes"
+	"context"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
@@ -29,6 +31,7 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 	loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
 	checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
 	checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
+	timeout := f.Duration("timeout", 10*time.Second, "timeout for http requests")
 	err := f.Parse(args)
 	if err == flag.ErrHelp {
 		return 0
@@ -37,11 +40,14 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 		return 2
 	}
 
+	ctx := context.Background()
+
 	logger := ctxlog.New(stdout, "text", *loglevel)
 	logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
 
 	infof := logger.Infof
 	warnf := logger.Warnf
+	debugf := logger.Debugf
 	var errors []string
 	errorf := func(f string, args ...interface{}) {
 		logger.Errorf(f, args...)
@@ -324,5 +330,100 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
 		}()
 	}
 
+	var vm arvados.VirtualMachine
+	var vmlist arvados.VirtualMachineList
+	testname = "getting list of virtual machines"
+	logger.Info(testname)
+	err = client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+	if err != nil {
+		errorf("%s: %s", testname, err)
+	} else if len(vmlist.Items) < 1 {
+		errorf("%s: none found", testname)
+	} else {
+		vm = vmlist.Items[0]
+		infof("%s: ok", testname)
+	}
+
+	testname = "getting workbench1 webshell page"
+	logger.Info(testname)
+	func() {
+		if vm.UUID == "" {
+			errorf("%s: skipping, no vm available", testname)
+			return
+		}
+		webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
+		debugf("%s: url %s", testname, webshelltermurl)
+		req, err := http.NewRequest("GET", webshelltermurl, nil)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			return
+		}
+		req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			return
+		}
+		defer resp.Body.Close()
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			errorf("%s: error reading response: %s", testname, err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			errorf("%s: unexpected response status: %s %q", testname, resp.Status, body)
+			return
+		}
+		infof("%s: ok", testname)
+	}()
+
+	testname = "connecting to webshell service"
+	logger.Info(testname)
+	func() {
+		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*timeout))
+		defer cancel()
+		if vm.UUID == "" {
+			errorf("%s: skipping, no vm available", testname)
+			return
+		}
+		u := cluster.Services.WebShell.ExternalURL
+		webshellurl := u.String() + vm.Hostname + "?"
+		if strings.HasPrefix(u.Host, "*") {
+			u.Host = vm.Hostname + u.Host[1:]
+			webshellurl = u.String() + "?"
+		}
+		debugf("%s: url %s", testname, webshellurl)
+		req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
+			"width":   {"80"},
+			"height":  {"25"},
+			"session": {"xyzzy"},
+			"rooturl": {webshellurl},
+		}.Encode()))
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			return
+		}
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			return
+		}
+		defer resp.Body.Close()
+		debugf("%s: response status %s", testname, resp.Status)
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			errorf("%s: error reading response: %s", testname, err)
+		}
+		debugf("%s: response body %q", testname, body)
+		// We don't speak the protocol, so we get a 400 error
+		// from the webshell server even if everything is
+		// OK. Anything else (404, 502, ???) indicates a
+		// problem.
+		if resp.StatusCode != http.StatusBadRequest {
+			errorf("%s: unexpected response status: %s, %q", testname, resp.Status, body)
+			return
+		}
+		infof("%s: ok", testname)
+	}()
 	return 0
 }

commit 751cdf33d5519c393323baa15233ca55b7c7a9a1
Author: Tom Clegg <tom at curii.com>
Date:   Mon May 31 16:52:36 2021 -0400

    17609: Add diagnostics command.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index aefcce79a..cb1546211 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -11,6 +11,7 @@ import (
 	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/lib/costanalyzer"
 	"git.arvados.org/arvados.git/lib/deduplicationreport"
+	"git.arvados.org/arvados.git/lib/diagnostics"
 	"git.arvados.org/arvados.git/lib/mount"
 )
 
@@ -59,6 +60,7 @@ var (
 		"costanalyzer":         costanalyzer.Command,
 		"shell":                shellCommand{},
 		"connect-ssh":          connectSSHCommand{},
+		"diagnostics":          diagnostics.Command{},
 	})
 )
 
diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go
index b7d918739..63d7576b4 100644
--- a/lib/cmd/cmd.go
+++ b/lib/cmd/cmd.go
@@ -16,6 +16,8 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+
+	"github.com/sirupsen/logrus"
 )
 
 type Handler interface {
@@ -153,3 +155,9 @@ func SubcommandToFront(args []string, flagset FlagSet) []string {
 	copy(newargs[flagargs+1:], args[flagargs+1:])
 	return newargs
 }
+
+type NoPrefixFormatter struct{}
+
+func (NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	return []byte(entry.Message + "\n"), nil
+}
diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go
index 9b0685225..633e95e29 100644
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@ -7,33 +7,26 @@ package costanalyzer
 import (
 	"io"
 
+	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
-	"github.com/sirupsen/logrus"
 )
 
 var Command command
 
 type command struct{}
 
-type NoPrefixFormatter struct{}
-
-func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
-	return []byte(entry.Message), nil
-}
-
 // RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
 func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	var err error
 	logger := ctxlog.New(stderr, "text", "info")
+	logger.SetFormatter(cmd.NoPrefixFormatter{})
 	defer func() {
 		if err != nil {
 			logger.Error("\n" + err.Error() + "\n")
 		}
 	}()
 
-	logger.SetFormatter(new(NoPrefixFormatter))
-
 	loader := config.NewLoader(stdin, logger)
 	loader.SkipLegacy = true
 
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
new file mode 100644
index 000000000..26bbdfe1d
--- /dev/null
+++ b/lib/diagnostics/cmd.go
@@ -0,0 +1,328 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package diagnostics
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+type Command struct {
+	projectName string
+}
+
+func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	f := flag.NewFlagSet(prog, flag.ContinueOnError)
+	f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
+	loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
+	checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
+	checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
+	err := f.Parse(args)
+	if err == flag.ErrHelp {
+		return 0
+	} else if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+
+	logger := ctxlog.New(stdout, "text", *loglevel)
+	logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+
+	infof := logger.Infof
+	warnf := logger.Warnf
+	var errors []string
+	errorf := func(f string, args ...interface{}) {
+		logger.Errorf(f, args...)
+		errors = append(errors, fmt.Sprintf(f, args...))
+	}
+	defer func() {
+		if len(errors) == 0 {
+			logger.Info("--- no errors ---")
+		} else {
+			fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
+			for _, e := range errors {
+				logger.Error(e)
+			}
+		}
+	}()
+
+	client := arvados.NewClientFromEnv()
+
+	var dd arvados.DiscoveryDocument
+	ddpath := "discovery/v1/apis/arvados/v1/rest"
+	testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
+	logger.Info(testname)
+	err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+	if err != nil {
+		errorf("%s: %s", testname, err)
+	} else {
+		infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
+	}
+
+	var cluster arvados.Cluster
+	cfgpath := "arvados/v1/config"
+	testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
+	logger.Info(testname)
+	err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+	if err != nil {
+		errorf("%s: %s", testname, err)
+	} else {
+		infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
+	}
+
+	var user arvados.User
+	testname = "getting current user record"
+	logger.Info(testname)
+	err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+	if err != nil {
+		errorf("%s: %s", testname, err)
+		return 2
+	} else {
+		infof("%s: ok, uuid = %s", testname, user.UUID)
+	}
+
+	// uncomment to create some spurious errors
+	// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
+
+	// TODO: detect routing errors here, like finding wb2 at the
+	// wb1 address.
+	for _, svc := range []*arvados.Service{
+		&cluster.Services.Keepproxy,
+		&cluster.Services.WebDAV,
+		&cluster.Services.WebDAVDownload,
+		&cluster.Services.Websocket,
+		&cluster.Services.Workbench1,
+		&cluster.Services.Workbench2,
+	} {
+		testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
+		logger.Info(testname)
+		u := svc.ExternalURL
+		if strings.HasPrefix(u.Scheme, "ws") {
+			// We can do a real websocket test elsewhere,
+			// but for now we'll just check the https
+			// connection.
+			u.Scheme = "http" + u.Scheme[2:]
+		}
+		if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
+			u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
+		}
+		req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			continue
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			continue
+		}
+		resp.Body.Close()
+		infof("%s: ok", testname)
+	}
+
+	for _, url := range []string{
+		cluster.Services.Controller.ExternalURL.String(),
+		cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
+		cluster.Services.WebDAVDownload.ExternalURL.String(),
+	} {
+		testname = fmt.Sprintf("checking CORS headers at %s", url)
+		logger.Info(testname)
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			continue
+		}
+		req.Header.Set("Origin", "https://example.com")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			errorf("%s: %s", testname, err)
+			continue
+		}
+		if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
+			warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
+		} else {
+			infof("%s: ok", testname)
+		}
+	}
+
+	var keeplist arvados.KeepServiceList
+	testname = "checking internal/external client detection"
+	logger.Info(testname)
+	err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+	if err != nil {
+		errorf("%s: error getting keep services list: %s", testname, err)
+	} else if len(keeplist.Items) == 0 {
+		errorf("%s: controller did not return any keep services", testname)
+	} else {
+		found := map[string]int{}
+		for _, ks := range keeplist.Items {
+			found[ks.ServiceType]++
+		}
+		infof := infof
+		isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
+		isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
+		if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
+			infof = errorf
+		}
+		if isExternal {
+			infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
+		} else if isInternal {
+			infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
+		} else {
+			errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
+		}
+	}
+
+	var project arvados.Group
+	var grplist arvados.GroupList
+	testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
+	logger.Info(testname)
+	err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+		Filters: []arvados.Filter{
+			{"name", "=", diag.projectName},
+			{"group_class", "=", "project"},
+			{"owner_uuid", "=", user.UUID}},
+		Limit: -1})
+	if err != nil {
+		errorf("%s: list groups: %s", testname, err)
+	} else if len(grplist.Items) < 1 {
+		infof("%s: list groups: ok, no results", testname)
+		err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
+			"name":        diag.projectName,
+			"group_class": "project",
+		}})
+		if err != nil {
+			errorf("%s: create project: %s", testname, err)
+		} else {
+			infof("%s: created project, uuid = %s", testname, project.UUID)
+		}
+	} else {
+		project = grplist.Items[0]
+		infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
+	}
+
+	testname = "creating temporary collection"
+	logger.Info(testname)
+	var collection arvados.Collection
+	err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+		"ensure_unique_name": true,
+		"collection": map[string]interface{}{
+			"name":     "test collection",
+			"trash_at": time.Now().Add(time.Hour)}})
+	if err != nil {
+		errorf("%s: %s", testname, err)
+	} else {
+		infof("%s: ok, uuid = %s", testname, collection.UUID)
+		defer func() {
+			testname := "deleting temporary collection"
+			logger.Info(testname)
+			err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+			if err != nil {
+				errorf("%s: %s", testname, err)
+			} else {
+				infof("%s: ok", testname)
+			}
+		}()
+	}
+
+	testname = "uploading file via webdav"
+	logger.Info(testname)
+	func() {
+		if collection.UUID == "" {
+			infof("%s: skipping, no test collection", testname)
+			return
+		}
+		req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+		if err != nil {
+			errorf("%s: BUG? http.NewRequest: %s", testname, err)
+			return
+		}
+		req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			errorf("%s: error performing http request: %s", testname, err)
+			return
+		}
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusCreated {
+			errorf("%s: status %s", testname, resp.Status)
+			return
+		}
+		infof("%s: ok, status %s", testname, resp.Status)
+		err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
+		if err != nil {
+			errorf("%s: get updated collection: %s", testname, err)
+			return
+		}
+		infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
+	}()
+
+	davurl := cluster.Services.WebDAV.ExternalURL
+	testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
+	logger.Info(testname)
+	if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
+		infof("%s: looks ok", testname)
+	} else if davurl.Host == "" {
+		warnf("%s: host missing - content previews will not work", testname)
+	} else {
+		warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
+	}
+
+	for _, trial := range []struct {
+		status  int
+		fileurl string
+	}{
+		{http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+		{http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+		{http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+		{http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+		{http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+		{http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+	} {
+		func() {
+			testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
+			logger.Info(testname)
+			if collection.UUID == "" {
+				errorf("%s: skipping, no test collection", testname)
+				return
+			}
+			req, err := http.NewRequest("GET", trial.fileurl, nil)
+			if err != nil {
+				errorf("%s: %s", testname, err)
+				return
+			}
+			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				errorf("%s: %s", testname, err)
+				return
+			}
+			defer resp.Body.Close()
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				errorf("%s: error reading response: %s", testname, err)
+			}
+			if resp.StatusCode != trial.status {
+				errorf("%s: unexpected response status: %s", testname, resp.Status)
+			} else if trial.status == http.StatusOK && string(body) != "testfiledata" {
+				errorf("%s: unexpected response content: %q", testname, body)
+			} else {
+				infof("%s: ok", testname)
+			}
+		}()
+	}
+
+	return 0
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list