[ARVADOS] created: 2.1.0-915-g08c8b9cc4
Git user
git at public.arvados.org
Tue Jun 8 15:29:18 UTC 2021
at 08c8b9cc496627bc3fd3d87ae333fadce4797eaa (commit)
commit 08c8b9cc496627bc3fd3d87ae333fadce4797eaa
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 8 02:08:43 2021 -0400
17609: Tweak messages.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 1c7b98baf..b0241b3ae 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -72,15 +72,15 @@ type diagnoser struct {
}
func (diag *diagnoser) debugf(f string, args ...interface{}) {
- diag.logger.Debugf(f, args...)
+ diag.logger.Debugf(" ... "+f, args...)
}
func (diag *diagnoser) infof(f string, args ...interface{}) {
- diag.logger.Infof(f, args...)
+ diag.logger.Infof(" ... "+f, args...)
}
func (diag *diagnoser) warnf(f string, args ...interface{}) {
- diag.logger.Warnf(f, args...)
+ diag.logger.Warnf(" ... "+f, args...)
}
func (diag *diagnoser) errorf(f string, args ...interface{}) {
@@ -101,14 +101,14 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
}
diag.done[id] = true
- diag.infof("%4d: %s", id, title)
+ diag.logger.Infof("%4d: %s", id, title)
t0 := time.Now()
err := fn()
- elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
+ elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
if err != nil {
diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
} else {
- diag.debugf("%4d: %s (%s): ok", id, title, elapsed)
+ diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
}
}
@@ -533,7 +533,7 @@ func (diag *diagnoser) runtests() {
diag.dotest(160, "running a container", func() error {
if diag.priority < 1 {
- diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
+ diag.infof("skipping (use priority > 0 if you want to run a container)")
return nil
}
if project.UUID == "" {
commit 3975b33706d85857e8d7a8ec2677edbb35984a14
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 8 01:53:12 2021 -0400
17609: Behave better when project creation fails.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 2f43263e5..1c7b98baf 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -31,7 +31,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
- f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000)")
+ f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
err := f.Parse(args)
if err == flag.ErrHelp {
@@ -334,13 +334,17 @@ func (diag *diagnoser) runtests() {
var collection arvados.Collection
diag.dotest(90, "creating temporary collection", func() error {
+ if project.UUID == "" {
+ return fmt.Errorf("skipping, no project to work in")
+ }
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
"ensure_unique_name": true,
"collection": map[string]interface{}{
- "name": "test collection",
- "trash_at": time.Now().Add(time.Hour)}})
+ "owner_uuid": project.UUID,
+ "name": "test collection",
+ "trash_at": time.Now().Add(time.Hour)}})
if err != nil {
return err
}
@@ -532,6 +536,9 @@ func (diag *diagnoser) runtests() {
diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
return nil
}
+ if project.UUID == "" {
+ return fmt.Errorf("skipping, no project to work in")
+ }
var cr arvados.ContainerRequest
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
commit 93cf55af0be9ef96c79d17323d82ba85eb1b9968
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 8 01:45:55 2021 -0400
17609: Improve output formatting.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/go.mod b/go.mod
index aa289761b..9062c667d 100644
--- a/go.mod
+++ b/go.mod
@@ -55,13 +55,13 @@ require (
github.com/prometheus/common v0.7.0
github.com/satori/go.uuid v1.2.1-0.20180103174451-36e9d2ebbde5 // indirect
github.com/sergi/go-diff v1.0.0 // indirect
- github.com/sirupsen/logrus v1.4.2
+ github.com/sirupsen/logrus v1.8.1
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/net v0.0.0-20201021035429-f5854403a974
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
- golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+ golang.org/x/sys v0.0.0-20210603125802-9665404d3644
golang.org/x/tools v0.1.0 // indirect
google.golang.org/api v0.13.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
diff --git a/go.sum b/go.sum
index 006118ad9..23731f69a 100644
--- a/go.sum
+++ b/go.sum
@@ -234,6 +234,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg=
@@ -301,9 +303,12 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 4f3ad8b0a..2f43263e5 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -41,7 +41,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
return 2
}
diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
- diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+ diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
diag.runtests()
if len(diag.errors) == 0 {
diag.logger.Info("--- no errors ---")
@@ -101,14 +101,15 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
}
diag.done[id] = true
- diag.infof("%4d %s", id, title)
+ diag.infof("%4d: %s", id, title)
t0 := time.Now()
err := fn()
elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
if err != nil {
- diag.errorf("%s (%s): %s", title, elapsed, err)
+ diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
+ } else {
+ diag.debugf("%4d: %s (%s): ok", id, title, elapsed)
}
- diag.debugf("%4d %s (%s): ok", id, title, elapsed)
}
func (diag *diagnoser) runtests() {
commit 56d721569988522e5cca008a303b19d008b937dd
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 8 01:45:40 2021 -0400
17609: Add "run container" test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 54c78d07c..4f3ad8b0a 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -31,6 +31,7 @@ func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdou
f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+ f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000)")
f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
err := f.Parse(args)
if err == flag.ErrHelp {
@@ -60,6 +61,7 @@ type diagnoser struct {
stdout io.Writer
stderr io.Writer
logLevel string
+ priority int
projectName string
checkInternal bool
checkExternal bool
@@ -523,4 +525,80 @@ func (diag *diagnoser) runtests() {
}
return nil
})
+
+ diag.dotest(160, "running a container", func() error {
+ if diag.priority < 1 {
+ diag.debugf("skipping, caller requested priority<1 (%d)", diag.priority)
+ return nil
+ }
+
+ var cr arvados.ContainerRequest
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+
+ timestamp := time.Now().Format(time.RFC3339)
+ err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+ "owner_uuid": project.UUID,
+ "name": fmt.Sprintf("diagnostics container request %s", timestamp),
+ "container_image": "arvados/jobs",
+ "command": []string{"echo", timestamp},
+ "use_existing": false,
+ "output_path": "/mnt/output",
+ "output_name": fmt.Sprintf("diagnostics output %s", timestamp),
+ "priority": diag.priority,
+ "state": arvados.ContainerRequestStateCommitted,
+ "mounts": map[string]map[string]interface{}{
+ "/mnt/output": {
+ "kind": "collection",
+ "writable": true,
+ },
+ },
+ "runtime_constraints": arvados.RuntimeConstraints{
+ VCPUs: 1,
+ RAM: 1 << 26,
+ KeepCacheRAM: 1 << 26,
+ },
+ }})
+ if err != nil {
+ return err
+ }
+ diag.debugf("container request uuid = %s", cr.UUID)
+ diag.debugf("container uuid = %s", cr.ContainerUUID)
+
+ timeout := 10 * time.Minute
+ diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
+ defer cancel()
+
+ var c arvados.Container
+ for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
+ ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
+ defer cancel()
+
+ crStateWas := cr.State
+ err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
+ if err != nil {
+ return err
+ }
+ if cr.State != crStateWas {
+ diag.debugf("container request state = %s", cr.State)
+ }
+
+ cStateWas := c.State
+ err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
+ if err != nil {
+ return err
+ }
+ if c.State != cStateWas {
+ diag.debugf("container state = %s", c.State)
+ }
+ }
+
+ if c.State != arvados.ContainerStateComplete {
+ return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
+ } else if c.ExitCode != 0 {
+ return fmt.Errorf("container exited %d", c.ExitCode)
+ }
+ return nil
+ })
}
commit dd7268b61d18f37036f40479c67b4b9f446eb2a0
Author: Tom Clegg <tom at curii.com>
Date: Wed Jun 2 15:23:56 2021 -0400
17609: Improve log text alignment.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 3b3ed400b..54c78d07c 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -99,14 +99,14 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) {
}
diag.done[id] = true
- diag.infof("%d %s", id, title)
+ diag.infof("%4d %s", id, title)
t0 := time.Now()
err := fn()
elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
if err != nil {
diag.errorf("%s (%s): %s", title, elapsed, err)
}
- diag.debugf("%d %s (%s): ok", id, title, elapsed)
+ diag.debugf("%4d %s (%s): ok", id, title, elapsed)
}
func (diag *diagnoser) runtests() {
commit 6c517a23f8af60ecbb87099be26f8df7a7652f4f
Author: Tom Clegg <tom at curii.com>
Date: Wed Jun 2 15:17:13 2021 -0400
17609: Use timeout on all http requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 7021bb0ee..3b3ed400b 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -120,7 +120,9 @@ func (diag *diagnoser) runtests() {
var dd arvados.DiscoveryDocument
ddpath := "discovery/v1/apis/arvados/v1/rest"
diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
- err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
if err != nil {
return err
}
@@ -131,7 +133,9 @@ func (diag *diagnoser) runtests() {
var cluster arvados.Cluster
cfgpath := "arvados/v1/config"
diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
- err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
if err != nil {
return err
}
@@ -141,7 +145,9 @@ func (diag *diagnoser) runtests() {
var user arvados.User
diag.dotest(30, "getting current user record", func() error {
- err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return err
}
@@ -163,6 +169,8 @@ func (diag *diagnoser) runtests() {
&cluster.Services.Workbench2,
} {
diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
u := svc.ExternalURL
if strings.HasPrefix(u.Scheme, "ws") {
// We can do a real websocket test elsewhere,
@@ -173,7 +181,7 @@ func (diag *diagnoser) runtests() {
if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
}
- req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return err
}
@@ -192,7 +200,9 @@ func (diag *diagnoser) runtests() {
cluster.Services.WebDAVDownload.ExternalURL.String(),
} {
diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
- req, err := http.NewRequest("GET", url, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return err
}
@@ -210,7 +220,9 @@ func (diag *diagnoser) runtests() {
var keeplist arvados.KeepServiceList
diag.dotest(60, "checking internal/external client detection", func() error {
- err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
if err != nil {
return fmt.Errorf("error getting keep services list: %s", err)
} else if len(keeplist.Items) == 0 {
@@ -288,8 +300,10 @@ func (diag *diagnoser) runtests() {
var project arvados.Group
diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
var grplist arvados.GroupList
- err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+ err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
Filters: []arvados.Filter{
{"name", "=", diag.projectName},
{"group_class", "=", "project"},
@@ -304,7 +318,7 @@ func (diag *diagnoser) runtests() {
return nil
}
diag.debugf("list groups: ok, no results")
- err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
+ err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
"name": diag.projectName,
"group_class": "project",
}})
@@ -317,7 +331,9 @@ func (diag *diagnoser) runtests() {
var collection arvados.Collection
diag.dotest(90, "creating temporary collection", func() error {
- err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
"ensure_unique_name": true,
"collection": map[string]interface{}{
"name": "test collection",
@@ -332,16 +348,20 @@ func (diag *diagnoser) runtests() {
if collection.UUID != "" {
defer func() {
diag.dotest(9990, "deleting temporary collection", func() error {
- return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
})
}()
}
diag.dotest(100, "uploading file via webdav", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if collection.UUID == "" {
return fmt.Errorf("skipping, no test collection")
}
- req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+ req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
if err != nil {
return fmt.Errorf("BUG? http.NewRequest: %s", err)
}
@@ -355,7 +375,7 @@ func (diag *diagnoser) runtests() {
return fmt.Errorf("status %s", resp.Status)
}
diag.debugf("ok, status %s", resp.Status)
- err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
if err != nil {
return fmt.Errorf("get updated collection: %s", err)
}
@@ -387,10 +407,12 @@ func (diag *diagnoser) runtests() {
{true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
} {
diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if trial.needcoll && collection.UUID == "" {
return fmt.Errorf("skipping, no test collection")
}
- req, err := http.NewRequest("GET", trial.fileurl, nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
if err != nil {
return err
}
@@ -416,8 +438,10 @@ func (diag *diagnoser) runtests() {
var vm arvados.VirtualMachine
diag.dotest(130, "getting list of virtual machines", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
var vmlist arvados.VirtualMachineList
- err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+ err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
if err != nil {
return err
}
@@ -429,12 +453,14 @@ func (diag *diagnoser) runtests() {
})
diag.dotest(140, "getting workbench1 webshell page", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if vm.UUID == "" {
return fmt.Errorf("skipping, no vm available")
}
webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
diag.debugf("url %s", webshelltermurl)
- req, err := http.NewRequest("GET", webshelltermurl, nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
if err != nil {
return err
}
commit 6560ade0a1fca34182814e0881b8b543216ad328
Author: Tom Clegg <tom at curii.com>
Date: Wed Jun 2 15:12:58 2021 -0400
17609: Fix limit args to work with older API.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 60860e257..7021bb0ee 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -210,7 +210,7 @@ func (diag *diagnoser) runtests() {
var keeplist arvados.KeepServiceList
diag.dotest(60, "checking internal/external client detection", func() error {
- err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+ err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
if err != nil {
return fmt.Errorf("error getting keep services list: %s", err)
} else if len(keeplist.Items) == 0 {
@@ -294,7 +294,7 @@ func (diag *diagnoser) runtests() {
{"name", "=", diag.projectName},
{"group_class", "=", "project"},
{"owner_uuid", "=", user.UUID}},
- Limit: -1})
+ Limit: 999999})
if err != nil {
return fmt.Errorf("list groups: %s", err)
}
commit a98105b60cb4bd19ed0723bc14b7a5e4514e7a83
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 1 16:23:08 2021 -0400
17609: Refactor tests to add some structure.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 66b678a8e..60860e257 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -11,6 +11,7 @@ import (
"fmt"
"io"
"io/ioutil"
+ "net"
"net/http"
"net/url"
"strings"
@@ -21,17 +22,16 @@ import (
"github.com/sirupsen/logrus"
)
-type Command struct {
- projectName string
-}
+type Command struct{}
-func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var diag diagnoser
f := flag.NewFlagSet(prog, flag.ContinueOnError)
f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
- loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
- checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
- checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
- timeout := f.Duration("timeout", 10*time.Second, "timeout for http requests")
+ f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+ f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
+ f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+ f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
err := f.Parse(args)
if err == flag.ErrHelp {
return 0
@@ -39,72 +39,122 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
fmt.Fprintln(stderr, err)
return 2
}
+ diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
+ diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+ diag.runtests()
+ if len(diag.errors) == 0 {
+ diag.logger.Info("--- no errors ---")
+ return 0
+ } else {
+ if diag.logger.Level > logrus.ErrorLevel {
+ fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
+ for _, e := range diag.errors {
+ diag.logger.Error(e)
+ }
+ }
+ return 1
+ }
+}
- ctx := context.Background()
+type diagnoser struct {
+ stdout io.Writer
+ stderr io.Writer
+ logLevel string
+ projectName string
+ checkInternal bool
+ checkExternal bool
+ timeout time.Duration
+ logger *logrus.Logger
+ errors []string
+ done map[int]bool
+}
+
+func (diag *diagnoser) debugf(f string, args ...interface{}) {
+ diag.logger.Debugf(f, args...)
+}
+
+func (diag *diagnoser) infof(f string, args ...interface{}) {
+ diag.logger.Infof(f, args...)
+}
+
+func (diag *diagnoser) warnf(f string, args ...interface{}) {
+ diag.logger.Warnf(f, args...)
+}
- logger := ctxlog.New(stdout, "text", *loglevel)
- logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+func (diag *diagnoser) errorf(f string, args ...interface{}) {
+ diag.logger.Errorf(f, args...)
+ diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
+}
- infof := logger.Infof
- warnf := logger.Warnf
- debugf := logger.Debugf
- var errors []string
- errorf := func(f string, args ...interface{}) {
- logger.Errorf(f, args...)
- errors = append(errors, fmt.Sprintf(f, args...))
+// Run the given func, logging appropriate messages before and after,
+// adding timing info, etc.
+//
+// The id argument should be unique among tests, and shouldn't change
+// when other tests are added/removed.
+func (diag *diagnoser) dotest(id int, title string, fn func() error) {
+ if diag.done == nil {
+ diag.done = map[int]bool{}
+ } else if diag.done[id] {
+ diag.errorf("(bug) reused test id %d", id)
}
- defer func() {
- if len(errors) == 0 {
- logger.Info("--- no errors ---")
- } else {
- fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
- for _, e := range errors {
- logger.Error(e)
- }
- }
- }()
+ diag.done[id] = true
+
+ diag.infof("%d %s", id, title)
+ t0 := time.Now()
+ err := fn()
+ elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
+ if err != nil {
+ diag.errorf("%s (%s): %s", title, elapsed, err)
+ }
+ diag.debugf("%d %s (%s): ok", id, title, elapsed)
+}
+func (diag *diagnoser) runtests() {
client := arvados.NewClientFromEnv()
+ if client.APIHost == "" || client.AuthToken == "" {
+ diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
+ return
+ }
+
var dd arvados.DiscoveryDocument
ddpath := "discovery/v1/apis/arvados/v1/rest"
- testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
- logger.Info(testname)
- err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- } else {
- infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
- }
+ diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
+ err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+ if err != nil {
+ return err
+ }
+ diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
+ return nil
+ })
var cluster arvados.Cluster
cfgpath := "arvados/v1/config"
- testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
- logger.Info(testname)
- err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- } else {
- infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
- }
+ diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
+ err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+ if err != nil {
+ return err
+ }
+ diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
+ return nil
+ })
var user arvados.User
- testname = "getting current user record"
- logger.Info(testname)
- err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- return 2
- } else {
- infof("%s: ok, uuid = %s", testname, user.UUID)
- }
+ diag.dotest(30, "getting current user record", func() error {
+ err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return err
+ }
+ diag.debugf("user uuid = %s", user.UUID)
+ return nil
+ })
// uncomment to create some spurious errors
// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
// TODO: detect routing errors here, like finding wb2 at the
// wb1 address.
- for _, svc := range []*arvados.Service{
+ for i, svc := range []*arvados.Service{
&cluster.Services.Keepproxy,
&cluster.Services.WebDAV,
&cluster.Services.WebDAVDownload,
@@ -112,278 +162,303 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
&cluster.Services.Workbench1,
&cluster.Services.Workbench2,
} {
- testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
- logger.Info(testname)
- u := svc.ExternalURL
- if strings.HasPrefix(u.Scheme, "ws") {
- // We can do a real websocket test elsewhere,
- // but for now we'll just check the https
- // connection.
- u.Scheme = "http" + u.Scheme[2:]
- }
- if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
- u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
- }
- req, err := http.NewRequest(http.MethodGet, u.String(), nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- continue
- }
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- errorf("%s: %s", testname, err)
- continue
- }
- resp.Body.Close()
- infof("%s: ok", testname)
+ diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
+ u := svc.ExternalURL
+ if strings.HasPrefix(u.Scheme, "ws") {
+ // We can do a real websocket test elsewhere,
+ // but for now we'll just check the https
+ // connection.
+ u.Scheme = "http" + u.Scheme[2:]
+ }
+ if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
+ u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
+ }
+ req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ resp.Body.Close()
+ return nil
+ })
}
- for _, url := range []string{
+ for i, url := range []string{
cluster.Services.Controller.ExternalURL.String(),
cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
cluster.Services.WebDAVDownload.ExternalURL.String(),
} {
- testname = fmt.Sprintf("checking CORS headers at %s", url)
- logger.Info(testname)
- req, err := http.NewRequest("GET", url, nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- continue
- }
- req.Header.Set("Origin", "https://example.com")
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- errorf("%s: %s", testname, err)
- continue
- }
- if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
- warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
- } else {
- infof("%s: ok", testname)
- }
+ diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Origin", "https://example.com")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
+ return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
+ }
+ return nil
+ })
}
var keeplist arvados.KeepServiceList
- testname = "checking internal/external client detection"
- logger.Info(testname)
- err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
- if err != nil {
- errorf("%s: error getting keep services list: %s", testname, err)
- } else if len(keeplist.Items) == 0 {
- errorf("%s: controller did not return any keep services", testname)
- } else {
+ diag.dotest(60, "checking internal/external client detection", func() error {
+ err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+ if err != nil {
+ return fmt.Errorf("error getting keep services list: %s", err)
+ } else if len(keeplist.Items) == 0 {
+ return fmt.Errorf("controller did not return any keep services")
+ }
found := map[string]int{}
for _, ks := range keeplist.Items {
found[ks.ServiceType]++
}
- infof := infof
isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
- if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
- infof = errorf
- }
if isExternal {
- infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
+ diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
} else if isInternal {
- infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
- } else {
- errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
+ diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
+ }
+ if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
+ return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
+ }
+ return nil
+ })
+
+ for i, ks := range keeplist.Items {
+ u := url.URL{
+ Scheme: "http",
+ Host: net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
+ Path: "/",
+ }
+ if ks.ServiceSSLFlag {
+ u.Scheme = "https"
}
+ diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("reading response body: %s", err)
+ }
+ loc := strings.TrimSpace(string(body))
+ if !strings.HasPrefix(loc, "d41d8") {
+ return fmt.Errorf("unexpected response from write: %q", body)
+ }
+
+ req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ body, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("reading response body: %s", err)
+ }
+ if len(body) != 0 {
+ return fmt.Errorf("unexpected response from read: %q", body)
+ }
+
+ return nil
+ })
}
var project arvados.Group
- var grplist arvados.GroupList
- testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
- logger.Info(testname)
- err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
- Filters: []arvados.Filter{
- {"name", "=", diag.projectName},
- {"group_class", "=", "project"},
- {"owner_uuid", "=", user.UUID}},
- Limit: -1})
- if err != nil {
- errorf("%s: list groups: %s", testname, err)
- } else if len(grplist.Items) < 1 {
- infof("%s: list groups: ok, no results", testname)
+ diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
+ var grplist arvados.GroupList
+ err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+ Filters: []arvados.Filter{
+ {"name", "=", diag.projectName},
+ {"group_class", "=", "project"},
+ {"owner_uuid", "=", user.UUID}},
+ Limit: -1})
+ if err != nil {
+ return fmt.Errorf("list groups: %s", err)
+ }
+ if len(grplist.Items) > 0 {
+ project = grplist.Items[0]
+ diag.debugf("using existing project, uuid = %s", project.UUID)
+ return nil
+ }
+ diag.debugf("list groups: ok, no results")
err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
"name": diag.projectName,
"group_class": "project",
}})
if err != nil {
- errorf("%s: create project: %s", testname, err)
- } else {
- infof("%s: created project, uuid = %s", testname, project.UUID)
+ return fmt.Errorf("create project: %s", err)
}
- } else {
- project = grplist.Items[0]
- infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
- }
+ diag.debugf("created project, uuid = %s", project.UUID)
+ return nil
+ })
- testname = "creating temporary collection"
- logger.Info(testname)
var collection arvados.Collection
- err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
- "ensure_unique_name": true,
- "collection": map[string]interface{}{
- "name": "test collection",
- "trash_at": time.Now().Add(time.Hour)}})
- if err != nil {
- errorf("%s: %s", testname, err)
- } else {
- infof("%s: ok, uuid = %s", testname, collection.UUID)
+ diag.dotest(90, "creating temporary collection", func() error {
+ err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ "trash_at": time.Now().Add(time.Hour)}})
+ if err != nil {
+ return err
+ }
+ diag.debugf("ok, uuid = %s", collection.UUID)
+ return nil
+ })
+
+ if collection.UUID != "" {
defer func() {
- testname := "deleting temporary collection"
- logger.Info(testname)
- err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
- if err != nil {
- errorf("%s: %s", testname, err)
- } else {
- infof("%s: ok", testname)
- }
+ diag.dotest(9990, "deleting temporary collection", func() error {
+ return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ })
}()
}
- testname = "uploading file via webdav"
- logger.Info(testname)
- func() {
+ diag.dotest(100, "uploading file via webdav", func() error {
if collection.UUID == "" {
- infof("%s: skipping, no test collection", testname)
- return
+ return fmt.Errorf("skipping, no test collection")
}
req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
if err != nil {
- errorf("%s: BUG? http.NewRequest: %s", testname, err)
- return
+ return fmt.Errorf("BUG? http.NewRequest: %s", err)
}
req.Header.Set("Authorization", "Bearer "+client.AuthToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
- errorf("%s: error performing http request: %s", testname, err)
- return
+ return fmt.Errorf("error performing http request: %s", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
- errorf("%s: status %s", testname, resp.Status)
- return
+ return fmt.Errorf("status %s", resp.Status)
}
- infof("%s: ok, status %s", testname, resp.Status)
+ diag.debugf("ok, status %s", resp.Status)
err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
if err != nil {
- errorf("%s: get updated collection: %s", testname, err)
- return
+ return fmt.Errorf("get updated collection: %s", err)
}
- infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
- }()
+ diag.debugf("ok, pdh %s", collection.PortableDataHash)
+ return nil
+ })
davurl := cluster.Services.WebDAV.ExternalURL
- testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
- logger.Info(testname)
- if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
- infof("%s: looks ok", testname)
- } else if davurl.Host == "" {
- warnf("%s: host missing - content previews will not work", testname)
- } else {
- warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
- }
+ diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
+ if davurl.Host == "" {
+ return fmt.Errorf("host missing - content previews will not work")
+ }
+ if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
+ diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
+ }
+ return nil
+ })
- for _, trial := range []struct {
- status int
- fileurl string
+ for i, trial := range []struct {
+ needcoll bool
+ status int
+ fileurl string
}{
- {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
- {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
- {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
- {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
- {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
- {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+ {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+ {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+ {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+ {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+ {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+ {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
} {
- func() {
- testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
- logger.Info(testname)
- if collection.UUID == "" {
- errorf("%s: skipping, no test collection", testname)
- return
+ diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ if trial.needcoll && collection.UUID == "" {
+ return fmt.Errorf("skipping, no test collection")
}
req, err := http.NewRequest("GET", trial.fileurl, nil)
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
req.Header.Set("Authorization", "Bearer "+client.AuthToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- errorf("%s: error reading response: %s", testname, err)
+ return fmt.Errorf("reading response: %s", err)
}
if resp.StatusCode != trial.status {
- errorf("%s: unexpected response status: %s", testname, resp.Status)
- } else if trial.status == http.StatusOK && string(body) != "testfiledata" {
- errorf("%s: unexpected response content: %q", testname, body)
- } else {
- infof("%s: ok", testname)
+ return fmt.Errorf("unexpected response status: %s", resp.Status)
}
- }()
+ if trial.status == http.StatusOK && string(body) != "testfiledata" {
+ return fmt.Errorf("unexpected response content: %q", body)
+ }
+ return nil
+ })
}
var vm arvados.VirtualMachine
- var vmlist arvados.VirtualMachineList
- testname = "getting list of virtual machines"
- logger.Info(testname)
- err = client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
- if err != nil {
- errorf("%s: %s", testname, err)
- } else if len(vmlist.Items) < 1 {
- errorf("%s: none found", testname)
- } else {
+ diag.dotest(130, "getting list of virtual machines", func() error {
+ var vmlist arvados.VirtualMachineList
+ err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+ if err != nil {
+ return err
+ }
+ if len(vmlist.Items) < 1 {
+ return fmt.Errorf("no VMs found")
+ }
vm = vmlist.Items[0]
- infof("%s: ok", testname)
- }
+ return nil
+ })
- testname = "getting workbench1 webshell page"
- logger.Info(testname)
- func() {
+ diag.dotest(140, "getting workbench1 webshell page", func() error {
if vm.UUID == "" {
- errorf("%s: skipping, no vm available", testname)
- return
+ return fmt.Errorf("skipping, no vm available")
}
webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
- debugf("%s: url %s", testname, webshelltermurl)
+ diag.debugf("url %s", webshelltermurl)
req, err := http.NewRequest("GET", webshelltermurl, nil)
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
req.Header.Set("Authorization", "Bearer "+client.AuthToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- errorf("%s: error reading response: %s", testname, err)
+ return fmt.Errorf("reading response: %s", err)
}
if resp.StatusCode != http.StatusOK {
- errorf("%s: unexpected response status: %s %q", testname, resp.Status, body)
- return
+ return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
}
- infof("%s: ok", testname)
- }()
+ return nil
+ })
- testname = "connecting to webshell service"
- logger.Info(testname)
- func() {
- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*timeout))
+ diag.dotest(150, "connecting to webshell service", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- errorf("%s: skipping, no vm available", testname)
- return
+ return fmt.Errorf("skipping, no vm available")
}
u := cluster.Services.WebShell.ExternalURL
webshellurl := u.String() + vm.Hostname + "?"
@@ -391,7 +466,7 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
u.Host = vm.Hostname + u.Host[1:]
webshellurl = u.String() + "?"
}
- debugf("%s: url %s", testname, webshellurl)
+ diag.debugf("url %s", webshellurl)
req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
"width": {"80"},
"height": {"25"},
@@ -399,31 +474,27 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
"rooturl": {webshellurl},
}.Encode()))
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
resp, err := http.DefaultClient.Do(req)
if err != nil {
- errorf("%s: %s", testname, err)
- return
+ return err
}
defer resp.Body.Close()
- debugf("%s: response status %s", testname, resp.Status)
+ diag.debugf("response status %s", resp.Status)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- errorf("%s: error reading response: %s", testname, err)
+ return fmt.Errorf("reading response: %s", err)
}
- debugf("%s: response body %q", testname, body)
+ diag.debugf("response body %q", body)
// We don't speak the protocol, so we get a 400 error
// from the webshell server even if everything is
// OK. Anything else (404, 502, ???) indicates a
// problem.
if resp.StatusCode != http.StatusBadRequest {
- errorf("%s: unexpected response status: %s, %q", testname, resp.Status, body)
- return
+ return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
}
- infof("%s: ok", testname)
- }()
- return 0
+ return nil
+ })
}
commit 5b3f3eee0352885c8daaca67f9d42f4480c761dd
Author: Tom Clegg <tom at curii.com>
Date: Tue Jun 1 11:28:54 2021 -0400
17609: Add webshell test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 26bbdfe1d..66b678a8e 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -6,11 +6,13 @@ package diagnostics
import (
"bytes"
+ "context"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
+ "net/url"
"strings"
"time"
@@ -29,6 +31,7 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
+ timeout := f.Duration("timeout", 10*time.Second, "timeout for http requests")
err := f.Parse(args)
if err == flag.ErrHelp {
return 0
@@ -37,11 +40,14 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
return 2
}
+ ctx := context.Background()
+
logger := ctxlog.New(stdout, "text", *loglevel)
logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
infof := logger.Infof
warnf := logger.Warnf
+ debugf := logger.Debugf
var errors []string
errorf := func(f string, args ...interface{}) {
logger.Errorf(f, args...)
@@ -324,5 +330,100 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo
}()
}
+ var vm arvados.VirtualMachine
+ var vmlist arvados.VirtualMachineList
+ testname = "getting list of virtual machines"
+ logger.Info(testname)
+ err = client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ } else if len(vmlist.Items) < 1 {
+ errorf("%s: none found", testname)
+ } else {
+ vm = vmlist.Items[0]
+ infof("%s: ok", testname)
+ }
+
+ testname = "getting workbench1 webshell page"
+ logger.Info(testname)
+ func() {
+ if vm.UUID == "" {
+ errorf("%s: skipping, no vm available", testname)
+ return
+ }
+ webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
+ debugf("%s: url %s", testname, webshelltermurl)
+ req, err := http.NewRequest("GET", webshelltermurl, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ errorf("%s: error reading response: %s", testname, err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ errorf("%s: unexpected response status: %s %q", testname, resp.Status, body)
+ return
+ }
+ infof("%s: ok", testname)
+ }()
+
+ testname = "connecting to webshell service"
+ logger.Info(testname)
+ func() {
+ ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*timeout))
+ defer cancel()
+ if vm.UUID == "" {
+ errorf("%s: skipping, no vm available", testname)
+ return
+ }
+ u := cluster.Services.WebShell.ExternalURL
+ webshellurl := u.String() + vm.Hostname + "?"
+ if strings.HasPrefix(u.Host, "*") {
+ u.Host = vm.Hostname + u.Host[1:]
+ webshellurl = u.String() + "?"
+ }
+ debugf("%s: url %s", testname, webshellurl)
+ req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
+ "width": {"80"},
+ "height": {"25"},
+ "session": {"xyzzy"},
+ "rooturl": {webshellurl},
+ }.Encode()))
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ defer resp.Body.Close()
+ debugf("%s: response status %s", testname, resp.Status)
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ errorf("%s: error reading response: %s", testname, err)
+ }
+ debugf("%s: response body %q", testname, body)
+ // We don't speak the protocol, so we get a 400 error
+ // from the webshell server even if everything is
+ // OK. Anything else (404, 502, ???) indicates a
+ // problem.
+ if resp.StatusCode != http.StatusBadRequest {
+ errorf("%s: unexpected response status: %s, %q", testname, resp.Status, body)
+ return
+ }
+ infof("%s: ok", testname)
+ }()
return 0
}
commit 751cdf33d5519c393323baa15233ca55b7c7a9a1
Author: Tom Clegg <tom at curii.com>
Date: Mon May 31 16:52:36 2021 -0400
17609: Add diagnostics command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index aefcce79a..cb1546211 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -11,6 +11,7 @@ import (
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/costanalyzer"
"git.arvados.org/arvados.git/lib/deduplicationreport"
+ "git.arvados.org/arvados.git/lib/diagnostics"
"git.arvados.org/arvados.git/lib/mount"
)
@@ -59,6 +60,7 @@ var (
"costanalyzer": costanalyzer.Command,
"shell": shellCommand{},
"connect-ssh": connectSSHCommand{},
+ "diagnostics": diagnostics.Command{},
})
)
diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go
index b7d918739..63d7576b4 100644
--- a/lib/cmd/cmd.go
+++ b/lib/cmd/cmd.go
@@ -16,6 +16,8 @@ import (
"runtime"
"sort"
"strings"
+
+ "github.com/sirupsen/logrus"
)
type Handler interface {
@@ -153,3 +155,9 @@ func SubcommandToFront(args []string, flagset FlagSet) []string {
copy(newargs[flagargs+1:], args[flagargs+1:])
return newargs
}
+
+type NoPrefixFormatter struct{}
+
+func (NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ return []byte(entry.Message + "\n"), nil
+}
diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go
index 9b0685225..633e95e29 100644
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@ -7,33 +7,26 @@ package costanalyzer
import (
"io"
+ "git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/sirupsen/logrus"
)
var Command command
type command struct{}
-type NoPrefixFormatter struct{}
-
-func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
- return []byte(entry.Message), nil
-}
-
// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
+ logger.SetFormatter(cmd.NoPrefixFormatter{})
defer func() {
if err != nil {
logger.Error("\n" + err.Error() + "\n")
}
}()
- logger.SetFormatter(new(NoPrefixFormatter))
-
loader := config.NewLoader(stdin, logger)
loader.SkipLegacy = true
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
new file mode 100644
index 000000000..26bbdfe1d
--- /dev/null
+++ b/lib/diagnostics/cmd.go
@@ -0,0 +1,328 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package diagnostics
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+type Command struct {
+ projectName string
+}
+
+func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ f := flag.NewFlagSet(prog, flag.ContinueOnError)
+ f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
+ loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)")
+ checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client")
+ checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client")
+ err := f.Parse(args)
+ if err == flag.ErrHelp {
+ return 0
+ } else if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+
+ logger := ctxlog.New(stdout, "text", *loglevel)
+ logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+
+ infof := logger.Infof
+ warnf := logger.Warnf
+ var errors []string
+ errorf := func(f string, args ...interface{}) {
+ logger.Errorf(f, args...)
+ errors = append(errors, fmt.Sprintf(f, args...))
+ }
+ defer func() {
+ if len(errors) == 0 {
+ logger.Info("--- no errors ---")
+ } else {
+ fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
+ for _, e := range errors {
+ logger.Error(e)
+ }
+ }
+ }()
+
+ client := arvados.NewClientFromEnv()
+
+ var dd arvados.DiscoveryDocument
+ ddpath := "discovery/v1/apis/arvados/v1/rest"
+ testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath)
+ logger.Info(testname)
+ err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ } else {
+ infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL)
+ }
+
+ var cluster arvados.Cluster
+ cfgpath := "arvados/v1/config"
+ testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath)
+ logger.Info(testname)
+ err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ } else {
+ infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning)
+ }
+
+ var user arvados.User
+ testname = "getting current user record"
+ logger.Info(testname)
+ err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return 2
+ } else {
+ infof("%s: ok, uuid = %s", testname, user.UUID)
+ }
+
+ // uncomment to create some spurious errors
+ // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
+
+ // TODO: detect routing errors here, like finding wb2 at the
+ // wb1 address.
+ for _, svc := range []*arvados.Service{
+ &cluster.Services.Keepproxy,
+ &cluster.Services.WebDAV,
+ &cluster.Services.WebDAVDownload,
+ &cluster.Services.Websocket,
+ &cluster.Services.Workbench1,
+ &cluster.Services.Workbench2,
+ } {
+ testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL)
+ logger.Info(testname)
+ u := svc.ExternalURL
+ if strings.HasPrefix(u.Scheme, "ws") {
+ // We can do a real websocket test elsewhere,
+ // but for now we'll just check the https
+ // connection.
+ u.Scheme = "http" + u.Scheme[2:]
+ }
+ if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
+ u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
+ }
+ req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ continue
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ continue
+ }
+ resp.Body.Close()
+ infof("%s: ok", testname)
+ }
+
+ for _, url := range []string{
+ cluster.Services.Controller.ExternalURL.String(),
+ cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
+ cluster.Services.WebDAVDownload.ExternalURL.String(),
+ } {
+ testname = fmt.Sprintf("checking CORS headers at %s", url)
+ logger.Info(testname)
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ continue
+ }
+ req.Header.Set("Origin", "https://example.com")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ continue
+ }
+ if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
+ warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr)
+ } else {
+ infof("%s: ok", testname)
+ }
+ }
+
+ var keeplist arvados.KeepServiceList
+ testname = "checking internal/external client detection"
+ logger.Info(testname)
+ err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+ if err != nil {
+ errorf("%s: error getting keep services list: %s", testname, err)
+ } else if len(keeplist.Items) == 0 {
+ errorf("%s: controller did not return any keep services", testname)
+ } else {
+ found := map[string]int{}
+ for _, ks := range keeplist.Items {
+ found[ks.ServiceType]++
+ }
+ infof := infof
+ isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
+ isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
+ if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) {
+ infof = errorf
+ }
+ if isExternal {
+ infof("%s: controller returned only proxy services, this host is considered \"external\"", testname)
+ } else if isInternal {
+ infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname)
+ } else {
+ errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found)
+ }
+ }
+
+ var project arvados.Group
+ var grplist arvados.GroupList
+ testname = fmt.Sprintf("finding/creating %q project", diag.projectName)
+ logger.Info(testname)
+ err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+ Filters: []arvados.Filter{
+ {"name", "=", diag.projectName},
+ {"group_class", "=", "project"},
+ {"owner_uuid", "=", user.UUID}},
+ Limit: -1})
+ if err != nil {
+ errorf("%s: list groups: %s", testname, err)
+ } else if len(grplist.Items) < 1 {
+ infof("%s: list groups: ok, no results", testname)
+ err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
+ "name": diag.projectName,
+ "group_class": "project",
+ }})
+ if err != nil {
+ errorf("%s: create project: %s", testname, err)
+ } else {
+ infof("%s: created project, uuid = %s", testname, project.UUID)
+ }
+ } else {
+ project = grplist.Items[0]
+ infof("%s: ok, using existing project, uuid = %s", testname, project.UUID)
+ }
+
+ testname = "creating temporary collection"
+ logger.Info(testname)
+ var collection arvados.Collection
+ err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "name": "test collection",
+ "trash_at": time.Now().Add(time.Hour)}})
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ } else {
+ infof("%s: ok, uuid = %s", testname, collection.UUID)
+ defer func() {
+ testname := "deleting temporary collection"
+ logger.Info(testname)
+ err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ } else {
+ infof("%s: ok", testname)
+ }
+ }()
+ }
+
+ testname = "uploading file via webdav"
+ logger.Info(testname)
+ func() {
+ if collection.UUID == "" {
+ infof("%s: skipping, no test collection", testname)
+ return
+ }
+ req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+ if err != nil {
+ errorf("%s: BUG? http.NewRequest: %s", testname, err)
+ return
+ }
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: error performing http request: %s", testname, err)
+ return
+ }
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusCreated {
+ errorf("%s: status %s", testname, resp.Status)
+ return
+ }
+ infof("%s: ok, status %s", testname, resp.Status)
+ err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ if err != nil {
+ errorf("%s: get updated collection: %s", testname, err)
+ return
+ }
+ infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash)
+ }()
+
+ davurl := cluster.Services.WebDAV.ExternalURL
+ testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl)
+ logger.Info(testname)
+ if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") {
+ infof("%s: looks ok", testname)
+ } else if davurl.Host == "" {
+ warnf("%s: host missing - content previews will not work", testname)
+ } else {
+ warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname)
+ }
+
+ for _, trial := range []struct {
+ status int
+ fileurl string
+ }{
+ {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+ {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+ {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+ {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+ {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+ {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+ } {
+ func() {
+ testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl)
+ logger.Info(testname)
+ if collection.UUID == "" {
+ errorf("%s: skipping, no test collection", testname)
+ return
+ }
+ req, err := http.NewRequest("GET", trial.fileurl, nil)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ errorf("%s: %s", testname, err)
+ return
+ }
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ errorf("%s: error reading response: %s", testname, err)
+ }
+ if resp.StatusCode != trial.status {
+ errorf("%s: unexpected response status: %s", testname, resp.Status)
+ } else if trial.status == http.StatusOK && string(body) != "testfiledata" {
+ errorf("%s: unexpected response content: %q", testname, body)
+ } else {
+ infof("%s: ok", testname)
+ }
+ }()
+ }
+
+ return 0
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list