[ARVADOS] created: 2.1.0-724-g7e9dac21d
Git user
git at public.arvados.org
Thu Apr 22 20:20:54 UTC 2021
at 7e9dac21daa4aafee44fbb789abbdd7f5fcf5177 (commit)
commit 7e9dac21daa4aafee44fbb789abbdd7f5fcf5177
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 22 16:14:38 2021 -0400
17507: Fix calling Child() without lock.
Encountered in services/keep-web tests.
START: handler_test.go:654: IntegrationSuite.TestDirectoryListingWithNoAnonymousToken
START: server_test.go:431: IntegrationSuite.SetUpTest
[...]
fatal error: concurrent map iteration and map write
goroutine 2192 [running]:
runtime.throw(0xe545ab, 0x26)
/var/lib/arvados/go/src/runtime/panic.go:1112 +0x72 fp=0xc002544c78 sp=0xc002544c48 pc=0x4366b2
runtime.mapiternext(0xc002544d18)
/var/lib/arvados/go/src/runtime/map.go:853 +0x552 fp=0xc002544cf8 sp=0xc002544c78 pc=0x411442
git.arvados.org/arvados.git/sdk/go/arvados.(*treenode).MemorySize(0xc00202e480, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_base.go:336 +0x106 fp=0xc002544d98 sp=0xc002544cf8 pc=0x8c7de6
git.arvados.org/arvados.git/sdk/go/arvados.(*treenode).MemorySize(0xc0003a66c0, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_base.go:337 +0xe3 fp=0xc002544e38 sp=0xc002544d98 pc=0x8c7dc3
git.arvados.org/arvados.git/sdk/go/arvados.(*treenode).MemorySize(0xc0020c3680, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_base.go:337 +0xe3 fp=0xc002544ed8 sp=0xc002544e38 pc=0x8c7dc3
git.arvados.org/arvados.git/sdk/go/arvados.(*fileSystem).MemorySize(0xc001af0000, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_base.go:631 +0x33 fp=0xc002544ef8 sp=0xc002544ed8 pc=0x8ca1e3
git.arvados.org/arvados.git/services/keep-web.(*cache).collectionBytes(0xc00261da08, 0x3ff0000000000101)
/home/tom/arvados/services/keep-web/cache.go:448 +0x21c fp=0xc002544f70 sp=0xc002544ef8 pc=0xc2903c
git.arvados.org/arvados.git/services/keep-web.(*cache).updateGauges(0xc00261da08)
/home/tom/arvados/services/keep-web/cache.go:170 +0x2f fp=0xc002544f90 sp=0xc002544f70 pc=0xc269bf
git.arvados.org/arvados.git/services/keep-web.(*cache).setup.func1(0xc00261da08)
/home/tom/arvados/services/keep-web/cache.go:164 +0x6d fp=0xc002544fd8 sp=0xc002544f90 pc=0xc65dad
runtime.goexit()
/var/lib/arvados/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc002544fe0 sp=0xc002544fd8 pc=0x468dd1
created by git.arvados.org/arvados.git/services/keep-web.(*cache).setup
/home/tom/arvados/services/keep-web/cache.go:162 +0x1db
goroutine 2144 [runnable]:
git.arvados.org/arvados.git/sdk/go/arvados.(*treenode).Child(0xc00202e480, 0xc002c4e117, 0x9, 0xc0005c2070, 0x10c9680, 0xc0021d0240, 0x0, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_base.go:289 +0x171
git.arvados.org/arvados.git/sdk/go/arvados.(*lookupnode).Readdir(0xc00202e480, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_lookup.go:53 +0x27b
git.arvados.org/arvados.git/sdk/go/arvados.(*filehandle).Readdir(0xc00263c140, 0x0, 0xc002066360, 0xc00263c140, 0x10c4f40, 0xc00263c140, 0x0)
/home/tom/arvados/sdk/go/arvados/fs_filehandle.go:81 +0x1d2
golang.org/x/net/webdav.walkFS(0x10bbe00, 0xc002b09590, 0x10bd900, 0xc002af1e60, 0xffffffffffffffff, 0xc00161a574, 0xc, 0x10c17c0, 0xc00227c280, 0xc0005c2628, ...)
/home/tom/arvados/tmp/GOPATH/pkg/mod/golang.org/x/net at v0.0.0-20200202094626-16171245cfb2/webdav/file.go:772 +0x25e
golang.org/x/net/webdav.walkFS(0x10bbe00, 0xc002b09590, 0x10bd900, 0xc002af1e60, 0xffffffffffffffff, 0xe2f941, 0x5, 0x10c17c0, 0xc002b1a4c0, 0xc0005c2628, ...)
/home/tom/arvados/tmp/GOPATH/pkg/mod/golang.org/x/net at v0.0.0-20200202094626-16171245cfb2/webdav/file.go:786 +0x564
golang.org/x/net/webdav.walkFS(0x10bbe00, 0xc002b09590, 0x10bd900, 0xc002af1e60, 0xffffffffffffffff, 0xc00216be76, 0x0, 0x10c17c0, 0xc002b1a0c0, 0xc0005c2628, ...)
/home/tom/arvados/tmp/GOPATH/pkg/mod/golang.org/x/net at v0.0.0-20200202094626-16171245cfb2/webdav/file.go:786 +0x564
golang.org/x/net/webdav.(*Handler).handlePropfind(0xc002b1a080, 0x7f1c582d9120, 0xc0029e5400, 0xc002b04900, 0xc0022bc718, 0x415ad3, 0xc002b1a080)
/home/tom/arvados/tmp/GOPATH/pkg/mod/golang.org/x/net at v0.0.0-20200202094626-16171245cfb2/webdav/webdav.go:566 +0x3d7
golang.org/x/net/webdav.(*Handler).ServeHTTP(0xc002b1a080, 0x7f1c582d9120, 0xc0029e5400, 0xc002b04900)
/home/tom/arvados/tmp/GOPATH/pkg/mod/golang.org/x/net at v0.0.0-20200202094626-16171245cfb2/webdav/webdav.go:67 +0x556
git.arvados.org/arvados.git/services/keep-web.(*handler).serveSiteFS(0xc002221ae0, 0x7f1c582d9120, 0xc0029e5400, 0xc002b04900, 0xc002b22010, 0x1, 0x1, 0x101)
/home/tom/arvados/services/keep-web/handler.go:593 +0x80b
git.arvados.org/arvados.git/services/keep-web.(*handler).ServeHTTP(0xc002221ae0, 0x10b6ec0, 0xc002b094a0, 0xc002b04900)
/home/tom/arvados/services/keep-web/handler.go:330 +0x2a0b
git.arvados.org/arvados.git/sdk/go/httpserver.LogRequests.func1(0x7f1c58319c58, 0xc002b09470, 0xc002b04800)
/home/tom/arvados/sdk/go/httpserver/logger.go:56 +0x8d8
net/http.HandlerFunc.ServeHTTP(0xc002282460, 0x7f1c58319c58, 0xc002b09470, 0xc002b04800)
/var/lib/arvados/go/src/net/http/server.go:2012 +0x44
git.arvados.org/arvados.git/sdk/go/httpserver.AddRequestIDs.func1(0x7f1c58319c58, 0xc002b09470, 0xc002b04800)
/home/tom/arvados/sdk/go/httpserver/id_generator.go:57 +0x1a5
net/http.HandlerFunc.ServeHTTP(0xc002282480, 0x7f1c58319c58, 0xc002b09470, 0xc002b04800)
/var/lib/arvados/go/src/net/http/server.go:2012 +0x44
git.arvados.org/arvados.git/sdk/go/httpserver.HandlerWithContext.func1(0x7f1c58319c58, 0xc002b09470, 0xc002b04700)
/home/tom/arvados/sdk/go/httpserver/logger.go:30 +0x107
net/http.HandlerFunc.ServeHTTP(0xc002901b30, 0x7f1c58319c58, 0xc002b09470, 0xc002b04700)
/var/lib/arvados/go/src/net/http/server.go:2012 +0x44
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func1(0x10b8bc0, 0xc002b1a000, 0xc002b04700)
/home/tom/arvados/tmp/GOPATH/pkg/mod/github.com/prometheus/client_golang at v1.2.1/prometheus/promhttp/instrument_server.go:68 +0x11c
net/http.HandlerFunc.ServeHTTP(0xc002901e00, 0x10b8bc0, 0xc002b1a000, 0xc002b04700)
/var/lib/arvados/go/src/net/http/server.go:2012 +0x44
git.arvados.org/arvados.git/sdk/go/httpserver.(*metrics).ServeHTTP(0xc002113e00, 0x10b8bc0, 0xc002b1a000, 0xc002b04700)
/home/tom/arvados/sdk/go/httpserver/metrics.go:70 +0x51
git.arvados.org/arvados.git/services/keep-web.(*IntegrationSuite).testDirectoryListing(0xc000010138, 0xc0007d41e0)
/home/tom/arvados/services/keep-web/handler_test.go:864 +0x1a07
git.arvados.org/arvados.git/services/keep-web.(*IntegrationSuite).TestDirectoryListingWithNoAnonymousToken(0xc000010138, 0xc0007d41e0)
/home/tom/arvados/services/keep-web/handler_test.go:656 +0x67
reflect.Value.call(0xe2bbc0, 0xc000010138, 0x4613, 0xe2deb6, 0x4, 0xc000282f08, 0x1, 0x1, 0x171d800, 0xc000282e48, ...)
/var/lib/arvados/go/src/reflect/value.go:460 +0x8ab
reflect.Value.Call(0xe2bbc0, 0xc000010138, 0x4613, 0xc000282f08, 0x1, 0x1, 0xc0007d42d0, 0xc000128060, 0xc00287a720)
/var/lib/arvados/go/src/reflect/value.go:321 +0xb4
gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1(0xc0007d41e0)
/home/tom/arvados/tmp/GOPATH/pkg/mod/gopkg.in/check.v1 at v1.0.0-20161208181325-20d25e280405/check.go:772 +0x628
gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1(0xc000174c00, 0xc0007d41e0, 0xc00222b760)
/home/tom/arvados/tmp/GOPATH/pkg/mod/gopkg.in/check.v1 at v1.0.0-20161208181325-20d25e280405/check.go:666 +0x98
created by gopkg.in/check%2ev1.(*suiteRunner).forkCall
/home/tom/arvados/tmp/GOPATH/pkg/mod/gopkg.in/check.v1 at v1.0.0-20161208181325-20d25e280405/check.go:663 +0x1fb
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs_lookup.go
index 56b595323..021e8241c 100644
--- a/sdk/go/arvados/fs_lookup.go
+++ b/sdk/go/arvados/fs_lookup.go
@@ -50,9 +50,11 @@ func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
return nil, err
}
for _, child := range all {
+ ln.treenode.Lock()
_, err = ln.treenode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
return child, nil
})
+ ln.treenode.Unlock()
if err != nil {
return nil, err
}
commit bb610b46d803536da941c6d11cd1ebf92afb5c25
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 22 15:12:17 2021 -0400
17507: Support ListObjectsV2 API.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 620a21b88..0ef0e2dc4 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -7,6 +7,7 @@ package main
import (
"crypto/hmac"
"crypto/sha256"
+ "encoding/base64"
"encoding/xml"
"errors"
"fmt"
@@ -33,6 +34,42 @@ const (
s3MaxClockSkew = 5 * time.Minute
)
+type commonPrefix struct {
+ Prefix string
+}
+
+type listV1Resp struct {
+ XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+ s3.ListResp
+ // s3.ListResp marshals an empty tag when
+ // CommonPrefixes is nil, which confuses some clients.
+ // Fix by using this nested struct instead.
+ CommonPrefixes []commonPrefix
+ // Similarly, we need omitempty here, because an empty
+ // tag confuses some clients (e.g.,
+ // github.com/aws/aws-sdk-net never terminates its
+ // paging loop).
+ NextMarker string `xml:"NextMarker,omitempty"`
+ // ListObjectsV2 has a KeyCount response field.
+ KeyCount int
+}
+
+type listV2Resp struct {
+ XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+ IsTruncated bool
+ Contents []s3.Key
+ Name string
+ Prefix string
+ Delimiter string
+ MaxKeys int
+ CommonPrefixes []commonPrefix
+ EncodingType string `xml:",omitempty"`
+ KeyCount int
+ ContinuationToken string `xml:",omitempty"`
+ NextContinuationToken string `xml:",omitempty"`
+ StartAfter string `xml:",omitempty"`
+}
+
func hmacstring(msg string, key []byte) []byte {
h := hmac.New(sha256.New, key)
io.WriteString(h, msg)
@@ -559,19 +596,50 @@ var errDone = errors.New("done")
func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
var params struct {
- delimiter string
- marker string
- maxKeys int
- prefix string
+ v2 bool
+ delimiter string
+ maxKeys int
+ prefix string
+ marker string // decoded continuationToken (v2) or provided by client (v1)
+ startAfter string // v2
+ continuationToken string // v2
+ encodingTypeURL bool // v2
}
params.delimiter = r.FormValue("delimiter")
- params.marker = r.FormValue("marker")
if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
params.maxKeys = int(mk)
} else {
params.maxKeys = s3MaxKeys
}
params.prefix = r.FormValue("prefix")
+ switch r.FormValue("list-type") {
+ case "":
+ case "2":
+ params.v2 = true
+ default:
+ http.Error(w, "invalid list-type parameter", http.StatusBadRequest)
+ return
+ }
+ if params.v2 {
+ params.continuationToken = r.FormValue("continuation-token")
+ marker, err := base64.StdEncoding.DecodeString(params.continuationToken)
+ if err != nil {
+ http.Error(w, "invalid continuation token", http.StatusBadRequest)
+ return
+ }
+ params.marker = string(marker)
+ params.startAfter = r.FormValue("start-after")
+ switch r.FormValue("encoding-type") {
+ case "":
+ case "url":
+ params.encodingTypeURL = true
+ default:
+ http.Error(w, "invalid encoding-type parameter", http.StatusBadRequest)
+ return
+ }
+ } else {
+ params.marker = r.FormValue("marker")
+ }
bucketdir := "by_id/" + bucket
// walkpath is the directory (relative to bucketdir) we need
@@ -588,33 +656,16 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request,
walkpath = ""
}
- type commonPrefix struct {
- Prefix string
- }
- type listResp struct {
- XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
- s3.ListResp
- // s3.ListResp marshals an empty tag when
- // CommonPrefixes is nil, which confuses some clients.
- // Fix by using this nested struct instead.
- CommonPrefixes []commonPrefix
- // Similarly, we need omitempty here, because an empty
- // tag confuses some clients (e.g.,
- // github.com/aws/aws-sdk-net never terminates its
- // paging loop).
- NextMarker string `xml:"NextMarker,omitempty"`
- // ListObjectsV2 has a KeyCount response field.
- KeyCount int
- }
- resp := listResp{
- ListResp: s3.ListResp{
- Name: bucket,
- Prefix: params.prefix,
- Delimiter: params.delimiter,
- Marker: params.marker,
- MaxKeys: params.maxKeys,
- },
- }
+ resp := listV2Resp{
+ Name: bucket,
+ Prefix: params.prefix,
+ Delimiter: params.delimiter,
+ MaxKeys: params.maxKeys,
+ ContinuationToken: r.FormValue("continuation-token"),
+ StartAfter: params.startAfter,
+ }
+ nextMarker := ""
+
commonPrefixes := map[string]bool{}
err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
if path == bucketdir {
@@ -654,7 +705,7 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request,
return errDone
}
}
- if path < params.marker || path < params.prefix {
+ if path < params.marker || path < params.prefix || path <= params.startAfter {
return nil
}
if fi.IsDir() && !h.Config.cluster.Collections.S3FolderObjects {
@@ -678,8 +729,8 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request,
}
if len(resp.Contents)+len(commonPrefixes) >= params.maxKeys {
resp.IsTruncated = true
- if params.delimiter != "" {
- resp.NextMarker = path
+ if params.delimiter != "" || params.v2 {
+ nextMarker = path
}
return errDone
}
@@ -702,9 +753,56 @@ func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request,
sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
}
resp.KeyCount = len(resp.Contents)
+ var respV1orV2 interface{}
+
+ if params.v2 {
+ if params.encodingTypeURL {
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+ // "If you specify the encoding-type request
+ // parameter, Amazon S3 includes this element
+ // in the response, and returns encoded key
+ // name values in the following response
+ // elements:
+ //
+ // Delimiter, Prefix, Key, and StartAfter.
+ //
+ // Type: String
+ //
+ // Valid Values: url"
+ //
+ // We take this to mean "percent-encoded" as
+ // defined in RFC3986 (URI: Generic Syntax).
+ resp.EncodingType = "url"
+ resp.Delimiter = url.PathEscape(resp.Delimiter)
+ resp.Prefix = url.PathEscape(resp.Prefix)
+ resp.StartAfter = url.PathEscape(resp.StartAfter)
+ for i, ent := range resp.Contents {
+ ent.Key = url.PathEscape(ent.Key)
+ resp.Contents[i] = ent
+ }
+ }
+ resp.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextMarker))
+ respV1orV2 = resp
+ } else {
+ respV1orV2 = listV1Resp{
+ CommonPrefixes: resp.CommonPrefixes,
+ NextMarker: nextMarker,
+ KeyCount: resp.KeyCount,
+ ListResp: s3.ListResp{
+ IsTruncated: resp.IsTruncated,
+ Name: bucket,
+ Prefix: params.prefix,
+ Delimiter: params.delimiter,
+ Marker: params.marker,
+ MaxKeys: params.maxKeys,
+ Contents: resp.Contents,
+ },
+ }
+ }
+
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
- if err := xml.NewEncoder(w).Encode(resp); err != nil {
+ if err := xml.NewEncoder(w).Encode(respV1orV2); err != nil {
ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")
}
}
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index e60b55c93..5e582fc1d 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -8,6 +8,7 @@ import (
"bytes"
"crypto/rand"
"crypto/sha256"
+ "encoding/xml"
"fmt"
"io/ioutil"
"net/http"
@@ -886,6 +887,127 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
}
}
+func (s *IntegrationSuite) TestS3ListObjectsV2(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ dirs := 2
+ filesPerDir := 40
+ stage.writeBigDirs(c, dirs, filesPerDir)
+
+ for _, trial := range []struct {
+ prefix string
+ delimiter string
+ startAfter string
+ maxKeys int
+ expectKeys int
+ expectCommonPrefixes map[string]bool
+ }{
+ {
+ maxKeys: 15,
+ // Expect {filesPerDir plus the dir itself}
+ // for each dir, plus emptydir, emptyfile, and
+ // sailboat.txt.
+ expectKeys: (filesPerDir+1)*dirs + 3,
+ },
+ {
+ startAfter: "dir0/z",
+ maxKeys: 15,
+ // Expect {filesPerDir plus the dir itself}
+ // for each dir except dir0, plus emptydir,
+ // emptyfile, and sailboat.txt.
+ expectKeys: (filesPerDir+1)*(dirs-1) + 3,
+ },
+ {
+ maxKeys: 1,
+ delimiter: "/",
+ expectKeys: 2, // emptyfile, sailboat.txt
+ expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
+ },
+ {
+ startAfter: "dir0/z",
+ maxKeys: 15,
+ delimiter: "/",
+ expectKeys: 2, // emptyfile, sailboat.txt
+ expectCommonPrefixes: map[string]bool{"dir1/": true, "emptydir/": true},
+ },
+ {
+ startAfter: "dir0/file10.txt",
+ maxKeys: 15,
+ delimiter: "/",
+ expectKeys: 2,
+ expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true, "emptydir/": true},
+ },
+ {
+ startAfter: "dir0/file10.txt",
+ maxKeys: 15,
+ prefix: "d",
+ delimiter: "/",
+ expectKeys: 0,
+ expectCommonPrefixes: map[string]bool{"dir0/": true, "dir1/": true},
+ },
+ } {
+ c.Logf("[trial %+v]", trial)
+ params := url.Values{
+ "list-type": {"2"},
+ "prefix": {trial.prefix},
+ "delimiter": {trial.delimiter},
+ "start-after": {trial.startAfter},
+ "max-keys": {fmt.Sprintf("%d", trial.maxKeys)},
+ }
+ keySeen := map[string]bool{}
+ prefixSeen := map[string]bool{}
+ for {
+ req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = params.Encode()
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ var listResp listV2Resp
+ err = xml.Unmarshal(buf, &listResp)
+ c.Assert(err, check.IsNil)
+ c.Check(listResp.Name, check.Equals, stage.collbucket.Name)
+ c.Check(listResp.MaxKeys, check.Equals, trial.maxKeys)
+ c.Check(listResp.Prefix, check.Equals, trial.prefix)
+ c.Check(listResp.Delimiter, check.Equals, trial.delimiter)
+ c.Check(listResp.StartAfter, check.Equals, trial.startAfter)
+ c.Check(listResp.ContinuationToken, check.Equals, params.Get("continuation-token"))
+ c.Check(len(listResp.Contents) <= trial.maxKeys, check.Equals, true)
+ for _, ent := range listResp.Contents {
+ c.Check(ent.Key > trial.startAfter, check.Equals, true)
+ c.Check(keySeen[ent.Key], check.Equals, false, check.Commentf("dup key %q", ent.Key))
+ keySeen[ent.Key] = true
+ }
+ for _, ent := range listResp.CommonPrefixes {
+ c.Check(strings.HasSuffix(ent.Prefix, trial.delimiter), check.Equals, true, check.Commentf("bad CommonPrefix %q", ent.Prefix))
+ if strings.HasPrefix(trial.startAfter, ent.Prefix) {
+ // If we asked for
+ // startAfter=dir0/file10.txt,
+ // we expect dir0/ to be
+ // returned as a common prefix
+ } else {
+ c.Check(ent.Prefix > trial.startAfter, check.Equals, true)
+ }
+ c.Check(prefixSeen[ent.Prefix], check.Equals, false, check.Commentf("dup common prefix %q", ent.Prefix))
+ prefixSeen[ent.Prefix] = true
+ }
+ if listResp.IsTruncated {
+ c.Assert(listResp.NextContinuationToken, check.Not(check.Equals), "")
+ params["continuation-token"] = []string{listResp.NextContinuationToken}
+ } else {
+ break
+ }
+ }
+ c.Check(keySeen, check.HasLen, trial.expectKeys)
+ c.Check(prefixSeen, check.HasLen, len(trial.expectCommonPrefixes))
+ if len(trial.expectCommonPrefixes) > 0 {
+ c.Check(prefixSeen, check.DeepEquals, trial.expectCommonPrefixes)
+ }
+ }
+}
+
// TestS3cmd checks compatibility with the s3cmd command line tool, if
// it's installed. As of Debian buster, s3cmd is only in backports, so
// `arvados-server install` don't install it, and this test skips if
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list