[ARVADOS] created: 2.1.0-182-gf46eee810

[Git user]

        at  f46eee810702b655737007bdfecf91201cdb27ca (commit)


commit f46eee810702b655737007bdfecf91201cdb27ca
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Wed Dec 2 17:36:35 2020 -0500

    17009: Fix s3 ListObjects endpoint with vhost-style requests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 7fb90789a..97201d292 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -249,11 +249,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 	fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
 
 	var objectNameGiven bool
+	var bucketName string
 	fspath := "/by_id"
 	if id := parseCollectionIDFromDNSName(r.Host); id != "" {
 		fspath += "/" + id
+		bucketName = id
 		objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
 	} else {
+		bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
 		objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
 	}
 	fspath += r.URL.Path
@@ -268,7 +271,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 			fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
 		} else {
 			// ListObjects
-			h.s3list(w, r, fs)
+			h.s3list(bucketName, w, r, fs)
 		}
 		return true
 	case r.Method == http.MethodGet || r.Method == http.MethodHead:
@@ -504,15 +507,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path
 
 var errDone = errors.New("done")
 
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
 	var params struct {
-		bucket    string
 		delimiter string
 		marker    string
 		maxKeys   int
 		prefix    string
 	}
-	params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
 	params.delimiter = r.FormValue("delimiter")
 	params.marker = r.FormValue("marker")
 	if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
@@ -522,7 +523,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
 	}
 	params.prefix = r.FormValue("prefix")
 
-	bucketdir := "by_id/" + params.bucket
+	bucketdir := "by_id/" + bucket
 	// walkpath is the directory (relative to bucketdir) we need
 	// to walk: the innermost directory that is guaranteed to
 	// contain all paths that have the requested prefix. Examples:
@@ -557,7 +558,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
 	}
 	resp := listResp{
 		ListResp: s3.ListResp{
-			Name:      strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+			Name:      bucket,
 			Prefix:    params.prefix,
 			Delimiter: params.delimiter,
 			Marker:    params.marker,

commit 104427eeb7fc75ee9362307582d13f4051548bec
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Dec 3 16:24:37 2020 -0500

    17009: Test virtual host-style S3 requests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 562d5296b..a6aab357e 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
 	"os/exec"
@@ -440,6 +441,70 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 	c.Assert(fs.Sync(), check.IsNil)
 }
 
+func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
+	stage := s.s3setup(c)
+	defer stage.teardown(c)
+	for _, trial := range []struct {
+		url            string
+		method         string
+		body           string
+		responseCode   int
+		responseRegexp []string
+	}{
+		{
+			url:            "https://" + stage.collbucket.Name + ".example.com/",
+			method:         "GET",
+			responseCode:   http.StatusOK,
+			responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+		},
+		{
+			url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
+			method:         "GET",
+			responseCode:   http.StatusOK,
+			responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+		},
+		{
+			url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
+			method:         "GET",
+			responseCode:   http.StatusOK,
+			responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+		},
+		{
+			url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
+			method:         "GET",
+			responseCode:   http.StatusOK,
+			responseRegexp: []string{`⛵\n`},
+		},
+		{
+			url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+			method:       "PUT",
+			body:         "boop",
+			responseCode: http.StatusOK,
+		},
+		{
+			url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+			method:         "GET",
+			responseCode:   http.StatusOK,
+			responseRegexp: []string{`boop`},
+		},
+	} {
+		url, err := url.Parse(trial.url)
+		c.Assert(err, check.IsNil)
+		req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
+		c.Assert(err, check.IsNil)
+		req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+		rr := httptest.NewRecorder()
+		s.testServer.Server.Handler.ServeHTTP(rr, req)
+		resp := rr.Result()
+		c.Check(resp.StatusCode, check.Equals, trial.responseCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		c.Assert(err, check.IsNil)
+		for _, re := range trial.responseRegexp {
+			c.Check(string(body), check.Matches, re)
+		}
+	}
+}
+
 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 	stage := s.s3setup(c)
 	defer stage.teardown(c)

-----------------------------------------------------------------------


hooks/post-receive
--