[ARVADOS] created: 2.1.0-115-gbaeef76a2

[Git user]

        at  baeef76a2b3b60fb3613d01b1df2916397e8c589 (commit)


commit baeef76a2b3b60fb3613d01b1df2916397e8c589
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Wed Nov 18 17:35:29 2020 -0500

    17009: Support accessing S3 with virtual hosted-style URLs.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/doc/api/keep-s3.html.textile.liquid b/doc/api/keep-s3.html.textile.liquid
index 2cae81761..d5ad1dc60 100644
--- a/doc/api/keep-s3.html.textile.liquid
+++ b/doc/api/keep-s3.html.textile.liquid
@@ -21,7 +21,11 @@ To access Arvados S3 using an S3 client library, you must tell it to use the URL
 
 The "bucket name" is an Arvados collection uuid, portable data hash, or project uuid.
 
-The bucket name must be encoded as the first path segment of every request.  This is what the S3 documentation calls "Path-Style Requests".
+Path-style and virtual host-style requests are supported.
+* A path-style request uses the hostname indicated by @Services.WebDAVDownload.ExternalURL@, with the bucket name in the first path segment: @https://download.example.com/zzzzz-4zz18-asdfgasdfgasdfg/@.
+* A virtual host-style request uses the hostname pattern indicated by @Services.WebDAV.ExternalURL@, with a bucket name in place of the leading @*@: @https://zzzzz-4zz18-asdfgasdfgasdfg.collections.example.com/@.
+
+If you have wildcard DNS, TLS, and routing set up, an S3 client configured with endpoint @collections.example.com@ should work regardless of which request style it uses.
 
 h3. Supported Operations
 
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 49fb2456f..57c9d7efb 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -205,7 +205,15 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 	fs := client.SiteFileSystem(kc)
 	fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
 
-	objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+	var objectNameGiven bool
+	fspath := "/by_id"
+	if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+		fspath += "/" + id
+		objectNameGiven = true
+	} else {
+		objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+	}
+	fspath += r.URL.Path
 
 	switch {
 	case r.Method == http.MethodGet && !objectNameGiven:
@@ -221,7 +229,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case r.Method == http.MethodGet || r.Method == http.MethodHead:
-		fspath := "/by_id" + r.URL.Path
 		fi, err := fs.Stat(fspath)
 		if r.Method == "HEAD" && !objectNameGiven {
 			// HeadBucket
@@ -255,7 +262,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 			http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
 			return true
 		}
-		fspath := "by_id" + r.URL.Path
 		var objectIsDir bool
 		if strings.HasSuffix(fspath, "/") {
 			if !h.Config.cluster.Collections.S3FolderObjects {
@@ -350,7 +356,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
 			http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
 			return true
 		}
-		fspath := "by_id" + r.URL.Path
 		if strings.HasSuffix(fspath, "/") {
 			fspath = strings.TrimSuffix(fspath, "/")
 			fi, err := fs.Stat(fspath)
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 786e68afe..f8dc60086 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -700,3 +700,12 @@ func (s *IntegrationSuite) TestS3cmd(c *check.C) {
 	c.Check(err, check.IsNil)
 	c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
 }
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+	stage := s.s3setup(c)
+	defer stage.teardown(c)
+
+	hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+	c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+	c.Check(body, check.Equals, "⛵\n")
+}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index acdc11b30..43817b51f 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -257,12 +257,16 @@ func (s *IntegrationSuite) Test200(c *check.C) {
 }
 
 // Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 	curlArgs := []string{"--silent", "--show-error", "--include"}
 	testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 	curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
-	if token != "" {
-		curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+	if strings.Contains(auth, " ") {
+		// caller supplied entire Authorization header value
+		curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+	} else if auth != "" {
+		// caller supplied Arvados token
+		curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 	}
 	curlArgs = append(curlArgs, args...)
 	curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)

-----------------------------------------------------------------------


hooks/post-receive
--