[ARVADOS] created: 2.1.0-97-g70be08860

[Git user]

        at  70be08860db9e45d78a037d86b9a0420f1e392a1 (commit)


commit 70be08860db9e45d78a037d86b9a0420f1e392a1
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Mon Nov 16 21:25:12 2020 -0500

    17106: Test S3 with modified v2 token issued by LoginCluster.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 3da01ca68..1f28c877a 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
@@ -15,7 +16,10 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 
 	"git.arvados.org/arvados.git/lib/boot"
 	"git.arvados.org/arvados.git/lib/config"
@@ -280,6 +284,83 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
 	c.Check(coll.PortableDataHash, check.Equals, pdh)
 }
 
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+	testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+	conn1 := s.conn("z1111")
+	rootctx1, _, _ := s.rootClients("z1111")
+	userctx1, ac1, kc1, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+	conn3 := s.conn("z3333")
+	_, ac3, kc3 := s.clientsWithToken("z3333", ac1.AuthToken)
+
+	// Create a collection on z1111
+	var coll arvados.Collection
+	fs1, err := coll.FileSystem(ac1, kc1)
+	c.Assert(err, check.IsNil)
+	f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+	c.Assert(err, check.IsNil)
+	_, err = io.WriteString(f, testText)
+	c.Assert(err, check.IsNil)
+	err = f.Close()
+	c.Assert(err, check.IsNil)
+	mtxt, err := fs1.MarshalManifest(".")
+	c.Assert(err, check.IsNil)
+	coll1, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+		"manifest_text": mtxt,
+	}})
+	c.Assert(err, check.IsNil)
+
+	// Create same collection on z3333
+	fs3, err := coll.FileSystem(ac3, kc3)
+	c.Assert(err, check.IsNil)
+	f, err = fs3.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+	c.Assert(err, check.IsNil)
+	_, err = io.WriteString(f, testText)
+	c.Assert(err, check.IsNil)
+	err = f.Close()
+	c.Assert(err, check.IsNil)
+	mtxt, err = fs3.MarshalManifest(".")
+	c.Assert(err, check.IsNil)
+	coll3, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+		"manifest_text": mtxt,
+	}})
+	c.Assert(err, check.IsNil)
+
+	for _, trial := range []struct {
+		label string
+		conn  *rpc.Conn
+		coll  arvados.Collection
+	}{
+		{"z1111", conn1, coll1},
+		{"z3333", conn3, coll3},
+	} {
+		c.Logf("================ %s", trial.label)
+		cfgjson, err := trial.conn.ConfigGet(userctx1)
+		c.Assert(err, check.IsNil)
+		var cluster arvados.Cluster
+		err = json.Unmarshal(cfgjson, &cluster)
+		c.Assert(err, check.IsNil)
+
+		c.Logf("TokenV2 is %s", ac1.AuthToken)
+		mungedtoken := strings.Replace(ac1.AuthToken, "/", "_", -1)
+		host := cluster.Services.WebDAV.ExternalURL.Host
+		s3args := []string{
+			"--ssl", "--no-check-certificate",
+			"--host=" + host, "--host-bucket=" + host,
+			"--access_key=" + mungedtoken, "--secret_key=" + mungedtoken,
+		}
+		buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+trial.coll.UUID)...).CombinedOutput()
+		c.Check(err, check.IsNil)
+		c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+trial.coll.UUID+`/test.txt\n`)
+
+		buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+trial.coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+		// Command fails because we don't return Etag header.
+		// c.Check(err, check.IsNil)
+		flen := strconv.Itoa(len(testText))
+		c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
+	}
+}
+
 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
 	conn1 := s.conn("z1111")
 	conn3 := s.conn("z3333")

commit 0691937dd5e8e01c3f7db521aaae420eb23060ae
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Mon Nov 16 20:37:05 2020 -0500

    17106: Accept v2 token with / replaced by _ as s3 access/secret key.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 49fb2456f..ef3a16404 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -152,7 +152,14 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
 	} else {
 		// Access key and secret key are both an entire
 		// Arvados token or OIDC access token.
-		ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+		mungedKey := key
+		if strings.HasPrefix(key, "v2_") {
+			// Entire Arvados token, with "/" replaced by
+			// "_" to avoid colliding with the
+			// Authorization header format.
+			mungedKey = strings.Replace(key, "_", "/", -1)
+		}
+		ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+mungedKey)
 		err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
 		secret = key
 	}
@@ -170,7 +177,7 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
 	} else if expect != signature {
 		return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
 	}
-	return secret, nil
+	return aca.TokenV2(), nil
 }
 
 // serveS3 handles r and returns true if r is a request from an S3

-----------------------------------------------------------------------


hooks/post-receive
--