[ARVADOS] updated: 2.1.0-108-g78096170b
Git user
git at public.arvados.org
Mon Feb 22 19:09:20 UTC 2021
Summary of changes:
doc/install/install-api-server.html.textile.liquid | 12 ++-
lib/config/config.default.yml | 2 +
lib/config/generated_config.go | 2 +
lib/controller/integration_test.go | 77 +++++++++++++++++
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/python/arvados/keep.py | 6 +-
sdk/ruby/arvados.gemspec | 2 +-
services/keep-web/handler.go | 44 +++++-----
services/keep-web/handler_test.go | 19 +++++
services/keep-web/s3.go | 9 +-
services/keep-web/s3_test.go | 97 +++++++++++++++++++---
services/login-sync/arvados-login-sync.gemspec | 2 +-
12 files changed, 231 insertions(+), 43 deletions(-)
via 78096170b070a9eb17b37f913798397744fa1ff5 (commit)
via 3c4a9ebba460acf4bc6a29786ef9be6965a5ead3 (commit)
via ec67645272eecd27cedd04d7a79062d5d8f02f98 (commit)
via ea3f1b8246c27a6a44edfc561f13935ef377c1cb (commit)
via d4892015f8a24ba3e81bdcce7cd238afeb2d4ecf (commit)
via 276b0fa42a1d4bd766de2124a9f16799a3b65bc5 (commit)
via 43a2d6a0bc25d635b61aa3366ecdd53d0fda3bec (commit)
via 67e9e5cd1a09d3af3c2d11210d92d32d6b6609c1 (commit)
via fa2df0a62a589f33f1cfa1a193598875d941596e (commit)
from 1e818c0bd916e13b900a708da896bb00d4cb1f07 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 78096170b070a9eb17b37f913798397744fa1ff5
Author: Ward Vandewege <ward at curii.com>
Date: Fri Oct 23 13:01:55 2020 -0400
Fix author e-mail addresses in our gemspecs.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/cli/arvados-cli.gemspec b/sdk/cli/arvados-cli.gemspec
index 4096a2eb1..319703198 100644
--- a/sdk/cli/arvados-cli.gemspec
+++ b/sdk/cli/arvados-cli.gemspec
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
s.summary = "Arvados CLI tools"
s.description = "Arvados command line tools, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev at arvados.org'
+ s.email = 'packaging at arvados.org'
#s.bindir = '.'
s.licenses = ['Apache-2.0']
s.files = ["bin/arv", "bin/arv-tag", "LICENSE-2.0.txt"]
diff --git a/sdk/ruby/arvados.gemspec b/sdk/ruby/arvados.gemspec
index 019e156a5..0c2878bad 100644
--- a/sdk/ruby/arvados.gemspec
+++ b/sdk/ruby/arvados.gemspec
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
s.summary = "Arvados client library"
s.description = "Arvados client library, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev at curoverse.com'
+ s.email = 'packaging at arvados.org'
s.licenses = ['Apache-2.0']
s.files = ["lib/arvados.rb", "lib/arvados/google_api_client.rb",
"lib/arvados/collection.rb", "lib/arvados/keep.rb",
diff --git a/services/login-sync/arvados-login-sync.gemspec b/services/login-sync/arvados-login-sync.gemspec
index b45f8692b..6d782ca0e 100644
--- a/services/login-sync/arvados-login-sync.gemspec
+++ b/services/login-sync/arvados-login-sync.gemspec
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
s.summary = "Set up local login accounts for Arvados users"
s.description = "Creates and updates local login accounts for Arvados users. Built from git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev at curoverse.com'
+ s.email = 'packaging at arvados.org'
s.licenses = ['AGPL-3.0']
s.files = ["bin/arvados-login-sync", "agpl-3.0.txt"]
s.executables << "arvados-login-sync"
commit 3c4a9ebba460acf4bc6a29786ef9be6965a5ead3
Author: Tom Clegg <tom at curii.com>
Date: Wed Dec 9 15:44:57 2020 -0500
17202: Test avoiding redirect for cross-origin inline images.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 8e2e05c76..5291efeb8 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -583,6 +583,25 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+
+ // GET + Origin header is representative of both AJAX GET
+ // requests and inline images via <IMG crossorigin="anonymous"
+ // src="...">.
+ u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://origin.example"},
+ },
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
commit ec67645272eecd27cedd04d7a79062d5d8f02f98
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 9 09:34:14 2020 -0500
17202: Use explicit SameSite=Lax for 303-with-cookie.
This improves XSS protection on some browsers, including Safari and
Firefox for Android.
On most browsers, Lax is already the default.
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 8e4274038..2d6fb78f8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -773,6 +773,7 @@ func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, loc
Value: auth.EncodeTokenCookie([]byte(formToken)),
Path: "/",
HttpOnly: true,
+ SameSite: http.SameSiteLaxMode,
})
}
commit ea3f1b8246c27a6a44edfc561f13935ef377c1cb
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Dec 8 19:52:36 2020 -0500
17202: Bypass 303-with-token on cross-origin requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index ab1bc080b..8e4274038 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -296,27 +296,32 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
formToken := r.FormValue("api_token")
- if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
- // The client provided an explicit token in the POST
- // body. The Origin header indicates this *might* be
- // an AJAX request, in which case redirect-with-cookie
- // won't work: we should just serve the content in the
- // POST response. This is safe because:
+ origin := r.Header.Get("Origin")
+ cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
+ safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
+ safeAttachment := attachment && r.URL.Query().Get("api_token") == ""
+ if formToken == "" {
+ // No token to use or redact.
+ } else if safeAjax || safeAttachment {
+ // If this is a cross-origin request, the URL won't
+ // appear in the browser's address bar, so
+ // substituting a clipboard-safe URL is pointless.
+ // Redirect-with-cookie wouldn't work anyway, because
+ // it's not safe to allow third-party use of our
+ // cookie.
//
- // * We're supplying an attachment, not inline
- // content, so we don't need to convert the POST to
- // a GET and avoid the "really resubmit form?"
- // problem.
- //
- // * The token isn't embedded in the URL, so we don't
- // need to worry about bookmarks and copy/paste.
+ // If we're supplying an attachment, we don't need to
+ // convert POST to GET to avoid the "really resubmit
+ // form?" problem, so provided the token isn't
+ // embedded in the URL, there's no reason to do
+ // redirect-with-cookie in this case either.
reqTokens = append(reqTokens, formToken)
- } else if formToken != "" && browserMethod[r.Method] {
- // The client provided an explicit token in the query
- // string, or a form in POST body. We must put the
- // token in an HttpOnly cookie, and redirect to the
- // same URL with the query param redacted and method =
- // GET.
+ } else if browserMethod[r.Method] {
+ // If this is a page view, and the client provided a
+ // token via query string or POST body, we must put
+ // the token in an HttpOnly cookie, and redirect to an
+ // equivalent URL with the query param redacted and
+ // method = GET.
h.seeOtherWithCookie(w, r, "", credentialsOK)
return
}
commit d4892015f8a24ba3e81bdcce7cd238afeb2d4ecf
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 14 15:41:33 2020 -0500
17208: Update test for s3cmd's new console output.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 077493ffc..7189e99a9 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -8,13 +8,17 @@ import (
"bytes"
"context"
"encoding/json"
+ "fmt"
"io"
"math"
"net"
"net/http"
"net/url"
"os"
+ "os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/config"
@@ -251,6 +255,79 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not in PATH")
+ return
+ }
+
+ testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ conn3 := s.conn("z3333")
+
+ createColl := func(clusterID string) arvados.Collection {
+ _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken)
+ var coll arvados.Collection
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ return coll
+ }
+
+ for _, trial := range []struct {
+ clusterID string // create the collection on this cluster (then use z3333 to access it)
+ token string
+ }{
+ // Try the hardest test first: z3333 hasn't seen
+ // z1111's token yet, and we're just passing the
+ // opaque secret part, so z3333 has to guess that it
+ // belongs to z1111.
+ {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ } {
+ c.Logf("================ %v", trial)
+ coll := createColl(trial.clusterID)
+
+ cfgjson, err := conn3.ConfigGet(userctx1)
+ c.Assert(err, check.IsNil)
+ var cluster arvados.Cluster
+ err = json.Unmarshal(cfgjson, &cluster)
+ c.Assert(err, check.IsNil)
+
+ c.Logf("TokenV2 is %s", ac1.AuthToken)
+ host := cluster.Services.WebDAV.ExternalURL.Host
+ s3args := []string{
+ "--ssl", "--no-check-certificate",
+ "--host=" + host, "--host-bucket=" + host,
+ "--access_key=" + trial.token, "--secret_key=" + trial.token,
+ }
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
+
+ buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ // Command fails because we don't return Etag header.
+ flen := strconv.Itoa(len(testText))
+ c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
+ }
+}
+
func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
commit 276b0fa42a1d4bd766de2124a9f16799a3b65bc5
Author: Tom Clegg <tom at curii.com>
Date: Fri Dec 11 00:43:40 2020 -0500
17208: Add test case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index e2f378cbc..3a4c4b224 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -7,6 +7,7 @@ package main
import (
"bytes"
"crypto/rand"
+ "crypto/sha256"
"fmt"
"io/ioutil"
"net/http"
@@ -541,6 +542,38 @@ func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
}
}
+func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ rawPath string
+ normalizedPath string
+ }{
+ {"/foo", "/foo"}, // boring case
+ {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped
+ {"/foo%2fbar", "/foo/bar"}, // / must not be escaped
+ {"/(foo)", "/%28foo%29"}, // () must be escaped
+ {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
+ } {
+ date := time.Now().UTC().Format("20060102T150405Z")
+ scope := "20200202/fakeregion/S3/aws4_request"
+ canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
+ c.Logf("canonicalRequest %q", canonicalRequest)
+ expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
+ c.Logf("expected stringToSign %q", expect)
+
+ req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
+ req.Header.Set("X-Amz-Date", date)
+ req.Host = "host.example.com"
+
+ obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
+ if !c.Check(err, check.IsNil) {
+ continue
+ }
+ c.Check(obtained, check.Equals, expect)
+ }
+}
+
func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
commit 43a2d6a0bc25d635b61aa3366ecdd53d0fda3bec
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 10 16:41:41 2020 -0500
17208: Use normalized path to compute signatures.
Transparently clean paths containing "//".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index a6dfa9998..63135e297 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -74,6 +74,8 @@ func s3querystring(u *url.URL) string {
return strings.Join(keys, "&")
}
+var reMultipleSlashChars = regexp.MustCompile(`//+`)
+
func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) {
timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date")
if timestr == "" {
@@ -96,7 +98,10 @@ func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string,
}
}
- canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
+ normalizedURL := *r.URL
+ normalizedURL.RawPath = ""
+ normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+ canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
}
@@ -243,7 +248,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
}
- fspath += r.URL.Path
+ fspath += reMultipleSlashChars.ReplaceAllString(r.URL.Path, "/")
switch {
case r.Method == http.MethodGet && !objectNameGiven:
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 3435c8cbf..e2f378cbc 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -198,6 +198,11 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix
c.Check(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
c.Check(resp.ContentLength, check.Equals, int64(4))
+
+ // HeadObject with superfluous leading slashes
+ exists, err = bucket.Exists(prefix + "//sailboat.txt")
+ c.Check(err, check.IsNil)
+ c.Check(exists, check.Equals, true)
}
func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
@@ -224,6 +229,18 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
path: "newdir/newfile",
size: 1 << 26,
contentType: "application/octet-stream",
+ }, {
+ path: "/aaa",
+ size: 2,
+ contentType: "application/octet-stream",
+ }, {
+ path: "//bbb",
+ size: 2,
+ contentType: "application/octet-stream",
+ }, {
+ path: "ccc//",
+ size: 0,
+ contentType: "application/x-directory",
}, {
path: "newdir1/newdir2/newfile",
size: 0,
@@ -239,9 +256,14 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
objname := prefix + trial.path
_, err := bucket.GetReader(objname)
+ if !c.Check(err, check.NotNil) {
+ continue
+ }
c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
- c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
+ if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
+ continue
+ }
buf := make([]byte, trial.size)
rand.Read(buf)
@@ -359,14 +381,6 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
s.testServer.Config.cluster.Collections.S3FolderObjects = false
- // Can't use V4 signature for these tests, because
- // double-slash is incorrectly cleaned by the aws.V4Signature,
- // resulting in a "bad signature" error. (Cleaning the path is
- // appropriate for other services, but not in S3 where object
- // names "foo//bar" and "foo/bar" are semantically different.)
- bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour)))
- bucket.S3.Signature = aws.V2Signature
-
var wg sync.WaitGroup
for _, trial := range []struct {
path string
@@ -389,8 +403,6 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket,
path: "/",
}, {
path: "//",
- }, {
- path: "foo//bar",
}, {
path: "",
},
@@ -437,6 +449,17 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
c.Assert(fs.Sync(), check.IsNil)
}
+func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
+ scope := "20200202/region/service/aws4_request"
+ signedHeaders := "date"
+ req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
+ stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
+ c.Assert(err, check.IsNil)
+ sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
+}
+
func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
@@ -483,12 +506,29 @@ func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
responseCode: http.StatusOK,
responseRegexp: []string{`boop`},
},
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "GET",
+ responseCode: http.StatusNotFound,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "PUT",
+ body: "boop",
+ responseCode: http.StatusOK,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`boop`},
+ },
} {
url, err := url.Parse(trial.url)
c.Assert(err, check.IsNil)
req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
c.Assert(err, check.IsNil)
- req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
rr := httptest.NewRecorder()
s.testServer.Server.Handler.ServeHTTP(rr, req)
resp := rr.Result()
commit 67e9e5cd1a09d3af3c2d11210d92d32d6b6609c1
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Tue Dec 22 12:41:01 2020 -0300
17118: Changes the way exception raising is done on PySDK's KeepWriterThread.
This solves the OOM bug where some keepstores fail when uploading data.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index bc43b849c..bd0e5dc1e 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -648,7 +648,7 @@ class KeepClient(object):
class KeepWriterThread(threading.Thread):
- TaskFailed = RuntimeError()
+ class TaskFailed(RuntimeError): pass
def __init__(self, queue, data, data_hash, timeout=None):
super(KeepClient.KeepWriterThread, self).__init__()
@@ -667,7 +667,7 @@ class KeepClient(object):
try:
locator, copies = self.do_task(service, service_root)
except Exception as e:
- if e is not self.TaskFailed:
+ if not isinstance(e, self.TaskFailed):
_logger.exception("Exception in KeepWriterThread")
self.queue.write_fail(service)
else:
@@ -687,7 +687,7 @@ class KeepClient(object):
self.data_hash,
result['status_code'],
result['body'])
- raise self.TaskFailed
+ raise self.TaskFailed()
_logger.debug("KeepWriterThread %s succeeded %s+%i %s",
str(threading.current_thread()),
commit fa2df0a62a589f33f1cfa1a193598875d941596e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 2 17:14:37 2020 -0500
17161: Improve SystemRootToken docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index 2893111e3..647cd983a 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -49,20 +49,24 @@ h3. Tokens
<pre><code> SystemRootToken: <span class="userinput">"$system_root_token"</span>
ManagementToken: <span class="userinput">"$management_token"</span>
Collections:
- BlobSigningKey: <span class="userinput">"blob_signing_key"</span>
+ BlobSigningKey: <span class="userinput">"$blob_signing_key"</span>
</code></pre>
</notextile>
- at SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+These secret tokens are used to authenticate messages between Arvados components.
+* @SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+* @ManagementToken@ is used to authenticate access to system metrics.
+* @API.RailsSessionSecretToken@ is used to sign session cookies.
+* @Collections.BlobSigningKey@ is used to control access to Keep blocks.
@ManagementToken@ is used to authenticate access to system metrics.
@Collections.BlobSigningKey@ is used to control access to Keep blocks.
-You can generate a random token for each of these items at the command line like this:
+Each token should be a string of at least 50 alphanumeric characters. You can generate a suitable token with the following command:
<notextile>
-<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50; echo</span>
+<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50 ; echo</span>
</code></pre>
</notextile>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 2812fd2bb..3ecff5448 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -12,6 +12,8 @@
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 27bc2e4e0..cd752ab75 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -18,6 +18,8 @@ var DefaultYAML = []byte(`# Copyright (C) The Arvados Authors. All rights reserv
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list