[ARVADOS] updated: 1.3.0-919-g9bf075c29
Git user
git at public.curoverse.com
Tue May 21 15:41:51 UTC 2019
Summary of changes:
apps/workbench/app/models/arvados_api_client.rb | 4 +-
build/run-tests.sh | 14 +++--
lib/controller/federation_test.go | 5 +-
lib/controller/router/request.go | 8 +++
lib/controller/router/router.go | 43 +++++++++++--
lib/controller/router/router_test.go | 81 ++++++++++++++++---------
lib/controller/server_test.go | 5 +-
lib/service/cmd.go | 3 +-
sdk/go/arvados/api.go | 17 +++---
sdk/go/arvados/fs_backend.go | 1 -
sdk/go/arvados/fs_collection.go | 7 ++-
sdk/go/arvados/fs_project_test.go | 47 ++++++++------
sdk/go/httpserver/logger.go | 28 +++++----
sdk/go/httpserver/logger_test.go | 14 +++--
sdk/go/httpserver/metrics.go | 2 +-
sdk/python/tests/nginx.conf | 2 +-
sdk/python/tests/run_test_server.py | 1 +
services/keep-balance/server.go | 10 ++-
services/keep-web/cache.go | 6 +-
services/keep-web/cadaver_test.go | 3 +-
services/keep-web/handler_test.go | 21 ++++---
services/keep-web/server.go | 6 +-
services/keepproxy/keepproxy.go | 2 +-
23 files changed, 224 insertions(+), 106 deletions(-)
via 9bf075c292d938905f60f4ef303bee8bdb113cf9 (commit)
via 44155f3f75dfe196520fae09115588f277c62d38 (commit)
via a36bdfd874300f0afd4d4bffc11dbdd97bd6d210 (commit)
via 9a5e63bb094e474045415ef6692265a9e8abb010 (commit)
via cea4c0f2b9d80cd22355014090c68a8db227cf04 (commit)
via 40711540d5f82594dda6a476ecdf8f1fecf1a214 (commit)
via 274aa34f6098df959ded86272567f01e3a35eb6d (commit)
via d99c3d34dd752789c11538a7a13b9e41dc9254e1 (commit)
via 9ee7fdab1e90acb63e1941d5bde04615491e006e (commit)
via b98681c27aa33df769fc933dd0eeadd6bc208173 (commit)
via 4238b017e7332a700e644015889a4fcdf251595a (commit)
via 96bb342903bc651483f64bc7e5cd769b2ad49077 (commit)
from 486286dd282fe2c95701229d0872855bfd99c3bf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9bf075c292d938905f60f4ef303bee8bdb113cf9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 21 11:41:20 2019 -0400
14287: Enable new controller code in integration tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index e595a298a..c39e590e8 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -413,6 +413,7 @@ def run_controller():
f.write("""
Clusters:
zzzzz:
+ EnableBetaController14287: true
Logging:
Level: "{}"
ManagementToken: e687950a23c3a9bceec28c6223a06c79
commit 44155f3f75dfe196520fae09115588f277c62d38
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 21 11:40:39 2019 -0400
14287: Add debug logs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 06b583ddf..1c55029c8 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -6,6 +6,7 @@ package router
import (
"context"
+ "fmt"
"net/http"
"git.curoverse.com/arvados.git/lib/controller/federation"
@@ -13,6 +14,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/julienschmidt/httprouter"
+ "github.com/sirupsen/logrus"
)
type router struct {
@@ -186,19 +188,23 @@ func (rtr *router) addRoutes(cluster *arvados.Cluster) {
}
for _, method := range methods {
rtr.mux.HandlerFunc(method, "/"+route.endpoint.Path, func(w http.ResponseWriter, req *http.Request) {
+ logger := ctxlog.FromContext(req.Context())
params, err := rtr.loadRequestParams(req, route.endpoint.AttrsKey)
if err != nil {
+ logger.WithField("req", req).WithField("route", route).WithError(err).Debug("error loading request params")
rtr.sendError(w, err)
return
}
opts := route.defaultOpts()
err = rtr.transcode(params, opts)
if err != nil {
+ logger.WithField("params", params).WithError(err).Debugf("error transcoding params to %T", opts)
rtr.sendError(w, err)
return
}
respOpts, err := rtr.responseOptions(opts)
if err != nil {
+ logger.WithField("opts", opts).WithError(err).Debugf("error getting response options from %T", opts)
rtr.sendError(w, err)
return
}
@@ -214,9 +220,14 @@ func (rtr *router) addRoutes(cluster *arvados.Cluster) {
ctx := req.Context()
ctx = context.WithValue(ctx, auth.ContextKeyCredentials, creds)
ctx = arvados.ContextWithRequestID(ctx, req.Header.Get("X-Request-Id"))
+ logger.WithFields(logrus.Fields{
+ "apiEndpoint": route.endpoint,
+ "apiOptsType": fmt.Sprintf("%T", opts),
+ "apiOpts": opts,
+ }).Debug("exec")
resp, err := route.exec(ctx, opts)
if err != nil {
- ctxlog.FromContext(ctx).WithError(err).Debugf("returning error response for %#v", err)
+ logger.WithError(err).Debugf("returning error type %T", err)
rtr.sendError(w, err)
return
}
commit a36bdfd874300f0afd4d4bffc11dbdd97bd6d210
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 21 10:23:09 2019 -0400
14287: Use ctxlog for httpserver logging.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 43344c744..65a6b85c4 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -6,6 +6,7 @@ package controller
import (
"bytes"
+ "context"
"encoding/json"
"fmt"
"io"
@@ -72,7 +73,9 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
EnableBetaController14287: enableBetaController14287,
}, NodeProfile: &nodeProfile}
s.testServer = newServerFromIntegrationTestEnv(c)
- s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
+ s.testServer.Server.Handler = httpserver.HandlerWithContext(
+ ctxlog.Context(context.Background(), s.log),
+ httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler)))
s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
"zzzzz": {
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index e5fd41712..b591a1795 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -5,6 +5,7 @@
package controller
import (
+ "context"
"net/http"
"os"
"path/filepath"
@@ -47,7 +48,9 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
+ Handler: httpserver.HandlerWithContext(
+ ctxlog.Context(context.Background(), log),
+ httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
},
Addr: nodeProfile.Controller.Listen,
}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 4b7341d72..6ea22d30b 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -120,7 +120,8 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
}
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
+ Handler: httpserver.HandlerWithContext(ctx,
+ httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
},
Addr: listen,
}
diff --git a/sdk/go/httpserver/logger.go b/sdk/go/httpserver/logger.go
index 357daee26..f64708454 100644
--- a/sdk/go/httpserver/logger.go
+++ b/sdk/go/httpserver/logger.go
@@ -9,6 +9,7 @@ import (
"net/http"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/stats"
"github.com/sirupsen/logrus"
)
@@ -19,18 +20,23 @@ type contextKey struct {
var (
requestTimeContextKey = contextKey{"requestTime"}
- loggerContextKey = contextKey{"logger"}
)
+// HandlerWithContext returns an http.Handler that changes the request
+// context to ctx (replacing http.Server's default
+// context.Background()), then calls next.
+func HandlerWithContext(ctx context.Context, next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ next.ServeHTTP(w, r.WithContext(ctx))
+ })
+}
+
// LogRequests wraps an http.Handler, logging each request and
-// response via logger.
-func LogRequests(logger logrus.FieldLogger, h http.Handler) http.Handler {
- if logger == nil {
- logger = logrus.StandardLogger()
- }
+// response.
+func LogRequests(h http.Handler) http.Handler {
return http.HandlerFunc(func(wrapped http.ResponseWriter, req *http.Request) {
w := &responseTimer{ResponseWriter: WrapResponseWriter(wrapped)}
- lgr := logger.WithFields(logrus.Fields{
+ lgr := ctxlog.FromContext(req.Context()).WithFields(logrus.Fields{
"RequestID": req.Header.Get("X-Request-Id"),
"remoteAddr": req.RemoteAddr,
"reqForwardedFor": req.Header.Get("X-Forwarded-For"),
@@ -42,7 +48,7 @@ func LogRequests(logger logrus.FieldLogger, h http.Handler) http.Handler {
})
ctx := req.Context()
ctx = context.WithValue(ctx, &requestTimeContextKey, time.Now())
- ctx = context.WithValue(ctx, &loggerContextKey, lgr)
+ ctx = ctxlog.Context(ctx, lgr)
req = req.WithContext(ctx)
logRequest(w, req, lgr)
@@ -52,11 +58,7 @@ func LogRequests(logger logrus.FieldLogger, h http.Handler) http.Handler {
}
func Logger(req *http.Request) logrus.FieldLogger {
- if lgr, ok := req.Context().Value(&loggerContextKey).(logrus.FieldLogger); ok {
- return lgr
- } else {
- return logrus.StandardLogger()
- }
+ return ctxlog.FromContext(req.Context())
}
func logRequest(w *responseTimer, req *http.Request, lgr *logrus.Entry) {
diff --git a/sdk/go/httpserver/logger_test.go b/sdk/go/httpserver/logger_test.go
index 8386db927..3b2bc7758 100644
--- a/sdk/go/httpserver/logger_test.go
+++ b/sdk/go/httpserver/logger_test.go
@@ -6,12 +6,14 @@ package httpserver
import (
"bytes"
+ "context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -31,15 +33,19 @@ func (s *Suite) TestLogRequests(c *check.C) {
log.Formatter = &logrus.JSONFormatter{
TimestampFormat: time.RFC3339Nano,
}
+ ctx := ctxlog.Context(context.Background(), log)
+
+ h := AddRequestIDs(LogRequests(
+ http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ w.Write([]byte("hello world"))
+ })))
- h := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- w.Write([]byte("hello world"))
- })
req, err := http.NewRequest("GET", "https://foo.example/bar", nil)
req.Header.Set("X-Forwarded-For", "1.2.3.4:12345")
c.Assert(err, check.IsNil)
resp := httptest.NewRecorder()
- AddRequestIDs(LogRequests(log, h)).ServeHTTP(resp, req)
+
+ HandlerWithContext(ctx, h).ServeHTTP(resp, req)
dec := json.NewDecoder(captured)
diff --git a/sdk/go/httpserver/metrics.go b/sdk/go/httpserver/metrics.go
index 032093f8d..fab6c3f11 100644
--- a/sdk/go/httpserver/metrics.go
+++ b/sdk/go/httpserver/metrics.go
@@ -104,7 +104,7 @@ func (m *metrics) ServeAPI(token string, next http.Handler) http.Handler {
//
// For the metrics to be accurate, the caller must ensure every
// request passed to the Handler also passes through
-// LogRequests(logger, ...), and vice versa.
+// LogRequests(...), and vice versa.
//
// If registry is nil, a new registry is created.
//
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 894056c9f..e2f13a425 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -5,6 +5,7 @@
package main
import (
+ "context"
"fmt"
"net/http"
"os"
@@ -14,6 +15,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/sirupsen/logrus"
)
@@ -127,11 +129,13 @@ func (srv *Server) start() error {
if srv.config.Listen == "" {
return nil
}
+ ctx := ctxlog.Context(context.Background(), srv.Logger)
server := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.LogRequests(srv.Logger,
- auth.RequireLiteralToken(srv.config.ManagementToken,
- srv.metrics.Handler(srv.Logger))),
+ Handler: httpserver.HandlerWithContext(ctx,
+ httpserver.LogRequests(
+ auth.RequireLiteralToken(srv.config.ManagementToken,
+ srv.metrics.Handler(srv.Logger)))),
},
Addr: srv.config.Listen,
}
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index f70dd1a71..167fbbe5b 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -5,10 +5,13 @@
package main
import (
+ "context"
"net/http"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
type server struct {
@@ -20,7 +23,8 @@ func (srv *server) Start() error {
h := &handler{Config: srv.Config}
reg := prometheus.NewRegistry()
h.Config.Cache.registry = reg
- mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+ ctx := ctxlog.Context(context.Background(), logrus.StandardLogger())
+ mh := httpserver.Instrument(reg, nil, httpserver.HandlerWithContext(ctx, httpserver.AddRequestIDs(httpserver.LogRequests(h))))
h.MetricsAPI = mh.ServeAPI(h.Config.ManagementToken, http.NotFoundHandler())
srv.Handler = mh
srv.Addr = srv.Config.Listen
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index c6fd99b9d..f8aa6c4aa 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -182,7 +182,7 @@ func main() {
// Start serving requests.
router = MakeRESTRouter(!cfg.DisableGet, !cfg.DisablePut, kc, time.Duration(cfg.Timeout), cfg.ManagementToken)
- http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(nil, router)))
+ http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
log.Println("shutting down")
}
commit 9a5e63bb094e474045415ef6692265a9e8abb010
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 21 09:38:57 2019 -0400
14287: Handle CORS OPTIONS requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 3dba53edd..06b583ddf 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -16,20 +16,26 @@ import (
)
type router struct {
- mux *httprouter.Router
- fed federation.Interface
+ mux *httprouter.Router
+ muxOPTIONS *http.ServeMux
+ fed federation.Interface
}
func New(cluster *arvados.Cluster, np *arvados.NodeProfile) *router {
rtr := &router{
- mux: httprouter.New(),
- fed: federation.New(cluster, np),
+ mux: httprouter.New(),
+ muxOPTIONS: http.NewServeMux(),
+ fed: federation.New(cluster, np),
}
rtr.addRoutes(cluster)
return rtr
}
func (rtr *router) addRoutes(cluster *arvados.Cluster) {
+ rtr.muxOPTIONS.HandleFunc("/login", serveOPTIONSError)
+ rtr.muxOPTIONS.HandleFunc("/logout", serveOPTIONSError)
+ rtr.muxOPTIONS.HandleFunc("/auth/", serveOPTIONSError)
+ rtr.muxOPTIONS.HandleFunc("/", serveOPTIONSOK)
for _, route := range []struct {
endpoint arvados.APIEndpoint
defaultOpts func() interface{}
@@ -221,6 +227,10 @@ func (rtr *router) addRoutes(cluster *arvados.Cluster) {
}
func (rtr *router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "OPTIONS" {
+ rtr.muxOPTIONS.ServeHTTP(w, r)
+ return
+ }
r.ParseForm()
if m := r.FormValue("_method"); m != "" {
r2 := *r
@@ -229,3 +239,15 @@ func (rtr *router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
rtr.mux.ServeHTTP(w, r)
}
+
+func serveOPTIONSOK(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Access-Control-Allow-Origin", "*")
+ w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, PUT, POST, DELETE")
+ w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
+ w.Header().Set("Access-Control-Max-Age", "86486400")
+ w.WriteHeader(http.StatusOK)
+}
+
+func serveOPTIONSError(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+}
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 348216d18..377e6fb6a 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -46,21 +46,21 @@ func (s *RouterSuite) doRequest(c *check.C, token, method, path string, hdrs htt
req.Header[k] = v
}
req.Header.Set("Authorization", "Bearer "+token)
- rw := httptest.NewRecorder()
- s.rtr.ServeHTTP(rw, req)
- c.Logf("response body: %s", rw.Body.String())
+ rr := httptest.NewRecorder()
+ s.rtr.ServeHTTP(rr, req)
+ c.Logf("response body: %s", rr.Body.String())
var jresp map[string]interface{}
- err := json.Unmarshal(rw.Body.Bytes(), &jresp)
+ err := json.Unmarshal(rr.Body.Bytes(), &jresp)
c.Check(err, check.IsNil)
- return req, rw, jresp
+ return req, rr, jresp
}
func (s *RouterSuite) TestCollectionResponses(c *check.C) {
token := arvadostest.ActiveTokenV2
// Check "get collection" response has "kind" key
- _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
c.Check(jresp["items"].([]interface{})[0].(map[string]interface{})["kind"], check.Equals, "arvados#collection")
@@ -73,8 +73,8 @@ func (s *RouterSuite) TestCollectionResponses(c *check.C) {
`,"select":["name"]`,
`,"select":["uuid"]`,
} {
- _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"where":{"uuid":["`+arvadostest.FooCollection+`"]}`+selectj+`}`))
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"where":{"uuid":["`+arvadostest.FooCollection+`"]}`+selectj+`}`))
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
@@ -98,8 +98,8 @@ func (s *RouterSuite) TestCollectionResponses(c *check.C) {
}
// Check "create collection" response has "kind" key
- _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`ensure_unique_name=true`))
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`ensure_unique_name=true`))
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.FitsTypeOf, "")
c.Check(jresp["kind"], check.Equals, "arvados#collection")
}
@@ -107,14 +107,14 @@ func (s *RouterSuite) TestCollectionResponses(c *check.C) {
func (s *RouterSuite) TestContainerList(c *check.C) {
token := arvadostest.ActiveTokenV2
- _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=0`, nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp := s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=0`, nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
c.Check(jresp["items"], check.HasLen, 0)
- _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=2&select=["uuid","command"]`, nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers?limit=2&select=["uuid","command"]`, nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
c.Check(jresp["items"], check.HasLen, 2)
@@ -124,8 +124,8 @@ func (s *RouterSuite) TestContainerList(c *check.C) {
c.Check(item0["command"].([]interface{})[0], check.FitsTypeOf, "")
c.Check(item0["mounts"], check.IsNil)
- _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers`, nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp = s.doRequest(c, token, "GET", `/arvados/v1/containers`, nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
c.Check(jresp["items_available"].(float64) > 2, check.Equals, true)
avail := int(jresp["items_available"].(float64))
@@ -140,20 +140,20 @@ func (s *RouterSuite) TestContainerList(c *check.C) {
func (s *RouterSuite) TestContainerLock(c *check.C) {
uuid := arvadostest.QueuedContainerUUID
token := arvadostest.AdminToken
- _, rw, jresp := s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp := s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
c.Check(jresp["state"], check.Equals, "Locked")
- _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusUnprocessableEntity)
- c.Check(rw.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
- _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusUnprocessableEntity)
+ c.Check(rr.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
+ _, rr, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
c.Check(jresp["state"], check.Equals, "Queued")
c.Check(jresp["environment"], check.IsNil)
- _, rw, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusUnprocessableEntity)
+ _, rr, jresp = s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusUnprocessableEntity)
c.Check(jresp["uuid"], check.IsNil)
}
@@ -161,8 +161,8 @@ func (s *RouterSuite) TestFullTimestampsInResponse(c *check.C) {
uuid := arvadostest.CollectionReplicationDesired2Confirmed2UUID
token := arvadostest.ActiveTokenV2
- _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections/`+uuid, nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections/`+uuid, nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.Equals, uuid)
expectNS := map[string]int{
"created_at": 596506000, // fixture says 596506247, but truncated by postgresql
@@ -188,8 +188,8 @@ func (s *RouterSuite) TestSelectParam(c *check.C) {
} {
j, err := json.Marshal(sel)
c.Assert(err, check.IsNil)
- _, rw, resp := s.doRequest(c, token, "GET", "/arvados/v1/containers/"+uuid+"?select="+string(j), nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
+ _, rr, resp := s.doRequest(c, token, "GET", "/arvados/v1/containers/"+uuid+"?select="+string(j), nil, nil)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(resp["kind"], check.Equals, "arvados#container")
c.Check(resp["uuid"], check.HasLen, 27)
@@ -199,3 +199,24 @@ func (s *RouterSuite) TestSelectParam(c *check.C) {
c.Check(hasMounts, check.Equals, false)
}
}
+
+func (s *RouterSuite) TestCORS(c *check.C) {
+ token := arvadostest.ActiveTokenV2
+ req := (&testReq{
+ method: "OPTIONS",
+ path: "arvados/v1/collections/" + arvadostest.FooCollection,
+ header: http.Header{"Origin": {"https://example.com"}},
+ token: token,
+ }).Request()
+ rr := httptest.NewRecorder()
+ s.rtr.ServeHTTP(rr, req)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
+ c.Check(rr.Body.String(), check.HasLen, 0)
+ c.Check(rr.Result().Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
+ for _, hdr := range []string{"Authorization", "Content-Type"} {
+ c.Check(rr.Result().Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
+ }
+ for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
+ c.Check(rr.Result().Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
+ }
+}
commit cea4c0f2b9d80cd22355014090c68a8db227cf04
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 20 16:04:02 2019 -0400
14287: Enable readline history in run-tests.sh interactive mode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9a7b87646..a81f37ef7 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1218,19 +1218,21 @@ else
# assume emacs, or something, is offering a history buffer
# and pre-populating the command will only cause trouble
nextcmd=
- elif [[ "$nextcmd" != "install deps" ]]; then
- :
- elif [[ -e "$VENVDIR/bin/activate" ]]; then
- nextcmd="test lib/cmd"
- else
+ elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
nextcmd="install deps"
+ else
+ nextcmd=""
fi
}
echo
help_interactive
nextcmd="install deps"
setnextcmd
- while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+ HISTFILE="$WORKSPACE/tmp/.history"
+ history -r
+ while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+ history -s "$nextcmd"
+ history -w
read verb target opts <<<"${nextcmd}"
target="${target%/}"
target="${target/\/:/:}"
commit 40711540d5f82594dda6a476ecdf8f1fecf1a214
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 20 09:46:00 2019 -0400
f UpdateBody
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs_backend.go
index 9ae0fc3a5..c8308aea5 100644
--- a/sdk/go/arvados/fs_backend.go
+++ b/sdk/go/arvados/fs_backend.go
@@ -26,5 +26,4 @@ type keepClient interface {
type apiClient interface {
RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
}
commit 274aa34f6098df959ded86272567f01e3a35eb6d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri May 17 15:31:28 2019 -0400
14287: Ignore etag and unsigned_manifest_text in collection updates.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/request.go b/lib/controller/router/request.go
index de288a0a2..47d8bb110 100644
--- a/lib/controller/router/request.go
+++ b/lib/controller/router/request.go
@@ -97,6 +97,13 @@ func (rtr *router) loadRequestParams(req *http.Request, attrsKey string) (map[st
}
if v, ok := params[attrsKey]; ok && attrsKey != "" {
+ if v, ok := v.(map[string]interface{}); ok {
+ // Delete field(s) that appear in responses
+ // but not in update attrs, so clients can
+ // fetch-modify-update.
+ delete(v, "etag")
+ delete(v, "unsigned_manifest_text")
+ }
params["attrs"] = v
delete(params, attrsKey)
}
commit d99c3d34dd752789c11538a7a13b9e41dc9254e1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri May 17 15:30:20 2019 -0400
14287: Use map instead of UpdateBody to update specific attrs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 6644f4cfb..972b3979f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -131,7 +131,12 @@ func (fs *collectionFileSystem) Sync() error {
UUID: fs.uuid,
ManifestText: txt,
}
- err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": coll.ManifestText,
+ },
+ "select": []string{"uuid"},
+ })
if err != nil {
return fmt.Errorf("sync failed: update %s: %s", fs.uuid, err)
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 1a06ce146..0b8ce90b3 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -119,20 +119,24 @@ func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
}
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
- badCollection := Collection{
- Name: "bad/collection",
- OwnerUUID: arvadostest.AProjectUUID,
- }
- err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", s.client.UpdateBody(&badCollection), nil)
+ var badCollection Collection
+ err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": "bad/collection",
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
- badProject := Group{
- Name: "bad/project",
- GroupClass: "project",
- OwnerUUID: arvadostest.AProjectUUID,
- }
- err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", s.client.UpdateBody(&badProject), nil)
+ var badProject Group
+ err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", nil, map[string]interface{}{
+ "group": map[string]string{
+ "name": "bad/project",
+ "group_class": "project",
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
@@ -155,11 +159,13 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
- oob := Collection{
- Name: "oob",
- OwnerUUID: arvadostest.AProjectUUID,
- }
- err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+ var oob Collection
+ err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": "oob",
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
@@ -180,8 +186,13 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
c.Check(err, check.IsNil)
// Delete test.txt behind s.fs's back by updating the
- // collection record with the old (empty) ManifestText.
- err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+ // collection record with an empty ManifestText.
+ err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": "",
+ "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
+ },
+ })
c.Assert(err, check.IsNil)
err = project.Sync()
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 8336b78f9..b9a1f3069 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -157,7 +157,11 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad
}
var updated arvados.Collection
defer c.pdhs.Remove(coll.UUID)
- err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
+ err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": coll.ManifestText,
+ },
+ })
if err == nil {
c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
expire: time.Now().Add(time.Duration(c.TTL)),
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 44d0b0ffe..bf7a1942b 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"io/ioutil"
- "net/url"
"os"
"os/exec"
"path/filepath"
@@ -74,7 +73,7 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
- err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
+ err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{}})
c.Assert(err, check.IsNil)
readPath, writePath, pdhPath := pathFunc(newCollection)
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 7a015c91f..95535249b 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -465,8 +465,12 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
f.Close()
mtxt, err := fs.MarshalManifest(".")
c.Assert(err, check.IsNil)
- coll := arvados.Collection{ManifestText: mtxt}
- err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
c.Assert(err, check.IsNil)
u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
@@ -773,11 +777,14 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
arv := arvados.NewClientFromEnv()
var newCollection arvados.Collection
- err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", arv.UpdateBody(&arvados.Collection{
- OwnerUUID: arvadostest.ActiveUserUUID,
- ManifestText: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
- Name: "keep-web test collection",
- }), map[string]bool{"ensure_unique_name": true})
+ err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
+ "name": "keep-web test collection",
+ },
+ "ensure_unique_name": true,
+ })
c.Assert(err, check.IsNil)
defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
commit 9ee7fdab1e90acb63e1941d5bde04615491e006e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri May 17 15:28:31 2019 -0400
14287: Include x-request-id in test suite nginx access log.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 1ef3b00c6..a7b8bacdc 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -8,7 +8,7 @@ events {
}
http {
log_format customlog
- '[$time_local] $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
+ '[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
client_body_temp_path "{{TMPDIR}}";
commit b98681c27aa33df769fc933dd0eeadd6bc208173
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 16 16:04:30 2019 -0400
14287: Propagate include_trash flag.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 00d93367a..ebf44a822 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -40,14 +40,16 @@ type GetOptions struct {
}
type ListOptions struct {
- Select []string `json:"select"`
- Filters []Filter `json:"filters"`
- Where map[string]interface{} `json:"where"`
- Limit int `json:"limit"`
- Offset int `json:"offset"`
- Order []string `json:"order"`
- Distinct bool `json:"distinct"`
- Count string `json:"count"`
+ Select []string `json:"select"`
+ Filters []Filter `json:"filters"`
+ Where map[string]interface{} `json:"where"`
+ Limit int `json:"limit"`
+ Offset int `json:"offset"`
+ Order []string `json:"order"`
+ Distinct bool `json:"distinct"`
+ Count string `json:"count"`
+ IncludeTrash bool `json:"include_trash"`
+ IncludeOldVersions bool `json:"include_old_versions"`
}
type CreateOptions struct {
commit 4238b017e7332a700e644015889a4fcdf251595a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 16 14:09:48 2019 -0400
14287: Propagate "distinct" param.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/request.go b/lib/controller/router/request.go
index f9eb3e76d..de288a0a2 100644
--- a/lib/controller/router/request.go
+++ b/lib/controller/router/request.go
@@ -134,6 +134,7 @@ func (rtr *router) transcode(src interface{}, dst interface{}) error {
}
var boolParams = map[string]bool{
+ "distinct": true,
"ensure_unique_name": true,
"include_trash": true,
"include_old_versions": true,
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 597f47a95..00d93367a 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -40,13 +40,14 @@ type GetOptions struct {
}
type ListOptions struct {
- Select []string `json:"select"`
- Filters []Filter `json:"filters"`
- Where map[string]interface{} `json:"where"`
- Limit int `json:"limit"`
- Offset int `json:"offset"`
- Order []string `json:"order"`
- Count string `json:"count"`
+ Select []string `json:"select"`
+ Filters []Filter `json:"filters"`
+ Where map[string]interface{} `json:"where"`
+ Limit int `json:"limit"`
+ Offset int `json:"offset"`
+ Order []string `json:"order"`
+ Distinct bool `json:"distinct"`
+ Count string `json:"count"`
}
type CreateOptions struct {
commit 96bb342903bc651483f64bc7e5cd769b2ad49077
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 16 14:09:40 2019 -0400
14287: Don't send reader_tokens="[false]".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/apps/workbench/app/models/arvados_api_client.rb b/apps/workbench/app/models/arvados_api_client.rb
index 5a8fd518d..ce91cd305 100644
--- a/apps/workbench/app/models/arvados_api_client.rb
+++ b/apps/workbench/app/models/arvados_api_client.rb
@@ -113,11 +113,13 @@ class ArvadosApiClient
# Clean up /arvados/v1/../../discovery/v1 to /discovery/v1
url.sub! '/arvados/v1/../../', '/'
+ anon_tokens = [Rails.configuration.anonymous_user_token].select { |x| x && include_anon_token }
+
query = {
'reader_tokens' => ((tokens[:reader_tokens] ||
Thread.current[:reader_tokens] ||
[]) +
- (include_anon_token ? [Rails.configuration.anonymous_user_token] : [])).to_json,
+ anon_tokens).to_json,
}
if !data.nil?
data.each do |k,v|
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list