[ARVADOS] updated: 1.3.0-903-g8d60b124a
Git user
git at public.curoverse.com
Wed May 15 16:04:44 UTC 2019
Summary of changes:
build/rails-package-scripts/arvados-api-server.sh | 2 +-
build/run-library.sh | 2 +-
build/run-tests.sh | 17 +++
cmd/arvados-server/cmd.go | 3 +
lib/config/cmd.go | 118 +++++++++++++++
lib/config/cmd_test.go | 96 +++++++++++++
lib/config/config.default.yml | 6 +-
lib/config/deprecated.go | 82 +++++++++++
lib/config/deprecated_test.go | 53 +++++++
lib/config/generate.go | 72 ++++++++++
.../{config.default.yml => generated_config.go} | 17 ++-
lib/config/load.go | 149 +++++++++++++++++++
lib/config/load_test.go | 158 +++++++++++++++++++++
.../remove_file_api.js => lib/config/uptodate.go | 4 +-
lib/config/uptodate_test.go | 22 +++
lib/controller/fed_collections.go | 14 +-
lib/controller/fed_generic.go | 20 +--
lib/controller/federation/conn.go | 10 ++
lib/controller/federation_test.go | 8 +-
lib/controller/handler.go | 1 +
lib/controller/router/request_test.go | 155 ++++++++++++++++++--
lib/controller/router/response.go | 82 ++++++++++-
lib/controller/router/router.go | 14 ++
lib/controller/router/router_test.go | 51 +++++--
lib/controller/rpc/conn.go | 14 ++
lib/controller/semaphore.go | 14 ++
lib/service/cmd.go | 3 +-
sdk/go/arvados/api.go | 14 +-
sdk/go/arvados/collection.go | 1 +
sdk/go/arvados/config.go | 31 ++--
sdk/go/arvados/postgresql.go | 3 +
sdk/python/tests/run_test_server.py | 14 +-
services/api/Gemfile | 2 +
services/api/Gemfile.lock | 17 +++
services/api/app/assets/images/logo.png | Bin 9429 -> 66943 bytes
.../api/app/assets/stylesheets/application.css | 6 +-
.../controllers/arvados/v1/schema_controller.rb | 3 +-
.../app/controllers/arvados/v1/users_controller.rb | 68 ++++++---
.../app/controllers/user_sessions_controller.rb | 10 ++
.../api/app/views/layouts/application.html.erb | 7 +-
.../api/app/views/static/login_failure.html.erb | 6 +-
.../api/app/views/user_sessions/create.html.erb | 7 +
services/api/config/arvados_config.rb | 4 +-
.../functional/arvados/v1/users_controller_test.rb | 78 +++++++++-
services/keepstore/unix_volume.go | 16 ++-
services/nodemanager/doc/ec2.example.cfg | 11 +-
services/nodemanager/setup.py | 2 +-
tools/arvbox/bin/arvbox | 2 +-
.../lib/arvbox/docker/service/certificate/run | 6 +-
.../lib/arvbox/docker/service/controller/run | 39 ++++-
.../service/crunch-dispatch-local/run-service | 2 +-
.../arvbox/docker/service/workbench2/run-service | 21 +++
tools/keep-xref/keep-xref.py | 104 ++++++++++++++
vendor/vendor.json | 6 +
54 files changed, 1532 insertions(+), 135 deletions(-)
create mode 100644 lib/config/cmd.go
create mode 100644 lib/config/cmd_test.go
create mode 100644 lib/config/deprecated.go
create mode 100644 lib/config/deprecated_test.go
create mode 100644 lib/config/generate.go
copy lib/config/{config.default.yml => generated_config.go} (97%)
create mode 100644 lib/config/load.go
create mode 100644 lib/config/load_test.go
copy apps/workbench/test/support/remove_file_api.js => lib/config/uptodate.go (67%)
create mode 100644 lib/config/uptodate_test.go
create mode 100644 lib/controller/semaphore.go
create mode 100644 services/api/app/views/user_sessions/create.html.erb
create mode 100755 tools/keep-xref/keep-xref.py
via 8d60b124aba24724a0ace90872276e1f1c5a09f3 (commit)
via d74b9938429acae9f2459d81baa987ef22d75e3a (commit)
via d235817fee3a904eeff85381c1a5227474900852 (commit)
via 42b483f3a722ad8a506040160e54d0080a197318 (commit)
via 5d3c8552f70ddd6b1ed5397e3650020e88538224 (commit)
via 42bc78f2f1e9017fff31799e521b61730d53976f (commit)
via 016b112f01f1d1a2b312df5ab1b7af52f2d36d47 (commit)
via b4eff2bc2af25e3629100f45f7af77ec13f8ab57 (commit)
via 900e78f4d16da54796797db543b9777292ab5ec6 (commit)
via 4490616be768aeda32979995cedcb6c7ca79504e (commit)
via ab4fabf39c7cb1c48a2ba09ba2f45da45b6c4574 (commit)
via 7520945351c2bb42481354c57e66486f271734fc (commit)
via cce14cb6e78ee472cfde4629101e284b2cb63d1b (commit)
via 5fa8839f730408828a583f1e80c2538b23fa7005 (commit)
via 57abc6f7661db3dda9eee9ca66669f642d1cb4bd (commit)
via 20c5632277f5f87ea047ba51dca82f0ce8aed81d (commit)
via 7488f45ab70fef8d3591b8b34b68f1a66a5258e0 (commit)
via 9794db8c78656af88a8e539bafbb990476c0be07 (commit)
via 0f1476fae8e60066c536ae85937975b793d6fb08 (commit)
via cafd28756a1465381774ed2e1df500bd148a4a1f (commit)
via 9a251d966ae83934381840e4b47478e10221d265 (commit)
via 9eca0b4f14f8aefd58dba08bd735070fdac363df (commit)
via e2a30b016300357dcfc8ae0ea4bfd7111f7e9862 (commit)
via 6731fb679926da11149c5c287072a6c139046f10 (commit)
via ce6416c4bfaf33ac17815f070ae5693fe1b7f1ae (commit)
via 3d7e21293962aac93f195ab02bf29d60039387bb (commit)
via 0d77a226aca48ee76cc296bc5e94f30265869844 (commit)
via 3309c3663ccab0e4401384d8e159eb09e9a1670b (commit)
via 7881bce47e7bce5375778b185f598ba53b6a64c6 (commit)
via d23014650ec0f90a3ffdf748cde8ee10c7ba365b (commit)
via d6e0bfffc0e79ef129d89ae220dcbebb6dc474a7 (commit)
via 568954946c142635351c66f34b2da5432265c3b4 (commit)
via 7e1ff5f3b44cb5698e490297d08d708138518569 (commit)
via 163acd6733a4af2969875ef72ad8c3b225a11e02 (commit)
via 889cdaf5db96d87f14cff6a232408ab55e1226b6 (commit)
via 689901263ebfdd996da3711236615038e6245db3 (commit)
via 6bd68f542545ae8694da311c7a6757a8a7a6e7b5 (commit)
via cb13593decb097f501c0a1d64510a653b3233395 (commit)
via dac1fdef3e46dfab1f16da27b4f45613dd9b71e6 (commit)
via f7c4c3fa0d3d177982a7ff665325ea082a872c99 (commit)
via 771897a58ea5fb2980c040d29ef01232abecec81 (commit)
via 86074c13f4441fa0804e30a1d68781175ba32e0d (commit)
via 8a207287266e997b4e9b8d10a02ce68ce1cf7b69 (commit)
via 889c4f35d4b74d7c6f043790b04104fec9d8e37b (commit)
via 6a08de4e6d04ed9706ae5f6502c9ff3d26a8bc0e (commit)
via 50d3c3897f0fad1bfcc4fc86096155d15d25483e (commit)
via 90a750e42e27ee5cfdffa65ba675b19005cbb345 (commit)
via a5cce35e9cb84ef0487e13058702fc7368c8d546 (commit)
from 1f2f20ac03b2f11a0bdaa1e057f85e170e1994ed (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8d60b124aba24724a0ace90872276e1f1c5a09f3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 15 12:04:19 2019 -0400
14287: Fill in resp.items[].kind even if no uuid/pdh is selected.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 3eba61145..995fb01ff 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -65,18 +65,28 @@ func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts re
if respKind != "" {
tmp["kind"] = respKind
}
+ defaultItemKind := ""
+ if strings.HasSuffix(respKind, "List") {
+ defaultItemKind = strings.TrimSuffix(respKind, "List")
+ }
if items, ok := tmp["items"].([]interface{}); ok {
for i, item := range items {
- // Fill in "kind" by inspecting UUID
+ // Fill in "kind" by inspecting UUID/PDH if
+ // possible; fall back on assuming each
+ // Items[] entry in an "arvados#fooList"
+ // response should have kind="arvados#foo".
item, _ := item.(map[string]interface{})
- uuid, _ := item["uuid"].(string)
- if len(uuid) != 27 {
- // unsure whether this happens
- } else if t, ok := infixMap[uuid[6:11]]; !ok {
- // infix not listed in infixMap
- } else if k := kind(t); k != "" {
+ infix := ""
+ if uuid, _ := item["uuid"].(string); len(uuid) == 27 {
+ infix = uuid[6:11]
+ }
+ if k := kind(infixMap[infix]); k != "" {
item["kind"] = k
+ } else if pdh, _ := item["portable_data_hash"].(string); pdh != "" {
+ item["kind"] = "arvados#collection"
+ } else if defaultItemKind != "" {
+ item["kind"] = defaultItemKind
}
items[i] = applySelectParam(opts.Select, item)
}
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 2e354f925..348216d18 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -11,6 +11,7 @@ import (
"net/http"
"net/http/httptest"
"os"
+ "strings"
"testing"
"time"
@@ -54,27 +55,50 @@ func (s *RouterSuite) doRequest(c *check.C, token, method, path string, hdrs htt
return req, rw, jresp
}
-func (s *RouterSuite) TestCollectionParams(c *check.C) {
+func (s *RouterSuite) TestCollectionResponses(c *check.C) {
token := arvadostest.ActiveTokenV2
- _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections?include_trash=true`, nil, nil)
- c.Check(rw.Code, check.Equals, http.StatusOK)
- c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
- c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
- c.Check(jresp["items"].([]interface{})[0].(map[string]interface{})["kind"], check.Equals, "arvados#collection")
-
- _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
+ // Check "get collection" response has "kind" key
+ _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
c.Check(jresp["items"].([]interface{})[0].(map[string]interface{})["kind"], check.Equals, "arvados#collection")
- _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`ensure_unique_name=true`))
- c.Check(rw.Code, check.Equals, http.StatusOK)
- c.Check(jresp["uuid"], check.FitsTypeOf, "")
- c.Check(jresp["kind"], check.Equals, "arvados#collection")
+ // Check items in list response have a "kind" key regardless
+ // of whether a uuid/pdh is selected.
+ for _, selectj := range []string{
+ ``,
+ `,"select":["portable_data_hash"]`,
+ `,"select":["name"]`,
+ `,"select":["uuid"]`,
+ } {
+ _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"where":{"uuid":["`+arvadostest.FooCollection+`"]}`+selectj+`}`))
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
+ c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+ c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
+ item0 := jresp["items"].([]interface{})[0].(map[string]interface{})
+ c.Check(item0["kind"], check.Equals, "arvados#collection")
+ if selectj == "" || strings.Contains(selectj, "portable_data_hash") {
+ c.Check(item0["portable_data_hash"], check.Equals, arvadostest.FooCollectionPDH)
+ } else {
+ c.Check(item0["portable_data_hash"], check.IsNil)
+ }
+ if selectj == "" || strings.Contains(selectj, "name") {
+ c.Check(item0["name"], check.FitsTypeOf, "")
+ } else {
+ c.Check(item0["name"], check.IsNil)
+ }
+ if selectj == "" || strings.Contains(selectj, "uuid") {
+ c.Check(item0["uuid"], check.Equals, arvadostest.FooCollection)
+ } else {
+ c.Check(item0["uuid"], check.IsNil)
+ }
+ }
- _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections?ensure_unique_name=true`, nil, nil)
+ // Check "create collection" response has "kind" key
+ _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`ensure_unique_name=true`))
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.FitsTypeOf, "")
c.Check(jresp["kind"], check.Equals, "arvados#collection")
commit d74b9938429acae9f2459d81baa987ef22d75e3a
Merge: 900e78f4d d235817fe
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 15 11:12:30 2019 -0400
14287: Merge branch 'master' into 14287-controller-structure
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc lib/controller/federation_test.go
index 06c8f0086,c4aa33c15..43344c744
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@@ -65,11 -64,10 +65,11 @@@ func (s *FederationSuite) SetUpTest(c *
NodeProfiles: map[string]arvados.NodeProfile{
"*": nodeProfile,
},
- RequestLimits: arvados.RequestLimits{
- MaxItemsPerResponse: 1000,
- MultiClusterRequestConcurrency: 4,
+ API: arvados.API{
+ MaxItemsPerResponse: 1000,
+ MaxRequestAmplification: 4,
},
+ EnableBetaController14287: enableBetaController14287,
}, NodeProfile: &nodeProfile}
s.testServer = newServerFromIntegrationTestEnv(c)
s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
diff --cc lib/controller/handler.go
index c799b617f,775d29034..8df6ab4e5
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -73,14 -72,8 +73,15 @@@ func (h *Handler) setup()
mux.Handle("/_health/", &health.Handler{
Token: h.Cluster.ManagementToken,
Prefix: "/_health/",
+ Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
})
+
+ if h.Cluster.EnableBetaController14287 {
+ rtr := router.New(h.Cluster, h.NodeProfile)
+ mux.Handle("/arvados/v1/collections", rtr)
+ mux.Handle("/arvados/v1/collections/", rtr)
+ }
+
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
hs = h.setupProxyRemoteCluster(hs)
diff --cc sdk/go/arvados/config.go
index d309748f4,6b3150c6f..6d249b1df
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -68,11 -68,9 +68,11 @@@ type Cluster struct
HTTPRequestTimeout Duration
RemoteClusters map[string]RemoteCluster
PostgreSQL PostgreSQL
- RequestLimits RequestLimits
+ API API
Logging Logging
TLS TLS
+
+ EnableBetaController14287 bool
}
type Services struct {
diff --cc sdk/python/tests/run_test_server.py
index d52a28459,79767c2fa..e595a298a
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@@ -413,8 -413,7 +413,9 @@@ def run_controller()
f.write("""
Clusters:
zzzzz:
+ Logging:
+ Level: "{}"
+ ManagementToken: e687950a23c3a9bceec28c6223a06c79
HTTPRequestTimeout: 30s
PostgreSQL:
ConnectionPool: 32
commit 900e78f4d16da54796797db543b9777292ab5ec6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 14 11:20:49 2019 -0400
14287: Handle collection/.../provenance and .../used_by requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index ad46d8788..cdebde885 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -26,6 +26,8 @@ type Interface interface {
CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error)
CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error)
CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error)
+ CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error)
+ CollectionUsedBy(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error)
CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error)
ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error)
ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error)
@@ -240,6 +242,14 @@ func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOption
return conn.local.CollectionList(ctx, options)
}
+func (conn *Conn) CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
+ return conn.local.CollectionProvenance(ctx, options)
+}
+
+func (conn *Conn) CollectionUsedBy(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
+ return conn.local.CollectionUsedBy(ctx, options)
+}
+
func (conn *Conn) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
return conn.chooseBackend(options.UUID).CollectionDelete(ctx, options)
}
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 82ca5ef5e..3eba61145 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -53,15 +53,19 @@ func applySelectParam(selectParam []string, orig map[string]interface{}) map[str
}
func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts responseOptions) {
- respKind := kind(resp)
var tmp map[string]interface{}
+
err := rtr.transcode(resp, &tmp)
if err != nil {
rtr.sendError(w, err)
return
}
- tmp["kind"] = respKind
+ respKind := kind(resp)
+ if respKind != "" {
+ tmp["kind"] = respKind
+ }
+
if items, ok := tmp["items"].([]interface{}); ok {
for i, item := range items {
// Fill in "kind" by inspecting UUID
@@ -71,8 +75,8 @@ func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts re
// unsure whether this happens
} else if t, ok := infixMap[uuid[6:11]]; !ok {
// infix not listed in infixMap
- } else {
- item["kind"] = kind(t)
+ } else if k := kind(t); k != "" {
+ item["kind"] = k
}
items[i] = applySelectParam(opts.Select, item)
}
@@ -125,7 +129,11 @@ var infixMap = map[string]interface{}{
var mungeKind = regexp.MustCompile(`\..`)
func kind(resp interface{}) string {
- return mungeKind.ReplaceAllStringFunc(fmt.Sprintf("%T", resp), func(s string) string {
+ t := fmt.Sprintf("%T", resp)
+ if !strings.HasPrefix(t, "arvados.") {
+ return ""
+ }
+ return mungeKind.ReplaceAllStringFunc(t, func(s string) string {
// "arvados.CollectionList" => "arvados#collectionList"
return "#" + strings.ToLower(s[1:])
})
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index cd66e90d2..179436034 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -64,6 +64,20 @@ func (rtr *router) addRoutes(cluster *arvados.Cluster) {
},
},
{
+ arvados.EndpointCollectionProvenance,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionProvenance(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
+ arvados.EndpointCollectionUsedBy,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.fed.CollectionUsedBy(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+ {
arvados.EndpointCollectionDelete,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 9bb3eb33f..e74e870ad 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -140,6 +140,20 @@ func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOption
return resp, err
}
+func (conn *Conn) CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
+ ep := arvados.EndpointCollectionProvenance
+ var resp map[string]interface{}
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
+func (conn *Conn) CollectionUsedBy(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
+ ep := arvados.EndpointCollectionUsedBy
+ var resp map[string]interface{}
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
func (conn *Conn) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
ep := arvados.EndpointCollectionDelete
var resp arvados.Collection
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 84f73c5a2..874e9e517 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -16,6 +16,8 @@ var (
EndpointCollectionUpdate = APIEndpoint{"PATCH", "arvados/v1/collections/:uuid", "collection"}
EndpointCollectionGet = APIEndpoint{"GET", "arvados/v1/collections/:uuid", ""}
EndpointCollectionList = APIEndpoint{"GET", "arvados/v1/collections", ""}
+ EndpointCollectionProvenance = APIEndpoint{"GET", "arvados/v1/collections/:uuid/provenance", ""}
+ EndpointCollectionUsedBy = APIEndpoint{"GET", "arvados/v1/collections/:uuid/used_by", ""}
EndpointCollectionDelete = APIEndpoint{"DELETE", "arvados/v1/collections/:uuid", ""}
EndpointSpecimenCreate = APIEndpoint{"POST", "arvados/v1/specimens", "specimen"}
EndpointSpecimenUpdate = APIEndpoint{"PATCH", "arvados/v1/specimens/:uuid", "specimen"}
commit 4490616be768aeda32979995cedcb6c7ca79504e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 14 11:20:10 2019 -0400
14287: Test request formatting variations.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/request_test.go b/lib/controller/router/request_test.go
index cffdccc90..02cc9ce3f 100644
--- a/lib/controller/router/request_test.go
+++ b/lib/controller/router/request_test.go
@@ -6,24 +6,163 @@ package router
import (
"bytes"
+ "encoding/json"
+ "io"
+ "net/http"
"net/http/httptest"
+ "net/url"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
+type testReq struct {
+ method string
+ path string
+ token string // default is ActiveTokenV2; use noToken to omit
+ param map[string]interface{}
+ attrs map[string]interface{}
+ attrsKey string
+ header http.Header
+
+ // variations on request formatting
+ json bool
+ jsonAttrsTop bool
+ jsonStringParam bool
+ tokenInBody bool
+ tokenInQuery bool
+ noContentType bool
+
+ body *bytes.Buffer
+}
+
+const noToken = "(no token)"
+
+func (tr *testReq) Request() *http.Request {
+ param := map[string]interface{}{}
+ for k, v := range tr.param {
+ param[k] = v
+ }
+
+ if tr.body != nil {
+ // caller provided a buffer
+ } else if tr.json {
+ if tr.jsonAttrsTop {
+ for k, v := range tr.attrs {
+ param[k] = v
+ }
+ } else if tr.attrs != nil {
+ param[tr.attrsKey] = tr.attrs
+ }
+ tr.body = bytes.NewBuffer(nil)
+ err := json.NewEncoder(tr.body).Encode(param)
+ if err != nil {
+ panic(err)
+ }
+ } else {
+ values := make(url.Values)
+ for k, v := range param {
+ if vs, ok := v.(string); ok && !tr.jsonStringParam {
+ values.Set(k, vs)
+ } else {
+ jv, err := json.Marshal(v)
+ if err != nil {
+ panic(err)
+ }
+ values.Set(k, string(jv))
+ }
+ }
+ if tr.attrs != nil {
+ jattrs, err := json.Marshal(tr.attrs)
+ if err != nil {
+ panic(err)
+ }
+ values.Set(tr.attrsKey, string(jattrs))
+ }
+ tr.body = bytes.NewBuffer(nil)
+ io.WriteString(tr.body, values.Encode())
+ }
+ method := tr.method
+ if method == "" {
+ method = "GET"
+ }
+ path := tr.path
+ if path == "" {
+ path = "example/test/path"
+ }
+ req := httptest.NewRequest(method, "https://an.example/"+path, tr.body)
+ token := tr.token
+ if token == "" {
+ token = arvadostest.ActiveTokenV2
+ }
+ if token != noToken {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ if tr.json {
+ req.Header.Set("Content-Type", "application/json")
+ } else {
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ }
+ for k, v := range tr.header {
+ req.Header[k] = append([]string(nil), v...)
+ }
+ return req
+}
+
+func (tr *testReq) bodyContent() string {
+ return string(tr.body.Bytes())
+}
+
func (s *RouterSuite) TestAttrsInBody(c *check.C) {
- for _, body := range []string{
- `{"foo":"bar"}`,
- `{"model_name": {"foo":"bar"}}`,
+ attrs := map[string]interface{}{"foo": "bar"}
+ for _, tr := range []testReq{
+ {attrsKey: "model_name", json: true, attrs: attrs},
+ {attrsKey: "model_name", json: true, attrs: attrs, jsonAttrsTop: true},
} {
- c.Logf("body: %s", body)
- req := httptest.NewRequest("POST", "https://an.example/ctrl", bytes.NewBufferString(body))
- req.Header.Set("Content-Type", "application/json")
- params, err := s.rtr.loadRequestParams(req, "model_name")
- c.Assert(err, check.IsNil)
+ c.Logf("tr: %#v", tr)
+ req := tr.Request()
+ params, err := s.rtr.loadRequestParams(req, tr.attrsKey)
c.Logf("params: %#v", params)
+ c.Assert(err, check.IsNil)
c.Check(params, check.NotNil)
c.Assert(params["attrs"], check.FitsTypeOf, map[string]interface{}{})
c.Check(params["attrs"].(map[string]interface{})["foo"], check.Equals, "bar")
}
}
+
+func (s *RouterSuite) TestBoolParam(c *check.C) {
+ testKey := "ensure_unique_name"
+
+ for i, tr := range []testReq{
+ {method: "POST", param: map[string]interface{}{testKey: false}, json: true},
+ {method: "POST", param: map[string]interface{}{testKey: false}},
+ {method: "POST", param: map[string]interface{}{testKey: "false"}},
+ {method: "POST", param: map[string]interface{}{testKey: "0"}},
+ {method: "POST", param: map[string]interface{}{testKey: ""}},
+ } {
+ c.Logf("#%d, tr: %#v", i, tr)
+ req := tr.Request()
+ c.Logf("tr.body: %s", tr.bodyContent())
+ params, err := s.rtr.loadRequestParams(req, tr.attrsKey)
+ c.Logf("params: %#v", params)
+ c.Assert(err, check.IsNil)
+ c.Check(params, check.NotNil)
+ c.Check(params[testKey], check.Equals, false)
+ }
+
+ for i, tr := range []testReq{
+ {method: "POST", param: map[string]interface{}{testKey: true}, json: true},
+ {method: "POST", param: map[string]interface{}{testKey: true}},
+ {method: "POST", param: map[string]interface{}{testKey: "true"}},
+ {method: "POST", param: map[string]interface{}{testKey: "1"}},
+ } {
+ c.Logf("#%d, tr: %#v", i, tr)
+ req := tr.Request()
+ c.Logf("tr.body: %s", tr.bodyContent())
+ params, err := s.rtr.loadRequestParams(req, tr.attrsKey)
+ c.Logf("params: %#v", params)
+ c.Assert(err, check.IsNil)
+ c.Check(params, check.NotNil)
+ c.Check(params[testKey], check.Equals, true)
+ }
+}
commit 7520945351c2bb42481354c57e66486f271734fc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 13 15:58:43 2019 -0400
14287: Set controller log level=debug in tests if ARVADOS_DEBUG set.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 6687ca491..d52a28459 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -413,6 +413,8 @@ def run_controller():
f.write("""
Clusters:
zzzzz:
+ Logging:
+ Level: "{}"
HTTPRequestTimeout: 30s
PostgreSQL:
ConnectionPool: 32
@@ -430,6 +432,7 @@ Clusters:
TLS: true
Insecure: true
""".format(
+ ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
_dbconfig('host'),
_dbconfig('database'),
_dbconfig('username'),
commit cce14cb6e78ee472cfde4629101e284b2cb63d1b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 13 15:58:01 2019 -0400
14287: Propagate etag in collection records in responses.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 136159a7e..5b919bea7 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -16,6 +16,7 @@ import (
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid"`
+ Etag string `json:"etag"`
OwnerUUID string `json:"owner_uuid"`
TrashAt *time.Time `json:"trash_at"`
ManifestText string `json:"manifest_text"`
commit 5fa8839f730408828a583f1e80c2538b23fa7005
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 13 15:57:36 2019 -0400
14287: Propagate where param in list requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index d53907308..84f73c5a2 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -38,12 +38,13 @@ type GetOptions struct {
}
type ListOptions struct {
- Select []string `json:"select"`
- Filters []Filter `json:"filters"`
- Limit int `json:"limit"`
- Offset int `json:"offset"`
- Order string `json:"order"`
- Count string `json:"count"`
+ Select []string `json:"select"`
+ Filters []Filter `json:"filters"`
+ Where map[string]interface{} `json:"where"`
+ Limit int `json:"limit"`
+ Offset int `json:"offset"`
+ Order string `json:"order"`
+ Count string `json:"count"`
}
type CreateOptions struct {
commit 57abc6f7661db3dda9eee9ca66669f642d1cb4bd
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 13 15:55:54 2019 -0400
14287: Remove extra zeroes from items[] entries too.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 4536380fd..82ca5ef5e 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -35,22 +35,54 @@ func (rtr *router) responseOptions(opts interface{}) (responseOptions, error) {
return rOpts, nil
}
+func applySelectParam(selectParam []string, orig map[string]interface{}) map[string]interface{} {
+ if len(selectParam) == 0 {
+ return orig
+ }
+ selected := map[string]interface{}{}
+ for _, attr := range selectParam {
+ if v, ok := orig[attr]; ok {
+ selected[attr] = v
+ }
+ }
+ // Preserve "kind" even if not requested
+ if v, ok := orig["kind"]; ok {
+ selected["kind"] = v
+ }
+ return selected
+}
+
func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts responseOptions) {
+ respKind := kind(resp)
var tmp map[string]interface{}
err := rtr.transcode(resp, &tmp)
if err != nil {
rtr.sendError(w, err)
return
}
- if len(opts.Select) > 0 {
- selected := map[string]interface{}{}
- for _, attr := range opts.Select {
- if v, ok := tmp[attr]; ok {
- selected[attr] = v
+
+ tmp["kind"] = respKind
+ if items, ok := tmp["items"].([]interface{}); ok {
+ for i, item := range items {
+ // Fill in "kind" by inspecting UUID
+ item, _ := item.(map[string]interface{})
+ uuid, _ := item["uuid"].(string)
+ if len(uuid) != 27 {
+ // unsure whether this happens
+ } else if t, ok := infixMap[uuid[6:11]]; !ok {
+ // infix not listed in infixMap
+ } else {
+ item["kind"] = kind(t)
}
+ items[i] = applySelectParam(opts.Select, item)
+ }
+ if opts.Count == "none" {
+ delete(tmp, "items_available")
}
- tmp = selected
+ } else {
+ tmp = applySelectParam(opts.Select, tmp)
}
+
// Format non-nil timestamps as rfc3339NanoFixed (by default
// they will have been encoded to time.RFC3339Nano, which
// omits trailing zeroes).
@@ -74,7 +106,6 @@ func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts re
tmp[k] = t.Format(rfc3339NanoFixed)
}
}
- tmp["kind"] = kind(resp)
json.NewEncoder(w).Encode(tmp)
}
@@ -86,6 +117,11 @@ func (rtr *router) sendError(w http.ResponseWriter, err error) {
httpserver.Error(w, err.Error(), code)
}
+var infixMap = map[string]interface{}{
+ "4zz18": arvados.Collection{},
+ "j7d0g": arvados.Group{},
+}
+
var mungeKind = regexp.MustCompile(`\..`)
func kind(resp interface{}) string {
commit 20c5632277f5f87ea047ba51dca82f0ce8aed81d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 13 15:51:51 2019 -0400
14287: Remove zero/missing values when req uses select or count=none.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 9a2891140..4536380fd 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -20,6 +20,7 @@ const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
type responseOptions struct {
Select []string
+ Count string
}
func (rtr *router) responseOptions(opts interface{}) (responseOptions, error) {
@@ -27,6 +28,9 @@ func (rtr *router) responseOptions(opts interface{}) (responseOptions, error) {
switch opts := opts.(type) {
case *arvados.GetOptions:
rOpts.Select = opts.Select
+ case *arvados.ListOptions:
+ rOpts.Select = opts.Select
+ rOpts.Count = opts.Count
}
return rOpts, nil
}
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index a1c790680..d53907308 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -43,6 +43,7 @@ type ListOptions struct {
Limit int `json:"limit"`
Offset int `json:"offset"`
Order string `json:"order"`
+ Count string `json:"count"`
}
type CreateOptions struct {
commit 9a251d966ae83934381840e4b47478e10221d265
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri May 10 16:00:37 2019 -0400
14287: Add "kind" key to controller responses.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index ddbeee666..9a2891140 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -6,7 +6,9 @@ package router
import (
"encoding/json"
+ "fmt"
"net/http"
+ "regexp"
"strings"
"time"
@@ -68,6 +70,7 @@ func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts re
tmp[k] = t.Format(rfc3339NanoFixed)
}
}
+ tmp["kind"] = kind(resp)
json.NewEncoder(w).Encode(tmp)
}
@@ -78,3 +81,12 @@ func (rtr *router) sendError(w http.ResponseWriter, err error) {
}
httpserver.Error(w, err.Error(), code)
}
+
+var mungeKind = regexp.MustCompile(`\..`)
+
+func kind(resp interface{}) string {
+ return mungeKind.ReplaceAllStringFunc(fmt.Sprintf("%T", resp), func(s string) string {
+ // "arvados.CollectionList" => "arvados#collectionList"
+ return "#" + strings.ToLower(s[1:])
+ })
+}
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 686b8933a..2e354f925 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -60,18 +60,24 @@ func (s *RouterSuite) TestCollectionParams(c *check.C) {
_, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections?include_trash=true`, nil, nil)
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+ c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
+ c.Check(jresp["items"].([]interface{})[0].(map[string]interface{})["kind"], check.Equals, "arvados#collection")
_, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
+ c.Check(jresp["kind"], check.Equals, "arvados#collectionList")
+ c.Check(jresp["items"].([]interface{})[0].(map[string]interface{})["kind"], check.Equals, "arvados#collection")
_, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, bytes.NewBufferString(`ensure_unique_name=true`))
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.FitsTypeOf, "")
+ c.Check(jresp["kind"], check.Equals, "arvados#collection")
_, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections?ensure_unique_name=true`, nil, nil)
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.FitsTypeOf, "")
+ c.Check(jresp["kind"], check.Equals, "arvados#collection")
}
func (s *RouterSuite) TestContainerList(c *check.C) {
@@ -161,6 +167,7 @@ func (s *RouterSuite) TestSelectParam(c *check.C) {
_, rw, resp := s.doRequest(c, token, "GET", "/arvados/v1/containers/"+uuid+"?select="+string(j), nil, nil)
c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(resp["kind"], check.Equals, "arvados#container")
c.Check(resp["uuid"], check.HasLen, 27)
c.Check(resp["command"], check.HasLen, 2)
c.Check(resp["mounts"], check.IsNil)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list