[ARVADOS] updated: 2.1.0-1501-g3e9284a47
Git user
git at public.arvados.org
Wed Oct 27 01:01:57 UTC 2021
Summary of changes:
doc/_includes/_metadata_vocabulary_example.liquid | 4 +-
lib/config/export.go | 6 ++
lib/controller/federation/conn.go | 6 ++
lib/controller/handler.go | 1 +
lib/controller/handler_test.go | 41 ++++++++++++
lib/controller/router/router.go | 7 ++
lib/controller/rpc/conn.go | 7 ++
sdk/go/arvados/api.go | 2 +
sdk/go/arvados/vocabulary.go | 80 +++++++++++++++++++++--
sdk/go/arvados/vocabulary_test.go | 78 +++++++++++++++++++++-
10 files changed, 222 insertions(+), 10 deletions(-)
via 3e9284a47aaaac14cf2ebb4104ffcc0d3960fcfe (commit)
via 2c45047246d7a98558b9107332a47e2e8f6aada7 (commit)
from 670026b664b5e7de5346c863d07ee4b03b2fdaa3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3e9284a47aaaac14cf2ebb4104ffcc0d3960fcfe
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Oct 26 22:01:29 2021 -0300
17944: Adds vocabulary check, with tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 585e5932a..e978b3755 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"reflect"
+ "strings"
)
type Vocabulary struct {
@@ -49,6 +50,9 @@ func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
}
func (v *Vocabulary) Validate() error {
+ if v == nil {
+ return nil
+ }
tagKeys := map[string]bool{}
// Checks for Vocabulary strictness
if v.StrictTags && len(v.Tags) == 0 {
@@ -61,10 +65,11 @@ func (v *Vocabulary) Validate() error {
}
tagKeys[key] = true
for _, lbl := range v.Tags[key].Labels {
- if tagKeys[lbl.Label] {
- return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", lbl.Label, key)
+ label := strings.ToLower(lbl.Label)
+ if tagKeys[label] {
+ return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", label, key)
}
- tagKeys[lbl.Label] = true
+ tagKeys[label] = true
}
// Checks for value strictness
if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 {
@@ -78,10 +83,73 @@ func (v *Vocabulary) Validate() error {
}
tagValues[val] = true
for _, tagLbl := range v.Tags[key].Values[val].Labels {
- if tagValues[tagLbl.Label] {
- return fmt.Errorf("tag value label %q for value %q[%q] already seen as a value key or label", tagLbl.Label, key, val)
+ label := strings.ToLower(tagLbl.Label)
+ if tagValues[label] {
+ return fmt.Errorf("tag value label %q for pair (%q:%q) already seen as a value key or label", label, key, val)
}
- tagValues[tagLbl.Label] = true
+ tagValues[label] = true
+ }
+ }
+ }
+ return nil
+}
+
+func (v *Vocabulary) getLabelsToKeys() (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ for key, val := range v.Tags {
+ for _, lbl := range val.Labels {
+ label := strings.ToLower(lbl.Label)
+ labels[label] = key
+ }
+ }
+ return labels
+}
+
+func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ if _, ok := v.Tags[key]; ok {
+ for val := range v.Tags[key].Values {
+ for _, tagLbl := range v.Tags[key].Values[val].Labels {
+ label := strings.ToLower(tagLbl.Label)
+ labels[label] = val
+ }
+ }
+ }
+ return labels
+}
+
+// Check validates the given data against the vocabulary.
+func (v *Vocabulary) Check(data map[string]interface{}) error {
+ if v == nil {
+ return nil
+ }
+ for key, val := range data {
+ // Checks for key validity
+ if _, ok := v.Tags[key]; !ok {
+ lcKey := strings.ToLower(key)
+ alias, ok := v.getLabelsToKeys()[lcKey]
+ if ok {
+ return fmt.Errorf("tag key %q is not defined but is an alias for %q", key, alias)
+ } else if v.StrictTags {
+ return fmt.Errorf("tag key %q is not defined", key)
+ }
+ // If the key is not defined, we don't need to check the value
+ return nil
+ }
+ // Checks for value validity -- key is defined
+ if _, ok := v.Tags[key].Values[val.(string)]; !ok {
+ lcVal := strings.ToLower(val.(string))
+ alias, ok := v.getLabelsToValues(key)[lcVal]
+ if ok {
+ return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
+ } else if v.Tags[key].Strict {
+ return fmt.Errorf("tag value %q is not defined", val)
}
}
}
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 45ef3dbd1..3a3e4f1b5 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -5,13 +5,89 @@
package arvados
import (
+ "encoding/json"
+
check "gopkg.in/check.v1"
)
-type VocabularySuite struct{}
+type VocabularySuite struct {
+ testVoc *Vocabulary
+}
var _ = check.Suite(&VocabularySuite{})
+func (s *VocabularySuite) SetUpTest(c *check.C) {
+ s.testVoc = &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+ },
+ },
+ },
+ "IDTAGIMPORTANCE": {
+ Strict: true,
+ Labels: []VocabularyLabel{{Label: "Importance"}, {Label: "Priority"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVAL3": {
+ Labels: []VocabularyLabel{{Label: "Low"}, {Label: "Low priority"}},
+ },
+ "IDVAL2": {
+ Labels: []VocabularyLabel{{Label: "Medium"}, {Label: "Medium priority"}},
+ },
+ "IDVAL1": {
+ Labels: []VocabularyLabel{{Label: "High"}, {Label: "High priority"}},
+ },
+ },
+ },
+ "IDTAGCOMMENT": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Comment"}},
+ },
+ },
+ }
+ err := s.testVoc.Validate()
+ c.Assert(err, check.IsNil)
+}
+
+func (s *VocabularySuite) TestCheck(c *check.C) {
+ tests := []struct {
+ name string
+ strictVoc bool
+ props string
+ expectSuccess bool
+ }{
+ {"Unknown key to non-strict vocabulary", false, `{"foo":"bar"}`, true},
+ {"Unknown key to strict vocabulary", true, `{"foo":"bar"}`, false},
+ {"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
+ {"Known non-strict key, unknown value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false},
+ {"Known strict key, unknown value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
+ {"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ s.testVoc.StrictTags = tt.strictVoc
+
+ var data map[string]interface{}
+ err := json.Unmarshal([]byte(tt.props), &data)
+ c.Assert(err, check.IsNil)
+ err = s.testVoc.Check(data)
+ if tt.expectSuccess {
+ c.Assert(err, check.IsNil)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
tests := []struct {
name string
commit 2c45047246d7a98558b9107332a47e2e8f6aada7
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Oct 26 14:06:30 2021 -0300
17944: Exports JSON-encoded vocabulary to /arvados/v1/vocabulary endpoint.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/doc/_includes/_metadata_vocabulary_example.liquid b/doc/_includes/_metadata_vocabulary_example.liquid
index 016b48c6a..fb8e57725 100644
--- a/doc/_includes/_metadata_vocabulary_example.liquid
+++ b/doc/_includes/_metadata_vocabulary_example.liquid
@@ -2,9 +2,7 @@
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{
+{% endcomment %}{
"strict_tags": false,
"tags": {
"IDTAGANIMALS": {
diff --git a/lib/config/export.go b/lib/config/export.go
index f2c15b0ee..fe33c73d3 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -14,6 +14,12 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
)
+// ExportVocabularyJSON writes a JSON object with the loaded vocabulary
+// to w.
+func ExportVocabularyJSON(w io.Writer, cluster *arvados.Cluster) error {
+ return json.NewEncoder(w).Encode(cluster.API.Vocabulary)
+}
+
// ExportJSON writes a JSON object with the safe (non-secret) portions
// of the cluster config to w.
func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index aa05cb1e6..495c87aee 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -192,6 +192,12 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return json.RawMessage(buf.Bytes()), err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+ var buf bytes.Buffer
+ err := config.ExportVocabularyJSON(&buf, conn.cluster)
+ return json.RawMessage(buf.Bytes()), err
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID {
// defer entire login procedure to designated cluster
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index a35d00301..51c72b282 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -97,6 +97,7 @@ func (h *Handler) setup() {
WrapCalls: api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls),
})
mux.Handle("/arvados/v1/config", rtr)
+ mux.Handle("/arvados/v1/vocabulary", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
mux.Handle("/arvados/v1/collections", rtr)
mux.Handle("/arvados/v1/collections/", rtr)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 9b71c349a..063523c55 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -88,6 +88,47 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
}
}
+func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
+ s.cluster.API.Vocabulary = &arvados.Vocabulary{
+ Tags: map[string]arvados.VocabularyTag{
+ "IDTAGIMPORTANCE": {
+ Labels: []arvados.VocabularyLabel{{Label: "Importance"}},
+ Values: map[string]arvados.VocabularyTagValue{
+ "HIGH": {
+ Labels: []arvados.VocabularyLabel{{Label: "High"}},
+ },
+ "LOW": {
+ Labels: []arvados.VocabularyLabel{{Label: "Low"}},
+ },
+ },
+ },
+ },
+ }
+ err := s.cluster.API.Vocabulary.Validate()
+ c.Check(err, check.IsNil)
+ for _, method := range []string{"GET", "OPTIONS"} {
+ c.Log(c.TestName()+" ", method)
+ req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Log(resp.Body.String())
+ if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+ continue
+ }
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
+ if method == "OPTIONS" {
+ c.Check(resp.Body.String(), check.HasLen, 0)
+ continue
+ }
+ var voc *arvados.Vocabulary
+ err := json.Unmarshal(resp.Body.Bytes(), &voc)
+ c.Check(err, check.IsNil)
+ c.Check(voc, check.DeepEquals, s.cluster.API.Vocabulary)
+ }
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 9826c1e74..d04eccf68 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -65,6 +65,13 @@ func (rtr *router) addRoutes() {
return rtr.backend.ConfigGet(ctx)
},
},
+ {
+ arvados.EndpointVocabularyGet,
+ func() interface{} { return &struct{}{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.VocabularyGet(ctx)
+ },
+ },
{
arvados.EndpointLogin,
func() interface{} { return &arvados.LoginOptions{} },
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 640bbf1c2..bd9332b03 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -178,6 +178,13 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return resp, err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+ ep := arvados.EndpointVocabularyGet
+ var resp json.RawMessage
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, nil)
+ return resp, err
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
ep := arvados.EndpointLogin
var resp arvados.LoginResponse
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index b429e8008..63f784f15 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -23,6 +23,7 @@ type APIEndpoint struct {
var (
EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""}
+ EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
EndpointLogin = APIEndpoint{"GET", "login", ""}
EndpointLogout = APIEndpoint{"GET", "logout", ""}
EndpointCollectionCreate = APIEndpoint{"POST", "arvados/v1/collections", "collection"}
@@ -219,6 +220,7 @@ type BlockWriteResponse struct {
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
+ VocabularyGet(ctx context.Context) (json.RawMessage, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error)
CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list