[ARVADOS] updated: 2.1.0-1501-g3e9284a47

Git user git at public.arvados.org
Wed Oct 27 01:01:57 UTC 2021


Summary of changes:
 doc/_includes/_metadata_vocabulary_example.liquid |  4 +-
 lib/config/export.go                              |  6 ++
 lib/controller/federation/conn.go                 |  6 ++
 lib/controller/handler.go                         |  1 +
 lib/controller/handler_test.go                    | 41 ++++++++++++
 lib/controller/router/router.go                   |  7 ++
 lib/controller/rpc/conn.go                        |  7 ++
 sdk/go/arvados/api.go                             |  2 +
 sdk/go/arvados/vocabulary.go                      | 80 +++++++++++++++++++++--
 sdk/go/arvados/vocabulary_test.go                 | 78 +++++++++++++++++++++-
 10 files changed, 222 insertions(+), 10 deletions(-)

       via  3e9284a47aaaac14cf2ebb4104ffcc0d3960fcfe (commit)
       via  2c45047246d7a98558b9107332a47e2e8f6aada7 (commit)
      from  670026b664b5e7de5346c863d07ee4b03b2fdaa3 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 3e9284a47aaaac14cf2ebb4104ffcc0d3960fcfe
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date:   Tue Oct 26 22:01:29 2021 -0300

    17944: Adds vocabulary check, with tests.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>

diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 585e5932a..e978b3755 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"strings"
 )
 
 type Vocabulary struct {
@@ -49,6 +50,9 @@ func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
 }
 
 func (v *Vocabulary) Validate() error {
+	if v == nil {
+		return nil
+	}
 	tagKeys := map[string]bool{}
 	// Checks for Vocabulary strictness
 	if v.StrictTags && len(v.Tags) == 0 {
@@ -61,10 +65,11 @@ func (v *Vocabulary) Validate() error {
 		}
 		tagKeys[key] = true
 		for _, lbl := range v.Tags[key].Labels {
-			if tagKeys[lbl.Label] {
-				return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", lbl.Label, key)
+			label := strings.ToLower(lbl.Label)
+			if tagKeys[label] {
+				return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", label, key)
 			}
-			tagKeys[lbl.Label] = true
+			tagKeys[label] = true
 		}
 		// Checks for value strictness
 		if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 {
@@ -78,10 +83,73 @@ func (v *Vocabulary) Validate() error {
 			}
 			tagValues[val] = true
 			for _, tagLbl := range v.Tags[key].Values[val].Labels {
-				if tagValues[tagLbl.Label] {
-					return fmt.Errorf("tag value label %q for value %q[%q] already seen as a value key or label", tagLbl.Label, key, val)
+				label := strings.ToLower(tagLbl.Label)
+				if tagValues[label] {
+					return fmt.Errorf("tag value label %q for pair (%q:%q) already seen as a value key or label", label, key, val)
 				}
-				tagValues[tagLbl.Label] = true
+				tagValues[label] = true
+			}
+		}
+	}
+	return nil
+}
+
+func (v *Vocabulary) getLabelsToKeys() (labels map[string]string) {
+	if v == nil {
+		return
+	}
+	labels = make(map[string]string)
+	for key, val := range v.Tags {
+		for _, lbl := range val.Labels {
+			label := strings.ToLower(lbl.Label)
+			labels[label] = key
+		}
+	}
+	return labels
+}
+
+func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) {
+	if v == nil {
+		return
+	}
+	labels = make(map[string]string)
+	if _, ok := v.Tags[key]; ok {
+		for val := range v.Tags[key].Values {
+			for _, tagLbl := range v.Tags[key].Values[val].Labels {
+				label := strings.ToLower(tagLbl.Label)
+				labels[label] = val
+			}
+		}
+	}
+	return labels
+}
+
+// Check validates the given data against the vocabulary.
+func (v *Vocabulary) Check(data map[string]interface{}) error {
+	if v == nil {
+		return nil
+	}
+	for key, val := range data {
+		// Checks for key validity
+		if _, ok := v.Tags[key]; !ok {
+			lcKey := strings.ToLower(key)
+			alias, ok := v.getLabelsToKeys()[lcKey]
+			if ok {
+				return fmt.Errorf("tag key %q is not defined but is an alias for %q", key, alias)
+			} else if v.StrictTags {
+				return fmt.Errorf("tag key %q is not defined", key)
+			}
+			// If the key is not defined, we don't need to check the value
+			return nil
+		}
+		// Checks for value validity -- key is defined
+		if _, ok := v.Tags[key].Values[val.(string)]; !ok {
+			lcVal := strings.ToLower(val.(string))
+			alias, ok := v.getLabelsToValues(key)[lcVal]
+			if ok {
+				return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
+			} else if v.Tags[key].Strict {
+				return fmt.Errorf("tag value %q is not defined", val)
 			}
 		}
 	}
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 45ef3dbd1..3a3e4f1b5 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -5,13 +5,89 @@
 package arvados
 
 import (
+	"encoding/json"
+
 	check "gopkg.in/check.v1"
 )
 
-type VocabularySuite struct{}
+type VocabularySuite struct {
+	testVoc *Vocabulary
+}
 
 var _ = check.Suite(&VocabularySuite{})
 
+func (s *VocabularySuite) SetUpTest(c *check.C) {
+	s.testVoc = &Vocabulary{
+		StrictTags: false,
+		Tags: map[string]VocabularyTag{
+			"IDTAGANIMALS": {
+				Strict: false,
+				Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+				Values: map[string]VocabularyTagValue{
+					"IDVALANIMAL1": {
+						Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+					},
+					"IDVALANIMAL2": {
+						Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+					},
+				},
+			},
+			"IDTAGIMPORTANCE": {
+				Strict: true,
+				Labels: []VocabularyLabel{{Label: "Importance"}, {Label: "Priority"}},
+				Values: map[string]VocabularyTagValue{
+					"IDVAL3": {
+						Labels: []VocabularyLabel{{Label: "Low"}, {Label: "Low priority"}},
+					},
+					"IDVAL2": {
+						Labels: []VocabularyLabel{{Label: "Medium"}, {Label: "Medium priority"}},
+					},
+					"IDVAL1": {
+						Labels: []VocabularyLabel{{Label: "High"}, {Label: "High priority"}},
+					},
+				},
+			},
+			"IDTAGCOMMENT": {
+				Strict: false,
+				Labels: []VocabularyLabel{{Label: "Comment"}},
+			},
+		},
+	}
+	err := s.testVoc.Validate()
+	c.Assert(err, check.IsNil)
+}
+
+func (s *VocabularySuite) TestCheck(c *check.C) {
+	tests := []struct {
+		name          string
+		strictVoc     bool
+		props         string
+		expectSuccess bool
+	}{
+		{"Unknown key to non-strict vocabulary", false, `{"foo":"bar"}`, true},
+		{"Unknown key to strict vocabulary", true, `{"foo":"bar"}`, false},
+		{"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
+		{"Known non-strict key, unknown value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+		{"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false},
+		{"Known strict key, unknown value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
+		{"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false},
+	}
+	for _, tt := range tests {
+		c.Log(c.TestName()+" ", tt.name)
+		s.testVoc.StrictTags = tt.strictVoc
+
+		var data map[string]interface{}
+		err := json.Unmarshal([]byte(tt.props), &data)
+		c.Assert(err, check.IsNil)
+		err = s.testVoc.Check(data)
+		if tt.expectSuccess {
+			c.Assert(err, check.IsNil)
+		} else {
+			c.Assert(err, check.NotNil)
+		}
+	}
+}
+
 func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
 	tests := []struct {
 		name       string

commit 2c45047246d7a98558b9107332a47e2e8f6aada7
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date:   Tue Oct 26 14:06:30 2021 -0300

    17944: Exports JSON-encoded vocabulary to /arvados/v1/vocabulary endpoint.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>

diff --git a/doc/_includes/_metadata_vocabulary_example.liquid b/doc/_includes/_metadata_vocabulary_example.liquid
index 016b48c6a..fb8e57725 100644
--- a/doc/_includes/_metadata_vocabulary_example.liquid
+++ b/doc/_includes/_metadata_vocabulary_example.liquid
@@ -2,9 +2,7 @@
 Copyright (C) The Arvados Authors. All rights reserved.
 
 SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{
+{% endcomment %}{
     "strict_tags": false,
     "tags": {
         "IDTAGANIMALS": {
diff --git a/lib/config/export.go b/lib/config/export.go
index f2c15b0ee..fe33c73d3 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -14,6 +14,12 @@ import (
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
+// ExportVocabularyJSON writes a JSON object with the loaded vocabulary
+// to w.
+func ExportVocabularyJSON(w io.Writer, cluster *arvados.Cluster) error {
+	return json.NewEncoder(w).Encode(cluster.API.Vocabulary)
+}
+
 // ExportJSON writes a JSON object with the safe (non-secret) portions
 // of the cluster config to w.
 func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index aa05cb1e6..495c87aee 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -192,6 +192,12 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
 	return json.RawMessage(buf.Bytes()), err
 }
 
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+	var buf bytes.Buffer
+	err := config.ExportVocabularyJSON(&buf, conn.cluster)
+	return json.RawMessage(buf.Bytes()), err
+}
+
 func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
 	if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID {
 		// defer entire login procedure to designated cluster
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index a35d00301..51c72b282 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -97,6 +97,7 @@ func (h *Handler) setup() {
 		WrapCalls:      api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls),
 	})
 	mux.Handle("/arvados/v1/config", rtr)
+	mux.Handle("/arvados/v1/vocabulary", rtr)
 	mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
 	mux.Handle("/arvados/v1/collections", rtr)
 	mux.Handle("/arvados/v1/collections/", rtr)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 9b71c349a..063523c55 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -88,6 +88,47 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
 	}
 }
 
+func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
+	s.cluster.API.Vocabulary = &arvados.Vocabulary{
+		Tags: map[string]arvados.VocabularyTag{
+			"IDTAGIMPORTANCE": {
+				Labels: []arvados.VocabularyLabel{{Label: "Importance"}},
+				Values: map[string]arvados.VocabularyTagValue{
+					"HIGH": {
+						Labels: []arvados.VocabularyLabel{{Label: "High"}},
+					},
+					"LOW": {
+						Labels: []arvados.VocabularyLabel{{Label: "Low"}},
+					},
+				},
+			},
+		},
+	}
+	err := s.cluster.API.Vocabulary.Validate()
+	c.Check(err, check.IsNil)
+	for _, method := range []string{"GET", "OPTIONS"} {
+		c.Log(c.TestName()+" ", method)
+		req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
+		resp := httptest.NewRecorder()
+		s.handler.ServeHTTP(resp, req)
+		c.Log(resp.Body.String())
+		if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+			continue
+		}
+		c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
+		c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
+		c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
+		if method == "OPTIONS" {
+			c.Check(resp.Body.String(), check.HasLen, 0)
+			continue
+		}
+		var voc *arvados.Vocabulary
+		err := json.Unmarshal(resp.Body.Bytes(), &voc)
+		c.Check(err, check.IsNil)
+		c.Check(voc, check.DeepEquals, s.cluster.API.Vocabulary)
+	}
+}
+
 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 	req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 	resp := httptest.NewRecorder()
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 9826c1e74..d04eccf68 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -65,6 +65,13 @@ func (rtr *router) addRoutes() {
 				return rtr.backend.ConfigGet(ctx)
 			},
 		},
+		{
+			arvados.EndpointVocabularyGet,
+			func() interface{} { return &struct{}{} },
+			func(ctx context.Context, opts interface{}) (interface{}, error) {
+				return rtr.backend.VocabularyGet(ctx)
+			},
+		},
 		{
 			arvados.EndpointLogin,
 			func() interface{} { return &arvados.LoginOptions{} },
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 640bbf1c2..bd9332b03 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -178,6 +178,13 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
 	return resp, err
 }
 
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+	ep := arvados.EndpointVocabularyGet
+	var resp json.RawMessage
+	err := conn.requestAndDecode(ctx, &resp, ep, nil, nil)
+	return resp, err
+}
+
 func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
 	ep := arvados.EndpointLogin
 	var resp arvados.LoginResponse
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index b429e8008..63f784f15 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -23,6 +23,7 @@ type APIEndpoint struct {
 
 var (
 	EndpointConfigGet                     = APIEndpoint{"GET", "arvados/v1/config", ""}
+	EndpointVocabularyGet                 = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
 	EndpointLogin                         = APIEndpoint{"GET", "login", ""}
 	EndpointLogout                        = APIEndpoint{"GET", "logout", ""}
 	EndpointCollectionCreate              = APIEndpoint{"POST", "arvados/v1/collections", "collection"}
@@ -219,6 +220,7 @@ type BlockWriteResponse struct {
 
 type API interface {
 	ConfigGet(ctx context.Context) (json.RawMessage, error)
+	VocabularyGet(ctx context.Context) (json.RawMessage, error)
 	Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
 	Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error)
 	CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list