[ARVADOS] created: 2.1.0-1512-g1d7f02105
Git user
git at public.arvados.org
Mon Nov 1 20:48:13 UTC 2021
at 1d7f021057d8cbfcd4390eb62a62c66dd65dc151 (commit)
commit 1d7f021057d8cbfcd4390eb62a62c66dd65dc151
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Nov 1 17:12:14 2021 -0300
17944: Forces vocabulary checking at startup time.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 51c72b282..638591307 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"net/http"
+ "net/http/httptest"
"net/url"
"strings"
"sync"
@@ -74,7 +75,21 @@ func (h *Handler) CheckHealth() error {
return err
}
_, _, err = railsproxy.FindRailsAPI(h.Cluster)
- return err
+ if err != nil {
+ return err
+ }
+ if h.Cluster.API.VocabularyPath != "" {
+ req, err := http.NewRequest("GET", "/arvados/v1/vocabulary", nil)
+ if err != nil {
+ return err
+ }
+ var resp httptest.ResponseRecorder
+ h.handlerStack.ServeHTTP(&resp, req)
+ if resp.Result().StatusCode != http.StatusOK {
+ return fmt.Errorf("%d %s", resp.Result().StatusCode, resp.Result().Status)
+ }
+ }
+ return nil
}
func (h *Handler) Done() <-chan struct{} {
commit 590a2f655f6a33aec7462b03101b4facfdcb9950
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Nov 1 16:02:20 2021 -0300
17944: Vocabulary check errors return status 400 instead of 500.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index c99faba73..728c760af 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -138,6 +138,54 @@ func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
}
}
+func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
+ voc := `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCE": {
+ "strict": true,
+ "labels": [{"label": "Importance"}],
+ "values": {
+ "HIGH": {
+ "labels": [{"label": "High"}]
+ },
+ "LOW": {
+ "labels": [{"label": "Low"}]
+ }
+ }
+ }
+ }
+ }`
+ f, err := os.CreateTemp("", "test-vocabulary-*.json")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(f.Name())
+ _, err = f.WriteString(voc)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ s.cluster.API.VocabularyPath = f.Name()
+
+ req := httptest.NewRequest("POST", "/arvados/v1/collections",
+ strings.NewReader(`{
+ "collection": {
+ "properties": {
+ "IDTAGIMPORTANCE": "Critical"
+ }
+ }
+ }`))
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ req.Header.Set("Content-type", "application/json")
+
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Log(resp.Body.String())
+ c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
+ var jresp httpserver.ErrorResponse
+ err = json.Unmarshal(resp.Body.Bytes(), &jresp)
+ c.Check(err, check.IsNil)
+ c.Assert(len(jresp.Errors), check.Equals, 1)
+ c.Check(jresp.Errors[0], check.Matches, `.*tag value.*for key.*is not listed as valid.*`)
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index bec9354b1..e3f89ebf4 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -8,6 +8,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "net/http"
"os"
"strings"
@@ -15,6 +16,7 @@ import (
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
)
@@ -60,7 +62,7 @@ func (conn *Conn) checkProperties(ctx context.Context, properties interface{}) e
if err != nil {
return err
}
- return voc.Check(props)
+ return httpErrorf(http.StatusBadRequest, voc.Check(props).Error())
}
func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
@@ -205,3 +207,7 @@ func (conn *Conn) GroupContents(ctx context.Context, options arvados.GroupConten
return conn.railsProxy.GroupContents(ctx, options)
}
+
+func httpErrorf(code int, format string, args ...interface{}) error {
+ return httpserver.ErrorWithStatus(fmt.Errorf(format, args...), code)
+}
commit e01b0608e49fbb159c809db346079293e5dc27cb
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Fri Oct 29 19:14:59 2021 -0300
17944: Moves vocabulary loading & caching to localdb.Conn
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/export.go b/lib/config/export.go
index fe33c73d3..f2c15b0ee 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -14,12 +14,6 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
)
-// ExportVocabularyJSON writes a JSON object with the loaded vocabulary
-// to w.
-func ExportVocabularyJSON(w io.Writer, cluster *arvados.Cluster) error {
- return json.NewEncoder(w).Encode(cluster.API.Vocabulary)
-}
-
// ExportJSON writes a JSON object with the safe (non-secret) portions
// of the cluster config to w.
func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
diff --git a/lib/config/load.go b/lib/config/load.go
index 684d7bbe6..248960beb 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -17,7 +17,6 @@ import (
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "github.com/fsnotify/fsnotify"
"github.com/ghodss/yaml"
"github.com/imdario/mergo"
"github.com/sirupsen/logrus"
@@ -28,7 +27,6 @@ var ErrNoClustersDefined = errors.New("config does not define any clusters")
type Loader struct {
Stdin io.Reader
Logger logrus.FieldLogger
- LoadVocabulary bool // Load the vocabulary from API.VocabularyPath
SkipDeprecated bool // Don't load deprecated config keys
SkipLegacy bool // Don't load legacy config files
SkipAPICalls bool // Don't do checks that call RailsAPI/controller
@@ -271,9 +269,6 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
ldr.loadOldKeepBalanceConfig,
)
}
- if ldr.LoadVocabulary {
- loadFuncs = append(loadFuncs, ldr.loadVocabulary)
- }
loadFuncs = append(loadFuncs, ldr.setImplicitStorageClasses)
for _, f := range loadFuncs {
err = f(&cfg)
@@ -394,84 +389,6 @@ func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
return nil
}
-func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
- cc, err := cfg.GetCluster("")
- if err != nil {
- return err
- }
- if cc.API.VocabularyPath == "" {
- return nil
- }
- ldr.Logger.Info("Loading vocabulary")
- voc, err := ldr.vocabularyFileLoader(cc.API.VocabularyPath, cc.Collections.ManagedProperties)
- if err != nil {
- return err
- }
- cc.API.Vocabulary = voc
-
- go watchVocabulary(ldr.Logger, cc.API.VocabularyPath, func() {
- ldr.Logger.Info("Reloading vocabulary")
- voc, err := ldr.vocabularyFileLoader(cc.API.VocabularyPath, cc.Collections.ManagedProperties)
- if err != nil {
- ldr.Logger.Error("Error reloading vocabulary: %v", err)
- }
- cc.API.Vocabulary = voc
- })
-
- return nil
-}
-
-func (ldr *Loader) vocabularyFileLoader(path string, mp arvados.ManagedProperties) (*arvados.Vocabulary, error) {
- vf, err := os.ReadFile(path)
- if err != nil {
- return nil, fmt.Errorf("couldn't read vocabulary file %q: %v", path, err)
- }
- // Managed properties' keys loading
- mk := make([]string, 0, len(mp))
- for k := range mp {
- mk = append(mk, k)
- }
- voc, err := arvados.NewVocabulary(vf, mk)
- if err != nil {
- return nil, fmt.Errorf("while loading vocabulary file %q: %s", path, err)
- }
- ldr.Logger.Info("Vocabulary loading succeeded")
- return voc, nil
-}
-
-func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
- watcher, err := fsnotify.NewWatcher()
- if err != nil {
- logger.WithError(err).Error("vocabulary fsnotify setup failed")
- return
- }
- defer watcher.Close()
-
- err = watcher.Add(vocPath)
- if err != nil {
- logger.WithError(err).Error("vocabulary file watcher failed")
- return
- }
-
- for {
- select {
- case err, ok := <-watcher.Errors:
- if !ok {
- return
- }
- logger.WithError(err).Warn("vocabulary file watcher error")
- case _, ok := <-watcher.Events:
- if !ok {
- return
- }
- for len(watcher.Events) > 0 {
- <-watcher.Events
- }
- fn()
- }
- }
-}
-
func checkKeyConflict(label string, m map[string]string) error {
saw := map[string]bool{}
for k := range m {
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 495c87aee..972941622 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -192,10 +192,8 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return json.RawMessage(buf.Bytes()), err
}
-func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
- var buf bytes.Buffer
- err := config.ExportVocabularyJSON(&buf, conn.cluster)
- return json.RawMessage(buf.Bytes()), err
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ return conn.chooseBackend(conn.cluster.ClusterID).VocabularyGet(ctx)
}
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 063523c55..c99faba73 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -89,23 +89,30 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
}
func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
- s.cluster.API.Vocabulary = &arvados.Vocabulary{
- Tags: map[string]arvados.VocabularyTag{
+ voc := `{
+ "strict_tags": false,
+ "tags": {
"IDTAGIMPORTANCE": {
- Labels: []arvados.VocabularyLabel{{Label: "Importance"}},
- Values: map[string]arvados.VocabularyTagValue{
+ "strict": false,
+ "labels": [{"label": "Importance"}],
+ "values": {
"HIGH": {
- Labels: []arvados.VocabularyLabel{{Label: "High"}},
+ "labels": [{"label": "High"}]
},
"LOW": {
- Labels: []arvados.VocabularyLabel{{Label: "Low"}},
- },
- },
- },
- },
- }
- err := s.cluster.API.Vocabulary.Validate()
- c.Check(err, check.IsNil)
+ "labels": [{"label": "Low"}]
+ }
+ }
+ }
+ }
+ }`
+ f, err := os.CreateTemp("", "test-vocabulary-*.json")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(f.Name())
+ _, err = f.WriteString(voc)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ s.cluster.API.VocabularyPath = f.Name()
for _, method := range []string{"GET", "OPTIONS"} {
c.Log(c.TestName()+" ", method)
req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
@@ -122,10 +129,12 @@ func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
c.Check(resp.Body.String(), check.HasLen, 0)
continue
}
- var voc *arvados.Vocabulary
- err := json.Unmarshal(resp.Body.Bytes(), &voc)
+ var expectedVoc, receivedVoc *arvados.Vocabulary
+ err := json.Unmarshal([]byte(voc), &expectedVoc)
+ c.Check(err, check.IsNil)
+ err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
c.Check(err, check.IsNil)
- c.Check(voc, check.DeepEquals, s.cluster.API.Vocabulary)
+ c.Check(receivedVoc, check.DeepEquals, expectedVoc)
}
}
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
index 6f99e57c4..96c89252e 100644
--- a/lib/controller/localdb/collection.go
+++ b/lib/controller/localdb/collection.go
@@ -51,7 +51,7 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions)
// CollectionCreate defers to railsProxy for everything except blob
// signatures and vocabulary checking.
func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.Collection{}, err
}
@@ -72,7 +72,7 @@ func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptio
// CollectionUpdate defers to railsProxy for everything except blob
// signatures and vocabulary checking.
func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.Collection{}, err
}
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 09a8dfbe1..ae996d27b 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -68,7 +68,8 @@ func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
- s.cluster.API.Vocabulary = voc
+ s.cluster.API.VocabularyPath = "foo"
+ s.localdb.vocabularyCache = voc
}
func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
@@ -94,7 +95,6 @@ func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(coll.Properties, check.DeepEquals, tt.props)
} else {
@@ -128,7 +128,6 @@ func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(coll.Properties, check.DeepEquals, tt.props)
} else {
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index a26381f74..bec9354b1 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -8,18 +8,24 @@ import (
"context"
"encoding/json"
"fmt"
+ "os"
"strings"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/fsnotify/fsnotify"
+ "github.com/sirupsen/logrus"
)
type railsProxy = rpc.Conn
type Conn struct {
- cluster *arvados.Cluster
- *railsProxy // handles API methods that aren't defined on Conn itself
+ cluster *arvados.Cluster
+ *railsProxy // handles API methods that aren't defined on Conn itself
+ vocabularyCache *arvados.Vocabulary
+ reloadVocabulary bool
loginController
}
@@ -34,7 +40,7 @@ func NewConn(cluster *arvados.Cluster) *Conn {
return &conn
}
-func (conn *Conn) checkProperties(properties interface{}) error {
+func (conn *Conn) checkProperties(ctx context.Context, properties interface{}) error {
if properties == nil {
return nil
}
@@ -50,7 +56,91 @@ func (conn *Conn) checkProperties(properties interface{}) error {
default:
return fmt.Errorf("unexpected properties type %T", properties)
}
- return conn.cluster.API.Vocabulary.Check(props)
+ voc, err := conn.VocabularyGet(ctx)
+ if err != nil {
+ return err
+ }
+ return voc.Check(props)
+}
+
+func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ logger.WithError(err).Error("vocabulary fsnotify setup failed")
+ return
+ }
+ defer watcher.Close()
+
+ err = watcher.Add(vocPath)
+ if err != nil {
+ logger.WithError(err).Error("vocabulary file watcher failed")
+ return
+ }
+
+ for {
+ select {
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ logger.WithError(err).Warn("vocabulary file watcher error")
+ case _, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ for len(watcher.Events) > 0 {
+ <-watcher.Events
+ }
+ fn()
+ }
+ }
+}
+
+func (conn *Conn) loadVocabularyFile() error {
+ vf, err := os.ReadFile(conn.cluster.API.VocabularyPath)
+ if err != nil {
+ return fmt.Errorf("couldn't read vocabulary file %q: %v", conn.cluster.API.VocabularyPath, err)
+ }
+ mk := make([]string, 0, len(conn.cluster.Collections.ManagedProperties))
+ for k := range conn.cluster.Collections.ManagedProperties {
+ mk = append(mk, k)
+ }
+ voc, err := arvados.NewVocabulary(vf, mk)
+ if err != nil {
+ return fmt.Errorf("while loading vocabulary file %q: %s", conn.cluster.API.VocabularyPath, err)
+ }
+ conn.vocabularyCache = voc
+ return nil
+}
+
+// VocabularyGet refreshes the vocabulary cache if necessary and returns it.
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ if conn.cluster.API.VocabularyPath == "" {
+ return arvados.Vocabulary{}, nil
+ }
+ logger := ctxlog.FromContext(ctx)
+ if conn.vocabularyCache == nil {
+ // Initial load of vocabulary file.
+ err := conn.loadVocabularyFile()
+ if err != nil {
+ logger.WithError(err).Error("error loading vocabulary file")
+ return arvados.Vocabulary{}, err
+ }
+ go watchVocabulary(logger, conn.cluster.API.VocabularyPath, func() {
+ logger.Info("vocabulary file changed, it'll be reloaded next time it's needed")
+ conn.reloadVocabulary = true
+ })
+ } else if conn.reloadVocabulary {
+ // Requested reload of vocabulary file.
+ conn.reloadVocabulary = false
+ err := conn.loadVocabularyFile()
+ if err != nil {
+ logger.WithError(err).Error("error reloading vocabulary file - ignoring")
+ } else {
+ logger.Info("vocabulary file reloaded successfully")
+ }
+ }
+ return *conn.vocabularyCache, nil
}
// Logout handles the logout of conn giving to the appropriate loginController
diff --git a/lib/controller/localdb/container_request.go b/lib/controller/localdb/container_request.go
index b1200be83..5b2ce95da 100644
--- a/lib/controller/localdb/container_request.go
+++ b/lib/controller/localdb/container_request.go
@@ -13,7 +13,7 @@ import (
// ContainerRequestCreate defers to railsProxy for everything except
// vocabulary checking.
func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.ContainerRequest, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.ContainerRequest{}, err
}
@@ -27,7 +27,7 @@ func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.Creat
// ContainerRequestUpdate defers to railsProxy for everything except
// vocabulary checking.
func (conn *Conn) ContainerRequestUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.ContainerRequest, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.ContainerRequest{}, err
}
diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go
index ab9d5b093..c231e3ca7 100644
--- a/lib/controller/localdb/container_request_test.go
+++ b/lib/controller/localdb/container_request_test.go
@@ -65,7 +65,8 @@ func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary strin
voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
- s.cluster.API.Vocabulary = voc
+ s.localdb.vocabularyCache = voc
+ s.cluster.API.VocabularyPath = "foo"
}
func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) {
@@ -106,7 +107,6 @@ func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(cnt.Properties, check.DeepEquals, tt.props)
} else {
@@ -158,7 +158,6 @@ func (s *ContainerRequestSuite) TestCRUpdateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(cnt.Properties, check.DeepEquals, tt.props)
} else {
diff --git a/lib/controller/localdb/group.go b/lib/controller/localdb/group.go
index d9617d8bc..0d77bdbd9 100644
--- a/lib/controller/localdb/group.go
+++ b/lib/controller/localdb/group.go
@@ -13,7 +13,7 @@ import (
// GroupCreate defers to railsProxy for everything except vocabulary
// checking.
func (conn *Conn) GroupCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Group, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.Group{}, err
}
@@ -27,7 +27,7 @@ func (conn *Conn) GroupCreate(ctx context.Context, opts arvados.CreateOptions) (
// GroupUpdate defers to railsProxy for everything except vocabulary
// checking.
func (conn *Conn) GroupUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Group, error) {
- err := conn.checkProperties(opts.Attrs["properties"])
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
if err != nil {
return arvados.Group{}, err
}
diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go
index 37bddfbd0..0991f3b72 100644
--- a/lib/controller/localdb/group_test.go
+++ b/lib/controller/localdb/group_test.go
@@ -65,7 +65,8 @@ func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) {
voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
- s.cluster.API.Vocabulary = voc
+ s.localdb.vocabularyCache = voc
+ s.cluster.API.VocabularyPath = "foo"
}
func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) {
@@ -92,7 +93,6 @@ func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(grp.Properties, check.DeepEquals, tt.props)
} else {
@@ -130,7 +130,6 @@ func (s *GroupSuite) TestGroupUpdateWithProperties(c *check.C) {
"properties": tt.props,
}})
if tt.success {
- c.Assert(err, check.IsNil)
c.Assert(err, check.IsNil)
c.Assert(grp.Properties, check.DeepEquals, tt.props)
} else {
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index bd9332b03..1acddfab7 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -178,9 +178,9 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return resp, err
}
-func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
ep := arvados.EndpointVocabularyGet
- var resp json.RawMessage
+ var resp arvados.Vocabulary
err := conn.requestAndDecode(ctx, &resp, ep, nil, nil)
return resp, err
}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 71c4399f7..e67c24f65 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -94,10 +94,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
// process _is_ the controller: we haven't started an
// http server yet.
loader.SkipAPICalls = true
- // The vocabulary file is expected to be present only
- // in the controller node, so it doesn't make sense to
- // try loading it elsewhere.
- loader.LoadVocabulary = true
}
cfg, err := loader.Load()
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 63f784f15..41727beea 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -220,7 +220,7 @@ type BlockWriteResponse struct {
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
- VocabularyGet(ctx context.Context) (json.RawMessage, error)
+ VocabularyGet(ctx context.Context) (Vocabulary, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error)
CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error)
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 87b71bfbe..1cd002082 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -108,7 +108,6 @@ type Cluster struct {
WebsocketClientEventQueue int
WebsocketServerEventQueue int
KeepServiceRequestTimeout Duration
- Vocabulary *Vocabulary `json:"-"`
VocabularyPath string
}
AuditLogs struct {
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index 64f7235f3..2cb35366c 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -33,9 +33,9 @@ func (as *APIStub) ConfigGet(ctx context.Context) (json.RawMessage, error) {
as.appendCall(ctx, as.ConfigGet, nil)
return nil, as.Error
}
-func (as *APIStub) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+func (as *APIStub) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
as.appendCall(ctx, as.VocabularyGet, nil)
- return nil, as.Error
+ return arvados.Vocabulary{}, as.Error
}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
as.appendCall(ctx, as.Login, options)
commit 64a87a49d8fa3219b08b402dc7971c27d105eedb
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Fri Oct 29 14:17:53 2021 -0300
17944: Refactors vocabulary file loader.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index 8462aa6e8..684d7bbe6 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -403,36 +403,40 @@ func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
return nil
}
ldr.Logger.Info("Loading vocabulary")
- err = ldr.vocabularyFileLoader(cc)
+ voc, err := ldr.vocabularyFileLoader(cc.API.VocabularyPath, cc.Collections.ManagedProperties)
if err != nil {
return err
}
+ cc.API.Vocabulary = voc
+
go watchVocabulary(ldr.Logger, cc.API.VocabularyPath, func() {
ldr.Logger.Info("Reloading vocabulary")
- err = ldr.vocabularyFileLoader(cc)
+ voc, err := ldr.vocabularyFileLoader(cc.API.VocabularyPath, cc.Collections.ManagedProperties)
if err != nil {
ldr.Logger.Error("Error reloading vocabulary: %v", err)
}
+ cc.API.Vocabulary = voc
})
+
return nil
}
-func (ldr *Loader) vocabularyFileLoader(cc *arvados.Cluster) error {
- vf, err := os.ReadFile(cc.API.VocabularyPath)
+func (ldr *Loader) vocabularyFileLoader(path string, mp arvados.ManagedProperties) (*arvados.Vocabulary, error) {
+ vf, err := os.ReadFile(path)
if err != nil {
- return fmt.Errorf("couldn't read vocabulary file %q: %v", cc.API.VocabularyPath, err)
+ return nil, fmt.Errorf("couldn't read vocabulary file %q: %v", path, err)
}
- mk := make([]string, 0, len(cc.Collections.ManagedProperties))
- for k := range cc.Collections.ManagedProperties {
+ // Managed properties' keys loading
+ mk := make([]string, 0, len(mp))
+ for k := range mp {
mk = append(mk, k)
}
voc, err := arvados.NewVocabulary(vf, mk)
if err != nil {
- return fmt.Errorf("while loading vocabulary file %q: %s", cc.API.VocabularyPath, err)
+ return nil, fmt.Errorf("while loading vocabulary file %q: %s", path, err)
}
- cc.API.Vocabulary = voc
ldr.Logger.Info("Vocabulary loading succeeded")
- return nil
+ return voc, nil
}
func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 50babd5f3..87b71bfbe 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -77,6 +77,12 @@ type UploadDownloadRolePermissions struct {
Admin UploadDownloadPermission
}
+type ManagedProperties map[string]struct {
+ Value interface{}
+ Function string
+ Protected bool
+}
+
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
@@ -111,23 +117,19 @@ type Cluster struct {
UnloggedAttributes StringSet
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- BlobTrash bool
- BlobTrashLifetime Duration
- BlobTrashCheckInterval Duration
- BlobTrashConcurrency int
- BlobDeleteConcurrency int
- BlobReplicateConcurrency int
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]struct {
- Value interface{}
- Function string
- Protected bool
- }
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ BlobTrash bool
+ BlobTrashLifetime Duration
+ BlobTrashCheckInterval Duration
+ BlobTrashConcurrency int
+ BlobDeleteConcurrency int
+ BlobReplicateConcurrency int
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties ManagedProperties
PreserveVersionIfIdle Duration
TrashSweepInterval Duration
TrustAllContent bool
commit 11cb8f51469f6e00d072ad0fb2158364f254f4e8
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Fri Oct 29 11:06:46 2021 -0300
17944: Adds vocabulary file auto-reloader.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index ee1000d14..8462aa6e8 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -17,6 +17,7 @@ import (
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/fsnotify/fsnotify"
"github.com/ghodss/yaml"
"github.com/imdario/mergo"
"github.com/sirupsen/logrus"
@@ -401,6 +402,22 @@ func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
if cc.API.VocabularyPath == "" {
return nil
}
+ ldr.Logger.Info("Loading vocabulary")
+ err = ldr.vocabularyFileLoader(cc)
+ if err != nil {
+ return err
+ }
+ go watchVocabulary(ldr.Logger, cc.API.VocabularyPath, func() {
+ ldr.Logger.Info("Reloading vocabulary")
+ err = ldr.vocabularyFileLoader(cc)
+ if err != nil {
+ ldr.Logger.Error("Error reloading vocabulary: %v", err)
+ }
+ })
+ return nil
+}
+
+func (ldr *Loader) vocabularyFileLoader(cc *arvados.Cluster) error {
vf, err := os.ReadFile(cc.API.VocabularyPath)
if err != nil {
return fmt.Errorf("couldn't read vocabulary file %q: %v", cc.API.VocabularyPath, err)
@@ -414,9 +431,43 @@ func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
return fmt.Errorf("while loading vocabulary file %q: %s", cc.API.VocabularyPath, err)
}
cc.API.Vocabulary = voc
+ ldr.Logger.Info("Vocabulary loading succeeded")
return nil
}
+func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ logger.WithError(err).Error("vocabulary fsnotify setup failed")
+ return
+ }
+ defer watcher.Close()
+
+ err = watcher.Add(vocPath)
+ if err != nil {
+ logger.WithError(err).Error("vocabulary file watcher failed")
+ return
+ }
+
+ for {
+ select {
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ logger.WithError(err).Warn("vocabulary file watcher error")
+ case _, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ for len(watcher.Events) > 0 {
+ <-watcher.Events
+ }
+ fn()
+ }
+ }
+}
+
func checkKeyConflict(label string, m map[string]string) error {
saw := map[string]bool{}
for k := range m {
commit 0111d00698155e5703fed8ad73ec03ac744862b7
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Oct 28 17:47:48 2021 -0300
17944: Adds reserved keys support on vocabulary.
Managed properties & other system-level property keys are specially treated
so that they'll be accepted on strict vocabularies.
Also, don't accept a vocabulary that attempts to define one of these keys to
avoid surprising the users that any value is allowed.
onArvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index a5b626a06..ee1000d14 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -405,7 +405,11 @@ func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
if err != nil {
return fmt.Errorf("couldn't read vocabulary file %q: %v", cc.API.VocabularyPath, err)
}
- voc, err := arvados.NewVocabulary(vf)
+ mk := make([]string, 0, len(cc.Collections.ManagedProperties))
+ for k := range cc.Collections.ManagedProperties {
+ mk = append(mk, k)
+ }
+ voc, err := arvados.NewVocabulary(vf, mk)
if err != nil {
return fmt.Errorf("while loading vocabulary file %q: %s", cc.API.VocabularyPath, err)
}
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index d7395d750..09a8dfbe1 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -65,7 +65,7 @@ func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
}
}`
}
- voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
s.cluster.API.Vocabulary = voc
diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go
index ce55d650b..ab9d5b093 100644
--- a/lib/controller/localdb/container_request_test.go
+++ b/lib/controller/localdb/container_request_test.go
@@ -62,7 +62,7 @@ func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary strin
}
}`
}
- voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
s.cluster.API.Vocabulary = voc
diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go
index d6b739c41..37bddfbd0 100644
--- a/lib/controller/localdb/group_test.go
+++ b/lib/controller/localdb/group_test.go
@@ -62,7 +62,7 @@ func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) {
}
}`
}
- voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
c.Assert(err, check.IsNil)
c.Assert(voc.Validate(), check.IsNil)
s.cluster.API.Vocabulary = voc
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 336927282..cb1106e9b 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -13,8 +13,9 @@ import (
)
type Vocabulary struct {
- StrictTags bool `json:"strict_tags"`
- Tags map[string]VocabularyTag `json:"tags"`
+ reservedTagKeys map[string]bool `json:"-"`
+ StrictTags bool `json:"strict_tags"`
+ Tags map[string]VocabularyTag `json:"tags"`
}
type VocabularyTag struct {
@@ -23,6 +24,20 @@ type VocabularyTag struct {
Values map[string]VocabularyTagValue `json:"values"`
}
+// Cannot have a constant map in Go, so we have to use a function
+func (v *Vocabulary) systemTagKeys() map[string]bool {
+ return map[string]bool{
+ "type": true,
+ "template_uuid": true,
+ "groups": true,
+ "username": true,
+ "image_timestamp": true,
+ "docker-image-repo-tag": true,
+ "filters": true,
+ "container_request": true,
+ }
+}
+
type VocabularyLabel struct {
Label string `json:"label"`
}
@@ -31,7 +46,7 @@ type VocabularyTagValue struct {
Labels []VocabularyLabel `json:"labels"`
}
-func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
+func NewVocabulary(data []byte, managedTagKeys []string) (voc *Vocabulary, err error) {
if r := bytes.Compare(data, []byte("")); r == 0 {
return &Vocabulary{}, nil
}
@@ -42,6 +57,13 @@ func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
if reflect.DeepEqual(voc, &Vocabulary{}) {
return nil, fmt.Errorf("JSON data provided doesn't match Vocabulary format: %q", data)
}
+ voc.reservedTagKeys = make(map[string]bool)
+ for _, managedKey := range managedTagKeys {
+ voc.reservedTagKeys[managedKey] = true
+ }
+ for systemKey := range voc.systemTagKeys() {
+ voc.reservedTagKeys[systemKey] = true
+ }
err = voc.Validate()
if err != nil {
return nil, err
@@ -60,6 +82,9 @@ func (v *Vocabulary) Validate() error {
}
// Checks for duplicate tag keys
for key := range v.Tags {
+ if v.reservedTagKeys[key] {
+ return fmt.Errorf("tag key %q is reserved", key)
+ }
if tagKeys[key] {
return fmt.Errorf("duplicate tag key %q", key)
}
@@ -144,6 +169,11 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
}
for key, val := range data {
// Checks for key validity
+ if v.reservedTagKeys[key] {
+ // Allow reserved keys to be used even if they are not defined in
+ // the vocabulary no matter its strictness.
+ continue
+ }
if _, ok := v.Tags[key]; !ok {
lcKey := strings.ToLower(key)
alias, ok := v.getLabelsToKeys()[lcKey]
@@ -153,7 +183,7 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
return fmt.Errorf("tag key %q is not defined", key)
}
// If the key is not defined, we don't need to check the value
- return nil
+ continue
}
// Checks for value validity -- key is defined
switch val := val.(type) {
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 043c37139..b2748c7be 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -18,6 +18,9 @@ var _ = check.Suite(&VocabularySuite{})
func (s *VocabularySuite) SetUpTest(c *check.C) {
s.testVoc = &Vocabulary{
+ reservedTagKeys: map[string]bool{
+ "reservedKey": true,
+ },
StrictTags: false,
Tags: map[string]VocabularyTag{
"IDTAGANIMALS": {
@@ -68,6 +71,7 @@ func (s *VocabularySuite) TestCheck(c *check.C) {
{"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
{"Unknown non-alias key on non-strict vocabulary", false, `{"foo":"bar"}`, true},
{"Known non-strict key, unknown non-alias value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Undefined but reserved key on strict vocabulary", true, `{"reservedKey":"bar"}`, true},
{"Known key, list of known values", false, `{"IDTAGANIMALS":["IDVALANIMAL1","IDVALANIMAL2"]}`, true},
{"Known non-strict key, list of unknown non-alias values", false, `{"IDTAGCOMMENT":["hello world","lorem ipsum"]}`, true},
// Check fails
@@ -120,6 +124,16 @@ func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
}}`,
true, "",
&Vocabulary{
+ reservedTagKeys: map[string]bool{
+ "type": true,
+ "template_uuid": true,
+ "groups": true,
+ "username": true,
+ "image_timestamp": true,
+ "docker-image-repo-tag": true,
+ "filters": true,
+ "container_request": true,
+ },
StrictTags: false,
Tags: map[string]VocabularyTag{
"IDTAGANIMALS": {
@@ -137,11 +151,21 @@ func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
},
},
},
+ {
+ "Valid data, but uses reserved key",
+ `{"tags":{
+ "type":{
+ "strict": false,
+ "labels": [{"label": "Type"}]
+ }
+ }}`,
+ false, "tag key.*is reserved", nil,
+ },
}
for _, tt := range tests {
c.Log(c.TestName()+" ", tt.name)
- voc, err := NewVocabulary([]byte(tt.data))
+ voc, err := NewVocabulary([]byte(tt.data), []string{})
if tt.isValid {
c.Assert(err, check.IsNil)
} else {
commit 0937d26624fe0236c31cf9f787bdb28ef1e7d390
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Oct 28 11:18:41 2021 -0300
17944: Fixes tests
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/localdb/container_request.go b/lib/controller/localdb/container_request.go
index 940c3cf84..b1200be83 100644
--- a/lib/controller/localdb/container_request.go
+++ b/lib/controller/localdb/container_request.go
@@ -10,7 +10,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
)
-// ContainerCreate defers to railsProxy for everything except
+// ContainerRequestCreate defers to railsProxy for everything except
// vocabulary checking.
func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.ContainerRequest, error) {
err := conn.checkProperties(opts.Attrs["properties"])
@@ -24,7 +24,7 @@ func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.Creat
return resp, nil
}
-// ContainerUpdate defers to railsProxy for everything except
+// ContainerRequestUpdate defers to railsProxy for everything except
// vocabulary checking.
func (conn *Conn) ContainerRequestUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.ContainerRequest, error) {
err := conn.checkProperties(opts.Attrs["properties"])
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index 8bf01693c..64f7235f3 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -33,6 +33,10 @@ func (as *APIStub) ConfigGet(ctx context.Context) (json.RawMessage, error) {
as.appendCall(ctx, as.ConfigGet, nil)
return nil, as.Error
}
+func (as *APIStub) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+ as.appendCall(ctx, as.VocabularyGet, nil)
+ return nil, as.Error
+}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
as.appendCall(ctx, as.Login, options)
return arvados.LoginResponse{}, as.Error
commit 8eb0a5fa1a226a8b8adc66d8c49422daa9445191
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Oct 27 22:08:39 2021 -0300
17944: Adds group & container_request vocabulary checking.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 44584fec0..d7395d750 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,7 +6,6 @@ package localdb
import (
"context"
- "encoding/json"
"regexp"
"strconv"
"time"
@@ -78,13 +77,13 @@ func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
tests := []struct {
name string
- props string
+ props map[string]interface{}
success bool
}{
- {"Invalid prop key", `{"Priority":"IDVALIMPORTANCES1"}`, false},
- {"Invalid prop value", `{"IDTAGIMPORTANCES": "high"}`, false},
- {"Valid prop key & value", `{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}`, true},
- {"Empty properties", "{}", true},
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
}
for _, tt := range tests {
c.Log(c.TestName()+" ", tt.name)
@@ -96,10 +95,8 @@ func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
}})
if tt.success {
c.Assert(err, check.IsNil)
- var wantedProps map[string]interface{}
- err = json.Unmarshal([]byte(tt.props), &wantedProps)
c.Assert(err, check.IsNil)
- c.Assert(coll.Properties, check.DeepEquals, wantedProps)
+ c.Assert(coll.Properties, check.DeepEquals, tt.props)
} else {
c.Assert(err, check.NotNil)
}
@@ -112,13 +109,13 @@ func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
tests := []struct {
name string
- props string
+ props map[string]interface{}
success bool
}{
- {"Invalid prop key", `{"Priority":"IDVALIMPORTANCES1"}`, false},
- {"Invalid prop value", `{"IDTAGIMPORTANCES": "high"}`, false},
- {"Valid prop key & value", `{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}`, true},
- {"Empty properties", "{}", true},
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
}
for _, tt := range tests {
c.Log(c.TestName()+" ", tt.name)
@@ -132,10 +129,8 @@ func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
}})
if tt.success {
c.Assert(err, check.IsNil)
- var wantedProps map[string]interface{}
- err = json.Unmarshal([]byte(tt.props), &wantedProps)
c.Assert(err, check.IsNil)
- c.Assert(coll.Properties, check.DeepEquals, wantedProps)
+ c.Assert(coll.Properties, check.DeepEquals, tt.props)
} else {
c.Assert(err, check.NotNil)
}
diff --git a/lib/controller/localdb/container_request.go b/lib/controller/localdb/container_request.go
new file mode 100644
index 000000000..940c3cf84
--- /dev/null
+++ b/lib/controller/localdb/container_request.go
@@ -0,0 +1,39 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// ContainerCreate defers to railsProxy for everything except
+// vocabulary checking.
+func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ resp, err := conn.railsProxy.ContainerRequestCreate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
+
+// ContainerUpdate defers to railsProxy for everything except
+// vocabulary checking.
+func (conn *Conn) ContainerRequestUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ resp, err := conn.railsProxy.ContainerRequestUpdate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go
new file mode 100644
index 000000000..ce55d650b
--- /dev/null
+++ b/lib/controller/localdb/container_request_test.go
@@ -0,0 +1,168 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ContainerRequestSuite{})
+
+type ContainerRequestSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *ContainerRequestSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *ContainerRequestSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *ContainerRequestSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.cluster.API.Vocabulary = voc
+}
+
+func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", "foo"},
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "output_path": "/out",
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ },
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(cnt.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *ContainerRequestSuite) TestCRUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", "foo"},
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "output_path": "/out",
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+ cnt, err = s.localdb.ContainerRequestUpdate(ctx, arvados.UpdateOptions{
+ UUID: cnt.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(cnt.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
diff --git a/lib/controller/localdb/group.go b/lib/controller/localdb/group.go
new file mode 100644
index 000000000..d9617d8bc
--- /dev/null
+++ b/lib/controller/localdb/group.go
@@ -0,0 +1,39 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// GroupCreate defers to railsProxy for everything except vocabulary
+// checking.
+func (conn *Conn) GroupCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Group, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Group{}, err
+ }
+ resp, err := conn.railsProxy.GroupCreate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
+
+// GroupUpdate defers to railsProxy for everything except vocabulary
+// checking.
+func (conn *Conn) GroupUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Group, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Group{}, err
+ }
+ resp, err := conn.railsProxy.GroupUpdate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go
new file mode 100644
index 000000000..d6b739c41
--- /dev/null
+++ b/lib/controller/localdb/group_test.go
@@ -0,0 +1,140 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&GroupSuite{})
+
+type GroupSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *GroupSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *GroupSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *GroupSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.cluster.API.Vocabulary = voc
+}
+
+func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "group_class": "project",
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(grp.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *GroupSuite) TestGroupUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "group_class": "project",
+ },
+ })
+ c.Assert(err, check.IsNil)
+ grp, err = s.localdb.GroupUpdate(ctx, arvados.UpdateOptions{
+ UUID: grp.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(grp.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
commit fe7b23bfd7d3e75bce23f73a9f2384f4e793a294
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Oct 27 18:13:03 2021 -0300
17944: Fixes tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index b6bdf0de6..a26381f74 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -38,11 +38,17 @@ func (conn *Conn) checkProperties(properties interface{}) error {
if properties == nil {
return nil
}
- // Check provided properties against the vocabulary.
var props map[string]interface{}
- err := json.Unmarshal([]byte(properties.(string)), &props)
- if err != nil {
- return err
+ switch properties := properties.(type) {
+ case string:
+ err := json.Unmarshal([]byte(properties), &props)
+ if err != nil {
+ return err
+ }
+ case map[string]interface{}:
+ props = properties
+ default:
+ return fmt.Errorf("unexpected properties type %T", properties)
}
return conn.cluster.API.Vocabulary.Check(props)
}
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index cf40eb6ff..336927282 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -168,11 +168,11 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
return err
}
default:
- return fmt.Errorf("tag value %q for key %q is not a valid type (%v)", singleVal, key, reflect.TypeOf(singleVal))
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", singleVal, key, singleVal)
}
}
default:
- return fmt.Errorf("tag value %q for key %q is not a valid type (%v)", val, key, reflect.TypeOf(val))
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", val, key, val)
}
}
return nil
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 3730e997e..043c37139 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -203,7 +203,7 @@ func (s *VocabularySuite) TestValidationErrors(c *check.C) {
},
},
},
- "tag value label.*for value.*already seen.*",
+ "tag value label.*for pair.*already seen.*",
},
{
"Strict key, no values",
commit bec3e4b91eb5234e990b78759794af15654aecee
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Oct 27 15:07:58 2021 -0300
17944: Adds vocabulary checking on collection's create & update.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
index d81dd812b..6f99e57c4 100644
--- a/lib/controller/localdb/collection.go
+++ b/lib/controller/localdb/collection.go
@@ -49,8 +49,12 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions)
}
// CollectionCreate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
@@ -66,8 +70,12 @@ func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptio
}
// CollectionUpdate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 4a4494964..44584fec0 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,6 +6,7 @@ package localdb
import (
"context"
+ "encoding/json"
"regexp"
"strconv"
"time"
@@ -48,6 +49,99 @@ func (s *CollectionSuite) TearDownTest(c *check.C) {
s.railsSpy.Close()
}
+func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary))
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.cluster.API.Vocabulary = voc
+}
+
+func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props string
+ success bool
+ }{
+ {"Invalid prop key", `{"Priority":"IDVALIMPORTANCES1"}`, false},
+ {"Invalid prop value", `{"IDTAGIMPORTANCES": "high"}`, false},
+ {"Valid prop key & value", `{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}`, true},
+ {"Empty properties", "{}", true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ var wantedProps map[string]interface{}
+ err = json.Unmarshal([]byte(tt.props), &wantedProps)
+ c.Assert(err, check.IsNil)
+ c.Assert(coll.Properties, check.DeepEquals, wantedProps)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props string
+ success bool
+ }{
+ {"Invalid prop key", `{"Priority":"IDVALIMPORTANCES1"}`, false},
+ {"Invalid prop value", `{"IDTAGIMPORTANCES": "high"}`, false},
+ {"Valid prop key & value", `{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}`, true},
+ {"Empty properties", "{}", true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
+ c.Assert(err, check.IsNil)
+ coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: coll.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ var wantedProps map[string]interface{}
+ err = json.Unmarshal([]byte(tt.props), &wantedProps)
+ c.Assert(err, check.IsNil)
+ c.Assert(coll.Properties, check.DeepEquals, wantedProps)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
func (s *CollectionSuite) TestSignatures(c *check.C) {
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index a90deded5..b6bdf0de6 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -6,6 +6,7 @@ package localdb
import (
"context"
+ "encoding/json"
"fmt"
"strings"
@@ -25,8 +26,7 @@ type Conn struct {
func NewConn(cluster *arvados.Cluster) *Conn {
railsProxy := railsproxy.NewConn(cluster)
railsProxy.RedactHostInErrors = true
- var conn Conn
- conn = Conn{
+ conn := Conn{
cluster: cluster,
railsProxy: railsProxy,
}
@@ -34,6 +34,19 @@ func NewConn(cluster *arvados.Cluster) *Conn {
return &conn
}
+func (conn *Conn) checkProperties(properties interface{}) error {
+ if properties == nil {
+ return nil
+ }
+ // Check provided properties against the vocabulary.
+ var props map[string]interface{}
+ err := json.Unmarshal([]byte(properties.(string)), &props)
+ if err != nil {
+ return err
+ }
+ return conn.cluster.API.Vocabulary.Check(props)
+}
+
// Logout handles the logout of conn giving to the appropriate loginController
func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
return conn.loginController.Logout(ctx, opts)
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 97d19d78d..cf40eb6ff 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -131,7 +131,7 @@ func (v *Vocabulary) checkValue(key, val string) error {
if ok {
return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
} else if v.Tags[key].Strict {
- return fmt.Errorf("tag value %q is not defined", val)
+ return fmt.Errorf("tag value %q for key %q is not listed as valid", val, key)
}
}
return nil
commit 6cceddfbb75ee59093ef133df4429be076dfacc4
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Oct 27 09:15:08 2021 -0300
17944: Adds list of strings support on vocabulary values.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index e978b3755..97d19d78d 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -124,6 +124,19 @@ func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) {
return labels
}
+func (v *Vocabulary) checkValue(key, val string) error {
+ if _, ok := v.Tags[key].Values[val]; !ok {
+ lcVal := strings.ToLower(val)
+ alias, ok := v.getLabelsToValues(key)[lcVal]
+ if ok {
+ return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
+ } else if v.Tags[key].Strict {
+ return fmt.Errorf("tag value %q is not defined", val)
+ }
+ }
+ return nil
+}
+
// Check validates the given data against the vocabulary.
func (v *Vocabulary) Check(data map[string]interface{}) error {
if v == nil {
@@ -143,14 +156,23 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
return nil
}
// Checks for value validity -- key is defined
- if _, ok := v.Tags[key].Values[val.(string)]; !ok {
- lcVal := strings.ToLower(val.(string))
- alias, ok := v.getLabelsToValues(key)[lcVal]
- if ok {
- return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
- } else if v.Tags[key].Strict {
- return fmt.Errorf("tag value %q is not defined", val)
+ switch val := val.(type) {
+ case string:
+ return v.checkValue(key, val)
+ case []interface{}:
+ for _, singleVal := range val {
+ switch singleVal := singleVal.(type) {
+ case string:
+ err := v.checkValue(key, singleVal)
+ if err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%v)", singleVal, key, reflect.TypeOf(singleVal))
+ }
}
+ default:
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%v)", val, key, reflect.TypeOf(val))
}
}
return nil
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 3a3e4f1b5..3730e997e 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -64,13 +64,19 @@ func (s *VocabularySuite) TestCheck(c *check.C) {
props string
expectSuccess bool
}{
- {"Unknown key to non-strict vocabulary", false, `{"foo":"bar"}`, true},
- {"Unknown key to strict vocabulary", true, `{"foo":"bar"}`, false},
+ // Check succeeds
{"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
- {"Known non-strict key, unknown value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Unknown non-alias key on non-strict vocabulary", false, `{"foo":"bar"}`, true},
+ {"Known non-strict key, unknown non-alias value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Known key, list of known values", false, `{"IDTAGANIMALS":["IDVALANIMAL1","IDVALANIMAL2"]}`, true},
+ {"Known non-strict key, list of unknown non-alias values", false, `{"IDTAGCOMMENT":["hello world","lorem ipsum"]}`, true},
+ // Check fails
+ {"Unknown non-alias key on strict vocabulary", true, `{"foo":"bar"}`, false},
{"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false},
- {"Known strict key, unknown value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
+ {"Known strict key, unknown non-alias value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
{"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false},
+ {"Known strict key, list of known alias values", false, `{"IDTAGIMPORTANCE":["Unimportant","High"]}`, false},
+ {"Known strict key, list of unknown non-alias values", false, `{"IDTAGIMPORTANCE":["foo","bar"]}`, false},
}
for _, tt := range tests {
c.Log(c.TestName()+" ", tt.name)
commit a6693f98843eca32b064165c180a7dfe87697d89
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Oct 26 22:01:29 2021 -0300
17944: Adds vocabulary check, with tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 585e5932a..e978b3755 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"reflect"
+ "strings"
)
type Vocabulary struct {
@@ -49,6 +50,9 @@ func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
}
func (v *Vocabulary) Validate() error {
+ if v == nil {
+ return nil
+ }
tagKeys := map[string]bool{}
// Checks for Vocabulary strictness
if v.StrictTags && len(v.Tags) == 0 {
@@ -61,10 +65,11 @@ func (v *Vocabulary) Validate() error {
}
tagKeys[key] = true
for _, lbl := range v.Tags[key].Labels {
- if tagKeys[lbl.Label] {
- return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", lbl.Label, key)
+ label := strings.ToLower(lbl.Label)
+ if tagKeys[label] {
+ return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", label, key)
}
- tagKeys[lbl.Label] = true
+ tagKeys[label] = true
}
// Checks for value strictness
if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 {
@@ -78,10 +83,73 @@ func (v *Vocabulary) Validate() error {
}
tagValues[val] = true
for _, tagLbl := range v.Tags[key].Values[val].Labels {
- if tagValues[tagLbl.Label] {
- return fmt.Errorf("tag value label %q for value %q[%q] already seen as a value key or label", tagLbl.Label, key, val)
+ label := strings.ToLower(tagLbl.Label)
+ if tagValues[label] {
+ return fmt.Errorf("tag value label %q for pair (%q:%q) already seen as a value key or label", label, key, val)
}
- tagValues[tagLbl.Label] = true
+ tagValues[label] = true
+ }
+ }
+ }
+ return nil
+}
+
+func (v *Vocabulary) getLabelsToKeys() (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ for key, val := range v.Tags {
+ for _, lbl := range val.Labels {
+ label := strings.ToLower(lbl.Label)
+ labels[label] = key
+ }
+ }
+ return labels
+}
+
+func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ if _, ok := v.Tags[key]; ok {
+ for val := range v.Tags[key].Values {
+ for _, tagLbl := range v.Tags[key].Values[val].Labels {
+ label := strings.ToLower(tagLbl.Label)
+ labels[label] = val
+ }
+ }
+ }
+ return labels
+}
+
+// Check validates the given data against the vocabulary.
+func (v *Vocabulary) Check(data map[string]interface{}) error {
+ if v == nil {
+ return nil
+ }
+ for key, val := range data {
+ // Checks for key validity
+ if _, ok := v.Tags[key]; !ok {
+ lcKey := strings.ToLower(key)
+ alias, ok := v.getLabelsToKeys()[lcKey]
+ if ok {
+ return fmt.Errorf("tag key %q is not defined but is an alias for %q", key, alias)
+ } else if v.StrictTags {
+ return fmt.Errorf("tag key %q is not defined", key)
+ }
+ // If the key is not defined, we don't need to check the value
+ return nil
+ }
+ // Checks for value validity -- key is defined
+ if _, ok := v.Tags[key].Values[val.(string)]; !ok {
+ lcVal := strings.ToLower(val.(string))
+ alias, ok := v.getLabelsToValues(key)[lcVal]
+ if ok {
+ return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
+ } else if v.Tags[key].Strict {
+ return fmt.Errorf("tag value %q is not defined", val)
}
}
}
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 45ef3dbd1..3a3e4f1b5 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -5,13 +5,89 @@
package arvados
import (
+ "encoding/json"
+
check "gopkg.in/check.v1"
)
-type VocabularySuite struct{}
+type VocabularySuite struct {
+ testVoc *Vocabulary
+}
var _ = check.Suite(&VocabularySuite{})
+func (s *VocabularySuite) SetUpTest(c *check.C) {
+ s.testVoc = &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+ },
+ },
+ },
+ "IDTAGIMPORTANCE": {
+ Strict: true,
+ Labels: []VocabularyLabel{{Label: "Importance"}, {Label: "Priority"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVAL3": {
+ Labels: []VocabularyLabel{{Label: "Low"}, {Label: "Low priority"}},
+ },
+ "IDVAL2": {
+ Labels: []VocabularyLabel{{Label: "Medium"}, {Label: "Medium priority"}},
+ },
+ "IDVAL1": {
+ Labels: []VocabularyLabel{{Label: "High"}, {Label: "High priority"}},
+ },
+ },
+ },
+ "IDTAGCOMMENT": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Comment"}},
+ },
+ },
+ }
+ err := s.testVoc.Validate()
+ c.Assert(err, check.IsNil)
+}
+
+func (s *VocabularySuite) TestCheck(c *check.C) {
+ tests := []struct {
+ name string
+ strictVoc bool
+ props string
+ expectSuccess bool
+ }{
+ {"Unknown key to non-strict vocabulary", false, `{"foo":"bar"}`, true},
+ {"Unknown key to strict vocabulary", true, `{"foo":"bar"}`, false},
+ {"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
+ {"Known non-strict key, unknown value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false},
+ {"Known strict key, unknown value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
+ {"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ s.testVoc.StrictTags = tt.strictVoc
+
+ var data map[string]interface{}
+ err := json.Unmarshal([]byte(tt.props), &data)
+ c.Assert(err, check.IsNil)
+ err = s.testVoc.Check(data)
+ if tt.expectSuccess {
+ c.Assert(err, check.IsNil)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
tests := []struct {
name string
commit a2ef7d5532cced0e150721f996fda8ccd7629463
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Oct 26 14:06:30 2021 -0300
17944: Exports JSON-encoded vocabulary to /arvados/v1/vocabulary endpoint.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/doc/_includes/_metadata_vocabulary_example.liquid b/doc/_includes/_metadata_vocabulary_example.liquid
index 016b48c6a..fb8e57725 100644
--- a/doc/_includes/_metadata_vocabulary_example.liquid
+++ b/doc/_includes/_metadata_vocabulary_example.liquid
@@ -2,9 +2,7 @@
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{
+{% endcomment %}{
"strict_tags": false,
"tags": {
"IDTAGANIMALS": {
diff --git a/lib/config/export.go b/lib/config/export.go
index f2c15b0ee..fe33c73d3 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -14,6 +14,12 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
)
+// ExportVocabularyJSON writes a JSON object with the loaded vocabulary
+// to w.
+func ExportVocabularyJSON(w io.Writer, cluster *arvados.Cluster) error {
+ return json.NewEncoder(w).Encode(cluster.API.Vocabulary)
+}
+
// ExportJSON writes a JSON object with the safe (non-secret) portions
// of the cluster config to w.
func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index aa05cb1e6..495c87aee 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -192,6 +192,12 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return json.RawMessage(buf.Bytes()), err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+ var buf bytes.Buffer
+ err := config.ExportVocabularyJSON(&buf, conn.cluster)
+ return json.RawMessage(buf.Bytes()), err
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID {
// defer entire login procedure to designated cluster
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index a35d00301..51c72b282 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -97,6 +97,7 @@ func (h *Handler) setup() {
WrapCalls: api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls),
})
mux.Handle("/arvados/v1/config", rtr)
+ mux.Handle("/arvados/v1/vocabulary", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
mux.Handle("/arvados/v1/collections", rtr)
mux.Handle("/arvados/v1/collections/", rtr)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 9b71c349a..063523c55 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -88,6 +88,47 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
}
}
+func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
+ s.cluster.API.Vocabulary = &arvados.Vocabulary{
+ Tags: map[string]arvados.VocabularyTag{
+ "IDTAGIMPORTANCE": {
+ Labels: []arvados.VocabularyLabel{{Label: "Importance"}},
+ Values: map[string]arvados.VocabularyTagValue{
+ "HIGH": {
+ Labels: []arvados.VocabularyLabel{{Label: "High"}},
+ },
+ "LOW": {
+ Labels: []arvados.VocabularyLabel{{Label: "Low"}},
+ },
+ },
+ },
+ },
+ }
+ err := s.cluster.API.Vocabulary.Validate()
+ c.Check(err, check.IsNil)
+ for _, method := range []string{"GET", "OPTIONS"} {
+ c.Log(c.TestName()+" ", method)
+ req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Log(resp.Body.String())
+ if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+ continue
+ }
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
+ if method == "OPTIONS" {
+ c.Check(resp.Body.String(), check.HasLen, 0)
+ continue
+ }
+ var voc *arvados.Vocabulary
+ err := json.Unmarshal(resp.Body.Bytes(), &voc)
+ c.Check(err, check.IsNil)
+ c.Check(voc, check.DeepEquals, s.cluster.API.Vocabulary)
+ }
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 9826c1e74..d04eccf68 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -65,6 +65,13 @@ func (rtr *router) addRoutes() {
return rtr.backend.ConfigGet(ctx)
},
},
+ {
+ arvados.EndpointVocabularyGet,
+ func() interface{} { return &struct{}{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.VocabularyGet(ctx)
+ },
+ },
{
arvados.EndpointLogin,
func() interface{} { return &arvados.LoginOptions{} },
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 640bbf1c2..bd9332b03 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -178,6 +178,13 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
return resp, err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (json.RawMessage, error) {
+ ep := arvados.EndpointVocabularyGet
+ var resp json.RawMessage
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, nil)
+ return resp, err
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
ep := arvados.EndpointLogin
var resp arvados.LoginResponse
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index b429e8008..63f784f15 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -23,6 +23,7 @@ type APIEndpoint struct {
var (
EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""}
+ EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
EndpointLogin = APIEndpoint{"GET", "login", ""}
EndpointLogout = APIEndpoint{"GET", "logout", ""}
EndpointCollectionCreate = APIEndpoint{"POST", "arvados/v1/collections", "collection"}
@@ -219,6 +220,7 @@ type BlockWriteResponse struct {
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
+ VocabularyGet(ctx context.Context) (json.RawMessage, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error)
CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error)
commit 93e936f86e31281f35630ab22279db0cb39f5c1f
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Oct 25 22:29:48 2021 -0300
17944: Loads vocabulary, checks its validity.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index 959bb7f5f..a5b626a06 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -27,6 +27,7 @@ var ErrNoClustersDefined = errors.New("config does not define any clusters")
type Loader struct {
Stdin io.Reader
Logger logrus.FieldLogger
+ LoadVocabulary bool // Load the vocabulary from API.VocabularyPath
SkipDeprecated bool // Don't load deprecated config keys
SkipLegacy bool // Don't load legacy config files
SkipAPICalls bool // Don't do checks that call RailsAPI/controller
@@ -269,6 +270,9 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
ldr.loadOldKeepBalanceConfig,
)
}
+ if ldr.LoadVocabulary {
+ loadFuncs = append(loadFuncs, ldr.loadVocabulary)
+ }
loadFuncs = append(loadFuncs, ldr.setImplicitStorageClasses)
for _, f := range loadFuncs {
err = f(&cfg)
@@ -389,9 +393,11 @@ func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
return nil
}
-// CheckVocabularyFile will be called only by interested components as the file
-// isn't expected to be present on every node.
-func (ldr *Loader) CheckVocabularyFile(cc arvados.Cluster) error {
+func (ldr *Loader) loadVocabulary(cfg *arvados.Config) error {
+ cc, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
if cc.API.VocabularyPath == "" {
return nil
}
@@ -399,11 +405,11 @@ func (ldr *Loader) CheckVocabularyFile(cc arvados.Cluster) error {
if err != nil {
return fmt.Errorf("couldn't read vocabulary file %q: %v", cc.API.VocabularyPath, err)
}
- var jsonData map[string]json.RawMessage
- err = json.Unmarshal(vf, &jsonData)
+ voc, err := arvados.NewVocabulary(vf)
if err != nil {
- return fmt.Errorf("invalid JSON data in vocabulary file %q", cc.API.VocabularyPath)
+ return fmt.Errorf("while loading vocabulary file %q: %s", cc.API.VocabularyPath, err)
}
+ cc.API.Vocabulary = voc
return nil
}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 9ab9371ab..71c4399f7 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -94,6 +94,10 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
// process _is_ the controller: we haven't started an
// http server yet.
loader.SkipAPICalls = true
+ // The vocabulary file is expected to be present only
+ // in the controller node, so it doesn't make sense to
+ // try loading it elsewhere.
+ loader.LoadVocabulary = true
}
cfg, err := loader.Load()
@@ -105,16 +109,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
return 1
}
- if strings.HasSuffix(prog, "controller") {
- // The vocabulary file is expected to be present only
- // in the controller node, so it doesn't make sense to
- // check it elsewhere.
- err = loader.CheckVocabularyFile(*cluster)
- if err != nil {
- return 1
- }
- }
-
// Now that we've read the config, replace the bootstrap
// logger with a new one according to the logging config.
log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 558aa68a6..50babd5f3 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -102,6 +102,7 @@ type Cluster struct {
WebsocketClientEventQueue int
WebsocketServerEventQueue int
KeepServiceRequestTimeout Duration
+ Vocabulary *Vocabulary `json:"-"`
VocabularyPath string
}
AuditLogs struct {
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
new file mode 100644
index 000000000..585e5932a
--- /dev/null
+++ b/sdk/go/arvados/vocabulary.go
@@ -0,0 +1,89 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "reflect"
+)
+
+type Vocabulary struct {
+ StrictTags bool `json:"strict_tags"`
+ Tags map[string]VocabularyTag `json:"tags"`
+}
+
+type VocabularyTag struct {
+ Strict bool `json:"strict"`
+ Labels []VocabularyLabel `json:"labels"`
+ Values map[string]VocabularyTagValue `json:"values"`
+}
+
+type VocabularyLabel struct {
+ Label string `json:"label"`
+}
+
+type VocabularyTagValue struct {
+ Labels []VocabularyLabel `json:"labels"`
+}
+
+func NewVocabulary(data []byte) (voc *Vocabulary, err error) {
+ if r := bytes.Compare(data, []byte("")); r == 0 {
+ return &Vocabulary{}, nil
+ }
+ err = json.Unmarshal(data, &voc)
+ if err != nil {
+ return nil, fmt.Errorf("invalid JSON format error: %q", err)
+ }
+ if reflect.DeepEqual(voc, &Vocabulary{}) {
+ return nil, fmt.Errorf("JSON data provided doesn't match Vocabulary format: %q", data)
+ }
+ err = voc.Validate()
+ if err != nil {
+ return nil, err
+ }
+ return voc, nil
+}
+
+func (v *Vocabulary) Validate() error {
+ tagKeys := map[string]bool{}
+ // Checks for Vocabulary strictness
+ if v.StrictTags && len(v.Tags) == 0 {
+ return fmt.Errorf("vocabulary is strict but no tags are defined")
+ }
+ // Checks for duplicate tag keys
+ for key := range v.Tags {
+ if tagKeys[key] {
+ return fmt.Errorf("duplicate tag key %q", key)
+ }
+ tagKeys[key] = true
+ for _, lbl := range v.Tags[key].Labels {
+ if tagKeys[lbl.Label] {
+ return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", lbl.Label, key)
+ }
+ tagKeys[lbl.Label] = true
+ }
+ // Checks for value strictness
+ if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 {
+ return fmt.Errorf("tag key %q is configured as strict but doesn't provide values", key)
+ }
+ // Checks for value duplication within a key
+ tagValues := map[string]bool{}
+ for val := range v.Tags[key].Values {
+ if tagValues[val] {
+ return fmt.Errorf("duplicate tag value %q for tag %q", val, key)
+ }
+ tagValues[val] = true
+ for _, tagLbl := range v.Tags[key].Values[val].Labels {
+ if tagValues[tagLbl.Label] {
+ return fmt.Errorf("tag value label %q for value %q[%q] already seen as a value key or label", tagLbl.Label, key, val)
+ }
+ tagValues[tagLbl.Label] = true
+ }
+ }
+ }
+ return nil
+}
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
new file mode 100644
index 000000000..45ef3dbd1
--- /dev/null
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -0,0 +1,146 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ check "gopkg.in/check.v1"
+)
+
+type VocabularySuite struct{}
+
+var _ = check.Suite(&VocabularySuite{})
+
+func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
+ tests := []struct {
+ name string
+ data string
+ isValid bool
+ errMatches string
+ expect *Vocabulary
+ }{
+ {"Empty data", "", true, "", &Vocabulary{}},
+ {"Invalid JSON", "foo", false, "invalid JSON format.*", nil},
+ {"Valid, empty JSON", "{}", false, ".*doesn't match Vocabulary format.*", nil},
+ {"Valid JSON, wrong data", `{"foo":"bar"}`, false, ".*doesn't match Vocabulary format.*", nil},
+ {
+ "Simple valid example",
+ `{"tags":{
+ "IDTAGANIMALS":{
+ "strict": false,
+ "labels": [{"label": "Animal"}, {"label": "Creature"}],
+ "values": {
+ "IDVALANIMAL1":{"labels":[{"label":"Human"}, {"label":"Homo sapiens"}]},
+ "IDVALANIMAL2":{"labels":[{"label":"Elephant"}, {"label":"Loxodonta"}]}
+ }
+ }
+ }}`,
+ true, "",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ voc, err := NewVocabulary([]byte(tt.data))
+ if tt.isValid {
+ c.Assert(err, check.IsNil)
+ } else {
+ c.Assert(err, check.NotNil)
+ if tt.errMatches != "" {
+ c.Assert(err, check.ErrorMatches, tt.errMatches)
+ }
+ }
+ c.Assert(voc, check.DeepEquals, tt.expect)
+ }
+}
+
+func (s *VocabularySuite) TestValidationErrors(c *check.C) {
+ tests := []struct {
+ name string
+ voc *Vocabulary
+ errMatches string
+ }{
+ {
+ "Strict vocabulary, no keys",
+ &Vocabulary{
+ StrictTags: true,
+ },
+ "vocabulary is strict but no tags are defined",
+ },
+ {
+ "Duplicated tag keys",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ },
+ "IDTAGCOMMENT": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Comment"}, {Label: "Animal"}},
+ },
+ },
+ },
+ "tag label.*for key.*already seen.*",
+ },
+ {
+ "Duplicated tag values",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Mammal"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Mammal"}},
+ },
+ },
+ },
+ },
+ },
+ "tag value label.*for value.*already seen.*",
+ },
+ {
+ "Strict key, no values",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: true,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ },
+ },
+ },
+ "tag key.*is configured as strict but doesn't provide values",
+ },
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ err := tt.voc.Validate()
+ c.Assert(err, check.NotNil)
+ c.Assert(err, check.ErrorMatches, tt.errMatches)
+ }
+}
commit 97a3eb09ffbb80befedbbf078e245afb2a2fc57f
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Oct 25 13:05:31 2021 -0300
17944: Initial vocabulary validation on controller service.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index 248960beb..959bb7f5f 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -389,6 +389,24 @@ func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
return nil
}
+// CheckVocabularyFile will be called only by interested components as the file
+// isn't expected to be present on every node.
+func (ldr *Loader) CheckVocabularyFile(cc arvados.Cluster) error {
+ if cc.API.VocabularyPath == "" {
+ return nil
+ }
+ vf, err := os.ReadFile(cc.API.VocabularyPath)
+ if err != nil {
+ return fmt.Errorf("couldn't read vocabulary file %q: %v", cc.API.VocabularyPath, err)
+ }
+ var jsonData map[string]json.RawMessage
+ err = json.Unmarshal(vf, &jsonData)
+ if err != nil {
+ return fmt.Errorf("invalid JSON data in vocabulary file %q", cc.API.VocabularyPath)
+ }
+ return nil
+}
+
func checkKeyConflict(label string, m map[string]string) error {
saw := map[string]bool{}
for k := range m {
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index 144d41c21..cd69727ec 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -121,8 +121,6 @@ func (h *Handler) setupProxyRemoteCluster(next http.Handler) http.Handler {
mux.ServeHTTP(w, req)
})
-
- return mux
}
type CurrentUser struct {
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index e67c24f65..9ab9371ab 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -105,6 +105,16 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
return 1
}
+ if strings.HasSuffix(prog, "controller") {
+ // The vocabulary file is expected to be present only
+ // in the controller node, so it doesn't make sense to
+ // check it elsewhere.
+ err = loader.CheckVocabularyFile(*cluster)
+ if err != nil {
+ return 1
+ }
+ }
+
// Now that we've read the config, replace the bootstrap
// logger with a new one according to the logging config.
log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
commit 1d482b718aa486cabefb05957b0fedc7ea8d5176
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Oct 21 15:16:14 2021 -0300
17944: Updates config knobs and documentation.
Workbench.VocabularyURL will be auto-exported providing the new vocabulary
endpoint, for backwards compatibility.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index 6f1c90d9b..b7760e3b3 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -194,7 +194,7 @@ navbar:
- admin/keep-balance.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
- - admin/workbench2-vocabulary.html.textile.liquid
+ - admin/metadata-vocabulary.html.textile.liquid
- admin/storage-classes.html.textile.liquid
- admin/keep-recovering-data.html.textile.liquid
- admin/keep-measuring-deduplication.html.textile.liquid
diff --git a/doc/_includes/_wb2_vocabulary_example.liquid b/doc/_includes/_metadata_vocabulary_example.liquid
similarity index 90%
rename from doc/_includes/_wb2_vocabulary_example.liquid
rename to doc/_includes/_metadata_vocabulary_example.liquid
index ee2ac97ef..016b48c6a 100644
--- a/doc/_includes/_wb2_vocabulary_example.liquid
+++ b/doc/_includes/_metadata_vocabulary_example.liquid
@@ -1,3 +1,9 @@
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
{
"strict_tags": false,
"tags": {
diff --git a/doc/admin/workbench2-vocabulary.html.textile.liquid b/doc/admin/metadata-vocabulary.html.textile.liquid
similarity index 79%
rename from doc/admin/workbench2-vocabulary.html.textile.liquid
rename to doc/admin/metadata-vocabulary.html.textile.liquid
index 9a8d7fcd0..e615fa3c8 100644
--- a/doc/admin/workbench2-vocabulary.html.textile.liquid
+++ b/doc/admin/metadata-vocabulary.html.textile.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: admin
-title: User properties vocabulary
+title: Metadata vocabulary
...
{% comment %}
@@ -12,17 +12,19 @@ SPDX-License-Identifier: CC-BY-SA-3.0
Many Arvados objects (like collections and projects) can store metadata as properties that in turn can be used in searches allowing a flexible way of organizing data inside the system.
-The Workbench2 user interface enables the site adminitrator to set up a properties vocabulary formal definition so that users can select from predefined key/value pairs of properties, offering the possibility to add different terms for the same concept.
+Arvados enables the site administrator to set up a formal metadata vocabulary definition so that users can select from predefined key/value pairs of properties, offering the possibility to add different terms for the same concept on clients' UI such as workbench2.
-h2. Workbench2 configuration
+The Controller service loads and caches the configured vocabulary file in memory at startup time, exporting it on a particular endpoint. From time to time, it'll check for updates in the local copy and refresh its cache if validation passes.
-Workbench2 retrieves the vocabulary file URL from the cluster config as shown:
+h2. Configuration
+
+The site administrator should place the JSON vocabulary file on the same host as the controller service and set up the config file as follows:
<notextile>
<pre><code>Cluster:
zzzzz:
- Workbench:
- VocabularyURL: <span class="userinput">https://site.example.com/vocabulary.json</span>
+ API:
+ VocabularyPath: <span class="userinput">/etc/arvados/vocabulary.json</span>
</code></pre>
</notextile>
@@ -35,10 +37,10 @@ Keys and values are indexed by identifiers so that the concept of a term is pres
The following is an example of a vocabulary definition:
{% codeblock as json %}
-{% include 'wb2_vocabulary_example' %}
+{% include 'metadata_vocabulary_example' %}
{% endcodeblock %}
-If the @strict_tags@ flag at the root level is @true@, it will restrict the users from saving property keys other than the ones defined in the vocabulary. Take notice that this restriction is at the client level on Workbench2, it doesn't limit the user's ability to set any arbitrary property via other means (e.g. Python SDK or CLI commands)
+If the @strict_tags@ flag at the root level is @true@, it will restrict the users from saving property keys other than the ones defined in the vocabulary. This restriction is enforced at the backend level to ensure consistency across different clients.
Inside the @tags@ member, IDs are defined (@IDTAGANIMALS@, @IDTAGCOMMENT@, @IDTAGIMPORTANCES@) and can have any format that the current application requires. Every key will declare at least a @labels@ list with zero or more label objects.
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 399ec35d2..4eaa43098 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -281,7 +281,7 @@ Workbench 2 is now ready for regular use. Follow the instructions to "install w
h3. New property vocabulary format for Workbench2
-(feature "#14151":https://dev.arvados.org/issues/14151) Workbench2 supports a new vocabulary format and it isn't compatible with the previous one, please read the "workbench2 vocabulary format admin page":{{site.baseurl}}/admin/workbench2-vocabulary.html for more information.
+(feature "#14151":https://dev.arvados.org/issues/14151) Workbench2 supports a new vocabulary format and it isn't compatible with the previous one, please read the "metadata vocabulary format admin page":{{site.baseurl}}/admin/metadata-vocabulary.html for more information.
h3. Cloud installations only: node manager replaced by arvados-dispatch-cloud
diff --git a/doc/install/install-workbench2-app.html.textile.liquid b/doc/install/install-workbench2-app.html.textile.liquid
index f3a320b10..c9a1c7012 100644
--- a/doc/install/install-workbench2-app.html.textile.liquid
+++ b/doc/install/install-workbench2-app.html.textile.liquid
@@ -75,7 +75,7 @@ server {
h2. Vocabulary configuration (optional)
-Workbench2 can load a vocabulary file which lists available metadata properties for groups and collections. To configure the property vocabulary definition, please visit the "Workbench2 Vocabulary Format":{{site.baseurl}}/admin/workbench2-vocabulary.html page in the Admin section.
+Workbench2 can load a vocabulary file which lists available metadata properties for groups and collections. To configure the property vocabulary definition, please visit the "Metadata Vocabulary Format":{{site.baseurl}}/admin/metadata-vocabulary.html page in the Admin section.
{% assign arvados_component = 'arvados-workbench2' %}
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 4e2a0e26d..da08a2596 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -234,6 +234,12 @@ Clusters:
# Timeout on requests to internal Keep services.
KeepServiceRequestTimeout: 15s
+ # Vocabulary file path, local to the node running the controller.
+ # This JSON file should contain the description of what's allowed
+ # as object's metadata. Its format is described at:
+ # https://doc.arvados.org/admin/metadata-vocabulary.html
+ VocabularyPath: ""
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
@@ -1521,7 +1527,6 @@ Clusters:
DefaultOpenIdPrefix: "https://www.google.com/accounts/o8/id"
# Workbench2 configs
- VocabularyURL: ""
FileViewersConfigURL: ""
# Idle time after which the user's session will be auto closed.
diff --git a/lib/config/export.go b/lib/config/export.go
index 92e2d7b4d..f2c15b0ee 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -72,6 +72,7 @@ var whitelist = map[string]bool{
"API.MaxTokenLifetime": false,
"API.RequestTimeout": true,
"API.SendTimeout": true,
+ "API.VocabularyPath": false,
"API.WebsocketClientEventQueue": false,
"API.WebsocketServerEventQueue": false,
"AuditLogs": false,
@@ -274,7 +275,6 @@ var whitelist = map[string]bool{
"Workbench.UserProfileFormFields.*.*": true,
"Workbench.UserProfileFormFields.*.*.*": true,
"Workbench.UserProfileFormMessage": true,
- "Workbench.VocabularyURL": true,
"Workbench.WelcomePageHTML": true,
}
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 875939a3e..02f2e88c5 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -240,6 +240,12 @@ Clusters:
# Timeout on requests to internal Keep services.
KeepServiceRequestTimeout: 15s
+ # Vocabulary file path, local to the node running the controller.
+ # This JSON file should contain the description of what's allowed
+ # as object's metadata. Its format is described at:
+ # https://doc.arvados.org/admin/metadata-vocabulary.html
+ VocabularyPath: ""
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
@@ -1527,7 +1533,6 @@ Clusters:
DefaultOpenIdPrefix: "https://www.google.com/accounts/o8/id"
# Workbench2 configs
- VocabularyURL: ""
FileViewersConfigURL: ""
# Idle time after which the user's session will be auto closed.
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index f1d27b8dc..558aa68a6 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -102,6 +102,7 @@ type Cluster struct {
WebsocketClientEventQueue int
WebsocketServerEventQueue int
KeepServiceRequestTimeout Duration
+ VocabularyPath string
}
AuditLogs struct {
MaxAge Duration
@@ -273,7 +274,6 @@ type Cluster struct {
Options map[string]struct{}
}
UserProfileFormMessage string
- VocabularyURL string
WelcomePageHTML string
InactivePageHTML string
SSHHelpPageHTML string
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list