[arvados] created: 2.5.0-178-g4fb349d7d
git repository hosting
git at public.arvados.org
Mon Feb 20 16:57:19 UTC 2023
at 4fb349d7d91cb507e4872687916b1de2033b746b (commit)
commit 4fb349d7d91cb507e4872687916b1de2033b746b
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Feb 20 11:55:32 2023 -0500
19980: Expand documentation of reserved properties
* Note they will be prefixed with `arv:` from now on.
* Explain why and how these are set.
* Explain these can be set even with a strict vocabulary.
* Add the arvados-cwl-runner Git properties.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/api/properties.html.textile.liquid b/doc/api/properties.html.textile.liquid
index bf4b05cdc..780f7a1ff 100644
--- a/doc/api/properties.html.textile.liquid
+++ b/doc/api/properties.html.textile.liquid
@@ -15,7 +15,23 @@ Searching for records using properties is described in "Filtering on subproperti
h2. Reserved properties
-The following properties are set by Arvados components.
+Components that ship with Arvados may automatically set properties on objects. These usually help track provenance or provide additional link metadata. These properties usually have a key that starts with @arv:@, and can always be set even when the system is configured with a strict vocabulary.
+
+table(table table-bordered table-condensed).
+|_. Property name|_. Appears on|_. Value type|_.Description|
+{% comment %}
+The arv:git* container properties, and the associated Git commands, primarily come from arvados_cwl.executor.ArvCwlExecutor.get_git_info.
+{% endcomment -%}
+|arv:gitBranch|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name of the branch checked out (the output of @git rev-parse --abbrev-ref HEAD@)|
+|arv:gitCommitter|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name and email address of the committer of the most recent commit (the output of @git log --format='%cn <%ce>' -n1 HEAD@)|
+|arv:gitCommit|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the full checksum of the most recent commit (the output of @git log --format='%H' -n1 HEAD@)|
+|arv:gitDate|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the commit date of the most recent commit in RFC 2822 format (the output of @git log --format='%cD' -n1 HEAD@)|
+|arv:gitDescribe|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name of the most recent tag that is reachable from the most recent commit (the output of @git describe --always --tags@)|
+|arv:gitOrigin|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the URL of the remote named @origin@, if set (the output of @git remote get-url origin@)|
+|arv:gitPath|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the absolute path of the checkout on the filesystem|
+|arv:gitStatus|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with a machine-readable summary of files modified in the checkout since the most recent commit (the output of @git status --untracked-files=no --porcelain@)|
+
+The following system properties predate the @arv:@ key prefix, but are still reserved and can always be set.
table(table table-bordered table-condensed).
|_. Property name|_. Appears on|_. Value type|_.Description|
commit 4935a70d264ae8085d754dd899a6ca6cf25d6644
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Feb 20 10:53:06 2023 -0500
19980: Recognize the arv: prefix for all system properties
arvados-cwl-runner currently uses this prefix to record Git information,
and we want to use it for all future system properties.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 16f80db07..caab6e8fb 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -10,10 +10,13 @@ import (
"errors"
"fmt"
"reflect"
+ "regexp"
"strconv"
"strings"
)
+const systemKeyPattern = `^arv:[a-zA-Z]`
+
type Vocabulary struct {
reservedTagKeys map[string]bool `json:"-"`
StrictTags bool `json:"strict_tags"`
@@ -26,7 +29,10 @@ type VocabularyTag struct {
Values map[string]VocabularyTagValue `json:"values"`
}
-// Cannot have a constant map in Go, so we have to use a function
+// Cannot have a constant map in Go, so we have to use a function.
+// If you are adding a new system property, it SHOULD match `systemKeyPattern`
+// above, and Check will allow it. This map is for historical exceptions that
+// predate standardizing on this prefix.
func (v *Vocabulary) systemTagKeys() map[string]bool {
return map[string]bool{
// Collection keys - set by arvados-cwl-runner
@@ -265,9 +271,13 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
if v == nil {
return nil
}
+ systemKeyRegexp, err := regexp.Compile(systemKeyPattern)
+ if err != nil {
+ return err
+ }
for key, val := range data {
// Checks for key validity
- if v.reservedTagKeys[key] {
+ if systemKeyRegexp.MatchString(key) || v.reservedTagKeys[key] {
// Allow reserved keys to be used even if they are not defined in
// the vocabulary no matter its strictness.
continue
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 2025f97b3..c06efda3e 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -6,6 +6,7 @@ package arvados
import (
"encoding/json"
+ "fmt"
"regexp"
"strings"
@@ -301,6 +302,44 @@ func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
}
}
+func (s *VocabularySuite) TestValidSystemProperties(c *check.C) {
+ s.testVoc.StrictTags = true
+ properties := map[string]interface{}{
+ "arv:gitBranch": "main",
+ "arv:OK": true,
+ "arv:cost": 123,
+ }
+ c.Check(s.testVoc.Check(properties), check.IsNil)
+}
+
+func (s *VocabularySuite) TestSystemPropertiesFirstCharacterAlphabetic(c *check.C) {
+ s.testVoc.StrictTags = true
+ properties := map[string]interface{}{"arv:": "value"}
+ c.Check(s.testVoc.Check(properties), check.NotNil)
+ // If we expand the list of allowed characters in the future, these lists
+ // may need adjustment to match.
+ for _, prefix := range []string{" ", ".", "_", "-", "1"} {
+ for _, suffix := range []string{"", "invalid"} {
+ key := fmt.Sprintf("arv:%s%s", prefix, suffix)
+ properties := map[string]interface{}{key: "value"}
+ c.Check(s.testVoc.Check(properties), check.NotNil)
+ }
+ }
+}
+
+func (s *VocabularySuite) TestSystemPropertiesPrefixTypo(c *check.C) {
+ s.testVoc.StrictTags = true
+ for _, key := range []string{
+ "arv :foo",
+ "arvados",
+ "arvados:foo",
+ "Arv:foo",
+ } {
+ properties := map[string]interface{}{key: "value"}
+ c.Check(s.testVoc.Check(properties), check.NotNil)
+ }
+}
+
func (s *VocabularySuite) TestValidationErrors(c *check.C) {
tests := []struct {
name string
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list