[arvados] created: 2.5.0-179-gf41bb69fd

git repository hosting git at public.arvados.org
Mon Feb 20 20:03:34 UTC 2023


        at  f41bb69fdd60627f6e4880f51ed48f69e79af760 (commit)


commit f41bb69fdd60627f6e4880f51ed48f69e79af760
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Feb 20 11:55:32 2023 -0500

    19980: Expand documentation of reserved properties
    
    * Note they will be prefixed with `arv:` from now on.
    * Explain why and how these are set.
    * Explain these can be set even with a strict vocabulary.
    * Add the arvados-cwl-runner Git properties.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/doc/api/properties.html.textile.liquid b/doc/api/properties.html.textile.liquid
index bf4b05cdc..780f7a1ff 100644
--- a/doc/api/properties.html.textile.liquid
+++ b/doc/api/properties.html.textile.liquid
@@ -15,7 +15,23 @@ Searching for records using properties is described in "Filtering on subproperti
 
 h2. Reserved properties
 
-The following properties are set by Arvados components.
+Components that ship with Arvados may automatically set properties on objects. These usually help track provenance or provide additional link metadata. These properties usually have a key that starts with @arv:@, and can always be set even when the system is configured with a strict vocabulary.
+
+table(table table-bordered table-condensed).
+|_. Property name|_. Appears on|_. Value type|_.Description|
+{% comment %}
+The arv:git* container properties, and the associated Git commands, primarily come from arvados_cwl.executor.ArvCwlExecutor.get_git_info.
+{% endcomment -%}
+|arv:gitBranch|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name of the branch checked out (the output of @git rev-parse --abbrev-ref HEAD@)|
+|arv:gitCommitter|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name and email address of the committer of the most recent commit (the output of @git log --format='%cn <%ce>' -n1 HEAD@)|
+|arv:gitCommit|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the full checksum of the most recent commit (the output of @git log --format='%H' -n1 HEAD@)|
+|arv:gitDate|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the commit date of the most recent commit in RFC 2822 format (the output of @git log --format='%cD' -n1 HEAD@)|
+|arv:gitDescribe|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the name of the most recent tag that is reachable from the most recent commit (the output of @git describe --always --tags@)|
+|arv:gitOrigin|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the URL of the remote named @origin@, if set (the output of @git remote get-url origin@)|
+|arv:gitPath|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with the absolute path of the checkout on the filesystem|
+|arv:gitStatus|container|string|When @arvados-cwl-runner@ is run from a Git checkout, this property is set with a machine-readable summary of files modified in the checkout since the most recent commit (the output of @git status --untracked-files=no --porcelain@)|
+
+The following system properties predate the @arv:@ key prefix, but are still reserved and can always be set.
 
 table(table table-bordered table-condensed).
 |_. Property name|_. Appears on|_. Value type|_.Description|

commit d5888bae24963b7e4808267a832024c930fbc1b0
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Feb 20 10:53:06 2023 -0500

    19980: Recognize the arv: prefix for all system properties
    
    arvados-cwl-runner currently uses this prefix to record Git information,
    and we want to use it for all future system properties.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go
index 16f80db07..caab6e8fb 100644
--- a/sdk/go/arvados/vocabulary.go
+++ b/sdk/go/arvados/vocabulary.go
@@ -10,10 +10,13 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 )
 
+const systemKeyPattern = `^arv:[a-zA-Z]`
+
 type Vocabulary struct {
 	reservedTagKeys map[string]bool          `json:"-"`
 	StrictTags      bool                     `json:"strict_tags"`
@@ -26,7 +29,10 @@ type VocabularyTag struct {
 	Values map[string]VocabularyTagValue `json:"values"`
 }
 
-// Cannot have a constant map in Go, so we have to use a function
+// Cannot have a constant map in Go, so we have to use a function.
+// If you are adding a new system property, it SHOULD match `systemKeyPattern`
+// above, and Check will allow it. This map is for historical exceptions that
+// predate standardizing on this prefix.
 func (v *Vocabulary) systemTagKeys() map[string]bool {
 	return map[string]bool{
 		// Collection keys - set by arvados-cwl-runner
@@ -265,9 +271,13 @@ func (v *Vocabulary) Check(data map[string]interface{}) error {
 	if v == nil {
 		return nil
 	}
+	systemKeyRegexp, err := regexp.Compile(systemKeyPattern)
+	if err != nil {
+		return err
+	}
 	for key, val := range data {
 		// Checks for key validity
-		if v.reservedTagKeys[key] {
+		if systemKeyRegexp.MatchString(key) || v.reservedTagKeys[key] {
 			// Allow reserved keys to be used even if they are not defined in
 			// the vocabulary no matter its strictness.
 			continue
diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go
index 2025f97b3..c06efda3e 100644
--- a/sdk/go/arvados/vocabulary_test.go
+++ b/sdk/go/arvados/vocabulary_test.go
@@ -6,6 +6,7 @@ package arvados
 
 import (
 	"encoding/json"
+	"fmt"
 	"regexp"
 	"strings"
 
@@ -301,6 +302,44 @@ func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
 	}
 }
 
+func (s *VocabularySuite) TestValidSystemProperties(c *check.C) {
+	s.testVoc.StrictTags = true
+	properties := map[string]interface{}{
+		"arv:gitBranch": "main",
+		"arv:OK":        true,
+		"arv:cost":      123,
+	}
+	c.Check(s.testVoc.Check(properties), check.IsNil)
+}
+
+func (s *VocabularySuite) TestSystemPropertiesFirstCharacterAlphabetic(c *check.C) {
+	s.testVoc.StrictTags = true
+	properties := map[string]interface{}{"arv:": "value"}
+	c.Check(s.testVoc.Check(properties), check.NotNil)
+	// If we expand the list of allowed characters in the future, these lists
+	// may need adjustment to match.
+	for _, prefix := range []string{" ", ".", "_", "-", "1"} {
+		for _, suffix := range []string{"", "invalid"} {
+			key := fmt.Sprintf("arv:%s%s", prefix, suffix)
+			properties := map[string]interface{}{key: "value"}
+			c.Check(s.testVoc.Check(properties), check.NotNil)
+		}
+	}
+}
+
+func (s *VocabularySuite) TestSystemPropertiesPrefixTypo(c *check.C) {
+	s.testVoc.StrictTags = true
+	for _, key := range []string{
+		"arv :foo",
+		"arvados",
+		"arvados:foo",
+		"Arv:foo",
+	} {
+		properties := map[string]interface{}{key: "value"}
+		c.Check(s.testVoc.Check(properties), check.NotNil)
+	}
+}
+
 func (s *VocabularySuite) TestValidationErrors(c *check.C) {
 	tests := []struct {
 		name       string

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list