[ARVADOS] updated: 2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca

git at public.curoverse.com git at public.curoverse.com
Tue Nov 4 16:29:01 EST 2014


Summary of changes:
 crunch_scripts/run-command                      | 49 ++++++++++-----
 doc/user/topics/run-command.html.textile.liquid | 83 ++++++++++++++++---------
 2 files changed, 86 insertions(+), 46 deletions(-)

       via  2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca (commit)
      from  51de3bf2f50cae3fce8a6ffdb3528e96afe67245 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Nov 4 16:28:54 2014 -0500

    3609: Documentation improvements.  More error checking on run-command list function evaluation.

diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 28adb74..fc3134f 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -119,25 +119,35 @@ def add_to_group(gr, match):
         gr[m] = []
     gr[m].append(match.group(0))
 
+class EvaluationError(Exception):
+    pass
+
 # Return the name of variable ('var') that will take on each value in 'items'
 # when performing an inner substitution
 def var_items(p, c, key):
+    if key not in c:
+        raise EvaluationError("'%s' was expected in 'p' but is missing" % key)
+
     if "var" in c:
+        if not isinstance(c["var"], basestring):
+            raise EvaluationError("Value of 'var' must be a string")
         # Var specifies the variable name for inner parameter substitution
         return (c["var"], get_items(p, c[key]))
     else:
         # The component function ('key') value is a list, so return the list
-        # directly with no parameter substition.
+        # directly with no parameter selected.
         if isinstance(c[key], list):
             return (None, get_items(p, c[key]))
-
-        # check if c[key] is a string that looks like a parameter
-        m = re.match("^\$\((.*)\)$", c[key])
-        if m and m.group(1) in p:
-            return (m.group(1), get_items(p, c[key]))
+        elif isinstance(c[key], basestring):
+            # check if c[key] is a string that looks like a parameter
+            m = re.match("^\$\((.*)\)$", c[key])
+            if m and m.group(1) in p:
+                return (m.group(1), get_items(p, c[key]))
+            else:
+                # backwards compatible, foreach specifies bare parameter name to use
+                return (c[key], get_items(p, p[c[key]]))
         else:
-            # backwards compatible, foreach specifies bare parameter name to use
-            return (c[key], get_items(p, p[c[key]]))
+            raise EvaluationError("Value of '%s' must be a string or list" % key)
 
 # "p" is the parameter scope, "c" is the item to be expanded.
 # If "c" is a dict, apply function expansion.
@@ -147,24 +157,28 @@ def expand_item(p, c):
     if isinstance(c, dict):
         if "foreach" in c and "command" in c:
             var, items = var_items(p, c, "foreach")
+            if var is None:
+                raise EvaluationError("Must specify 'var' in foreach")
             r = []
             for i in items:
                 params = copy.copy(p)
                 params[var] = i
                 r.append(expand_item(params, c["command"]))
             return r
-        if "list" in c and "index" in c and "command" in c:
+        elif "list" in c and "index" in c and "command" in c:
             var, items = var_items(p, c, "list")
+            if var is None:
+                raise EvaluationError("Must specify 'var' in list")
             params = copy.copy(p)
             params[var] = items[int(c["index"])]
             return expand_item(params, c["command"])
-        if "regex" in c:
+        elif "regex" in c:
             pattern = re.compile(c["regex"])
             if "filter" in c:
-                var, items = var_items(p, c, "filter")
+                _, items = var_items(p, c, "filter")
                 return [i for i in items if pattern.match(i)]
             elif "group" in c:
-                var, items = var_items(p, c, "group")
+                _, items = var_items(p, c, "group")
                 groups = {}
                 for i in items:
                     match = pattern.match(i)
@@ -172,20 +186,21 @@ def expand_item(p, c):
                         add_to_group(groups, match)
                 return [groups[k] for k in groups]
             elif "extract" in c:
-                var, items = var_items(p, c, "extract")
+                _, items = var_items(p, c, "extract")
                 r = []
                 for i in items:
                     match = pattern.match(i)
                     if match:
                         r.append(list(match.groups()))
                 return r
-        if "batch" in c and "size" in c:
-            var, items = var_items(p, c, "batch")
+        elif "batch" in c and "size" in c:
+            _, items = var_items(p, c, "batch")
             sz = int(c["size"])
             r = []
             for j in xrange(0, len(items), sz):
                 r.append(items[j:j+sz])
             return r
+        raise EvaluationError("Missing valid list context function")
     elif isinstance(c, list):
         return [expand_item(p, arg) for arg in c]
     elif isinstance(c, basestring):
@@ -195,7 +210,7 @@ def expand_item(p, c):
         else:
             return subst.do_substitution(p, c)
 
-    raise Exception("expand_item() unexpected parameter type %s" % (type(c))
+    raise EvaluationError("expand_item() unexpected parameter type %s" % (type(c))
 
 # Evaluate in a list context
 # "p" is the parameter scope, "value" will be evaluated
@@ -216,7 +231,7 @@ def get_items(p, value):
                 with open(value) as f:
                     items = [line.rstrip("\r\n") for line in f]
             return items
-    raise Exception("get_items did not yield a list")
+    raise EvaluationError("get_items did not yield a list")
 
 stdoutname = None
 stdoutfile = None
diff --git a/doc/user/topics/run-command.html.textile.liquid b/doc/user/topics/run-command.html.textile.liquid
index 7d92876..52097f8 100644
--- a/doc/user/topics/run-command.html.textile.liquid
+++ b/doc/user/topics/run-command.html.textile.liquid
@@ -53,8 +53,8 @@ The "command" list can include parameter substitutions.  Substitutions are enclo
 
 <pre>
 {
-  "command": ["echo", "$(file $(a))"],
-  "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2"
+  "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2",
+  "command": ["echo", "$(file $(a))"]
 }
 </pre>
 
@@ -67,40 +67,45 @@ table(table table-bordered table-condensed).
 
 h2. List context
 
-When a parameter is evaluated in a list context, that means its value should evaluate to a list instead of a string.  Parameter values can be a static list (as demonstrated above), a path to a file, a path to a directory, or a JSON object describing a list context function.
+Where specified by the documentation, parameters may be evaluated in a "list context".  That means the value will evaluate to a list instead of a string.  Parameter values can be a static list, a path to a file, a path to a directory, or a JSON object describing a list context function.
 
-If the value is a static list, it will evaluate the list items for parameter substation and list functions.
+If the value is a string, it is interpreted as a path.  If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped).  If the path specifies a directory, produce a list containing all of the entries in the directory.  Note that parameter expansion is not performed on list items produced this way.
 
-If the value is a string, it is interpreted as a path.  If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped).  If the path specifies a directory, produce a list containing all of the entries in the directory.  Note that parameter expansion is not performed on lists produced this way.
+If the value is a static list, it will evaluate each item and return the expanded list.  Each item may be a string (evaluated for parameter substitution), a list (recursively evaluated), or a JSON object (indicating a list function, described below).
 
 If the value is a JSON object, it is evaluated as a list function described below.
 
 h2. List functions
 
-When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions.  In the following functions, you can either specify the name of a user parameter to act on or provide list value directly in line, for example, the following two fragments yield the same result:
+When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions.  In the following functions, you can either specify the name of a user parameter to act on (@"$(a)"@ in the first example) or provide list value directly in line, for example, the following two fragments yield the same result:
 
 <pre>
 {
-  "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
-  "a": ["alice", "bob"]
+  "a": ["alice", "bob"],
+  "command": ["echo", {"foreach": "$(a)",
+                       "var": "a_var",
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
 <pre>
 {
-  "command": ["echo", {"foreach": ["alice", "bob"], "var":"a", "command": ["--something", "$(a)"]}],
+  "command": ["echo", {"foreach": ["alice", "bob"],
+                       "var": "a_var",
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
 Note: when you provide the list inline with "foreach" or "index", you must include the "var" parameter to specify the substitution variable name to use when evaluating the command fragment.
 
-You can also nest functions:
+You can also nest functions.  This filters @["alice", "bob"]@ on the regular expression @"b.*"@ to get the list @["bob"]@, assigns @a_var@ to each value of the list, then expands @"command"@ to get @["--something", "bob"]@.
 
 <pre>
 {
-  "command": ["echo", {"foreach": {"filter": ["alice", "bob"], "regex": "b.*"},
-                       "var":"a",
-                       "command": ["--something", "$(a)"]}]
+  "command": ["echo", {"foreach": {"filter": ["alice", "bob"],
+                                   "regex": "b.*"},
+                       "var": "a_var",
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
@@ -110,8 +115,10 @@ The @foreach@ list item function (not to be confused with the @task.foreach@ dir
 
 <pre>
 {
-  "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
-  "a": ["alice", "bob"]
+  "a": ["alice", "bob"],
+  "command": ["echo", {"foreach": "$(a)",
+                       "var": "a_var",
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
@@ -121,8 +128,11 @@ This function extracts a single item from a list.  The value of @index@ is zero-
 
 <pre>
 {
-  "command": ["echo", {"list": "$(a)", "var":"a", "index": 1, "command": ["--something", "$(a)"]}],
-  "a": ["alice", "bob"]
+  "a": ["alice", "bob"],
+  "command": ["echo", {"list": "$(a)",
+                       "var": "a_var",
+                       "index": 1,
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
@@ -132,20 +142,27 @@ Filter the list so that it only includes items that match a regular expression.
 
 <pre>
 {
-  "command": ["echo", {"filter": "$(a)", "regex": "b.*"}],
-  "a": ["alice", "bob"]
+  "a": ["alice", "bob"],
+  "command": ["echo", {"filter": "$(a)",
+                       "regex": "b.*"}]
 }
 </pre>
 
 h3. group
 
-Generate a list of lists, where items are grouped on common subexpression match.  Items which don't match the regular expression are excluded.  The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
+Generate a list of lists, where items are grouped on common subexpression match.  Items which don't match the regular expression are excluded.  In the following example, the subexpression is @(a?)@, resulting in two groups, strings that contain the letter 'a' and strings that do not.  The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
 
 <pre>
 {
-  "command": ["echo", {"foreach": "$(b)", "command":["--group", {"foreach": "b", "command":"$(b)"}]}],
   "a": ["alice", "bob", "carol", "dave"],
-  "b": {"group": "a", "regex": "[^a]*(a?).*"}
+  "b": {"group": "$(a)",
+        "regex": "[^a]*(a?).*"},
+  "command": ["echo", {"foreach": "$(b)",
+                       "var": "b_var",
+                       "command": ["--group",
+                                   {"foreach": "$(b_var)",
+                                    "var": "c_var",
+                                    "command": "$(c_var)"}]}]
 }
 </pre>
 
@@ -155,9 +172,14 @@ Generate a list of lists, where items are split by subexpression match.  Items w
 
 <pre>
 {
-  "command": ["echo", {"foreach": "$(b)", "command":[{"foreach": "$(b)", "command":"$(b)"}]}],
   "a": ["alice", "bob", "carol", "dave"],
-  "b": {"extract": "a", "regex": "(.+)(a)(.*)"}
+  "b": {"extract": "$(a)",
+        "regex": "(.+)(a)(.*)"},
+  "command": ["echo", {"foreach": "$(b)",
+                       "var": "b_var",
+                       "command": [{"foreach": "$(b_var)",
+                                    "var": "c_var",
+                                    "command": "$(c_var)"}]}]
 }
 </pre>
 
@@ -167,8 +189,11 @@ Generate a list of lists, where items are split into batch size.  If the list do
 
 <pre>
 {
-  "command": ["echo", {"foreach":{"batch": "$(a)", "size": 2}, "var":"a", "command":["--something", "$(a)"]}],
-  "a": ["alice", "bob", "carol", "dave"]
+  "a": ["alice", "bob", "carol", "dave"],
+  "command": ["echo", {"foreach":{"batch": "$(a)",
+                                  "size": 2},
+                       "var": "a_var",
+                       "command": ["--something", "$(a_var)"]}]
 }
 </pre>
 
@@ -225,10 +250,10 @@ You can also specify multiple parameters:
 
 <pre>
 {
-  "command": ["echo", "$(a)", "$(b)"],
-  "task.foreach": ["a", "b"],
   "a": ["alice", "bob"],
-  "b": ["carol", "dave"]
+  "b": ["carol", "dave"],
+  "task.foreach": ["a", "b"],
+  "command": ["echo", "$(a)", "$(b)"]
 }
 </pre>
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list