[ARVADOS] updated: 2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 4 16:29:01 EST 2014
Summary of changes:
crunch_scripts/run-command | 49 ++++++++++-----
doc/user/topics/run-command.html.textile.liquid | 83 ++++++++++++++++---------
2 files changed, 86 insertions(+), 46 deletions(-)
via 2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca (commit)
from 51de3bf2f50cae3fce8a6ffdb3528e96afe67245 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2fe1e71c5cc17dbf06fd7b1e188fd0279c07d3ca
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 4 16:28:54 2014 -0500
3609: Documentation improvements. More error checking on run-command list function evaluation.
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 28adb74..fc3134f 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -119,25 +119,35 @@ def add_to_group(gr, match):
gr[m] = []
gr[m].append(match.group(0))
+class EvaluationError(Exception):
+ pass
+
# Return the name of variable ('var') that will take on each value in 'items'
# when performing an inner substitution
def var_items(p, c, key):
+ if key not in c:
+ raise EvaluationError("'%s' was expected in 'p' but is missing" % key)
+
if "var" in c:
+ if not isinstance(c["var"], basestring):
+ raise EvaluationError("Value of 'var' must be a string")
# Var specifies the variable name for inner parameter substitution
return (c["var"], get_items(p, c[key]))
else:
# The component function ('key') value is a list, so return the list
- # directly with no parameter substition.
+ # directly with no parameter selected.
if isinstance(c[key], list):
return (None, get_items(p, c[key]))
-
- # check if c[key] is a string that looks like a parameter
- m = re.match("^\$\((.*)\)$", c[key])
- if m and m.group(1) in p:
- return (m.group(1), get_items(p, c[key]))
+ elif isinstance(c[key], basestring):
+ # check if c[key] is a string that looks like a parameter
+ m = re.match("^\$\((.*)\)$", c[key])
+ if m and m.group(1) in p:
+ return (m.group(1), get_items(p, c[key]))
+ else:
+ # backwards compatible, foreach specifies bare parameter name to use
+ return (c[key], get_items(p, p[c[key]]))
else:
- # backwards compatible, foreach specifies bare parameter name to use
- return (c[key], get_items(p, p[c[key]]))
+ raise EvaluationError("Value of '%s' must be a string or list" % key)
# "p" is the parameter scope, "c" is the item to be expanded.
# If "c" is a dict, apply function expansion.
@@ -147,24 +157,28 @@ def expand_item(p, c):
if isinstance(c, dict):
if "foreach" in c and "command" in c:
var, items = var_items(p, c, "foreach")
+ if var is None:
+ raise EvaluationError("Must specify 'var' in foreach")
r = []
for i in items:
params = copy.copy(p)
params[var] = i
r.append(expand_item(params, c["command"]))
return r
- if "list" in c and "index" in c and "command" in c:
+ elif "list" in c and "index" in c and "command" in c:
var, items = var_items(p, c, "list")
+ if var is None:
+ raise EvaluationError("Must specify 'var' in list")
params = copy.copy(p)
params[var] = items[int(c["index"])]
return expand_item(params, c["command"])
- if "regex" in c:
+ elif "regex" in c:
pattern = re.compile(c["regex"])
if "filter" in c:
- var, items = var_items(p, c, "filter")
+ _, items = var_items(p, c, "filter")
return [i for i in items if pattern.match(i)]
elif "group" in c:
- var, items = var_items(p, c, "group")
+ _, items = var_items(p, c, "group")
groups = {}
for i in items:
match = pattern.match(i)
@@ -172,20 +186,21 @@ def expand_item(p, c):
add_to_group(groups, match)
return [groups[k] for k in groups]
elif "extract" in c:
- var, items = var_items(p, c, "extract")
+ _, items = var_items(p, c, "extract")
r = []
for i in items:
match = pattern.match(i)
if match:
r.append(list(match.groups()))
return r
- if "batch" in c and "size" in c:
- var, items = var_items(p, c, "batch")
+ elif "batch" in c and "size" in c:
+ _, items = var_items(p, c, "batch")
sz = int(c["size"])
r = []
for j in xrange(0, len(items), sz):
r.append(items[j:j+sz])
return r
+ raise EvaluationError("Missing valid list context function")
elif isinstance(c, list):
return [expand_item(p, arg) for arg in c]
elif isinstance(c, basestring):
@@ -195,7 +210,7 @@ def expand_item(p, c):
else:
return subst.do_substitution(p, c)
- raise Exception("expand_item() unexpected parameter type %s" % (type(c))
+ raise EvaluationError("expand_item() unexpected parameter type %s" % (type(c))
# Evaluate in a list context
# "p" is the parameter scope, "value" will be evaluated
@@ -216,7 +231,7 @@ def get_items(p, value):
with open(value) as f:
items = [line.rstrip("\r\n") for line in f]
return items
- raise Exception("get_items did not yield a list")
+ raise EvaluationError("get_items did not yield a list")
stdoutname = None
stdoutfile = None
diff --git a/doc/user/topics/run-command.html.textile.liquid b/doc/user/topics/run-command.html.textile.liquid
index 7d92876..52097f8 100644
--- a/doc/user/topics/run-command.html.textile.liquid
+++ b/doc/user/topics/run-command.html.textile.liquid
@@ -53,8 +53,8 @@ The "command" list can include parameter substitutions. Substitutions are enclo
<pre>
{
- "command": ["echo", "$(file $(a))"],
- "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2"
+ "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2",
+ "command": ["echo", "$(file $(a))"]
}
</pre>
@@ -67,40 +67,45 @@ table(table table-bordered table-condensed).
h2. List context
-When a parameter is evaluated in a list context, that means its value should evaluate to a list instead of a string. Parameter values can be a static list (as demonstrated above), a path to a file, a path to a directory, or a JSON object describing a list context function.
+Where specified by the documentation, parameters may be evaluated in a "list context". That means the value will evaluate to a list instead of a string. Parameter values can be a static list, a path to a file, a path to a directory, or a JSON object describing a list context function.
-If the value is a static list, it will evaluate the list items for parameter substation and list functions.
+If the value is a string, it is interpreted as a path. If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped). If the path specifies a directory, produce a list containing all of the entries in the directory. Note that parameter expansion is not performed on list items produced this way.
-If the value is a string, it is interpreted as a path. If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped). If the path specifies a directory, produce a list containing all of the entries in the directory. Note that parameter expansion is not performed on lists produced this way.
+If the value is a static list, it will evaluate each item and return the expanded list. Each item may be a string (evaluated for parameter substitution), a list (recursively evaluated), or a JSON object (indicating a list function, described below).
If the value is a JSON object, it is evaluated as a list function described below.
h2. List functions
-When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions. In the following functions, you can either specify the name of a user parameter to act on or provide list value directly in line, for example, the following two fragments yield the same result:
+When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions. In the following functions, you can either specify the name of a user parameter to act on (@"$(a)"@ in the first example) or provide list value directly in line, for example, the following two fragments yield the same result:
<pre>
{
- "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"foreach": "$(a)",
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", {"foreach": ["alice", "bob"], "var":"a", "command": ["--something", "$(a)"]}],
+ "command": ["echo", {"foreach": ["alice", "bob"],
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
Note: when you provide the list inline with "foreach" or "index", you must include the "var" parameter to specify the substitution variable name to use when evaluating the command fragment.
-You can also nest functions:
+You can also nest functions. This filters @["alice", "bob"]@ on the regular expression @"b.*"@ to get the list @["bob"]@, assigns @a_var@ to each value of the list, then expands @"command"@ to get @["--something", "bob"]@.
<pre>
{
- "command": ["echo", {"foreach": {"filter": ["alice", "bob"], "regex": "b.*"},
- "var":"a",
- "command": ["--something", "$(a)"]}]
+ "command": ["echo", {"foreach": {"filter": ["alice", "bob"],
+ "regex": "b.*"},
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
@@ -110,8 +115,10 @@ The @foreach@ list item function (not to be confused with the @task.foreach@ dir
<pre>
{
- "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"foreach": "$(a)",
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
@@ -121,8 +128,11 @@ This function extracts a single item from a list. The value of @index@ is zero-
<pre>
{
- "command": ["echo", {"list": "$(a)", "var":"a", "index": 1, "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"list": "$(a)",
+ "var": "a_var",
+ "index": 1,
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
@@ -132,20 +142,27 @@ Filter the list so that it only includes items that match a regular expression.
<pre>
{
- "command": ["echo", {"filter": "$(a)", "regex": "b.*"}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"filter": "$(a)",
+ "regex": "b.*"}]
}
</pre>
h3. group
-Generate a list of lists, where items are grouped on common subexpression match. Items which don't match the regular expression are excluded. The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
+Generate a list of lists, where items are grouped on common subexpression match. Items which don't match the regular expression are excluded. In the following example, the subexpression is @(a?)@, resulting in two groups, strings that contain the letter 'a' and strings that do not. The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
<pre>
{
- "command": ["echo", {"foreach": "$(b)", "command":["--group", {"foreach": "b", "command":"$(b)"}]}],
"a": ["alice", "bob", "carol", "dave"],
- "b": {"group": "a", "regex": "[^a]*(a?).*"}
+ "b": {"group": "$(a)",
+ "regex": "[^a]*(a?).*"},
+ "command": ["echo", {"foreach": "$(b)",
+ "var": "b_var",
+ "command": ["--group",
+ {"foreach": "$(b_var)",
+ "var": "c_var",
+ "command": "$(c_var)"}]}]
}
</pre>
@@ -155,9 +172,14 @@ Generate a list of lists, where items are split by subexpression match. Items w
<pre>
{
- "command": ["echo", {"foreach": "$(b)", "command":[{"foreach": "$(b)", "command":"$(b)"}]}],
"a": ["alice", "bob", "carol", "dave"],
- "b": {"extract": "a", "regex": "(.+)(a)(.*)"}
+ "b": {"extract": "$(a)",
+ "regex": "(.+)(a)(.*)"},
+ "command": ["echo", {"foreach": "$(b)",
+ "var": "b_var",
+ "command": [{"foreach": "$(b_var)",
+ "var": "c_var",
+ "command": "$(c_var)"}]}]
}
</pre>
@@ -167,8 +189,11 @@ Generate a list of lists, where items are split into batch size. If the list do
<pre>
{
- "command": ["echo", {"foreach":{"batch": "$(a)", "size": 2}, "var":"a", "command":["--something", "$(a)"]}],
- "a": ["alice", "bob", "carol", "dave"]
+ "a": ["alice", "bob", "carol", "dave"],
+ "command": ["echo", {"foreach":{"batch": "$(a)",
+ "size": 2},
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
@@ -225,10 +250,10 @@ You can also specify multiple parameters:
<pre>
{
- "command": ["echo", "$(a)", "$(b)"],
- "task.foreach": ["a", "b"],
"a": ["alice", "bob"],
- "b": ["carol", "dave"]
+ "b": ["carol", "dave"],
+ "task.foreach": ["a", "b"],
+ "command": ["echo", "$(a)", "$(b)"]
}
</pre>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list