[ARVADOS] updated: 1.3.0-2727-gbda96db7a
Git user
git at public.arvados.org
Thu Jun 25 20:10:55 UTC 2020
Summary of changes:
apps/workbench/Gemfile.lock | 2 +-
doc/user/reference/api-tokens.html.textile.liquid | 2 +-
docker/jobs/Dockerfile | 4 +-
docker/jobs/apt.arvados.org-dev.list | 2 +-
docker/jobs/apt.arvados.org-stable.list | 2 +-
docker/jobs/apt.arvados.org-testing.list | 2 +-
lib/config/config.default.yml | 7 +++
lib/config/export.go | 1 +
lib/config/generated_config.go | 7 +++
lib/controller/federation_test.go | 1 +
lib/controller/handler_test.go | 1 +
lib/controller/proxy.go | 4 +-
lib/recovercollection/cmd.go | 8 +--
sdk/go/arvados/client.go | 70 ++++++++++++++++-------
sdk/go/arvados/config.go | 1 +
sdk/go/arvados/keep_service.go | 14 ++---
sdk/go/arvados/keep_service_test.go | 3 +-
services/api/Gemfile.lock | 2 +-
services/api/lib/fix_roles_projects.rb | 2 +
services/api/test/fixtures/links.yml | 14 +++++
services/keep-balance/balance.go | 45 ++++++++++-----
services/keep-balance/collection.go | 5 +-
services/keep-balance/collection_test.go | 3 +-
services/keep-balance/keep_service.go | 13 +++--
24 files changed, 150 insertions(+), 65 deletions(-)
discards 4d3e80d1823594a6b1e2cb6cb34d7b098da379c6 (commit)
discards 426e66b37ecfa31ca580eeefb95aa4f0ccda809d (commit)
via bda96db7a411b1a16d6111c46f9732379c8c8426 (commit)
via 8a2cfa63d23bc54ae1298f3bdb3c1aaf7c978ef1 (commit)
via 72bac44ada0faa8bf039fb0ffea0f069d7c59310 (commit)
via 233335acf0dcbdc96f9653c0a9b3040d93707ec6 (commit)
via cd3966ee048de85447418f00869eec59b38fd7b2 (commit)
via 32f0ac87cddf0ad953eae8c25d8761b83b038a36 (commit)
via 798129804332041aaf8213b96e5675048bce43f8 (commit)
via e37223349bd391fdd66bf5d256329e3f6ad70879 (commit)
via 3703e7f4603d7bd2af696f1e1fd39bbf9ee06ae5 (commit)
via 405b13d50e203958968427a2642bc18026a0c227 (commit)
via b96d5caa0056472fe67b82bd5305448d85c7d0cd (commit)
via fd080b34a321cbd6593d69f427b9eaeab890712f (commit)
via 2bc1a7a89597ab02aaeef84b82fdc51f8e375b79 (commit)
via 42dc22d84081091e9bb4c1d42ecc0e69ff7bc67a (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (4d3e80d1823594a6b1e2cb6cb34d7b098da379c6)
\
N -- N -- N (bda96db7a411b1a16d6111c46f9732379c8c8426)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit bda96db7a411b1a16d6111c46f9732379c8c8426
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jun 25 14:23:04 2020 -0400
16382: Extra handling for secondaryFiles containing expressions
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ea4ecf2b6..a47b34acc 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -169,21 +169,47 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
#
# Found a file, check for secondaryFiles
#
- primary["secondaryFiles"] = []
+ specs = []
+ primary["secondaryFiles"] = secondaryspec
for i, sf in enumerate(aslist(secondaryspec)):
pattern = builder.do_eval(sf["pattern"], context=primary)
if pattern is None:
continue
+ if isinstance(pattern, list):
+ specs.extend(pattern)
+ elif isinstance(pattern, dict):
+ specs.append(pattern)
+ elif isinstance(pattern, str):
+ specs.append({"pattern": pattern})
+ else:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Expression must return list, object, string or null")
+
+ found = []
+ for i, sf in enumerate(specs):
+ if isinstance(sf, dict):
+ if sf.get("class") == "File":
+ pattern = sf["basename"]
+ else:
+ pattern = sf["pattern"]
+ required = sf.get("required")
+ elif isinstance(sf, str):
+ pattern = sf
+ required = True
+ else:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Expression must return list, object, string or null")
+
sfpath = substitute(primary["location"], pattern)
- required = builder.do_eval(sf.get("required"), context=primary)
+ required = builder.do_eval(required, context=primary)
if fsaccess.exists(sfpath):
- primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+ found.append({"location": sfpath, "class": "File"})
elif required:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Required secondary file '%s' does not exist" % sfpath)
- primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+ primary["secondaryFiles"] = cmap(found)
if discovered is not None:
discovered[primary["location"]] = primary["secondaryFiles"]
elif inputschema["type"] not in primitive_types_set:
@@ -434,9 +460,8 @@ def packed_workflow(arvrunner, tool, merged_map):
def visit(v, cur_id):
if isinstance(v, dict):
if v.get("class") in ("CommandLineTool", "Workflow"):
- if "id" not in v:
- raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
- cur_id = rewrite_to_orig.get(v["id"], v["id"])
+ if "id" in v:
+ cur_id = rewrite_to_orig.get(v["id"], v["id"])
if "path" in v and "location" not in v:
v["location"] = v["path"]
del v["path"]
commit 8a2cfa63d23bc54ae1298f3bdb3c1aaf7c978ef1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jun 24 16:29:42 2020 -0400
16382: When packing workflow convert "path" to "location"
Previously it was being modified in-place and the modification was
propagated to the packed version, that no longer happens so it needs
to re-apply the path->location fix here.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7bb66a158..ea4ecf2b6 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -437,6 +437,9 @@ def packed_workflow(arvrunner, tool, merged_map):
if "id" not in v:
raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
cur_id = rewrite_to_orig.get(v["id"], v["id"])
+ if "path" in v and "location" not in v:
+ v["location"] = v["path"]
+ del v["path"]
if "location" in v and not v["location"].startswith("keep:"):
v["location"] = merged_map[cur_id].resolved[v["location"]]
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list