[ARVADOS] updated: 1.3.0-2727-gbda96db7a

Git user git at public.arvados.org
Thu Jun 25 20:10:55 UTC 2020


Summary of changes:
 apps/workbench/Gemfile.lock                       |  2 +-
 doc/user/reference/api-tokens.html.textile.liquid |  2 +-
 docker/jobs/Dockerfile                            |  4 +-
 docker/jobs/apt.arvados.org-dev.list              |  2 +-
 docker/jobs/apt.arvados.org-stable.list           |  2 +-
 docker/jobs/apt.arvados.org-testing.list          |  2 +-
 lib/config/config.default.yml                     |  7 +++
 lib/config/export.go                              |  1 +
 lib/config/generated_config.go                    |  7 +++
 lib/controller/federation_test.go                 |  1 +
 lib/controller/handler_test.go                    |  1 +
 lib/controller/proxy.go                           |  4 +-
 lib/recovercollection/cmd.go                      |  8 +--
 sdk/go/arvados/client.go                          | 70 ++++++++++++++++-------
 sdk/go/arvados/config.go                          |  1 +
 sdk/go/arvados/keep_service.go                    | 14 ++---
 sdk/go/arvados/keep_service_test.go               |  3 +-
 services/api/Gemfile.lock                         |  2 +-
 services/api/lib/fix_roles_projects.rb            |  2 +
 services/api/test/fixtures/links.yml              | 14 +++++
 services/keep-balance/balance.go                  | 45 ++++++++++-----
 services/keep-balance/collection.go               |  5 +-
 services/keep-balance/collection_test.go          |  3 +-
 services/keep-balance/keep_service.go             | 13 +++--
 24 files changed, 150 insertions(+), 65 deletions(-)

  discards  4d3e80d1823594a6b1e2cb6cb34d7b098da379c6 (commit)
  discards  426e66b37ecfa31ca580eeefb95aa4f0ccda809d (commit)
       via  bda96db7a411b1a16d6111c46f9732379c8c8426 (commit)
       via  8a2cfa63d23bc54ae1298f3bdb3c1aaf7c978ef1 (commit)
       via  72bac44ada0faa8bf039fb0ffea0f069d7c59310 (commit)
       via  233335acf0dcbdc96f9653c0a9b3040d93707ec6 (commit)
       via  cd3966ee048de85447418f00869eec59b38fd7b2 (commit)
       via  32f0ac87cddf0ad953eae8c25d8761b83b038a36 (commit)
       via  798129804332041aaf8213b96e5675048bce43f8 (commit)
       via  e37223349bd391fdd66bf5d256329e3f6ad70879 (commit)
       via  3703e7f4603d7bd2af696f1e1fd39bbf9ee06ae5 (commit)
       via  405b13d50e203958968427a2642bc18026a0c227 (commit)
       via  b96d5caa0056472fe67b82bd5305448d85c7d0cd (commit)
       via  fd080b34a321cbd6593d69f427b9eaeab890712f (commit)
       via  2bc1a7a89597ab02aaeef84b82fdc51f8e375b79 (commit)
       via  42dc22d84081091e9bb4c1d42ecc0e69ff7bc67a (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (4d3e80d1823594a6b1e2cb6cb34d7b098da379c6)
            \
             N -- N -- N (bda96db7a411b1a16d6111c46f9732379c8c8426)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit bda96db7a411b1a16d6111c46f9732379c8c8426
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Thu Jun 25 14:23:04 2020 -0400

    16382: Extra handling for secondaryFiles containing expressions
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ea4ecf2b6..a47b34acc 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -169,21 +169,47 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         #
         # Found a file, check for secondaryFiles
         #
-        primary["secondaryFiles"] = []
+        specs = []
+        primary["secondaryFiles"] = secondaryspec
         for i, sf in enumerate(aslist(secondaryspec)):
             pattern = builder.do_eval(sf["pattern"], context=primary)
             if pattern is None:
                 continue
+            if isinstance(pattern, list):
+                specs.extend(pattern)
+            elif isinstance(pattern, dict):
+                specs.append(pattern)
+            elif isinstance(pattern, str):
+                specs.append({"pattern": pattern})
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
+        found = []
+        for i, sf in enumerate(specs):
+            if isinstance(sf, dict):
+                if sf.get("class") == "File":
+                    pattern = sf["basename"]
+                else:
+                    pattern = sf["pattern"]
+                    required = sf.get("required")
+            elif isinstance(sf, str):
+                pattern = sf
+                required = True
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
             sfpath = substitute(primary["location"], pattern)
-            required = builder.do_eval(sf.get("required"), context=primary)
+            required = builder.do_eval(required, context=primary)
 
             if fsaccess.exists(sfpath):
-                primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+                found.append({"location": sfpath, "class": "File"})
             elif required:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Required secondary file '%s' does not exist" % sfpath)
 
-        primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+        primary["secondaryFiles"] = cmap(found)
         if discovered is not None:
             discovered[primary["location"]] = primary["secondaryFiles"]
     elif inputschema["type"] not in primitive_types_set:
@@ -434,9 +460,8 @@ def packed_workflow(arvrunner, tool, merged_map):
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
-                if "id" not in v:
-                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
-                cur_id = rewrite_to_orig.get(v["id"], v["id"])
+                if "id" in v:
+                    cur_id = rewrite_to_orig.get(v["id"], v["id"])
             if "path" in v and "location" not in v:
                 v["location"] = v["path"]
                 del v["path"]

commit 8a2cfa63d23bc54ae1298f3bdb3c1aaf7c978ef1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jun 24 16:29:42 2020 -0400

    16382: When packing workflow convert "path" to "location"
    
    Previously it was being modified in-place and the modification was
    propagated to the packed version, that no longer happens so it needs
    to re-apply the path->location fix here.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7bb66a158..ea4ecf2b6 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -437,6 +437,9 @@ def packed_workflow(arvrunner, tool, merged_map):
                 if "id" not in v:
                     raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
                 cur_id = rewrite_to_orig.get(v["id"], v["id"])
+            if "path" in v and "location" not in v:
+                v["location"] = v["path"]
+                del v["path"]
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]
             if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list