[arvados] updated: 2.5.0-14-g07c70cc59

git repository hosting git at public.arvados.org
Fri Jan 13 17:44:23 UTC 2023


Summary of changes:
 sdk/cwl/arvados_cwl/arvworkflow.py                 |  4 +-
 sdk/cwl/arvados_cwl/runner.py                      | 20 +++++----
 sdk/cwl/tests/test_submit.py                       | 47 +++++++++-------------
 sdk/cwl/tests/wf/submit_wf.cwl                     |  2 +-
 sdk/cwl/tests/wf/submit_wf_no_reuse.cwl            |  4 +-
 sdk/cwl/tests/wf/submit_wf_process_properties.cwl  |  2 +-
 sdk/cwl/tests/wf/submit_wf_runner_resources.cwl    |  2 +-
 ...ct_upload_wrapper.cwl => submit_wf_wrapper.cwl} | 30 +-------------
 8 files changed, 41 insertions(+), 70 deletions(-)
 copy sdk/cwl/tests/wf/{expect_upload_wrapper.cwl => submit_wf_wrapper.cwl} (54%)

       via  07c70cc59ef9f17ee3baf98d69be76d9a6585235 (commit)
      from  50bd5e3f71a1eb18c7721fc44a934413463baa7c (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 07c70cc59ef9f17ee3baf98d69be76d9a6585235
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Fri Jan 13 12:44:08 2023 -0500

    19385: More test fixing
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 614530c3e..8f48bcd9d 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -239,6 +239,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
         update_refs(result, w, tool.doc_loader.expand_url, merged_map, set_block_style, runtimeContext)
 
         with col.open(w[n+1:], "wt") as f:
+            #print(yamlloader.dump(result, stream=sys.stdout))
             yamlloader.dump(result, stream=f)
 
     for w in include_files:
@@ -352,7 +353,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
     if main.get("requirements"):
         wrapper["requirements"].extend(main["requirements"])
     if hints:
-        wrapper["hints"] = main["hints"]
+        wrapper["hints"] = hints
 
     doc = {"cwlVersion": "v1.2", "$graph": [wrapper]}
 
@@ -546,6 +547,7 @@ class ArvadosWorkflow(Workflow):
 
         discover_secondary_files(self.arvrunner.fs_access, builder,
                                  self.tool["inputs"], joborder)
+
         normalizeFilesDirs(joborder)
 
         with Perf(metrics, "subworkflow upload_deps"):
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 026eb3099..80986a94d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -69,7 +69,7 @@ from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 import arvados_cwl.arvdocker
-from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
+from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern, MapperEnt
 from ._version import __version__
 from . import done
 from . context import ArvRuntimeContext
@@ -447,6 +447,9 @@ def upload_dependencies(arvrunner, name, document_loader,
                                single_collection=True,
                                optional_deps=optional_deps)
 
+    for k, v in uuid_map.items():
+        mapper._pathmap["keep:"+k] = MapperEnt(v, "", "", False)
+
     keeprefs = set()
     def addkeepref(k):
         if k.startswith("keep:"):
@@ -626,17 +629,18 @@ def setloc(mapper, p):
 
     if collectionUUID in p:
         uuid = p[collectionUUID]
-        if uuid not in uuid_map:
+        keepuuid = "keep:"+uuid
+        if keepuuid not in mapper:
             raise SourceLine(p, collectionUUID, validate.ValidationException).makeError(
                 "Collection uuid %s not found" % uuid)
         gp = collection_pdh_pattern.match(loc)
-        if gp and uuid_map[uuid] != gp.groups()[0]:
+        if gp and mapper.mapper(keepuuid).resolved != gp.groups()[0]:
             # This file entry has both collectionUUID and a PDH
             # location. If the PDH doesn't match the one returned
             # the API server, raise an error.
             raise SourceLine(p, "location", validate.ValidationException).makeError(
                 "Expected collection uuid %s to be %s but API server reported %s" % (
-                    uuid, gp.groups()[0], uuid_map[p[collectionUUID]]))
+                    uuid, gp.groups()[0], mapper.mapper(keepuuid).resolved))
 
     gp = collection_uuid_pattern.match(loc)
     if not gp:
@@ -644,13 +648,13 @@ def setloc(mapper, p):
         return
 
     uuid = gp.groups()[0]
-    if uuid not in uuid_map:
+    keepuuid = "keep:"+uuid
+    if keepuuid not in mapper:
         raise SourceLine(p, "location", validate.ValidationException).makeError(
             "Collection uuid %s not found" % uuid)
-    p["location"] = "keep:%s%s" % (uuid_map[uuid], gp.groups()[1] if gp.groups()[1] else "")
+    p["location"] = "keep:%s%s" % (mapper.mapper(keepuuid).resolved, gp.groups()[1] if gp.groups()[1] else "")
     p[collectionUUID] = uuid
 
-
 def update_from_mapper(workflowobj, mapper):
     with Perf(metrics, "setloc"):
         visit_class(workflowobj, ("File", "Directory"), partial(setloc, mapper))
@@ -706,6 +710,8 @@ def upload_job_order(arvrunner, name, tool, job_order, runtimeContext):
 
     update_from_mapper(job_order, jobmapper)
 
+    #print(json.dumps(job_order, indent=2))
+
     return job_order
 
 FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 1280095a8..2289f13c2 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -69,7 +69,8 @@ def stubs(wfdetails=('submit_wf.cwl', None)):
 
             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
-                                 "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
+                                 "df80736f-f14d-4b10-b2e3-03aa27f034b6"]
 
             determine_image_id.return_value = None
 
@@ -393,14 +394,6 @@ class TestSubmit(unittest.TestCase):
         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
         expect_container["use_existing"] = False
         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
-                {
-                    "class": "LoadListingRequirement",
-                    "loadListing": "deep_listing"
-                },
-                {
-                    "class": "NetworkAccess",
-                    "networkAccess": True
-                },
             {
                 "class": "WorkReuse",
                 "enableReuse": False,
@@ -410,7 +403,7 @@ class TestSubmit(unittest.TestCase):
                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
                 }
         ]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:40eac42ef5535aca47ca5e47b5786f58+137/wf/submit_wf_no_reuse.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl"
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -655,6 +648,13 @@ class TestSubmit(unittest.TestCase):
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+                {
+                    "acrContainerImage": "999999999999999999999999999999d3+99",
+                    "class": "http://arvados.org/cwl#WorkflowRunnerResources",
+                    "ramMin": 2048
+                }
+        ]
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -968,6 +968,12 @@ class TestSubmit(unittest.TestCase):
             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
+        stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+                {
+                    "acrContainerImage": "999999999999999999999999999999d5+99",
+                    "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+                }
+        ]
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         stubs.api.container_requests().create.assert_called_with(
@@ -1006,14 +1012,6 @@ class TestSubmit(unittest.TestCase):
             "ram": (2000+512) * 2**20
         }
         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
-                {
-                    "class": "LoadListingRequirement",
-                    "loadListing": "deep_listing"
-                },
-                {
-                    "class": "NetworkAccess",
-                    "networkAccess": True
-                },
             {
                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
                 "acrContainerImage": "999999999999999999999999999999d3+99",
@@ -1026,7 +1024,7 @@ class TestSubmit(unittest.TestCase):
         #    "arv": "http://arvados.org/cwl#",
         #}
         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:48136b8a3e0b5768ea179729309a365e+145/wf/submit_wf_runner_resources.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl"
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
@@ -1353,6 +1351,7 @@ class TestSubmit(unittest.TestCase):
                 m.execute.return_value = {"items": []}
             return m
         stubs.api.collections().list.side_effect = list_side_effect
+        collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
 
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1439,14 +1438,6 @@ class TestSubmit(unittest.TestCase):
         expect_container = copy.deepcopy(stubs.expect_container_spec)
 
         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
-                {
-                    "class": "LoadListingRequirement",
-                    "loadListing": "deep_listing"
-                },
-                {
-                    "class": "NetworkAccess",
-                    "networkAccess": True
-                },
             {
                 "class": "http://arvados.org/cwl#ProcessProperties",
                 "processProperties": [
@@ -1471,7 +1462,7 @@ class TestSubmit(unittest.TestCase):
         #    "arv": "http://arvados.org/cwl#"
         #}
 
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:a068c0b781383b8a27ec5c04a355295b+147/wf/submit_wf_process_properties.cwl"
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl"
 
         expect_container["properties"].update({
             "baz": "blorp.txt",
diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
index 6856e5474..405f99bcb 100644
--- a/sdk/cwl/tests/wf/submit_wf.cwl
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -8,7 +8,7 @@
 # (e.g. submit_tool.cwl) and uploading to Keep works as intended.
 
 class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
 inputs:
   - id: x
     type: File
diff --git a/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl b/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
index 636b850b7..d3ebbb978 100644
--- a/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
@@ -5,7 +5,7 @@
 # Test case for arvados-cwl-runner. Disables job/container reuse.
 
 class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
 $namespaces:
   arv: "http://arvados.org/cwl#"
   cwltool: "http://commonwl.org/cwltool#"
@@ -24,5 +24,5 @@ steps:
     out: []
     run: ../tool/submit_tool.cwl
 hints:
-  arv:ReuseRequirement:
+  WorkReuse:
     enableReuse: false
diff --git a/sdk/cwl/tests/wf/submit_wf_process_properties.cwl b/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
index 0d669272f..ebea61aea 100644
--- a/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
@@ -11,7 +11,7 @@ $namespaces:
   arv: "http://arvados.org/cwl#"
 
 class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
 
 hints:
   arv:ProcessProperties:
diff --git a/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
index 814cd07ab..6fa841d45 100644
--- a/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
@@ -8,7 +8,7 @@
 # (e.g. submit_tool.cwl) and uploading to Keep works as intended.
 
 class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
 $namespaces:
   arv: "http://arvados.org/cwl#"
 hints:
diff --git a/sdk/cwl/tests/wf/submit_wf_wrapper.cwl b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
new file mode 100644
index 000000000..53df05efc
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
@@ -0,0 +1,61 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+    "$graph": [
+        {
+            "class": "Workflow",
+            "hints": [
+                {
+                    "acrContainerImage": "999999999999999999999999999999d3+99",
+                    "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+                }
+            ],
+            "id": "#main",
+            "inputs": [
+                {
+                    "id": "#main/x",
+                    "type": "File"
+                },
+                {
+                    "id": "#main/y",
+                    "type": "Directory"
+                },
+                {
+                    "id": "#main/z",
+                    "type": "Directory"
+                }
+            ],
+            "outputs": [],
+            "requirements": [
+                {
+                    "class": "SubworkflowFeatureRequirement"
+                }
+            ],
+            "steps": [
+                {
+                    "id": "#main/submit_wf.cwl",
+                    "in": [
+                        {
+                            "id": "#main/step/x",
+                            "source": "#main/x"
+                        },
+                        {
+                            "id": "#main/step/y",
+                            "source": "#main/y"
+                        },
+                        {
+                            "id": "#main/step/z",
+                            "source": "#main/z"
+                        }
+                    ],
+                    "label": "submit_wf.cwl",
+                    "out": [],
+                    "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
+                }
+            ]
+        }
+    ],
+    "cwlVersion": "v1.2"
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list