[arvados] updated: 2.5.0-14-g07c70cc59
git repository hosting
git at public.arvados.org
Fri Jan 13 17:44:23 UTC 2023
Summary of changes:
sdk/cwl/arvados_cwl/arvworkflow.py | 4 +-
sdk/cwl/arvados_cwl/runner.py | 20 +++++----
sdk/cwl/tests/test_submit.py | 47 +++++++++-------------
sdk/cwl/tests/wf/submit_wf.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_no_reuse.cwl | 4 +-
sdk/cwl/tests/wf/submit_wf_process_properties.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_runner_resources.cwl | 2 +-
...ct_upload_wrapper.cwl => submit_wf_wrapper.cwl} | 30 +-------------
8 files changed, 41 insertions(+), 70 deletions(-)
copy sdk/cwl/tests/wf/{expect_upload_wrapper.cwl => submit_wf_wrapper.cwl} (54%)
via 07c70cc59ef9f17ee3baf98d69be76d9a6585235 (commit)
from 50bd5e3f71a1eb18c7721fc44a934413463baa7c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 07c70cc59ef9f17ee3baf98d69be76d9a6585235
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jan 13 12:44:08 2023 -0500
19385: More test fixing
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 614530c3e..8f48bcd9d 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -239,6 +239,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
update_refs(result, w, tool.doc_loader.expand_url, merged_map, set_block_style, runtimeContext)
with col.open(w[n+1:], "wt") as f:
+ #print(yamlloader.dump(result, stream=sys.stdout))
yamlloader.dump(result, stream=f)
for w in include_files:
@@ -352,7 +353,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
if main.get("requirements"):
wrapper["requirements"].extend(main["requirements"])
if hints:
- wrapper["hints"] = main["hints"]
+ wrapper["hints"] = hints
doc = {"cwlVersion": "v1.2", "$graph": [wrapper]}
@@ -546,6 +547,7 @@ class ArvadosWorkflow(Workflow):
discover_secondary_files(self.arvrunner.fs_access, builder,
self.tool["inputs"], joborder)
+
normalizeFilesDirs(joborder)
with Perf(metrics, "subworkflow upload_deps"):
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 026eb3099..80986a94d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -69,7 +69,7 @@ from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap, CommentedSeq
import arvados_cwl.arvdocker
-from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
+from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern, MapperEnt
from ._version import __version__
from . import done
from . context import ArvRuntimeContext
@@ -447,6 +447,9 @@ def upload_dependencies(arvrunner, name, document_loader,
single_collection=True,
optional_deps=optional_deps)
+ for k, v in uuid_map.items():
+ mapper._pathmap["keep:"+k] = MapperEnt(v, "", "", False)
+
keeprefs = set()
def addkeepref(k):
if k.startswith("keep:"):
@@ -626,17 +629,18 @@ def setloc(mapper, p):
if collectionUUID in p:
uuid = p[collectionUUID]
- if uuid not in uuid_map:
+ keepuuid = "keep:"+uuid
+ if keepuuid not in mapper:
raise SourceLine(p, collectionUUID, validate.ValidationException).makeError(
"Collection uuid %s not found" % uuid)
gp = collection_pdh_pattern.match(loc)
- if gp and uuid_map[uuid] != gp.groups()[0]:
+ if gp and mapper.mapper(keepuuid).resolved != gp.groups()[0]:
# This file entry has both collectionUUID and a PDH
# location. If the PDH doesn't match the one returned
# the API server, raise an error.
raise SourceLine(p, "location", validate.ValidationException).makeError(
"Expected collection uuid %s to be %s but API server reported %s" % (
- uuid, gp.groups()[0], uuid_map[p[collectionUUID]]))
+ uuid, gp.groups()[0], mapper.mapper(keepuuid).resolved))
gp = collection_uuid_pattern.match(loc)
if not gp:
@@ -644,13 +648,13 @@ def setloc(mapper, p):
return
uuid = gp.groups()[0]
- if uuid not in uuid_map:
+ keepuuid = "keep:"+uuid
+ if keepuuid not in mapper:
raise SourceLine(p, "location", validate.ValidationException).makeError(
"Collection uuid %s not found" % uuid)
- p["location"] = "keep:%s%s" % (uuid_map[uuid], gp.groups()[1] if gp.groups()[1] else "")
+ p["location"] = "keep:%s%s" % (mapper.mapper(keepuuid).resolved, gp.groups()[1] if gp.groups()[1] else "")
p[collectionUUID] = uuid
-
def update_from_mapper(workflowobj, mapper):
with Perf(metrics, "setloc"):
visit_class(workflowobj, ("File", "Directory"), partial(setloc, mapper))
@@ -706,6 +710,8 @@ def upload_job_order(arvrunner, name, tool, job_order, runtimeContext):
update_from_mapper(job_order, jobmapper)
+ #print(json.dumps(job_order, indent=2))
+
return job_order
FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 1280095a8..2289f13c2 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -69,7 +69,8 @@ def stubs(wfdetails=('submit_wf.cwl', None)):
uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
"df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
- "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
+ "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
+ "df80736f-f14d-4b10-b2e3-03aa27f034b6"]
determine_image_id.return_value = None
@@ -393,14 +394,6 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
expect_container["use_existing"] = False
expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
- {
- "class": "LoadListingRequirement",
- "loadListing": "deep_listing"
- },
- {
- "class": "NetworkAccess",
- "networkAccess": True
- },
{
"class": "WorkReuse",
"enableReuse": False,
@@ -410,7 +403,7 @@ class TestSubmit(unittest.TestCase):
"class": "http://arvados.org/cwl#WorkflowRunnerResources"
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:40eac42ef5535aca47ca5e47b5786f58+137/wf/submit_wf_no_reuse.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -655,6 +648,13 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "ramMin": 2048
+ }
+ ]
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -968,6 +968,12 @@ class TestSubmit(unittest.TestCase):
stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
+ stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+ {
+ "acrContainerImage": "999999999999999999999999999999d5+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ]
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
@@ -1006,14 +1012,6 @@ class TestSubmit(unittest.TestCase):
"ram": (2000+512) * 2**20
}
expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
- {
- "class": "LoadListingRequirement",
- "loadListing": "deep_listing"
- },
- {
- "class": "NetworkAccess",
- "networkAccess": True
- },
{
"class": "http://arvados.org/cwl#WorkflowRunnerResources",
"acrContainerImage": "999999999999999999999999999999d3+99",
@@ -1026,7 +1024,7 @@ class TestSubmit(unittest.TestCase):
# "arv": "http://arvados.org/cwl#",
#}
expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:48136b8a3e0b5768ea179729309a365e+145/wf/submit_wf_runner_resources.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -1353,6 +1351,7 @@ class TestSubmit(unittest.TestCase):
m.execute.return_value = {"items": []}
return m
stubs.api.collections().list.side_effect = list_side_effect
+ collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1439,14 +1438,6 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
- {
- "class": "LoadListingRequirement",
- "loadListing": "deep_listing"
- },
- {
- "class": "NetworkAccess",
- "networkAccess": True
- },
{
"class": "http://arvados.org/cwl#ProcessProperties",
"processProperties": [
@@ -1471,7 +1462,7 @@ class TestSubmit(unittest.TestCase):
# "arv": "http://arvados.org/cwl#"
#}
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:a068c0b781383b8a27ec5c04a355295b+147/wf/submit_wf_process_properties.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl"
expect_container["properties"].update({
"baz": "blorp.txt",
diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
index 6856e5474..405f99bcb 100644
--- a/sdk/cwl/tests/wf/submit_wf.cwl
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -8,7 +8,7 @@
# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
inputs:
- id: x
type: File
diff --git a/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl b/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
index 636b850b7..d3ebbb978 100644
--- a/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_no_reuse.cwl
@@ -5,7 +5,7 @@
# Test case for arvados-cwl-runner. Disables job/container reuse.
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
$namespaces:
arv: "http://arvados.org/cwl#"
cwltool: "http://commonwl.org/cwltool#"
@@ -24,5 +24,5 @@ steps:
out: []
run: ../tool/submit_tool.cwl
hints:
- arv:ReuseRequirement:
+ WorkReuse:
enableReuse: false
diff --git a/sdk/cwl/tests/wf/submit_wf_process_properties.cwl b/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
index 0d669272f..ebea61aea 100644
--- a/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_process_properties.cwl
@@ -11,7 +11,7 @@ $namespaces:
arv: "http://arvados.org/cwl#"
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
hints:
arv:ProcessProperties:
diff --git a/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
index 814cd07ab..6fa841d45 100644
--- a/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
@@ -8,7 +8,7 @@
# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.2
$namespaces:
arv: "http://arvados.org/cwl#"
hints:
diff --git a/sdk/cwl/tests/wf/submit_wf_wrapper.cwl b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
new file mode 100644
index 000000000..53df05efc
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
@@ -0,0 +1,61 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "SubworkflowFeatureRequirement"
+ }
+ ],
+ "steps": [
+ {
+ "id": "#main/submit_wf.cwl",
+ "in": [
+ {
+ "id": "#main/step/x",
+ "source": "#main/x"
+ },
+ {
+ "id": "#main/step/y",
+ "source": "#main/y"
+ },
+ {
+ "id": "#main/step/z",
+ "source": "#main/z"
+ }
+ ],
+ "label": "submit_wf.cwl",
+ "out": [],
+ "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.2"
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list