[arvados] updated: 2.5.0-26-gfb27923f7
git repository hosting
git at public.arvados.org
Wed Jan 25 02:05:34 UTC 2023
Summary of changes:
build/build-dev-docker-jobs-image.sh | 22 ++-
sdk/cwl/arvados_cwl/__init__.py | 5 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 7 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 158 +++++++++++++++++----
sdk/cwl/arvados_cwl/executor.py | 30 ++--
sdk/cwl/arvados_cwl/runner.py | 46 ++++--
sdk/cwl/setup.py | 4 +-
.../collection_per_tool/collection_per_tool.cwl | 4 +-
.../collection_per_tool_wrapper.cwl | 2 +-
sdk/cwl/tests/test_container.py | 3 +-
sdk/cwl/tests/wf/expect_upload_wrapper.cwl | 5 +-
sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl | 5 +-
sdk/dev-jobs.dockerfile | 3 +
13 files changed, 230 insertions(+), 64 deletions(-)
via fb27923f7ce22eddde0852d24d6c94323de41330 (commit)
via fcdf73b20710064a80807e3d8b579a11b629d193 (commit)
via cc095ef1f8c86b05a8e58f15c4086a2caa861ec5 (commit)
via 60da027b51ed0d29ae2aad263b9ebdf8cfb6f9b3 (commit)
via 5ed84016e9e9a210ecf26a01ad50d2d6a51694a5 (commit)
via 01f34b3dbf9d37a551f90c3025d42202b222713f (commit)
via d74058dabbd7bf7360ace73fd6b2874c6ae52809 (commit)
via d2464c017d342cda08842631f5aa6266f191fa41 (commit)
from 21c017bebaa9e8811170c113da77b9054def7117 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fb27923f7ce22eddde0852d24d6c94323de41330
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 24 21:05:12 2023 -0500
19385: Passing the v1.2.1 conformance tests!
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index bef05fed3..e850ebcb2 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -203,14 +203,15 @@ def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, set_block_style,
continue
if field in ("$include", "$import") and isinstance(d[field], str):
- d[field] = rel_ref(d[field], baseuri, urlexpander, {}, None)
+ d[field] = rel_ref(d[field], baseuri, urlexpander, {}, jobmapper)
continue
- if (field == "type" and
- isinstance(d["type"], str) and
- not is_basetype(d["type"])):
- d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map, jobmapper)
- continue
+ for t in ("type", "items"):
+ if (field == t and
+ isinstance(d[t], str) and
+ not is_basetype(d[t])):
+ d[t] = rel_ref(d[t], baseuri, urlexpander, merged_map, jobmapper)
+ continue
if field == "inputs" and isinstance(d["inputs"], MutableMapping):
for inp in d["inputs"]:
@@ -228,13 +229,13 @@ def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, set_block_style,
update_refs(d[field], baseuri, urlexpander, merged_map, jobmapper, set_block_style, runtimeContext, prefix, replacePrefix)
-def fix_schemadef(req, baseuri, urlexpander, merged_map, pdh):
+def fix_schemadef(req, baseuri, urlexpander, merged_map, jobmapper, pdh):
req = copy.deepcopy(req)
for f in req["types"]:
r = f["name"]
path, frag = urllib.parse.urldefrag(r)
- rel = rel_ref(r, baseuri, urlexpander, merged_map)
+ rel = rel_ref(r, baseuri, urlexpander, merged_map, jobmapper)
merged_map.setdefault(path, FileUpdates({}, {}))
#print("PPP", path, r, frag)
rename = "keep:%s/%s" %(pdh, rel)
@@ -455,7 +456,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
for i, r in enumerate(wrapper["requirements"]):
if r["class"] == "SchemaDefRequirement":
- wrapper["requirements"][i] = fix_schemadef(r, main["id"], tool.doc_loader.expand_url, merged_map, col.portable_data_hash())
+ wrapper["requirements"][i] = fix_schemadef(r, main["id"], tool.doc_loader.expand_url, merged_map, jobmapper, col.portable_data_hash())
# print()
# print("merrrrged maaap", merged_map)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index faee1a29f..1134a0a02 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -34,7 +34,7 @@ from arvados.errors import ApiError
import arvados_cwl.util
from .arvcontainer import RunnerContainer, cleanup_name_for_collection
-from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder
+from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder, update_from_merged_map
from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool
from .arvworkflow import ArvadosWorkflow, upload_workflow, new_upload_workflow, make_workflow_record
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
@@ -744,6 +744,8 @@ The 'jobs' API is no longer supported.
loadingContext.loader.idx["_:main"] = workflow_wrapper
#print("BBB", json.dumps(loadingContext.loader.idx["_:main"], indent=2))
+ if not submitting:
+ update_from_merged_map(tool, merged_map)
self.apply_reqs(job_order, tool)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index b921b7905..4d569df33 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -657,6 +657,33 @@ def update_from_mapper(workflowobj, mapper):
with Perf(metrics, "setloc"):
visit_class(workflowobj, ("File", "Directory"), partial(setloc, mapper))
+def apply_merged_map(merged_map, workflowobj):
+ def visit(v, cur_id):
+ if isinstance(v, dict):
+ if v.get("class") in ("CommandLineTool", "Workflow", "ExpressionTool"):
+ if "id" in v:
+ cur_id = v["id"]
+ if "path" in v and "location" not in v:
+ v["location"] = v["path"]
+ del v["path"]
+ if "location" in v and cur_id in merged_map:
+ if v["location"] in merged_map[cur_id].resolved:
+ v["location"] = merged_map[cur_id].resolved[v["location"]]
+ if v["location"] in merged_map[cur_id].secondaryFiles:
+ v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
+ #if v.get("class") == "DockerRequirement":
+ # v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
+ # runtimeContext)
+ for l in v:
+ visit(v[l], cur_id)
+ if isinstance(v, list):
+ for l in v:
+ visit(l, cur_id)
+ visit(workflowobj, None)
+
+def update_from_merged_map(tool, merged_map):
+ tool.visit(partial(apply_merged_map, merged_map))
+
def upload_job_order(arvrunner, name, tool, job_order, runtimeContext):
"""Upload local files referenced in the input object and return updated input
object with 'location' updated to the proper keep references.
commit fcdf73b20710064a80807e3d8b579a11b629d193
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 24 19:40:25 2023 -0500
19385: Down to just a few failing conformance tests, I hope
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index fc7982956..bef05fed3 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -12,6 +12,7 @@ import logging
import urllib
from io import StringIO
import sys
+import re
from typing import (MutableSequence, MutableMapping)
@@ -128,7 +129,7 @@ def make_wrapper_workflow(arvRunner, main, packed, project_uuid, name, git_info,
return json.dumps(doc, sort_keys=True, indent=4, separators=(',',': '))
-def rel_ref(s, baseuri, urlexpander, merged_map):
+def rel_ref(s, baseuri, urlexpander, merged_map, jobmapper):
if s.startswith("keep:"):
return s
@@ -141,17 +142,18 @@ def rel_ref(s, baseuri, urlexpander, merged_map):
fileuri = urllib.parse.urldefrag(baseuri)[0]
- #print("BBB", s, baseuri)
+ #print("BBB", s, baseuri, uri)
for u in (baseuri, fileuri):
if u in merged_map:
replacements = merged_map[u].resolved
- #print("RRR", uri, replacements)
- #print()
- #print(uri, replacements)
+ #print("RRR", u, uri, replacements)
if uri in replacements:
return replacements[uri]
+ if uri in jobmapper:
+ return jobmapper.mapper(uri).target
+
p1 = os.path.dirname(uri_file_path(fileuri))
p2 = os.path.dirname(uri_file_path(uri))
p3 = os.path.basename(uri_file_path(uri))
@@ -166,8 +168,15 @@ def rel_ref(s, baseuri, urlexpander, merged_map):
return os.path.join(r, p3)
+def is_basetype(tp):
+ basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory", "record", "array", "enum")
+ for b in basetypes:
+ if re.match(b+"(\[\])?\??", tp):
+ return True
+ return False
-def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix):
+
+def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, set_block_style, runtimeContext, prefix, replacePrefix):
if set_block_style and (isinstance(d, CommentedSeq) or isinstance(d, CommentedMap)):
d.fa.set_block_style()
@@ -177,11 +186,11 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
if s.startswith(prefix):
d[i] = replacePrefix+s[len(prefix):]
else:
- update_refs(s, baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
+ update_refs(s, baseuri, urlexpander, merged_map, jobmapper, set_block_style, runtimeContext, prefix, replacePrefix)
elif isinstance(d, MutableMapping):
if "id" in d:
baseuri = urlexpander(d["id"], baseuri, scoped_id=True)
- elif "name" in d:
+ elif "name" in d and isinstance(d["name"], str):
baseuri = urlexpander(d["name"], baseuri, scoped_id=True)
if d.get("class") == "DockerRequirement":
@@ -190,36 +199,33 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
for field in d:
if field in ("location", "run", "name") and isinstance(d[field], str):
- d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map)
+ d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map, jobmapper)
continue
if field in ("$include", "$import") and isinstance(d[field], str):
- d[field] = rel_ref(d[field], baseuri, urlexpander, {})
+ d[field] = rel_ref(d[field], baseuri, urlexpander, {}, None)
continue
- basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory", "record", "array", "enum")
-
if (field == "type" and
isinstance(d["type"], str) and
- d["type"] not in basetypes):
- #print("DDD ding", d["type"])
- d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map)
- #print("DDD dong", d["type"])
+ not is_basetype(d["type"])):
+ d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map, jobmapper)
continue
if field == "inputs" and isinstance(d["inputs"], MutableMapping):
for inp in d["inputs"]:
- if isinstance(d["inputs"][inp], str) and d["inputs"][inp] not in basetypes:
- #print("III", inp)
- d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map)
+ if isinstance(d["inputs"][inp], str) and not is_basetype(d["inputs"][inp]):
+ d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map, jobmapper)
+ if isinstance(d["inputs"][inp], MutableMapping):
+ update_refs(d["inputs"][inp], baseuri, urlexpander, merged_map, jobmapper, set_block_style, runtimeContext, prefix, replacePrefix)
continue
if field == "$schemas":
for n, s in enumerate(d["$schemas"]):
- d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map)
+ d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map, jobmapper)
continue
- update_refs(d[field], baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
+ update_refs(d[field], baseuri, urlexpander, merged_map, jobmapper, set_block_style, runtimeContext, prefix, replacePrefix)
def fix_schemadef(req, baseuri, urlexpander, merged_map, pdh):
@@ -242,9 +248,8 @@ def drop_ids(d):
for i, s in enumerate(d):
drop_ids(s)
elif isinstance(d, MutableMapping):
- for fixup in ("id", "name"):
- if fixup in d and d[fixup].startswith("file:"):
- del d[fixup]
+ if "id" in d and d["id"].startswith("file:"):
+ del d["id"]
for field in d:
drop_ids(d[field])
@@ -256,7 +261,8 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
submit_runner_ram=0, name=None, merged_map=None,
submit_runner_image=None,
git_info=None,
- set_defaults=False):
+ set_defaults=False,
+ jobmapper=None):
firstfile = None
workflow_files = set()
@@ -314,7 +320,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
# 2. find $import, $include, $schema, run, location
# 3. update field value
- update_refs(result, w, tool.doc_loader.expand_url, merged_map, set_block_style, runtimeContext, "", "")
+ update_refs(result, w, tool.doc_loader.expand_url, merged_map, jobmapper, set_block_style, runtimeContext, "", "")
with col.open(w[n+1:], "wt") as f:
#print(yamlloader.dump(result, stream=sys.stdout))
@@ -454,11 +460,11 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
# print()
# print("merrrrged maaap", merged_map)
# print()
- print("update_refs", main["id"], runfile)
+ #print("update_refs", main["id"], runfile)
#print(yamlloader.dump(wrapper, stream=sys.stdout))
- update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
+ update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, jobmapper, False, runtimeContext, main["id"]+"#", "#main/")
# Remove any lingering file references.
drop_ids(wrapper)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index e481a5f10..faee1a29f 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -657,7 +657,7 @@ The 'jobs' API is no longer supported.
# Upload local file references in the job order.
with Perf(metrics, "upload_job_order"):
- job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+ job_order, jobmapper = upload_job_order(self, "%s input" % runtimeContext.name,
updated_tool, job_order, runtimeContext)
# the last clause means: if it is a command line tool, and we
@@ -718,7 +718,8 @@ The 'jobs' API is no longer supported.
merged_map=merged_map,
submit_runner_image=runtimeContext.submit_runner_image,
git_info=git_info,
- set_defaults=(runtimeContext.update_workflow or runtimeContext.create_workflow))
+ set_defaults=(runtimeContext.update_workflow or runtimeContext.create_workflow),
+ jobmapper=jobmapper)
if runtimeContext.update_workflow or runtimeContext.create_workflow:
# Now create a workflow record and exit.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index aa8dac053..b921b7905 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -334,6 +334,8 @@ def upload_dependencies(arvrunner, name, document_loader,
if optional_deps:
sc_result.extend(optional_deps)
+ #print("BOING", uri, sc_result)
+
sc = []
uuids = {}
@@ -708,7 +710,7 @@ def upload_job_order(arvrunner, name, tool, job_order, runtimeContext):
#print(json.dumps(job_order, indent=2))
- return job_order
+ return job_order, jobmapper
FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
@@ -751,6 +753,9 @@ def upload_workflow_deps(arvrunner, tool, runtimeContext):
toolmap = {}
for k,v in pm.items():
toolmap[k] = v.resolved
+
+ #print("visited", deptool["id"], toolmap, discovered_secondaryfiles)
+
merged_map[deptool["id"]] = FileUpdates(toolmap, discovered_secondaryfiles)
return merged_map
commit cc095ef1f8c86b05a8e58f15c4086a2caa861ec5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 24 17:27:30 2023 -0500
19385: Finally passing schemadef test.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index ee0b10d14..3703fc621 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -544,6 +544,7 @@ class RunnerContainer(Runner):
container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
else:
main = self.loadingContext.loader.idx["_:main"]
+ #print(json.dumps(main, indent=2))
workflowpath = "/var/lib/cwl/workflow.json#main"
container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
"kind": "json",
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 03e6bd302..fc7982956 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -224,8 +224,6 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
def fix_schemadef(req, baseuri, urlexpander, merged_map, pdh):
req = copy.deepcopy(req)
- #if "id" in req:
- # del req["id"]
for f in req["types"]:
r = f["name"]
@@ -234,10 +232,24 @@ def fix_schemadef(req, baseuri, urlexpander, merged_map, pdh):
merged_map.setdefault(path, FileUpdates({}, {}))
#print("PPP", path, r, frag)
rename = "keep:%s/%s" %(pdh, rel)
+ #rename = "#%s" % frag
for mm in merged_map:
merged_map[mm].resolved[r] = rename
return req
+def drop_ids(d):
+ if isinstance(d, MutableSequence):
+ for i, s in enumerate(d):
+ drop_ids(s)
+ elif isinstance(d, MutableMapping):
+ for fixup in ("id", "name"):
+ if fixup in d and d[fixup].startswith("file:"):
+ del d[fixup]
+
+ for field in d:
+ drop_ids(d[field])
+
+
def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
runtimeContext,
uuid=None,
@@ -448,6 +460,9 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
+ # Remove any lingering file references.
+ drop_ids(wrapper)
+
#print("HHH")
#print(yamlloader.dump(wrapper, stream=sys.stdout))
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index a447f46b9..e481a5f10 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -705,6 +705,7 @@ The 'jobs' API is no longer supported.
#with Perf(metrics, "load_tool"):
# tool = load_tool(tool.tool, loadingContext)
+ workflow_wrapper = None
if submitting and not self.fast_submit:
# upload workflow and get back the workflow wrapper
@@ -736,9 +737,12 @@ The 'jobs' API is no longer supported.
#workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), "_:main", checklinks=True)
#tool = load_tool(workflow_wrapper[0], loadingContext)
- #print(json.dumps(workflow_wrapper, indent=2))
+ #print("AAA", json.dumps(loadingContext.loader.idx["_:main"], indent=2))
tool = load_tool(workflow_wrapper, loadingContext)
+ loadingContext.loader.idx["_:main"] = workflow_wrapper
+ #print("BBB", json.dumps(loadingContext.loader.idx["_:main"], indent=2))
+
self.apply_reqs(job_order, tool)
@@ -785,8 +789,8 @@ The 'jobs' API is no longer supported.
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
if submitting:
- tool = RunnerContainer(self, updated_tool,
- tool, loadingContext, runtimeContext.enable_reuse,
+ loadingContext.metadata = updated_tool.metadata.copy()
+ tool = RunnerContainer(self, tool, loadingContext, runtimeContext.enable_reuse,
self.output_name,
self.output_tags,
submit_runner_ram=runtimeContext.submit_runner_ram,
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9bd2152ff..aa8dac053 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -794,7 +794,7 @@ class Runner(Process):
"""Base class for runner processes, which submit an instance of
arvados-cwl-runner and wait for the final result."""
- def __init__(self, runner, updated_tool,
+ def __init__(self, runner,
tool, loadingContext, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
@@ -805,9 +805,8 @@ class Runner(Process):
git_info=None):
self.loadingContext = loadingContext.copy()
- self.loadingContext.metadata = updated_tool.metadata.copy()
- super(Runner, self).__init__(updated_tool.tool, loadingContext)
+ super(Runner, self).__init__(tool.tool, loadingContext)
self.arvrunner = runner
self.embedded_tool = tool
commit 60da027b51ed0d29ae2aad263b9ebdf8cfb6f9b3
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 24 17:08:58 2023 -0500
19385: more WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c3d8dc3ae..52a9a6c20 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -383,7 +383,8 @@ def main(args=sys.argv[1:],
if arvargs.workflow.startswith("arvwf:") or workflow_uuid_pattern.match(arvargs.workflow) or arvargs.workflow.startswith("keep:"):
executor.loadingContext.do_validate = False
- executor.fast_submit = True
+ if arvargs.submit:
+ executor.fast_submit = True
return cwltool.main.main(args=arvargs,
stdout=stdout,
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index bf5a9a9f1..68a344601 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -37,7 +37,7 @@ setup(name='arvados-cwl-runner',
# build.
install_requires=[
'cwltool==3.1.20230119183816',
- 'schema-salad>8.3.20220913105718',
+ 'schema-salad==8.3.20230109181937.dev6+gca19f37',
'arvados-python-client{}'.format(pysdk_dep),
'ciso8601 >= 2.0.0',
'networkx < 2.6',
commit 5ed84016e9e9a210ecf26a01ad50d2d6a51694a5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 24 16:03:56 2023 -0500
19385: WIP for complex schemadef + $import case
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 35cbf9435..03e6bd302 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -25,7 +25,7 @@ import arvados.collection
from cwltool.pack import pack
from cwltool.load_tool import fetch_document, resolve_and_validate_document
-from cwltool.process import shortname
+from cwltool.process import shortname, uniquename
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class, normalizeFilesDirs
from cwltool.context import LoadingContext
@@ -36,7 +36,7 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder, arvados_jobs_image)
+ make_builder, arvados_jobs_image, FileUpdates)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
from ._version import __version__
@@ -127,6 +127,7 @@ def make_wrapper_workflow(arvRunner, main, packed, project_uuid, name, git_info,
return json.dumps(doc, sort_keys=True, indent=4, separators=(',',': '))
+
def rel_ref(s, baseuri, urlexpander, merged_map):
if s.startswith("keep:"):
return s
@@ -135,11 +136,18 @@ def rel_ref(s, baseuri, urlexpander, merged_map):
uri = urlexpander(s, baseuri)
#print("CCC", uri)
+ if uri.startswith("keep:"):
+ return uri
+
fileuri = urllib.parse.urldefrag(baseuri)[0]
+ #print("BBB", s, baseuri)
+
for u in (baseuri, fileuri):
if u in merged_map:
replacements = merged_map[u].resolved
+ #print("RRR", uri, replacements)
+ #print()
#print(uri, replacements)
if uri in replacements:
return replacements[uri]
@@ -173,37 +181,62 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
elif isinstance(d, MutableMapping):
if "id" in d:
baseuri = urlexpander(d["id"], baseuri, scoped_id=True)
+ elif "name" in d:
+ baseuri = urlexpander(d["name"], baseuri, scoped_id=True)
if d.get("class") == "DockerRequirement":
dockerImageId = d.get("dockerImageId") or d.get("dockerPull")
d["http://arvados.org/cwl#dockerCollectionPDH"] = runtimeContext.cached_docker_lookups.get(dockerImageId)
- for s in d:
- for field in ("location", "run", "name"):
- if field in d and isinstance(d[field], str):
- d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map)
+ for field in d:
+ if field in ("location", "run", "name") and isinstance(d[field], str):
+ d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map)
+ continue
- for field in ("$include", "$import"):
- if field in d and isinstance(d[field], str):
- d[field] = rel_ref(d[field], baseuri, urlexpander, {})
+ if field in ("$include", "$import") and isinstance(d[field], str):
+ d[field] = rel_ref(d[field], baseuri, urlexpander, {})
+ continue
- basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory")
+ basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory", "record", "array", "enum")
- if ("type" in d and
+ if (field == "type" and
isinstance(d["type"], str) and
d["type"] not in basetypes):
+ #print("DDD ding", d["type"])
d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map)
+ #print("DDD dong", d["type"])
+ continue
- if "inputs" in d and isinstance(d["inputs"], MutableMapping):
+ if field == "inputs" and isinstance(d["inputs"], MutableMapping):
for inp in d["inputs"]:
if isinstance(d["inputs"][inp], str) and d["inputs"][inp] not in basetypes:
+ #print("III", inp)
d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map)
+ continue
- if "$schemas" in d:
+ if field == "$schemas":
for n, s in enumerate(d["$schemas"]):
d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map)
+ continue
+
+ update_refs(d[field], baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
+
+
+def fix_schemadef(req, baseuri, urlexpander, merged_map, pdh):
+ req = copy.deepcopy(req)
+ #if "id" in req:
+ # del req["id"]
- update_refs(d[s], baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
+ for f in req["types"]:
+ r = f["name"]
+ path, frag = urllib.parse.urldefrag(r)
+ rel = rel_ref(r, baseuri, urlexpander, merged_map)
+ merged_map.setdefault(path, FileUpdates({}, {}))
+ #print("PPP", path, r, frag)
+ rename = "keep:%s/%s" %(pdh, rel)
+ for mm in merged_map:
+ merged_map[mm].resolved[r] = rename
+ return req
def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
runtimeContext,
@@ -307,11 +340,13 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
# now construct the wrapper
+ runfile = "keep:%s/%s" % (col.portable_data_hash(), toolfile)
+
step = {
"id": "#main/" + toolname,
"in": [],
"out": [],
- "run": "keep:%s/%s" % (col.portable_data_hash(), toolfile),
+ "run": runfile,
"label": name
}
@@ -387,6 +422,10 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
if hints:
wrapper["hints"] = hints
+ # 1. check for SchemaDef
+ # 2. do what pack does
+ # 3. fix inputs
+
doc = {"cwlVersion": "v1.2", "$graph": [wrapper]}
if git_info:
@@ -396,8 +435,21 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
#print("MMM", main["id"])
#print(yamlloader.dump(wrapper, stream=sys.stdout))
+ for i, r in enumerate(wrapper["requirements"]):
+ if r["class"] == "SchemaDefRequirement":
+ wrapper["requirements"][i] = fix_schemadef(r, main["id"], tool.doc_loader.expand_url, merged_map, col.portable_data_hash())
+
+ # print()
+ # print("merrrrged maaap", merged_map)
+ # print()
+ print("update_refs", main["id"], runfile)
+
+ #print(yamlloader.dump(wrapper, stream=sys.stdout))
+
update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
+ #print("HHH")
+
#print(yamlloader.dump(wrapper, stream=sys.stdout))
return doc
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index ee8917b88..a447f46b9 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -726,12 +726,18 @@ The 'jobs' API is no longer supported.
self.stdout.write(uuid + "\n")
return (None, "success")
+ loadingContext.loader.idx.clear()
loadingContext.loader.idx["_:main"] = workflow_wrapper
+ workflow_wrapper["id"] = "_:main"
# Reload just the wrapper workflow.
self.fast_submit = True
- workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), tool.tool["id"])
- tool = load_tool(workflow_wrapper[0], loadingContext)
+ #print("bah bah", loadingContext.requirements)
+ #workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), "_:main", checklinks=True)
+
+ #tool = load_tool(workflow_wrapper[0], loadingContext)
+ #print(json.dumps(workflow_wrapper, indent=2))
+ tool = load_tool(workflow_wrapper, loadingContext)
self.apply_reqs(job_order, tool)
commit 01f34b3dbf9d37a551f90c3025d42202b222713f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jan 20 14:20:14 2023 -0500
19385: Reference rewriting fixups
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index af838d68e..bf1ab3418 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -16,6 +16,7 @@ Syntax:
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
+CWL_UTILS=path (optional) Path to cwl-utils git repository.
PYCMD=pythonexec (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
EOF
@@ -67,6 +68,14 @@ if [[ -n "$CWLTOOL" ]] ; then
cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
fi
+rm -rf sdk/cwl/cwlutils_dist
+mkdir -p sdk/cwl/cwlutils_dist
+if [[ -n "$CWL_UTILS" ]] ; then
+ (cd "$CWL_UTILS" && python3 setup.py sdist)
+ cwlutils=$(cd "$CWL_UTILS/dist" && ls -t cwl-utils-*.tar.gz | head -n1)
+ cp "$CWL_UTILS/dist/$cwlutils" $WORKSPACE/sdk/cwl/cwlutils_dist
+fi
+
. build/run-library.sh
# This defines python_sdk_version and cwl_runner_version with python-style
@@ -74,6 +83,17 @@ fi
calculate_python_sdk_cwl_package_versions
set -x
-docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --no-cache \
+ --build-arg sdk=$sdk \
+ --build-arg runner=$runner \
+ --build-arg salad=$salad \
+ --build-arg cwltool=$cwltool \
+ --build-arg pythoncmd=$py \
+ --build-arg pipcmd=$pipcmd \
+ --build-arg cwlutils=$cwlutils \
+ -f "$WORKSPACE/sdk/dev-jobs.dockerfile" \
+ -t arvados/jobs:$cwl_runner_version \
+ "$WORKSPACE/sdk"
+
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 3b15fcede..35cbf9435 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -128,22 +128,34 @@ def make_wrapper_workflow(arvRunner, main, packed, project_uuid, name, git_info,
return json.dumps(doc, sort_keys=True, indent=4, separators=(',',': '))
def rel_ref(s, baseuri, urlexpander, merged_map):
+ if s.startswith("keep:"):
+ return s
+
+ #print("BBB", s, baseuri)
uri = urlexpander(s, baseuri)
+ #print("CCC", uri)
+
fileuri = urllib.parse.urldefrag(baseuri)[0]
- if fileuri in merged_map:
- replacements = merged_map[fileuri].resolved
- if uri in replacements:
- return replacements[uri]
- if s.startswith("keep:"):
- return s
+ for u in (baseuri, fileuri):
+ if u in merged_map:
+ replacements = merged_map[u].resolved
+ #print(uri, replacements)
+ if uri in replacements:
+ return replacements[uri]
- p1 = os.path.dirname(uri_file_path(baseuri))
+ p1 = os.path.dirname(uri_file_path(fileuri))
p2 = os.path.dirname(uri_file_path(uri))
p3 = os.path.basename(uri_file_path(uri))
+
+ #print("PPP", p1, p2, p3)
+
r = os.path.relpath(p2, p1)
if r == ".":
r = ""
+
+ #print("RRR", r)
+
return os.path.join(r, p3)
@@ -167,10 +179,26 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
d["http://arvados.org/cwl#dockerCollectionPDH"] = runtimeContext.cached_docker_lookups.get(dockerImageId)
for s in d:
- for field in ("$include", "$import", "location", "run"):
+ for field in ("location", "run", "name"):
if field in d and isinstance(d[field], str):
d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map)
+ for field in ("$include", "$import"):
+ if field in d and isinstance(d[field], str):
+ d[field] = rel_ref(d[field], baseuri, urlexpander, {})
+
+ basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory")
+
+ if ("type" in d and
+ isinstance(d["type"], str) and
+ d["type"] not in basetypes):
+ d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map)
+
+ if "inputs" in d and isinstance(d["inputs"], MutableMapping):
+ for inp in d["inputs"]:
+ if isinstance(d["inputs"][inp], str) and d["inputs"][inp] not in basetypes:
+ d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map)
+
if "$schemas" in d:
for n, s in enumerate(d["$schemas"]):
d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map)
@@ -221,6 +249,8 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
col = arvados.collection.Collection(api_client=arvRunner.api)
+ #print(merged_map)
+
for w in workflow_files | import_files:
# 1. load YAML
@@ -363,8 +393,13 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
for g in git_info:
doc[g] = git_info[g]
+ #print("MMM", main["id"])
+ #print(yamlloader.dump(wrapper, stream=sys.stdout))
+
update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
+ #print(yamlloader.dump(wrapper, stream=sys.stdout))
+
return doc
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 053983b01..ee8917b88 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -23,7 +23,7 @@ import urllib
from cwltool.errors import WorkflowException
import cwltool.workflow
-from schema_salad.sourceline import SourceLine
+from schema_salad.sourceline import SourceLine, cmap
import schema_salad.validate as validate
from schema_salad.ref_resolver import file_uri, uri_file_path
@@ -705,7 +705,7 @@ The 'jobs' API is no longer supported.
#with Perf(metrics, "load_tool"):
# tool = load_tool(tool.tool, loadingContext)
- if runtimeContext.update_workflow or runtimeContext.create_workflow or (runtimeContext.submit and not self.fast_submit):
+ if submitting and not self.fast_submit:
# upload workflow and get back the workflow wrapper
workflow_wrapper = new_upload_workflow(self, tool, job_order,
@@ -730,7 +730,8 @@ The 'jobs' API is no longer supported.
# Reload just the wrapper workflow.
self.fast_submit = True
- tool = load_tool(workflow_wrapper, loadingContext)
+ workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), tool.tool["id"])
+ tool = load_tool(workflow_wrapper[0], loadingContext)
self.apply_reqs(job_order, tool)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index ae9f6d0d4..bf5a9a9f1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20221224142944',
+ 'cwltool==3.1.20230119183816',
'schema-salad>8.3.20220913105718',
'arvados-python-client{}'.format(pysdk_dep),
'ciso8601 >= 2.0.0',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 60db4a889..95b039eba 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -28,16 +28,19 @@ RUN apt-get update -q && apt-get install -qy --no-install-recommends \
ARG sdk
ARG runner
ARG salad
+ARG cwlutils
ARG cwltool
ADD python/dist/$sdk /tmp/
ADD cwl/salad_dist/$salad /tmp/
ADD cwl/cwltool_dist/$cwltool /tmp/
+ADD cwl/cwlutils_dist/$cwlutils /tmp/
ADD cwl/dist/$runner /tmp/
RUN $pipcmd install wheel
RUN cd /tmp/arvados-python-client-* && $pipcmd install .
RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
+RUN if test -d /tmp/cwl-utils-* ; then cd /tmp/cwl-utils-* && $pipcmd install . ; fi
RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
commit d74058dabbd7bf7360ace73fd6b2874c6ae52809
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jan 19 12:49:02 2023 -0500
19385: Unit tests pass
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 92ed3057a..ee0b10d14 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -250,11 +250,7 @@ class ArvadosContainer(JobBase):
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
runtimeContext.pull_image,
- runtimeContext.project_uuid,
- runtimeContext.force_docker_pull,
- runtimeContext.tmp_outdir_prefix,
- runtimeContext.match_local_docker,
- runtimeContext.copy_deps)
+ runtimeContext)
network_req, _ = self.get_requirement("NetworkAccess")
if network_req:
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index eb8a50dae..3b15fcede 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -300,7 +300,6 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
hints.append(wf_runner_resources)
- # uncomment me
wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner,
submit_runner_image or "arvados/jobs:"+__version__,
runtimeContext)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 80986a94d..9bd2152ff 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -589,11 +589,7 @@ def packed_workflow(arvrunner, tool, merged_map, runtimeContext, git_info):
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
if v.get("class") == "DockerRequirement":
v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
- runtimeContext.project_uuid,
- runtimeContext.force_docker_pull,
- runtimeContext.tmp_outdir_prefix,
- runtimeContext.match_local_docker,
- runtimeContext.copy_deps)
+ runtimeContext)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
@@ -719,7 +715,6 @@ FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
def upload_workflow_deps(arvrunner, tool, runtimeContext):
# Ensure that Docker images needed by this workflow are available
- # commented out for testing only, uncomment me
with Perf(metrics, "upload_docker"):
upload_docker(arvrunner, tool, runtimeContext)
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 5820b56b5..a2f404d7e 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1206,7 +1206,6 @@ class TestContainer(unittest.TestCase):
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_match_local_docker(self, keepdocker, determine_image_id):
arvados_cwl.add_arv_hints()
- #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
@@ -1280,7 +1279,7 @@ class TestContainer(unittest.TestCase):
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(container_request))
- #arv_docker_clear_cache()
+ runtimeContext.cached_docker_lookups.clear()
runtimeContext.match_local_docker = True
container_request['container_image'] = '99999999999999999999999999999993+99'
container_request['name'] = 'test_run_True_2'
commit d2464c017d342cda08842631f5aa6266f191fa41
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jan 19 11:22:32 2023 -0500
19385: Fixing tests WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e151dbf7b..c3d8dc3ae 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -381,7 +381,7 @@ def main(args=sys.argv[1:],
# unit tests.
stdout = None
- if arvargs.workflow.startswith("arvwf:") or workflow_uuid_pattern.match(arvargs.workflow):
+ if arvargs.workflow.startswith("arvwf:") or workflow_uuid_pattern.match(arvargs.workflow) or arvargs.workflow.startswith("keep:"):
executor.loadingContext.do_validate = False
executor.fast_submit = True
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 7e9840db6..053983b01 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -36,7 +36,7 @@ import arvados_cwl.util
from .arvcontainer import RunnerContainer, cleanup_name_for_collection
from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder
from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool
-from .arvworkflow import ArvadosWorkflow, upload_workflow, new_upload_workflow
+from .arvworkflow import ArvadosWorkflow, upload_workflow, new_upload_workflow, make_workflow_record
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
from .perf import Perf
from .pathmapper import NoFollowPathMapper
diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl
index f864f4920..a7159e3f8 100644
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
-cwlVersion: v1.0
+cwlVersion: v1.2
class: Workflow
inputs: []
outputs: []
@@ -14,4 +14,4 @@ steps:
step2:
in: []
out: []
- run: step2.cwl
\ No newline at end of file
+ run: step2.cwl
diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
index fda566c88..e43e1af0f 100644
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
@@ -26,7 +26,7 @@
"in": [],
"label": "collection_per_tool.cwl",
"out": [],
- "run": "keep:92045991f69a417f2f26660db67911ef+61/workflow.json#main"
+ "run": "keep:473135c3f4af514f85027e9e348cea45+179/collection_per_tool.cwl"
}
]
}
diff --git a/sdk/cwl/tests/wf/expect_upload_wrapper.cwl b/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
index 3821527bb..a7a30e68a 100644
--- a/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
+++ b/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
@@ -48,8 +48,7 @@
"nameroot": "renamed",
"size": 0
}
- ],
- "location": "_:df80736f-f14d-4b10-b2e3-03aa27f034b2"
+ ]
},
"id": "#main/z",
"type": "Directory"
@@ -80,7 +79,7 @@
],
"label": "submit_wf.cwl",
"out": [],
- "run": "keep:f1c2b0c514a5fb9b2a8b5b38a31bab66+61/workflow.json#main"
+ "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
}
]
}
diff --git a/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl b/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
index d486e5a76..3edb66d66 100644
--- a/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
+++ b/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
@@ -48,8 +48,7 @@
"nameroot": "renamed",
"size": 0
}
- ],
- "location": "_:df80736f-f14d-4b10-b2e3-03aa27f034b2"
+ ]
},
"id": "#main/z",
"type": "Directory"
@@ -80,7 +79,7 @@
],
"label": "testing 123",
"out": [],
- "run": "keep:f1c2b0c514a5fb9b2a8b5b38a31bab66+61/workflow.json#main"
+ "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
}
]
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list