[arvados] updated: 2.5.0-18-g21c017beb
git repository hosting
git at public.arvados.org
Wed Jan 18 22:15:06 UTC 2023
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 11 ++---
sdk/cwl/arvados_cwl/arvdocker.py | 8 ++--
sdk/cwl/arvados_cwl/arvworkflow.py | 45 ++++++++++---------
sdk/cwl/arvados_cwl/fsaccess.py | 5 ++-
sdk/cwl/tests/test_submit.py | 89 +++++++++++--------------------------
sdk/cwl/tests/wf/secret_wf.cwl | 2 +-
6 files changed, 64 insertions(+), 96 deletions(-)
via 21c017bebaa9e8811170c113da77b9054def7117 (commit)
via 347ece2d7c00703487b87998eab06c5edf8796db (commit)
via 91dc5cddefb9d5f09c261b5097165517ea5fb33a (commit)
via 165ec76d4d36b03449fcd756cb03261f688c1a75 (commit)
from 07c70cc59ef9f17ee3baf98d69be76d9a6585235 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 21c017bebaa9e8811170c113da77b9054def7117
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jan 18 17:14:47 2023 -0500
19385: Fix submit_secret
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 1bdda9b2f..eb8a50dae 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -204,21 +204,20 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
n = 7
allmatch = True
- while allmatch:
- n += 1
- for f in all_files:
- if len(f)-1 < n:
- n -= 1
- allmatch = False
- break
- if f[n] != firstfile[n]:
- allmatch = False
- break
-
- while firstfile[n] != "/":
- n -= 1
-
- prefix = firstfile[:n+1]
+ if firstfile:
+ while allmatch:
+ n += 1
+ for f in all_files:
+ if len(f)-1 < n:
+ n -= 1
+ allmatch = False
+ break
+ if f[n] != firstfile[n]:
+ allmatch = False
+ break
+
+ while firstfile[n] != "/":
+ n -= 1
col = arvados.collection.Collection(api_client=arvRunner.api)
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8609643bd..bb0485d8b 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -678,8 +678,6 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs()
def test_submit_keepref(self, stubs, tm, reader):
- raise Exception("broken")
-
with open("tests/wf/expect_arvworkflow.cwl") as f:
reader().open().__enter__().read.return_value = f.read()
@@ -1092,7 +1090,7 @@ class TestSubmit(unittest.TestCase):
@stubs()
def test_submit_secrets(self, stubs):
exited = arvados_cwl.main(
- ["--submit", "--no-wait", "--api=containers", "--debug",
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
"tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
@@ -1111,7 +1109,7 @@ class TestSubmit(unittest.TestCase):
'--thread-count=0',
"--enable-reuse",
"--collection-cache-size=256",
- '--output-name=Output from workflow secret_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
+ "--output-name=Output from workflow secret_wf.cwl",
"--debug",
"--on-error=continue",
"/var/lib/cwl/workflow.json#main",
@@ -1171,14 +1169,14 @@ class TestSubmit(unittest.TestCase):
],
"steps": [
{
- "id": "#main/secret_wf.cwl (2.5.0-15-g165ec76d4)",
+ "id": "#main/secret_wf.cwl",
"in": [
{
"id": "#main/step/pw",
"source": "#main/pw"
}
],
- "label": "secret_wf.cwl (2.5.0-15-g165ec76d4)",
+ "label": "secret_wf.cwl",
"out": [
{"id": "#main/step/out"}
],
@@ -1200,11 +1198,11 @@ class TestSubmit(unittest.TestCase):
"path": "/var/spool/cwl/cwl.output.json"
}
},
- "name": "secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
- "output_name": "Output from workflow secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
+ "name": "secret_wf.cwl",
+ "output_name": "Output from workflow secret_wf.cwl",
"output_path": "/var/spool/cwl",
"priority": 500,
- "properties": stubs.git_props,
+ "properties": {},
"runtime_constraints": {
"API": True,
"ram": 1342177280,
@@ -1220,8 +1218,6 @@ class TestSubmit(unittest.TestCase):
"use_existing": False
}
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"].update(stubs.git_info)
-
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
self.assertEqual(stubs.capture_stdout.getvalue(),
commit 347ece2d7c00703487b87998eab06c5edf8796db
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jan 18 16:55:57 2023 -0500
19385: update_refs fixes prefixes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 394895230..1bdda9b2f 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -155,7 +155,7 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
for i, s in enumerate(d):
if prefix and isinstance(s, str):
if s.startswith(prefix):
- d[i] = replacePrefix+s[len(prefix)+1:]
+ d[i] = replacePrefix+s[len(prefix):]
else:
update_refs(s, baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
elif isinstance(d, MutableMapping):
@@ -365,7 +365,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
for g in git_info:
doc[g] = git_info[g]
- update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"], "#main/")
+ update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
return doc
commit 91dc5cddefb9d5f09c261b5097165517ea5fb33a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jan 18 16:50:06 2023 -0500
19385: Fix submit secrets test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8f48bcd9d..394895230 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -147,13 +147,17 @@ def rel_ref(s, baseuri, urlexpander, merged_map):
return os.path.join(r, p3)
-def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeContext):
+def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix):
if set_block_style and (isinstance(d, CommentedSeq) or isinstance(d, CommentedMap)):
d.fa.set_block_style()
if isinstance(d, MutableSequence):
- for s in d:
- update_refs(s, baseuri, urlexpander, merged_map, set_block_style, runtimeContext)
+ for i, s in enumerate(d):
+ if prefix and isinstance(s, str):
+ if s.startswith(prefix):
+ d[i] = replacePrefix+s[len(prefix)+1:]
+ else:
+ update_refs(s, baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
elif isinstance(d, MutableMapping):
if "id" in d:
baseuri = urlexpander(d["id"], baseuri, scoped_id=True)
@@ -171,7 +175,7 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
for n, s in enumerate(d["$schemas"]):
d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map)
- update_refs(d[s], baseuri, urlexpander, merged_map, set_block_style, runtimeContext)
+ update_refs(d[s], baseuri, urlexpander, merged_map, set_block_style, runtimeContext, prefix, replacePrefix)
def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
runtimeContext,
@@ -236,7 +240,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
# 2. find $import, $include, $schema, run, location
# 3. update field value
- update_refs(result, w, tool.doc_loader.expand_url, merged_map, set_block_style, runtimeContext)
+ update_refs(result, w, tool.doc_loader.expand_url, merged_map, set_block_style, runtimeContext, "", "")
with col.open(w[n+1:], "wt") as f:
#print(yamlloader.dump(result, stream=sys.stdout))
@@ -361,7 +365,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
for g in git_info:
doc[g] = git_info[g]
- update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext)
+ update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"], "#main/")
return doc
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d1eafbf4d..8609643bd 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1131,62 +1131,23 @@ class TestSubmit(unittest.TestCase):
"/var/lib/cwl/workflow.json": {
"content": {
"$graph": [
- {
- "arguments": [
- "md5sum",
- "example.conf"
- ],
- "class": "CommandLineTool",
- "hints": [
- {
- "class": "http://commonwl.org/cwltool#Secrets",
- "secrets": [
- "#secret_job.cwl/pw"
- ]
- }
- ],
- "id": "#secret_job.cwl",
- "inputs": [
- {
- "id": "#secret_job.cwl/pw",
- "type": "string"
- }
- ],
- "outputs": [
- {
- "id": "#secret_job.cwl/out",
- "type": "File",
- "outputBinding": {
- "glob": "hashed_example.txt"
- }
- }
- ],
- "stdout": "hashed_example.txt",
- "requirements": [
- {
- "class": "InitialWorkDirRequirement",
- "listing": [
- {
- "entry": "username: user\npassword: $(inputs.pw)\n",
- "entryname": "example.conf"
- }
- ]
- }
- ]
- },
{
"class": "Workflow",
"hints": [
{
- "class": "DockerRequirement",
- "dockerPull": "debian:buster-slim",
- "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ "class": "DockerRequirement",
+ "dockerPull": "debian:buster-slim",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
},
{
"class": "http://commonwl.org/cwltool#Secrets",
"secrets": [
"#main/pw"
]
+ },
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
}
],
"id": "#main",
@@ -1199,31 +1160,34 @@ class TestSubmit(unittest.TestCase):
"outputs": [
{
"id": "#main/out",
- "outputSource": "#main/step1/out",
+ "outputSource": "#main/step/out",
"type": "File"
}
],
+ "requirements": [
+ {
+ "class": "SubworkflowFeatureRequirement"
+ }
+ ],
"steps": [
{
- "id": "#main/step1",
+ "id": "#main/secret_wf.cwl (2.5.0-15-g165ec76d4)",
"in": [
{
- "id": "#main/step1/pw",
+ "id": "#main/step/pw",
"source": "#main/pw"
}
],
+ "label": "secret_wf.cwl (2.5.0-15-g165ec76d4)",
"out": [
- "#main/step1/out"
+ {"id": "#main/step/out"}
],
- "run": "#secret_job.cwl"
+ "run": "keep:702edbfca88e907b62234556089dbce7+119/secret_wf.cwl"
}
]
}
],
- "$namespaces": {
- "cwltool": "http://commonwl.org/cwltool#"
- },
- "cwlVersion": "v1.0"
+ "cwlVersion": "v1.2"
},
"kind": "json"
},
diff --git a/sdk/cwl/tests/wf/secret_wf.cwl b/sdk/cwl/tests/wf/secret_wf.cwl
index 5d2e231ec..19d4262eb 100644
--- a/sdk/cwl/tests/wf/secret_wf.cwl
+++ b/sdk/cwl/tests/wf/secret_wf.cwl
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
-cwlVersion: v1.0
+cwlVersion: v1.2
class: Workflow
$namespaces:
cwltool: http://commonwl.org/cwltool#
commit 165ec76d4d36b03449fcd756cb03261f688c1a75
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jan 13 15:14:56 2023 -0500
19385: Continuing to fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index e0ee7285a..92ed3057a 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -17,7 +17,7 @@ import uuid
import math
import arvados_cwl.util
-import ruamel.yaml as yaml
+import ruamel.yaml
from cwltool.errors import WorkflowException
from cwltool.process import UnsupportedRequirement, shortname
@@ -536,10 +536,11 @@ class RunnerContainer(Runner):
"portable_data_hash": "%s" % workflowcollection
}
elif self.embedded_tool.tool.get("id", "").startswith("arvwf:"):
- workflowpath = "/var/lib/cwl/workflow.json#main"
- #record = self.arvrunner.api.workflows().get(uuid=self.embedded_tool.tool["id"][6:33]).execute(num_retries=self.arvrunner.num_retries)
- #packed = yaml.safe_load(record["definition"])
- packed = self.loadingContext.loader.idx[self.embedded_tool.tool["id"]]
+ uuid, frg = urllib.parse.urldefrag(self.embedded_tool.tool["id"])
+ workflowpath = "/var/lib/cwl/workflow.json#" + frg
+ packedtxt = self.loadingContext.loader.fetch_text(uuid)
+ yaml = ruamel.yaml.YAML(typ='safe', pure=True)
+ packed = yaml.load(packedtxt)
container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
"kind": "json",
"content": packed
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 9c20c0c50..f5e67a664 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -56,10 +56,10 @@ def determine_image_id(dockerImageId):
def arv_docker_get_image(api_client, dockerRequirement, pull_image, runtimeContext):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
- project_uuid = runtimeContext.project_uuid,
- force_pull = runtimeContext.force_docker_pull,
- tmp_outdir_prefix = runtimeContext.tmp_outdir_prefix,
- match_local_docker = runtimeContext.match_local_docker,
+ project_uuid = runtimeContext.project_uuid
+ force_pull = runtimeContext.force_docker_pull
+ tmp_outdir_prefix = runtimeContext.tmp_outdir_prefix
+ match_local_docker = runtimeContext.match_local_docker
copy_deps = runtimeContext.copy_deps
cached_lookups = runtimeContext.cached_docker_lookups
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 716cda335..a5e9db0cf 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -16,6 +16,7 @@ import re
import logging
import threading
from collections import OrderedDict
+from io import StringIO
import ruamel.yaml
@@ -238,7 +239,9 @@ class CollectionFetcher(DefaultFetcher):
yaml = ruamel.yaml.YAML(typ='rt', pure=True)
definition = yaml.load(record["definition"])
definition["label"] = record["name"]
- return yaml.round_trip_dump(definition)
+ stream = StringIO()
+ yaml.dump(definition, stream)
+ return stream.getvalue()
return super(CollectionFetcher, self).fetch_text(url)
def check_exists(self, url):
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 2289f13c2..d1eafbf4d 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -764,6 +764,7 @@ class TestSubmit(unittest.TestCase):
'kind': 'json',
'content': {
'cwlVersion': 'v1.0',
+ 'label': 'a test workflow',
'$graph': [
{
'id': '#main',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list