[ARVADOS] updated: 2.1.0-2485-g1e7856bff
Git user
git at public.arvados.org
Fri May 13 21:58:24 UTC 2022
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 4 +--
sdk/cwl/arvados_cwl/executor.py | 5 +--
.../{secondary/sub.cwl => 17004-output-props.cwl} | 21 ++++++------
sdk/cwl/tests/arvados-tests.sh | 4 +++
sdk/cwl/tests/test_set_output_prop.py | 37 ++++++++++++++++++++++
5 files changed, 57 insertions(+), 14 deletions(-)
copy sdk/cwl/tests/{secondary/sub.cwl => 17004-output-props.cwl} (53%)
create mode 100644 sdk/cwl/tests/test_set_output_prop.py
via 1e7856bffea0d0ecfcf940de90243dea0fbd3c2f (commit)
via fee9186b62c23aa8a112b65626a44fab6b34b3fd (commit)
from cf9afd31cf9bc682ea3f8c0d59012fa7080b0e9b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1e7856bffea0d0ecfcf940de90243dea0fbd3c2f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri May 13 17:51:06 2022 -0400
17004: Test for setting final output properties from inside container
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/17004-output-props.cwl b/sdk/cwl/tests/17004-output-props.cwl
new file mode 100644
index 000000000..4cf03bade
--- /dev/null
+++ b/sdk/cwl/tests/17004-output-props.cwl
@@ -0,0 +1,22 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.2
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+hints:
+ arv:OutputCollectionProperties:
+ outputProperties:
+ foo: bar
+ baz: $(inputs.inp.basename)
+inputs:
+ inp: File
+steps:
+ cat:
+ in:
+ inp: inp
+ run: cat.cwl
+ out: []
+outputs: []
diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
index 1bbaa505e..e3f162283 100755
--- a/sdk/cwl/tests/arvados-tests.sh
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -28,5 +28,9 @@ arvados-cwl-runner 18888-download_def.cwl --scripts scripts/
# integration test to check for the expected behavior.
python test_copy_deps.py
+# Test for #17004
+# Checks that the final output collection has the expected properties.
+python test_set_output_prop.py
+
# Run integration tests
exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum --api=containers
diff --git a/sdk/cwl/tests/test_set_output_prop.py b/sdk/cwl/tests/test_set_output_prop.py
new file mode 100644
index 000000000..3219eac98
--- /dev/null
+++ b/sdk/cwl/tests/test_set_output_prop.py
@@ -0,0 +1,37 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import subprocess
+
+api = arvados.api()
+
+def test_execute():
+ group = api.groups().create(body={"group": {"name": "test-17004-project", "group_class": "project"}}, ensure_unique_name=True).execute()
+ try:
+ contents = api.groups().contents(uuid=group["uuid"]).execute()
+ if len(contents["items"]) != 0:
+ raise Exception("Expected 0 items")
+
+ cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "17004-output-props.cwl", "--inp", "scripts/download_all_data.sh"]
+ print(" ".join(cmd))
+ subprocess.check_output(cmd)
+
+ contents = api.groups().contents(uuid=group["uuid"]).execute()
+
+ found = False
+ for c in contents["items"]:
+ if (c["kind"] == "arvados#collection" and
+ c["properties"].get("type") == "output" and
+ c["properties"].get("foo") == "bar" and
+ c["properties"].get("baz") == "download_all_data.sh"):
+ found = True
+ if not found:
+ raise Exception("Didn't find collection with properties")
+
+ finally:
+ api.groups().delete(uuid=group["uuid"]).execute()
+
+if __name__ == '__main__':
+ test_execute()
commit fee9186b62c23aa8a112b65626a44fab6b34b3fd
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri May 13 17:27:11 2022 -0400
17004: Fix setting properties on final collection
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index ec9c52c59..f3e122e60 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -349,8 +349,8 @@ class ArvadosContainer(JobBase):
for pr in output_properties_req["outputProperties"]:
container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
else:
- logger.warning("%s API server is too old to support setting properties on output collections.",
- self.arvrunner.label(self))
+ logger.warning("%s API revision is %s, revision %s is required to support setting properties on output collections.",
+ self.arvrunner.label(self), self.arvrunner.api._rootDesc["revision"], "20220510")
if runtimeContext.runnerjob.startswith("arvwf:"):
wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 7fdf07de2..fe078e322 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -32,7 +32,7 @@ from arvados.errors import ApiError
import arvados_cwl.util
from .arvcontainer import RunnerContainer
-from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
+from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder
from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool
from .arvworkflow import ArvadosWorkflow, upload_workflow
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
@@ -795,8 +795,9 @@ The 'jobs' API is no longer supported.
output_properties = {}
output_properties_req, _ = tool.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
if output_properties_req:
+ builder = make_builder(job_order, tool.hints, tool.requirements, runtimeContext, tool.metadata)
for pr in output_properties_req["outputProperties"]:
- output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+ output_properties[pr["propertyName"]] = builder.do_eval(pr["propertyValue"])
self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes,
self.output_tags, output_properties,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list