[arvados] updated: 2.6.0-569-gcf25e8bf1a

[git repository hosting]

Summary of changes:
 sdk/cwl/arvados_cwl/executor.py |  2 +-
 sdk/cwl/arvados_cwl/runner.py   | 30 +++++++++++++++++++-----------
 sdk/cwl/tests/test_submit.py    | 15 +++++++++++++++
 3 files changed, 35 insertions(+), 12 deletions(-)

       via  cf25e8bf1af840ddd304e18e789ccb2abc0afd1e (commit)
      from  e9d23ccbf78cacd454a8d4d3971032b21c2b1365 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit cf25e8bf1af840ddd304e18e789ccb2abc0afd1e
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Thu Oct 5 09:39:51 2023 -0400

    20933: Add --print-keep-deps test
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 330dba3dbe..677e10d265 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -729,7 +729,7 @@ The 'jobs' API is no longer supported.
 
             if runtimeContext.print_keep_deps:
                 # Just find and print out all the collection dependencies and exit
-                print_keep_deps(tool)
+                print_keep_deps(self, runtimeContext, merged_map, tool)
                 return (None, "success")
 
             # Did not register a workflow, we're going to submit
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 860f8a1b9a..554f137449 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -948,18 +948,19 @@ class Runner(Process):
             self.arvrunner.output_callback(outputs, processStatus)
 
 
-def print_keep_deps_visitor(references, doc_loader, tool):
+def print_keep_deps_visitor(api, runtimeContext, references, doc_loader, tool):
     def collect_locators(obj):
         loc = obj.get("location", "")
 
         g = arvados.util.keepuri_pattern.match(loc)
-        if g and g[1] not in references:
-            references.append(g[1])
-            return
+        if g:
+            references.add(g[1])
+
+        if obj.get("class") == "http://arvados.org/cwl#WorkflowRunnerResources" and "acrContainerImage" in obj:
+            references.add(obj["acrContainerImage"])
 
-        loc = obj.get("http://arvados.org/cwl#dockerCollectionPDH", "") or obj.get("acrContainerImage")
-        if loc:
-            references.append(loc)
+        if obj.get("class") == "DockerRequirement":
+            references.add(arvados_cwl.arvdocker.arv_docker_get_image(api, obj, False, runtimeContext))
 
     sc_result = scandeps(tool["id"], tool,
                          set(),
@@ -971,8 +972,15 @@ def print_keep_deps_visitor(references, doc_loader, tool):
     visit_class(tool, ("DockerRequirement", "http://arvados.org/cwl#WorkflowRunnerResources"), collect_locators)
 
 
-def print_keep_deps(tool):
-    references = []
+def print_keep_deps(arvRunner, runtimeContext, merged_map, tool):
+    references = set()
+
+    tool.visit(partial(print_keep_deps_visitor, arvRunner.api, runtimeContext, references, tool.doc_loader))
+
+    for mm in merged_map:
+        for k, v in merged_map[mm].resolved.items():
+            g = arvados.util.keepuri_pattern.match(v)
+            if g:
+                references.add(g[1])
 
-    tool.visit(partial(print_keep_deps_visitor, references, tool.doc_loader))
-    print(json.dumps(references))
+    arvRunner.stdout.write(json.dumps(sorted(list(references)))+"\n")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index b10d1159f9..c8bf127951 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1774,3 +1774,18 @@ class TestCreateWorkflow(unittest.TestCase):
         self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_workflow_uuid + '\n')
         self.assertEqual(exited, 0)
+
+
+class TestPrintKeepDeps(unittest.TestCase):
+    @stubs()
+    def test_print_keep_deps(self, stubs):
+        # test --print-keep-deps which is used by arv-copy
+
+        exited = arvados_cwl.main(
+            ["--print-keep-deps", "--debug",
+             "tests/wf/submit_wf_map.cwl"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
+        self.assertEqual(exited, 0)

-----------------------------------------------------------------------


hooks/post-receive
--