[ARVADOS] updated: 2.1.0-1432-g1c95615a4

Git user git at public.arvados.org
Tue Oct 5 01:33:18 UTC 2021


Summary of changes:
 sdk/cwl/arvados_cwl/__init__.py     |  2 +-
 sdk/cwl/arvados_cwl/arvcontainer.py |  4 ++--
 sdk/cwl/arvados_cwl/executor.py     | 14 +++++++++-----
 sdk/cwl/tests/test_container.py     |  9 +++++++++
 sdk/cwl/tests/test_submit.py        | 14 ++++++++------
 sdk/cwl/tests/wf/scatter2_subwf.cwl |  5 ++++-
 6 files changed, 33 insertions(+), 15 deletions(-)

       via  1c95615a417b6435dc86f80e3f02563196449611 (commit)
      from  c1872f15cdaf416ff9ccfc520085e8af1c5ed582 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 1c95615a417b6435dc86f80e3f02563196449611
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Mon Oct 4 21:33:00 2021 -0400

    Fix CWL tests refs #18238
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index ee636be37..71ef742e3 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -301,7 +301,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
             api_client.users().current().execute()
         if keep_client is None:
             keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
-        executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
+        executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout)
     except WorkflowException as e:
         logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False))
         return 1
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 560e8979e..ae3c66889 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -520,10 +520,10 @@ class RunnerContainer(Runner):
         if runtimeContext.debug:
             command.append("--debug")
 
-        if runtimeContext.storage_classes != "default":
+        if runtimeContext.storage_classes != "default" and runtimeContext.storage_classes:
             command.append("--storage-classes=" + runtimeContext.storage_classes)
 
-        if runtimeContext.intermediate_storage_classes != "default":
+        if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes:
             command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
 
         if self.on_error:
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index db1db2816..aa19633d8 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -99,7 +99,8 @@ class ArvCwlExecutor(object):
                  arvargs=None,
                  keep_client=None,
                  num_retries=4,
-                 thread_count=4):
+                 thread_count=4,
+                 stdout=sys.stdout):
 
         if arvargs is None:
             arvargs = argparse.Namespace()
@@ -132,6 +133,7 @@ class ArvCwlExecutor(object):
         self.should_estimate_cache_size = True
         self.fs_access = None
         self.secret_store = None
+        self.stdout = stdout
 
         if keep_client is not None:
             self.keep_client = keep_client
@@ -602,14 +604,15 @@ The 'jobs' API is no longer supported.
         if existing_uuid or runtimeContext.create_workflow:
             # Create a pipeline template or workflow record and exit.
             if self.work_api == "containers":
-                return (upload_workflow(self, tool, job_order,
+                uuid = upload_workflow(self, tool, job_order,
                                         self.project_uuid,
                                         uuid=existing_uuid,
                                         submit_runner_ram=runtimeContext.submit_runner_ram,
                                         name=runtimeContext.name,
                                         merged_map=merged_map,
-                                        submit_runner_image=runtimeContext.submit_runner_image),
-                        "success")
+                                        submit_runner_image=runtimeContext.submit_runner_image)
+                self.stdout.write(uuid + "\n")
+                return (None, "success")
 
         self.apply_reqs(job_order, tool)
 
@@ -679,7 +682,8 @@ The 'jobs' API is no longer supported.
         if runtimeContext.submit and not runtimeContext.wait:
             runnerjob = next(jobiter)
             runnerjob.run(runtimeContext)
-            return (runnerjob.uuid, "success")
+            self.stdout.write(runnerjob.uuid+"\n")
+            return (None, "success")
 
         current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
         if current_container:
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 8a380ff80..1a2bd112f 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -112,6 +112,7 @@ class TestContainer(unittest.TestCase):
             runner.ignore_docker_for_reuse = False
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
+            runner.api._rootDesc = {"revision": "20210628"}
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -175,6 +176,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 3600
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -270,6 +272,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -398,6 +401,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -622,6 +626,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -712,6 +717,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -808,6 +814,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -851,6 +858,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -924,6 +932,7 @@ class TestContainer(unittest.TestCase):
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
+        runner.api._rootDesc = {"revision": "20210628"}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8c0fcaf74..77f70851e 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -87,6 +87,7 @@ def stubs(func):
         stubs.api = mock.MagicMock()
         stubs.api._rootDesc = get_rootDesc()
         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
+        stubs.api._rootDesc["revision"] = "20210628"
 
         stubs.api.users().current().execute.return_value = {
             "uuid": stubs.fake_user_uuid,
@@ -446,7 +447,7 @@ class TestSubmit(unittest.TestCase):
                 "enableReuse": False,
             },
         ]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
             "arv": "http://arvados.org/cwl#",
             "cwltool": "http://commonwl.org/cwltool#"
         }
@@ -572,6 +573,7 @@ class TestSubmit(unittest.TestCase):
     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
         final_output_c = arvados.collection.Collection()
         make_output.return_value = ({},final_output_c)
+        stubs.api.config().get.return_value = {"default": {"Default": True}}
 
         def set_final_output(job_order, output_callback, runtimeContext):
             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
@@ -1032,7 +1034,7 @@ class TestSubmit(unittest.TestCase):
                 "keep_cache": 512
             }
         ]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
             "arv": "http://arvados.org/cwl#",
         }
         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -1126,9 +1128,6 @@ class TestSubmit(unittest.TestCase):
                     "content": {
                         "$graph": [
                             {
-                                "$namespaces": {
-                                    "cwltool": "http://commonwl.org/cwltool#"
-                                },
                                 "arguments": [
                                     "md5sum",
                                     "example.conf"
@@ -1217,6 +1216,9 @@ class TestSubmit(unittest.TestCase):
                                 ]
                             }
                         ],
+                        "$namespaces": {
+                            "cwltool": "http://commonwl.org/cwltool#"
+                        },
                         "cwlVersion": "v1.0"
                     },
                     "kind": "json"
@@ -1445,7 +1447,7 @@ class TestSubmit(unittest.TestCase):
                 ],
             }
         ]
-        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+        expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
             "arv": "http://arvados.org/cwl#"
         }
 
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index c54e1707f..218b0c501 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -86,5 +86,8 @@
       ]
     }
   ],
+  "$namespaces": {
+    "arv": "http://arvados.org/cwl#"
+  },
   "cwlVersion": "v1.0"
-}
\ No newline at end of file
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list