[ARVADOS] updated: 2.1.0-1432-g1c95615a4
Git user
git at public.arvados.org
Tue Oct 5 01:33:18 UTC 2021
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 4 ++--
sdk/cwl/arvados_cwl/executor.py | 14 +++++++++-----
sdk/cwl/tests/test_container.py | 9 +++++++++
sdk/cwl/tests/test_submit.py | 14 ++++++++------
sdk/cwl/tests/wf/scatter2_subwf.cwl | 5 ++++-
6 files changed, 33 insertions(+), 15 deletions(-)
via 1c95615a417b6435dc86f80e3f02563196449611 (commit)
from c1872f15cdaf416ff9ccfc520085e8af1c5ed582 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1c95615a417b6435dc86f80e3f02563196449611
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Oct 4 21:33:00 2021 -0400
Fix CWL tests refs #18238
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index ee636be37..71ef742e3 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -301,7 +301,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
+ executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout)
except WorkflowException as e:
logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False))
return 1
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 560e8979e..ae3c66889 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -520,10 +520,10 @@ class RunnerContainer(Runner):
if runtimeContext.debug:
command.append("--debug")
- if runtimeContext.storage_classes != "default":
+ if runtimeContext.storage_classes != "default" and runtimeContext.storage_classes:
command.append("--storage-classes=" + runtimeContext.storage_classes)
- if runtimeContext.intermediate_storage_classes != "default":
+ if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes:
command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
if self.on_error:
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index db1db2816..aa19633d8 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -99,7 +99,8 @@ class ArvCwlExecutor(object):
arvargs=None,
keep_client=None,
num_retries=4,
- thread_count=4):
+ thread_count=4,
+ stdout=sys.stdout):
if arvargs is None:
arvargs = argparse.Namespace()
@@ -132,6 +133,7 @@ class ArvCwlExecutor(object):
self.should_estimate_cache_size = True
self.fs_access = None
self.secret_store = None
+ self.stdout = stdout
if keep_client is not None:
self.keep_client = keep_client
@@ -602,14 +604,15 @@ The 'jobs' API is no longer supported.
if existing_uuid or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
if self.work_api == "containers":
- return (upload_workflow(self, tool, job_order,
+ uuid = upload_workflow(self, tool, job_order,
self.project_uuid,
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
merged_map=merged_map,
- submit_runner_image=runtimeContext.submit_runner_image),
- "success")
+ submit_runner_image=runtimeContext.submit_runner_image)
+ self.stdout.write(uuid + "\n")
+ return (None, "success")
self.apply_reqs(job_order, tool)
@@ -679,7 +682,8 @@ The 'jobs' API is no longer supported.
if runtimeContext.submit and not runtimeContext.wait:
runnerjob = next(jobiter)
runnerjob.run(runtimeContext)
- return (runnerjob.uuid, "success")
+ self.stdout.write(runnerjob.uuid+"\n")
+ return (None, "success")
current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current_container:
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 8a380ff80..1a2bd112f 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -112,6 +112,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -175,6 +176,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -270,6 +272,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -398,6 +401,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -622,6 +626,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -712,6 +717,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -808,6 +814,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -851,6 +858,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -924,6 +932,7 @@ class TestContainer(unittest.TestCase):
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8c0fcaf74..77f70851e 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -87,6 +87,7 @@ def stubs(func):
stubs.api = mock.MagicMock()
stubs.api._rootDesc = get_rootDesc()
stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
+ stubs.api._rootDesc["revision"] = "20210628"
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
@@ -446,7 +447,7 @@ class TestSubmit(unittest.TestCase):
"enableReuse": False,
},
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
"cwltool": "http://commonwl.org/cwltool#"
}
@@ -572,6 +573,7 @@ class TestSubmit(unittest.TestCase):
def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
final_output_c = arvados.collection.Collection()
make_output.return_value = ({},final_output_c)
+ stubs.api.config().get.return_value = {"default": {"Default": True}}
def set_final_output(job_order, output_callback, runtimeContext):
output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
@@ -1032,7 +1034,7 @@ class TestSubmit(unittest.TestCase):
"keep_cache": 512
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -1126,9 +1128,6 @@ class TestSubmit(unittest.TestCase):
"content": {
"$graph": [
{
- "$namespaces": {
- "cwltool": "http://commonwl.org/cwltool#"
- },
"arguments": [
"md5sum",
"example.conf"
@@ -1217,6 +1216,9 @@ class TestSubmit(unittest.TestCase):
]
}
],
+ "$namespaces": {
+ "cwltool": "http://commonwl.org/cwltool#"
+ },
"cwlVersion": "v1.0"
},
"kind": "json"
@@ -1445,7 +1447,7 @@ class TestSubmit(unittest.TestCase):
],
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#"
}
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index c54e1707f..218b0c501 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -86,5 +86,8 @@
]
}
],
+ "$namespaces": {
+ "arv": "http://arvados.org/cwl#"
+ },
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list