[ARVADOS] updated: 2.1.0-1203-g3a4719dd4
Git user
git at public.arvados.org
Fri Aug 13 14:34:51 UTC 2021
Summary of changes:
build/run-library.sh | 9 +++-
doc/_config.yml | 1 +
doc/admin/upgrading.html.textile.liquid | 8 +--
doc/install/singularity.html.textile.liquid | 42 +++++++++++++++
doc/user/cwl/cwl-run-options.html.textile.liquid | 10 ++--
doc/user/topics/arv-docker.html.textile.liquid | 8 +--
sdk/cwl/arvados_cwl/__init__.py | 4 ++
sdk/cwl/arvados_cwl/arvtool.py | 10 +++-
sdk/cwl/tests/test_submit.py | 16 ++++++
services/api/app/models/collection.rb | 4 +-
.../api/lib/tasks/manage_long_lived_tokens.rake | 60 +++++++++++++++-------
services/api/test/unit/collection_test.rb | 17 ++++++
12 files changed, 153 insertions(+), 36 deletions(-)
create mode 100644 doc/install/singularity.html.textile.liquid
discards 82c5c20ba8966a23fdfb040aac46a25ff74cfc7b (commit)
discards cb33e4c24bd396b223c455ad91a18691e0d59f72 (commit)
discards 0e8c57a68f5d69ce642cc27fb481495a318c4d1f (commit)
via 3a4719dd4ffcc1cd92d270d6b698c7ec13678c55 (commit)
via f509fecc2eef88be0a7b88d491ae0c0ac13d686b (commit)
via 559456537dc2365ff93910dae33e7dc3dda5772e (commit)
via 2b59be173b4a860c524c398a6c5f521015bf0939 (commit)
via 8ce476b7864f1a2d1f05cd0a770ae159eeb845eb (commit)
via b190f9b18bdbd38f167050058c70e014da0a9bdf (commit)
via 0ed4397416e2ccd0ba66eeb511b3f98b84ada15c (commit)
via be2e29a903a53d7565f1a52d0bebe5e5e337a1cd (commit)
via e99f026d040c6020dfcc51c6d988cf18d325a530 (commit)
via 1521969d1475ab6854d1f9987fdaa1f66a1cbaab (commit)
via 377467f5285670a797ec02b4ca6c5a129c289a39 (commit)
via 1d964c00b62c65eda7e6bd46b422e7f08c4843fb (commit)
via 48ef169e011a3335470e940cc0670bca27358dc1 (commit)
via 9214382fc7083d0310675ce3760144ad7b6cb070 (commit)
via f67c3d18e58d4d6f331763a9728f1ef1595558d7 (commit)
via c3481a62091153eb0f74cd2f769d46c552765fcb (commit)
via 5861d894b9b6614437fadfa261b7db1d4920d729 (commit)
via 12217911502706a6c541950d6569de47596b230a (commit)
via d29fa2efcb0afc764336045fff58c279d77cf30c (commit)
via 5bc8ad779b8d39c63df88c20e5a883f4fe15c6da (commit)
via 74aa79ebb951a915a8f5aba49852a0119835c579 (commit)
via 52111b8131b2a17a6f8ec485d571a3d144a74184 (commit)
via b74bc91079b0c2b5491e08f8ff677451c6d7e60e (commit)
via d069271ec828654f9a6b656b04b40f229cfefb27 (commit)
via 1c3fcbc97db70680109aba244287df0231285648 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (82c5c20ba8966a23fdfb040aac46a25ff74cfc7b)
\
N -- N -- N (3a4719dd4ffcc1cd92d270d6b698c7ec13678c55)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3a4719dd4ffcc1cd92d270d6b698c7ec13678c55
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Aug 13 10:33:34 2021 -0400
17390: Update arvados-cwl-runner docs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid
index 761d198ee..a1c102593 100644
--- a/doc/user/cwl/cwl-run-options.html.textile.liquid
+++ b/doc/user/cwl/cwl-run-options.html.textile.liquid
@@ -47,19 +47,19 @@ table(table table-bordered table-condensed).
|==--no-wait==| Submit workflow runner and exit.|
|==--log-timestamps==| Prefix logging lines with timestamp|
|==--no-log-timestamps==| No timestamp on logging lines|
-|==--api== {containers}|Select work submission API. Only supports 'containers'|
|==--compute-checksum==| Compute checksum of contents while collecting outputs|
|==--submit-runner-ram== SUBMIT_RUNNER_RAM|RAM (in MiB) required for the workflow runner (default 1024)|
|==--submit-runner-image== SUBMIT_RUNNER_IMAGE|Docker image for workflow runner|
|==--always-submit-runner==|When invoked with --submit --wait, always submit a runner to manage the workflow, even when only running a single CommandLineTool|
-|==--submit-request-uuid== UUID|Update and commit to supplied container request instead of creating a new one (containers API only).|
-|==--submit-runner-cluster== CLUSTER_ID|Submit workflow runner to a remote cluster (containers API only)|
+|==--submit-request-uuid== UUID|Update and commit to supplied container request instead of creating a new one.|
+|==--submit-runner-cluster== CLUSTER_ID|Submit workflow runner to a remote cluster|
|==--name NAME==|Name to use for workflow execution instance.|
|==--on-error== {stop,continue}|Desired workflow behavior when a step fails. One of 'stop' (do not submit any more steps) or 'continue' (may submit other steps that are not downstream from the error). Default is 'continue'.|
|==--enable-dev==|Enable loading and running development versions of CWL spec.|
-|==--storage-classes== STORAGE_CLASSES|Specify comma separated list of storage classes to be used when saving workflow output to Keep.|
+|==--storage-classes== STORAGE_CLASSES|Specify comma separated list of storage classes to be used when saving the final workflow output to Keep.|
+|==--intermediate-storage-classes== STORAGE_CLASSES|Specify comma separated list of storage classes to be used when intermediate workflow output to Keep.|
|==--intermediate-output-ttl== N|If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).|
-|==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower, containers api only)|
+|==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower)|
|==--thread-count== THREAD_COUNT|Number of threads to use for container submit and output collection.|
|==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).|
|==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.|
commit f509fecc2eef88be0a7b88d491ae0c0ac13d686b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Aug 12 15:24:14 2021 -0400
17390: Add unit tests for storage classes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 6d4f1bd52..edb9d5b52 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -42,7 +42,7 @@ from .context import ArvLoadingContext, ArvRuntimeContext
from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class, aslist
from cwltool.command_line_tool import compute_checksums
from cwltool.load_tool import load_tool
@@ -549,7 +549,7 @@ The 'jobs' API is no longer supported.
if runtimeContext.submit_request_uuid and self.work_api != "containers":
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
- default_storage_classes = [k for k,v in self.api.config()["StorageClasses"].items() if v.get("Default") is True].join(",")
+ default_storage_classes = ",".join([k for k,v in self.api.config()["StorageClasses"].items() if v.get("Default") is True])
if runtimeContext.storage_classes == "default":
runtimeContext.storage_classes = default_storage_classes
if runtimeContext.intermediate_storage_classes == "default":
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 09983f87a..2b46b89c6 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -163,7 +163,8 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
- 'secret_mounts': {}
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
}))
# The test passes some fields in builder.resources
@@ -250,7 +251,8 @@ class TestContainer(unittest.TestCase):
'partitions': ['blurb']
},
'properties': {},
- 'secret_mounts': {}
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
}
call_body = call_kwargs.get('body', None)
@@ -379,7 +381,8 @@ class TestContainer(unittest.TestCase):
'scheduling_parameters': {
},
'properties': {},
- 'secret_mounts': {}
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
}
call_body = call_kwargs.get('body', None)
@@ -463,7 +466,8 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
- 'secret_mounts': {}
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
}))
@mock.patch("arvados.collection.Collection")
@@ -696,7 +700,8 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
- 'secret_mounts': {}
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
}))
# The test passes no builder.resources
@@ -791,7 +796,8 @@ class TestContainer(unittest.TestCase):
"content": "username: user\npassword: blorp\n",
"kind": "text"
}
- }
+ },
+ 'output_storage_classes': ["default"]
}))
# The test passes no builder.resources
@@ -835,6 +841,79 @@ class TestContainer(unittest.TestCase):
self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_setting_storage_class(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://arvados.org/cwl#OutputStorageClass",
+ "finalStorageClass": ["baz_sc", "qux_sc"],
+ "intermediateStorageClass": ["foo_sc", "bar_sc"]
+ }
+ ]
+ })
+
+ loadingContext, runtimeContext = self.helper(runner, True)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['ls', '/var/spool/cwl'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["foo_sc", "bar_sc"]
+ }))
+
+
class TestWorkflow(unittest.TestCase):
def setUp(self):
cwltool.process._names = set()
@@ -972,7 +1051,8 @@ class TestWorkflow(unittest.TestCase):
"scheduling_parameters": {},
"secret_mounts": {},
"state": "Committed",
- "use_existing": True
+ "use_existing": True,
+ 'output_storage_classes': ["default"]
}))
mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
mockc.open().__enter__().write.assert_has_calls([mock.call(
@@ -1074,7 +1154,8 @@ class TestWorkflow(unittest.TestCase):
],
'use_existing': True,
'output_name': u'Output for step echo-subwf',
- 'cwd': '/var/spool/cwl'
+ 'cwd': '/var/spool/cwl',
+ 'output_storage_classes': ["default"]
}))
def test_default_work_api(self):
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 6cf59f25e..1b646a8e4 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -95,6 +95,11 @@ def stubs(func):
stubs.api.containers().current().execute.return_value = {
"uuid": stubs.fake_container_uuid,
}
+ stubs.api.config()["StorageClasses"].items.return_value = {
+ "default": {
+ "Default": True
+ }
+ }.items()
class CollectionExecute(object):
def __init__(self, exe):
@@ -342,14 +347,6 @@ class TestSubmit(unittest.TestCase):
cwltool.process._names = set()
arvados_cwl.arvdocker.arv_docker_clear_cache()
- @stubs
- def test_error_when_multiple_storage_classes_specified(self, stubs):
- storage_classes = "foo,bar"
- exited = arvados_cwl.main(
- ["--debug", "--storage-classes", storage_classes,
- "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdin, sys.stderr, api_client=stubs.api)
- self.assertEqual(exited, 1)
@mock.patch("time.sleep")
@stubs
@@ -526,6 +523,27 @@ class TestSubmit(unittest.TestCase):
stubs.expect_container_request_uuid + '\n')
self.assertEqual(exited, 0)
+ @stubs
+ def test_submit_multiple_storage_classes(self, stubs):
+ exited = arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate', '--disable-color',
+ '--eval-timeout=20', '--thread-count=0',
+ '--enable-reuse', "--collection-cache-size=256", "--debug",
+ "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
@mock.patch("cwltool.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
@@ -568,6 +586,27 @@ class TestSubmit(unittest.TestCase):
make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
self.assertEqual(exited, 0)
+ @mock.patch("cwltool.task_queue.TaskQueue")
+ @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
+ @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
+ @stubs
+ def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
+ final_output_c = arvados.collection.Collection()
+ make_output.return_value = ({},final_output_c)
+
+ def set_final_output(job_order, output_callback, runtimeContext):
+ output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
+ return []
+ job.side_effect = set_final_output
+
+ exited = arvados_cwl.main(
+ ["--debug", "--local",
+ "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+ self.assertEqual(exited, 0)
+
@stubs
def test_submit_container_output_ttl(self, stubs):
exited = arvados_cwl.main(
diff --git a/sdk/cwl/tests/wf/submit_storage_class_wf.cwl b/sdk/cwl/tests/wf/submit_storage_class_wf.cwl
new file mode 100644
index 000000000..c365d9648
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_storage_class_wf.cwl
@@ -0,0 +1,30 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a workflow file for dependencies
+# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+hints:
+ arv:OutputStorageClass:
+ finalStorageClass: [foo, bar]
+inputs:
+ - id: x
+ type: File
+ - id: y
+ type: Directory
+ - id: z
+ type: Directory
+outputs: []
+steps:
+ - id: step1
+ in:
+ - { id: x, source: "#x" }
+ out: []
+ run: ../tool/submit_tool.cwl
commit 559456537dc2365ff93910dae33e7dc3dda5772e
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Aug 10 16:59:21 2021 -0400
17390: Use cluster-configured default
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 6a565b10a..6d4f1bd52 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -549,6 +549,12 @@ The 'jobs' API is no longer supported.
if runtimeContext.submit_request_uuid and self.work_api != "containers":
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
+ default_storage_classes = [k for k,v in self.api.config()["StorageClasses"].items() if v.get("Default") is True].join(",")
+ if runtimeContext.storage_classes == "default":
+ runtimeContext.storage_classes = default_storage_classes
+ if runtimeContext.intermediate_storage_classes == "default":
+ runtimeContext.intermediate_storage_classes = default_storage_classes
+
if not runtimeContext.name:
runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"])
commit 2b59be173b4a860c524c398a6c5f521015bf0939
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Aug 10 16:54:40 2021 -0400
17390: setting intermediate/final storage classes WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 2e491fe5b..7bbb3b29e 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -179,7 +179,9 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="Enable loading and running development versions "
"of the CWL standards.", default=False)
parser.add_argument('--storage-classes', default="default",
- help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
+ help="Specify comma separated list of storage classes to be used when saving final workflow output to Keep.")
+ parser.add_argument('--intermediate-storage-classes', default="default",
+ help="Specify comma separated list of storage classes to be used when saving intermediate workflow output to Keep.")
parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
help="If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).",
@@ -246,7 +248,8 @@ def add_arv_hints():
"http://commonwl.org/cwltool#LoadListingRequirement",
"http://arvados.org/cwl#IntermediateOutput",
"http://arvados.org/cwl#ReuseRequirement",
- "http://arvados.org/cwl#ClusterTarget"
+ "http://arvados.org/cwl#ClusterTarget",
+ "http://arvados.org/cwl#OutputStorageClass"
])
def exit_signal_handler(sigcode, frame):
@@ -260,10 +263,6 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
job_order_object = None
arvargs = parser.parse_args(args)
- if len(arvargs.storage_classes.strip().split(',')) > 1:
- logger.error(str(u"Multiple storage classes are not supported currently."))
- return 1
-
arvargs.use_container = True
arvargs.relax_path_checks = True
arvargs.print_supported_versions = False
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index 8a3fa3173..2a2e857e0 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -266,3 +266,32 @@ $graph:
project_uuid:
type: string?
doc: The project that will own the container requests and intermediate collections
+
+
+- name: OutputStorageClass
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify the storage class to be used for intermediate and final output
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:StorageClassHint"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ intermediateStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
+ finalStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 95ed0a75b..fb14a63e3 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -210,3 +210,31 @@ $graph:
project_uuid:
type: string?
doc: The project that will own the container requests and intermediate collections
+
+- name: OutputStorageClass
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify the storage class to be used for intermediate and final output
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:StorageClassHint"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ intermediateStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
+ finalStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index 95ed0a75b..dd5919fc8 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -210,3 +210,32 @@ $graph:
project_uuid:
type: string?
doc: The project that will own the container requests and intermediate collections
+
+
+- name: OutputStorageClass
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify the storage class to be used for intermediate and final output
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:StorageClassHint"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ intermediateStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
+ finalStorageClass:
+ type:
+ - "null"
+ - string
+ - type: array
+ items: string
+ doc: One or more storages classes
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 72ef14f67..c9170c51b 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -273,6 +273,12 @@ class ArvadosContainer(JobBase):
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+ storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
+ if storage_class_req and storage_class_req.get("intermediateStorageClass"):
+ container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
+ else:
+ container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
+
if self.timelimit is not None and self.timelimit > 0:
scheduling_parameters["max_run_time"] = self.timelimit
@@ -495,6 +501,9 @@ class RunnerContainer(Runner):
if runtimeContext.storage_classes != "default":
command.append("--storage-classes=" + runtimeContext.storage_classes)
+ if runtimeContext.intermediate_storage_classes != "default":
+ command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
+
if self.on_error:
command.append("--on-error=" + self.on_error)
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 8cfe22ad7..77d4027cc 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -29,6 +29,7 @@ class ArvRuntimeContext(RuntimeContext):
self.wait = True
self.cwl_runner_job = None
self.storage_classes = "default"
+ self.intermediate_storage_classes = "default"
self.current_container = None
self.http_timeout = 300
self.submit_runner_cluster = None
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index f60c48087..6a565b10a 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -771,7 +771,13 @@ The 'jobs' API is no longer supported.
if self.output_tags is None:
self.output_tags = ""
- storage_classes = runtimeContext.storage_classes.strip().split(",")
+ storage_classes = ""
+ storage_class_req, _ = tool.get_requirement("http://arvados.org/cwl#OutputStorageClass")
+ if storage_class_req and storage_class_req.get("finalStorageClass"):
+ storage_classes = aslist(storage_class_req["finalStorageClass"])
+ else:
+ storage_classes = runtimeContext.storage_classes.strip().split(",")
+
self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
self.set_crunch_output()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list