[ARVADOS] created: 2.1.0-1281-g97e0090ec
Git user
git at public.arvados.org
Thu Sep 2 17:51:23 UTC 2021
at 97e0090ec440c3322053d7635d883cf8af6633ca (commit)
commit 97e0090ec440c3322053d7635d883cf8af6633ca
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Sep 2 13:51:04 2021 -0400
14018: Setting process properties WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index 17c37cf5c..09951a984 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -251,7 +251,7 @@ $graph:
type: string
doc: The property key
- name: propertyValue
- type: [cwl:Expression, Any]
+ type: [Any]
doc: The property value
@@ -271,6 +271,6 @@ $graph:
_type: "@vocab"
processProperties:
type: PropertyDef[]
- jsonldPredicate:
- mapSubject: propertyName
- mapPredicate: propertyValue
+ #jsonldPredicate:
+ # mapSubject: propertyName
+ # mapPredicate: propertyValue
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index c6e4577d0..3065e1a7c 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -28,7 +28,7 @@ import arvados.collection
from .arvdocker import arv_docker_get_image
from . import done
-from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
+from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields, make_builder
from .fsaccess import CollectionFetcher
from .pathmapper import NoFollowPathMapper, trim_listing
from .perf import Perf
@@ -482,8 +482,10 @@ class RunnerContainer(Runner):
properties_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#ProcessProperties")
if properties_req:
builder = make_builder(self.job_order, self.embedded_tool.hints, self.embedded_tool.requirements, runtimeContext, self.embedded_tool.metadata)
+ print("ZZZZZZ", properties_req["processProperties"])
for pr in properties_req["processProperties"]:
- container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+ print("ZZZZZZ", pr)
+ container_req["properties"][pr["propertyName"]] = builder.do_eval(pr["propertyValue"])
# --local means execute the workflow instead of submitting a container request
# --api=containers means use the containers API
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2b46b89c6..91283b0b6 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -914,6 +914,97 @@ class TestContainer(unittest.TestCase):
}))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_setting_process_properties(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ tool = cmap({
+ "inputs": [
+ {"id": "x", "type": "string"}],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://arvados.org/cwl#ProcessProperties",
+ "processProperties": [
+ {"propertyName": "foo",
+ "propertyValue": "bar"},
+ {"propertyName": "baz",
+ "propertyValue": "$(inputs.x)"},
+ {"propertyName": "quux",
+ "propertyValue": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+ ],
+ }
+ ]
+ })
+
+ loadingContext, runtimeContext = self.helper(runner, True)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['ls', '/var/spool/cwl'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {
+ "baz": "blorp",
+ "foo": "bar",
+ "quux": {
+ "q1": 1,
+ "q2": 2
+ }
+ },
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
+
+
class TestWorkflow(unittest.TestCase):
def setUp(self):
cwltool.process._names = set()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 1b646a8e4..2889ce331 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1419,6 +1419,51 @@ class TestSubmit(unittest.TestCase):
finally:
cwltool_logger.removeHandler(stderr_logger)
+ @stubs
+ def test_submit_set_process_properties(self, stubs):
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["name"] = "submit_wf_process_properties.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ {
+ "class": "http://arvados.org/cwl#ProcessProperties",
+ "processProperties": [
+ {"propertyName": "foo",
+ "propertyValue": "bar"},
+ {"propertyName": "baz",
+ "propertyValue": "$(inputs.x.basename)"},
+ {"propertyName": "quux",
+ "propertyValue": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+ ],
+ }
+ ]
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ "arv": "http://arvados.org/cwl#"
+ }
+
+ expect_container["properties"] = {
+ "baz": "blorp.txt",
+ "foo": "bar",
+ "quux": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
commit 44a8d4eeb0f61941786894ed761f068512216b31
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Sep 2 13:15:52 2021 -0400
14018: Setting process properties WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index dd5919fc8..17c37cf5c 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -239,3 +239,38 @@ $graph:
- type: array
items: string
doc: One or more storages classes
+
+
+- type: record
+ name: PropertyDef
+ doc: |
+ Define a property that will be set on the submitted container
+ request associated with this workflow or step.
+ fields:
+ - name: propertyName
+ type: string
+ doc: The property key
+ - name: propertyValue
+ type: [cwl:Expression, Any]
+ doc: The property value
+
+
+- name: ProcessProperties
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify metadata properties that will be set on the submitted
+ container request associated with this workflow or step.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:ProcessProperties"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ processProperties:
+ type: PropertyDef[]
+ jsonldPredicate:
+ mapSubject: propertyName
+ mapPredicate: propertyValue
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index c9170c51b..c6e4577d0 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -67,15 +67,21 @@ class ArvadosContainer(JobBase):
runtimeContext = self.job_runtime
- container_request = {
- "command": self.command_line,
- "name": self.name,
- "output_path": self.outdir,
- "cwd": self.outdir,
- "priority": runtimeContext.priority,
- "state": "Committed",
- "properties": {},
- }
+ if runtimeContext.submit_request_uuid:
+ container_request = self.arvrunner.api.container_requests().get(
+ uuid=runtimeContext.submit_request_uuid
+ ).execute(num_retries=self.arvrunner.num_retries)
+ else:
+ container_request = {}
+
+ container_request["command"] = self.command_line
+ container_request["name"] = self.name
+ container_request["output_path"] = self.outdir
+ container_request["cwd"] = self.outdir
+ container_request["priority"] = runtimeContext.priority
+ container_request["state"] = "Committed"
+ container_request.setdefault("properties", {})
+
runtime_constraints = {}
if runtimeContext.project_uuid:
@@ -303,6 +309,11 @@ class ArvadosContainer(JobBase):
enable_reuse = reuse_req["enableReuse"]
container_request["use_existing"] = enable_reuse
+ properties_req, _ = self.get_requirement("http://arvados.org/cwl#ProcessProperties")
+ if properties_req:
+ for pr in properties_req["processProperties"]:
+ container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+
if runtimeContext.runnerjob.startswith("arvwf:"):
wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
@@ -468,6 +479,11 @@ class RunnerContainer(Runner):
if self.embedded_tool.tool.get("id", "").startswith("arvwf:"):
container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
+ properties_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#ProcessProperties")
+ if properties_req:
+ builder = make_builder(self.job_order, self.embedded_tool.hints, self.embedded_tool.requirements, runtimeContext, self.embedded_tool.metadata)
+ for pr in properties_req["processProperties"]:
+ container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
# --local means execute the workflow instead of submitting a container request
# --api=containers means use the containers API
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list