[ARVADOS] created: 1.3.0-3023-g4ddeddd8d
Git user
git at public.arvados.org
Wed Aug 26 20:03:45 UTC 2020
at 4ddeddd8d58e502dd471213198f421c842e7a5c7 (commit)
commit 4ddeddd8d58e502dd471213198f421c842e7a5c7
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Aug 26 15:44:59 2020 -0400
16602: Workbench sets container_image from acrContainerImage
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index 8c4e5e7d9..ba2f66abe 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -117,6 +117,9 @@ class WorkUnitsController < ApplicationController
if hint[:keep_cache]
keep_cache = hint[:keep_cache]
end
+ if hint[:acrContainerImage]
+ attrs['container_image'] = hint[:acrContainerImage]
+ end
end
end
end
commit 18e2ef8679a7c1a3e1d9121d308e9b94fa7573e3
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Aug 26 15:40:33 2020 -0400
16602: Embed the desired workflow runner version in WorkflowRunnerResources
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index dce1bd4d0..8a3fa3173 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -240,6 +240,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index b9b9e6165..95ed0a75b 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -184,6 +184,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index b9b9e6165..95ed0a75b 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -184,6 +184,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 97c5fafe7..56c6f39e9 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -24,9 +24,10 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder)
+ make_builder, arvados_jobs_image)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
from .perf import Perf
@@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir
sum_res_pars = ("outdirMin", "outdirMax")
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
- submit_runner_ram=0, name=None, merged_map=None):
+ submit_runner_ram=0, name=None, merged_map=None,
+ submit_runner_image=None):
packed = packed_workflow(arvRunner, tool, merged_map)
@@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
+ wf_runner_resources = None
+
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ wf_runner_resources = h
+ found = True
+ break
+ if not found:
+ wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+ hints.append(wf_runner_resources)
+
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
if submit_runner_ram:
- hints = main.get("hints", [])
- found = False
- for h in hints:
- if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
- h["ramMin"] = submit_runner_ram
- found = True
- break
- if not found:
- hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
- "ramMin": submit_runner_ram})
- main["hints"] = hints
+ wf_runner_resources["ramMin"] = submit_runner_ram
+
+ main["hints"] = hints
body = {
"workflow": {
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index e8d1347dd..9ba798ec6 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -596,7 +596,8 @@ The 'jobs' API is no longer supported.
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
- merged_map=merged_map),
+ merged_map=merged_map,
+ submit_runner_image=runtimeContext.submit_runner_image),
"success")
self.apply_reqs(job_order, tool)
diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
index 9bf1c20aa..1054d8f29 100644
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
@@ -6,6 +6,12 @@
"$graph": [
{
"class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
"id": "#main",
"inputs": [],
"outputs": [],
@@ -82,4 +88,4 @@
}
],
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0698db70f..4da545bf3 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1354,7 +1354,7 @@ class TestSubmit(unittest.TestCase):
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
- open("tests/wf/expect_packed.cwl").read())
+ open("tests/wf/expect_upload_packed.cwl").read().rstrip())
@stubs
def test_create(self, stubs):
@@ -1472,7 +1472,7 @@ class TestCreateWorkflow(unittest.TestCase):
stubs.capture_stdout, sys.stderr, api_client=stubs.api)
toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
- expect_workflow = StripYAMLComments(open(toolfile).read())
+ expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
body = {
"workflow": {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list