[ARVADOS] updated: 1.1.4-560-g77cfd1ee2
Git user
git at public.curoverse.com
Mon Jul 9 08:48:34 EDT 2018
Summary of changes:
.../app/controllers/work_units_controller.rb | 29 ++++++++++++-----
apps/workbench/app/helpers/application_helper.rb | 13 +++++---
doc/user/cwl/cwl-extensions.html.textile.liquid | 13 ++++++++
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 22 +++++++++++++
sdk/cwl/arvados_cwl/arvcontainer.py | 2 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 13 +++++++-
sdk/cwl/arvados_cwl/runner.py | 18 +++++++++--
sdk/cwl/tests/test_submit.py | 36 ++++++++++++++++++++++
...ubmit_wf.cwl => submit_wf_runner_resources.cwl} | 6 ++++
10 files changed, 136 insertions(+), 18 deletions(-)
copy sdk/cwl/tests/wf/{submit_wf.cwl => submit_wf_runner_resources.cwl} (81%)
via 77cfd1ee241b4a2d408bd3ec5ebc69f34c7496f5 (commit)
via d3e2fb85a39562e38ea67f84e9f065815450294a (commit)
via b2a50a8539be09b73c5d65719e8eae4b2a15273b (commit)
via bd55c808de1d4d25fea7cc047956c45ec9752fa5 (commit)
via a8d2a183ac7246bf368d40dba1af226a4968a9c9 (commit)
via 69138b061aca4e0cbbb7dad080158205f6734e77 (commit)
via 535856c28a12bb07dc986b980b0f4ccfdfd25640 (commit)
from c584cb69cdcfb377deed94745785330562a54ae3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 77cfd1ee241b4a2d408bd3ec5ebc69f34c7496f5
Merge: c584cb69c d3e2fb85a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Jul 9 08:48:16 2018 -0400
Merge branch '13681-wb-acr-submit-ram' closes #13681
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
commit d3e2fb85a39562e38ea67f84e9f065815450294a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jul 6 15:41:46 2018 -0400
13681: Fix workbench for workflows that don't have "hints"
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index a229654c3..8527b4d48 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -110,13 +110,15 @@ class WorkUnitsController < ApplicationController
input_defaults[cwl_shortname(input[:id])] = input[:default]
end
end
- main[:hints].each do |hint|
- if hint[:class] == "http://arvados.org/cwl#WorkflowRunnerResources"
- if hint[:coresMin]
- runtime_constraints["vcpus"] = hint[:coresMin]
- end
- if hint[:ramMin]
- runtime_constraints["ram"] = hint[:ramMin] * 1024 * 1024
+ if main[:hints]
+ main[:hints].each do |hint|
+ if hint[:class] == "http://arvados.org/cwl#WorkflowRunnerResources"
+ if hint[:coresMin]
+ runtime_constraints["vcpus"] = hint[:coresMin]
+ end
+ if hint[:ramMin]
+ runtime_constraints["ram"] = hint[:ramMin] * 1024 * 1024
+ end
end
end
end
commit b2a50a8539be09b73c5d65719e8eae4b2a15273b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jul 6 13:48:34 2018 -0400
13681: Revert to yaml.round_trip_dump instead of json.dumps().
Also style tweaks.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f5afd387e..0706ef7db 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -52,8 +52,8 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
- hints = main["hints"] if "hints" in main else []
if submit_runner_ram:
+ hints = main.get("hints", [])
found = False
for h in hints:
if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
@@ -63,13 +63,13 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
if not found:
hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
"ramMin": submit_runner_ram})
- main["hints"] = hints
+ main["hints"] = hints
body = {
"workflow": {
"name": name,
"description": tool.tool.get("doc", ""),
- "definition": json.dumps(packed, indent=2, sort_keys=True)
+ "definition":yaml.round_trip_dump(packed)
}}
if project_uuid:
body["workflow"]["owner_uuid"] = project_uuid
commit bd55c808de1d4d25fea7cc047956c45ec9752fa5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Jul 5 16:28:13 2018 -0400
13681: Add new test file
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
new file mode 100644
index 000000000..9e2712194
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_wf_runner_resources.cwl
@@ -0,0 +1,31 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a workflow file for dependencies
+# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+hints:
+ arv:WorkflowRunnerResources:
+ ramMin: 2000
+ coresMin: 2
+inputs:
+ - id: x
+ type: File
+ - id: y
+ type: Directory
+ - id: z
+ type: Directory
+outputs: []
+steps:
+ - id: step1
+ in:
+ - { id: x, source: "#x" }
+ out: []
+ run: ../tool/submit_tool.cwl
commit a8d2a183ac7246bf368d40dba1af226a4968a9c9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Jul 5 15:15:16 2018 -0400
13681: Update documentation for arv:WorkflowRunnerResources
Also bring default allocation for cwl-runner launched by workbench in
line with a-c-r command line (1 GiB / 1 core).
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index 0fe68476e..a229654c3 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -98,7 +98,7 @@ class WorkUnitsController < ApplicationController
# runtime constriants
runtime_constraints = {
"vcpus" => 1,
- "ram" => 256000000,
+ "ram" => 1024 * 1024 * 1024,
"API" => true
}
diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index 26feb9938..f9ecf7a53 100644
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -40,6 +40,9 @@ hints:
secrets: [input1, input2]
cwltool:TimeLimit:
timelimit: 14400
+ arv:WorkflowRunnerResources:
+ ramMin: 2048
+ coresMin: 2
</pre>
The one exception to this is @arv:APIRequirement@, see note below.
@@ -114,6 +117,7 @@ table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|secrets|array<string>|Input parameters which are considered "secret". Must be strings.|
+
h2. cwltool:TimeLimit
Set an upper limit on the execution time of a CommandLineTool or ExpressionTool. A tool execution which exceeds the time limit may be preemptively terminated and considered failed. May also be used by batch systems to make scheduling decisions.
@@ -121,3 +125,12 @@ Set an upper limit on the execution time of a CommandLineTool or ExpressionTool.
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|timelimit|int|Execution time limit in seconds. If set to zero, no limit is enforced.|
+
+h2. arv:WorkflowRunnerResources
+
+Specify resource requirements for the workflow runner process (arvados-cwl-runner) that manages a workflow run. Must be applied to the top level workflow. Will also be set implicitly when using @--submit-runner-ram@ on the command line along with @--create-workflow@ or @--update-workflow at . Use this to adjust the runner's allocation if the workflow runner is getting "out of memory" exceptions or being killed by the out-of-memory (OOM) killer.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|ramMin|int|RAM, in mebibytes, to reserve for the arvados-cwl-runner process. Default 1 GiB|
+|coresMin|int|Number of cores to reserve to the arvados-cwl-runner process. Default 1 core.|
commit 69138b061aca4e0cbbb7dad080158205f6734e77
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Jul 5 15:03:41 2018 -0400
13681: Can use --submit-runner-ram with --create/update-workflow
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index b802e9e16..f5afd387e 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -52,13 +52,24 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
- # TODO nowhere for submit_runner_ram to go.
+ hints = main["hints"] if "hints" in main else []
+ if submit_runner_ram:
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ h["ramMin"] = submit_runner_ram
+ found = True
+ break
+ if not found:
+ hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "ramMin": submit_runner_ram})
+ main["hints"] = hints
body = {
"workflow": {
"name": name,
"description": tool.tool.get("doc", ""),
- "definition":yaml.round_trip_dump(packed)
+ "definition": json.dumps(packed, indent=2, sort_keys=True)
}}
if project_uuid:
body["workflow"]["owner_uuid"] = project_uuid
commit 535856c28a12bb07dc986b980b0f4ccfdfd25640
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Jul 5 14:39:13 2018 -0400
13681: Implement WorkflowRunnerResources hint
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index 0b0cdb4c3..0fe68476e 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -95,14 +95,31 @@ class WorkUnitsController < ApplicationController
attrs['cwd'] = "/var/spool/cwl"
attrs['output_path'] = "/var/spool/cwl"
+ # runtime constriants
+ runtime_constraints = {
+ "vcpus" => 1,
+ "ram" => 256000000,
+ "API" => true
+ }
+
input_defaults = {}
if wf_json
- inputs = get_cwl_inputs(wf_json)
- inputs.each do |input|
+ main = get_cwl_main(wf_json)
+ main[:inputs].each do |input|
if input[:default]
input_defaults[cwl_shortname(input[:id])] = input[:default]
end
end
+ main[:hints].each do |hint|
+ if hint[:class] == "http://arvados.org/cwl#WorkflowRunnerResources"
+ if hint[:coresMin]
+ runtime_constraints["vcpus"] = hint[:coresMin]
+ end
+ if hint[:ramMin]
+ runtime_constraints["ram"] = hint[:ramMin] * 1024 * 1024
+ end
+ end
+ end
end
# mounts
@@ -128,12 +145,6 @@ class WorkUnitsController < ApplicationController
end
attrs['mounts'] = mounts
- # runtime constriants
- runtime_constraints = {
- "vcpus" => 1,
- "ram" => 256000000,
- "API" => true
- }
attrs['runtime_constraints'] = runtime_constraints
else
raise ArgumentError, "Unsupported template uuid: #{template_uuid}"
diff --git a/apps/workbench/app/helpers/application_helper.rb b/apps/workbench/app/helpers/application_helper.rb
index 57b8d8780..106716a0f 100644
--- a/apps/workbench/app/helpers/application_helper.rb
+++ b/apps/workbench/app/helpers/application_helper.rb
@@ -426,18 +426,23 @@ module ApplicationHelper
lt
end
- def get_cwl_inputs(workflow)
- if workflow[:inputs]
- return workflow[:inputs]
+ def get_cwl_main(workflow)
+ if workflow[:"$graph"].nil?
+ return workflow
else
workflow[:"$graph"].each do |tool|
if tool[:id] == "#main"
- return tool[:inputs]
+ return tool
end
end
end
end
+ def get_cwl_inputs(workflow)
+ get_cwl_main(workflow)[:inputs]
+ end
+
+
def cwl_shortname(id)
if id[0] == "#"
id = id[1..-1]
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 062c7c2fa..131795ee2 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -714,7 +714,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
parser.add_argument("--submit-runner-ram", type=int,
help="RAM (in MiB) required for the workflow runner job (default 1024)",
- default=1024)
+ default=None)
parser.add_argument("--submit-runner-image", type=str,
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 4eaa8536c..4f762192a 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -211,3 +211,25 @@ $graph:
_type: "@vocab"
- name: enableReuse
type: boolean
+
+- name: WorkflowRunnerResources
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify memory or cores resource request for the CWL runner process itself.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:WorkflowRunnerResources'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ ramMin:
+ type: int?
+ doc: Minimum RAM, in mebibytes (2**20)
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/ramMin"
+ coresMin:
+ type: int?
+ doc: Minimum cores allocated to cwl-runner
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
\ No newline at end of file
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4ebcefb13..c0c6f7722 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -385,7 +385,7 @@ class RunnerContainer(Runner):
},
"secret_mounts": secret_mounts,
"runtime_constraints": {
- "vcpus": 1,
+ "vcpus": self.submit_runner_cores,
"ram": 1024*1024 * self.submit_runner_ram,
"API": True
},
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 12a847503..9c422a42f 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -377,13 +377,25 @@ class Runner(object):
self.priority = priority
self.secret_store = secret_store
+ self.submit_runner_cores = 1
+ self.submit_runner_ram = 1024 # defaut 1 GiB
+
+ runner_resource_req, _ = self.tool.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
+ if runner_resource_req:
+ if runner_resource_req.get("coresMin"):
+ self.submit_runner_cores = runner_resource_req["coresMin"]
+ if runner_resource_req.get("ramMin"):
+ self.submit_runner_ram = runner_resource_req["ramMin"]
+
if submit_runner_ram:
+ # Command line / initializer overrides default and/or spec from workflow
self.submit_runner_ram = submit_runner_ram
- else:
- self.submit_runner_ram = 3000
if self.submit_runner_ram <= 0:
- raise Exception("Value of --submit-runner-ram must be greater than zero")
+ raise Exception("Value of submit-runner-ram must be greater than zero")
+
+ if self.submit_runner_cores <= 0:
+ raise Exception("Value of submit-runner-cores must be greater than zero")
self.merged_map = merged_map or {}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8b7e4af81..cd4625130 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1131,6 +1131,42 @@ class TestSubmit(unittest.TestCase):
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_wf_runner_resources(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["runtime_constraints"] = {
+ "API": True,
+ "vcpus": 2,
+ "ram": 2000 * 2**20
+ }
+ expect_container["name"] = "submit_wf_runner_resources.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ {
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "coresMin": 2,
+ "ramMin": 2000
+ }
+ ]
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ "arv": "http://arvados.org/cwl#",
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list