[ARVADOS] created: e1b133b13f9ac50a87051d07c36a3904d6f01028
Git user
git at public.curoverse.com
Thu Dec 1 12:34:25 EST 2016
at e1b133b13f9ac50a87051d07c36a3904d6f01028 (commit)
commit e1b133b13f9ac50a87051d07c36a3904d6f01028
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Dec 1 12:34:20 2016 -0500
10651: Add --submit-runner-ram to specify amount of RAM that should be
allocated for workflow runner job or container.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 9cabedf..817efae 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -308,14 +308,16 @@ class ArvCwlRunner(object):
if self.work_api == "jobs":
tmpl = RunnerTemplate(self, tool, job_order,
kwargs.get("enable_reuse"),
- uuid=existing_uuid)
+ uuid=existing_uuid,
+ submit_runner_ram=kwargs.get("submit_runner_ram"))
tmpl.save()
# cwltool.main will write our return value to stdout.
return tmpl.uuid
else:
return upload_workflow(self, tool, job_order,
self.project_uuid,
- uuid=existing_uuid)
+ uuid=existing_uuid,
+ submit_runner_ram=kwargs.get("submit_runner_ram"))
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
@@ -346,9 +348,11 @@ class ArvCwlRunner(object):
self.output_callback,
**kwargs).next()
else:
- runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name, self.output_tags)
+ runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
+ self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"))
else:
- runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name, self.output_tags)
+ runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
+ self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"))
if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
# Create pipeline for local run
@@ -524,6 +528,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="Compute checksum of contents while collecting outputs",
dest="compute_checksum")
+ parser.add_argument("--submit-runner-ram", type=int,
+ help="RAM (in MiB) required for the workflow runner job.",
+ default=0)
+
parser.add_argument("workflow", type=str, nargs="?", default=None, help="The workflow to execute")
parser.add_argument("job_order", nargs=argparse.REMAINDER, help="The input object to the workflow.")
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1fda412..f83bd19 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -237,7 +237,7 @@ class RunnerContainer(Runner):
},
"runtime_constraints": {
"vcpus": 1,
- "ram": 1024*1024*256,
+ "ram": 1024*1024 * self.submit_runner_ram,
"API": True
}
}
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 1afb9af..93f84f8 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -257,7 +257,8 @@ class RunnerJob(Runner):
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": arvados_jobs_image(self.arvrunner)
+ "docker_image": arvados_jobs_image(self.arvrunner),
+ "min_ram_mb_per_node": self.submit_runner_ram
}
}
@@ -308,7 +309,7 @@ class RunnerTemplate(object):
'string': 'text',
}
- def __init__(self, runner, tool, job_order, enable_reuse, uuid):
+ def __init__(self, runner, tool, job_order, enable_reuse, uuid, submit_runner_ram=0):
self.runner = runner
self.tool = tool
self.job = RunnerJob(
@@ -317,7 +318,8 @@ class RunnerTemplate(object):
job_order=job_order,
enable_reuse=enable_reuse,
output_name=None,
- output_tags=None)
+ output_tags=None,
+ submit_runner_ram=submit_runner_ram)
self.uuid = uuid
def pipeline_component_spec(self):
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f425ae8..a9c3bd5 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -18,7 +18,7 @@ from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
-def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None):
+def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, submit_runner_ram=0):
upload_docker(arvRunner, tool)
document_loader, workflowobj, uri = (tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]), tool.tool["id"])
@@ -37,6 +37,8 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None):
upload_dependencies(arvRunner, name, document_loader,
packed, uri, False)
+ # TODO nowhere for submit_runner_ram to go.
+
body = {
"workflow": {
"name": tool.tool.get("label", name),
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3bbcb8b..373424c 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -161,7 +161,8 @@ def arvados_jobs_image(arvrunner):
return img
class Runner(object):
- def __init__(self, runner, tool, job_order, enable_reuse, output_name, output_tags):
+ def __init__(self, runner, tool, job_order, enable_reuse,
+ output_name, output_tags, submit_runner_ram=0):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -171,6 +172,10 @@ class Runner(object):
self.final_output = None
self.output_name = output_name
self.output_tags = output_tags
+ if submit_runner_ram:
+ self.submit_runner_ram = submit_runner_ram
+ else:
+ self.submit_runner_ram = 1024
def update_pipeline_component(self, record):
pass
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c4b0cea..e49a09f 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -98,7 +98,8 @@ def stubs(func):
}
stubs.expect_job_spec = {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'min_ram_mb_per_node': 1024
},
'script_parameters': {
'x': {
@@ -134,7 +135,7 @@ def stubs(func):
'owner_uuid': None,
"components": {
"cwl-runner": {
- 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
+ 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
'script_parameters': {
'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
@@ -193,7 +194,7 @@ def stubs(func):
'runtime_constraints': {
'API': True,
'vcpus': 1,
- 'ram': 268435456
+ 'ram': 1024*1024*1024
}
}
@@ -613,6 +614,7 @@ class TestTemplateInputs(unittest.TestCase):
"inputs_test.cwl": {
'runtime_constraints': {
'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'min_ram_mb_per_node': 1024
},
'script_parameters': {
'cwl:tool':
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list