[ARVADOS] created: 1.1.3-141-g6906707
Git user
git at public.curoverse.com
Tue Mar 6 11:42:45 EST 2018
at 6906707ffcbf0f063ab2a802b567f1f91a2fe84a (commit)
commit 6906707ffcbf0f063ab2a802b567f1f91a2fe84a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Mar 6 11:36:07 2018 -0500
12526: Add constant for default priority and check range.
Also fix a-c-r help text refs #12410
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index ee0a73d..e2607ce 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -54,6 +54,8 @@ arvados.log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
+DEFAULT_PRIORITY = 500
+
class ArvCwlRunner(object):
"""Execute a CWL tool or workflow, submit work (using either jobs or
containers API), wait for them to complete, and report output.
@@ -422,12 +424,15 @@ class ArvCwlRunner(object):
kwargs["tmpdir"] = "/tmp"
kwargs["docker_tmpdir"] = "/tmp"
elif self.work_api == "jobs":
- if kwargs["priority"] != 500:
+ if kwargs["priority"] != DEFAULT_PRIORITY:
raise Exception("--priority not implemented for jobs API.")
kwargs["outdir"] = "$(task.outdir)"
kwargs["docker_outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
+ if kwargs["priority"] < 1 or kwargs["priority"] > 1000:
+ raise Exception("--priority must be in the range 1..1000.")
+
runnerjob = None
if kwargs.get("submit"):
# Submit a runner job to run the workflow for us.
@@ -668,8 +673,8 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=0)
parser.add_argument("--priority", type=int,
- help="Workflow priority (range 1000 - 1, higher has precedence over lower, containers api only)",
- default=500)
+ help="Workflow priority (range 1..1000, higher has precedence over lower, containers api only)",
+ default=DEFAULT_PRIORITY)
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--trash-intermediate", action="store_true",
@@ -679,7 +684,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=False, dest="trash_intermediate",
help="Do not trash intermediate outputs (default).")
- parser.add_argument("workflow", type=str, nargs="?", default=None, help="The workflow to execute")
+ parser.add_argument("workflow", type=str, default=None, help="The workflow to execute")
parser.add_argument("job_order", nargs=argparse.REMAINDER, help="The input object to the workflow.")
return parser
commit 298fa8a8436596348086b42c8d31eba22609145a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 2 15:05:47 2018 -0500
12526: Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 4dbc659..ee0a73d 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -23,6 +23,7 @@ import cwltool.main
import cwltool.workflow
import cwltool.process
from schema_salad.sourceline import SourceLine
+import schema_salad.validate as validate
import arvados
import arvados.config
@@ -415,12 +416,14 @@ class ArvCwlRunner(object):
if self.work_api == "containers":
if self.ignore_docker_for_reuse:
- raise validate.ValidationException("--ignore-docker-for-reuse not supported with containers API.")
+ raise Exception("--ignore-docker-for-reuse not supported with containers API.")
kwargs["outdir"] = "/var/spool/cwl"
kwargs["docker_outdir"] = "/var/spool/cwl"
kwargs["tmpdir"] = "/tmp"
kwargs["docker_tmpdir"] = "/tmp"
elif self.work_api == "jobs":
+ if kwargs["priority"] != 500:
+ raise Exception("--priority not implemented for jobs API.")
kwargs["outdir"] = "$(task.outdir)"
kwargs["docker_outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 5c52d28..9f83822 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -47,7 +47,7 @@ class ArvadosContainer(object):
"name": self.name,
"output_path": self.outdir,
"cwd": self.outdir,
- "priority": 1,
+ "priority": kwargs.get("priority"),
"state": "Committed",
"properties": {},
}
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index fea21e9..cd555a7 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -55,7 +55,7 @@ class TestContainer(unittest.TestCase):
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=enable_reuse)
+ j.run(enable_reuse=enable_reuse, priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
@@ -68,7 +68,7 @@ class TestContainer(unittest.TestCase):
'ram': 1073741824
},
'use_existing': enable_reuse,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824
@@ -138,7 +138,7 @@ class TestContainer(unittest.TestCase):
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=True)
+ j.run(enable_reuse=True, priority=500)
call_args, call_kwargs = runner.api.container_requests().create.call_args
@@ -155,7 +155,7 @@ class TestContainer(unittest.TestCase):
'API': True
},
'use_existing': False,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 4194304000 },
@@ -247,7 +247,7 @@ class TestContainer(unittest.TestCase):
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
call_args, call_kwargs = runner.api.container_requests().create.call_args
@@ -267,7 +267,7 @@ class TestContainer(unittest.TestCase):
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824 },
@@ -346,7 +346,7 @@ class TestContainer(unittest.TestCase):
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
@@ -359,7 +359,7 @@ class TestContainer(unittest.TestCase):
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824 },
@@ -484,7 +484,7 @@ class TestContainer(unittest.TestCase):
}
for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
@@ -497,7 +497,7 @@ class TestContainer(unittest.TestCase):
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
"/keep/99999999999999999999999999999994+44": {
"kind": "collection",
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c0b74fe..298d6aa 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -194,7 +194,7 @@ def stubs(func):
expect_packed_workflow = yaml.round_trip_load(f)
stubs.expect_container_spec = {
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/var/spool/cwl': {
'writable': True,
@@ -709,7 +709,7 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(exited, 0)
expect_container = {
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/var/spool/cwl': {
'writable': True,
@@ -792,7 +792,7 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(exited, 0)
expect_container = {
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/var/spool/cwl': {
'writable': True,
@@ -980,6 +980,26 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_priority(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ stubs.expect_container_spec["priority"] = 669
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
commit f03f97c8d629667d460a95c746de4cdb7092a740
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 2 09:53:39 2018 -0500
12526: add --priority for --api=containers
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 628b6ae..4dbc659 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -443,7 +443,8 @@ class ArvCwlRunner(object):
on_error=kwargs.get("on_error"),
submit_runner_image=kwargs.get("submit_runner_image"),
intermediate_output_ttl=kwargs.get("intermediate_output_ttl"),
- merged_map=merged_map)
+ merged_map=merged_map,
+ priority=kwargs.get("priority"))
elif self.work_api == "jobs":
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
self.output_name,
@@ -663,6 +664,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).",
default=0)
+ parser.add_argument("--priority", type=int,
+ help="Workflow priority (range 1000 - 1, higher has precedence over lower, containers api only)",
+ default=500)
+
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--trash-intermediate", action="store_true",
default=False, dest="trash_intermediate",
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index a2aaa8d..5c52d28 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -311,7 +311,7 @@ class RunnerContainer(Runner):
"name": self.name,
"output_path": "/var/spool/cwl",
"cwd": "/var/spool/cwl",
- "priority": 1,
+ "priority": self.priority,
"state": "Committed",
"container_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
"mounts": {
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9b79d45..9716ca4 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -316,7 +316,7 @@ class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
- intermediate_output_ttl=0, merged_map=None):
+ intermediate_output_ttl=0, merged_map=None, priority=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -336,6 +336,7 @@ class Runner(object):
self.on_error = on_error
self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
self.intermediate_output_ttl = intermediate_output_ttl
+ self.priority = priority
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list