[ARVADOS] updated: 1.1.4-434-gc4b2d4323
Git user
git at public.curoverse.com
Mon Jun 18 05:04:04 EDT 2018
Summary of changes:
apps/workbench/test/integration_helper.rb | 2 +
build/build.list | 1 -
build/libcloud-pin.sh | 4 +-
build/run-build-packages.sh | 26 +-
build/run-tests.sh | 17 +-
doc/api/methods.html.textile.liquid | 2 +-
doc/api/methods/collections.html.textile.liquid | 1 +
.../methods/container_requests.html.textile.liquid | 2 +-
doc/api/methods/groups.html.textile.liquid | 1 +
doc/api/methods/links.html.textile.liquid | 2 +-
doc/api/methods/nodes.html.textile.liquid | 3 +-
lib/dispatchcloud/node_size.go | 20 +-
lib/dispatchcloud/node_size_test.go | 30 ++-
sdk/cwl/arvados_cwl/__init__.py | 33 +--
sdk/cwl/arvados_cwl/arvcontainer.py | 35 ++-
sdk/cwl/arvados_cwl/arvworkflow.py | 1 +
sdk/cwl/arvados_cwl/http.py | 151 +++++++++++
sdk/cwl/arvados_cwl/pathmapper.py | 27 +-
sdk/cwl/arvados_cwl/runner.py | 14 +-
sdk/cwl/arvados_version.py | 2 +-
sdk/cwl/setup.py | 8 +-
sdk/cwl/tests/test_container.py | 16 +-
sdk/cwl/tests/test_http.py | 286 +++++++++++++++++++++
sdk/cwl/tests/test_job.py | 6 +-
sdk/cwl/tests/test_pathmapper.py | 19 +-
sdk/cwl/tests/test_submit.py | 103 +++++++-
sdk/go/arvados/config.go | 1 +
sdk/go/arvados/container.go | 3 +-
sdk/go/keepclient/keepclient.go | 1 +
sdk/go/keepclient/keepclient_test.go | 23 +-
sdk/go/keepclient/support.go | 3 +
sdk/python/arvados/collection.py | 90 ++++++-
sdk/python/arvados/commands/run.py | 11 +-
sdk/python/setup.py | 2 +-
sdk/python/tests/test_collections.py | 38 ++-
.../arvados/v1/containers_controller.rb | 6 +-
services/api/app/models/container.rb | 8 +-
services/api/app/models/container_request.rb | 24 +-
services/api/app/models/group.rb | 3 +
services/api/config/application.default.yml | 5 +
.../migrate/20180607175050_properties_to_jsonb.rb | 32 +++
.../20180608123145_add_properties_to_groups.rb | 18 ++
services/api/db/structure.sql | 53 +++-
services/api/lib/update_priority.rb | 38 +++
services/api/test/unit/arvados_model_test.rb | 2 +-
services/api/test/unit/container_request_test.rb | 92 +++++++
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 18 +-
.../crunch-dispatch-slurm_test.go | 6 +-
services/fuse/setup.py | 2 +-
services/keep-balance/balance.go | 162 +++++++++---
services/keep-balance/balance_test.go | 206 ++++++++++++++-
services/keepproxy/keepproxy.go | 9 +
services/keepproxy/keepproxy_test.go | 46 +++-
services/keepstore/s3_volume.go | 4 +-
.../nodemanager/arvnodeman/computenode/__init__.py | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 7 +-
.../arvnodeman/computenode/driver/__init__.py | 2 +-
.../arvnodeman/computenode/driver/azure.py | 5 +-
.../arvnodeman/computenode/driver/ec2.py | 9 +
.../arvnodeman/computenode/driver/gce.py | 22 +-
services/nodemanager/arvnodeman/config.py | 46 +++-
services/nodemanager/arvnodeman/daemon.py | 6 +-
services/nodemanager/arvnodeman/jobqueue.py | 34 ++-
services/nodemanager/arvnodeman/launcher.py | 4 +-
services/nodemanager/arvnodeman/nodelist.py | 4 +-
.../nodemanager/arvnodeman/test/fake_driver.py | 19 +-
services/nodemanager/setup.py | 7 +-
services/nodemanager/tests/fake_azure.cfg.template | 8 +-
services/nodemanager/tests/fake_ec2.cfg.template | 8 +-
services/nodemanager/tests/fake_gce.cfg.template | 8 +-
services/nodemanager/tests/integration_test.py | 38 +--
services/nodemanager/tests/test_computenode.py | 6 +
.../nodemanager/tests/test_computenode_dispatch.py | 9 +
.../nodemanager/tests/test_computenode_driver.py | 2 +-
.../tests/test_computenode_driver_azure.py | 23 +-
.../tests/test_computenode_driver_ec2.py | 27 +-
.../tests/test_computenode_driver_gce.py | 11 +
services/nodemanager/tests/test_config.py | 24 +-
services/nodemanager/tests/test_daemon.py | 44 ++--
services/nodemanager/tests/test_failure.py | 5 +-
services/nodemanager/tests/test_nodelist.py | 1 +
services/nodemanager/tests/testutil.py | 4 +-
82 files changed, 1782 insertions(+), 321 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/http.py
create mode 100644 sdk/cwl/tests/test_http.py
create mode 100644 services/api/db/migrate/20180607175050_properties_to_jsonb.rb
create mode 100644 services/api/db/migrate/20180608123145_add_properties_to_groups.rb
create mode 100644 services/api/lib/update_priority.rb
via c4b2d43234205a0d9b1310e55b1f00dee9c20625 (commit)
via 3ac93fa684e01ee2e8ababa6d9cc617cdbd3cbea (commit)
via 94f2b439783a8e63d6d7b9ba2760f54fc642a8fb (commit)
via 0557b1dbb08d5328e29eac805824a8e21040f582 (commit)
via 4f1e8828caf40af5a80ca8189c46a6cdfe8d541f (commit)
via 0275a62826a50d84f8f633f475c930fde96af4a4 (commit)
via 9f768708156197decac1e6dc5dedf3a730a06fad (commit)
via e135f4e0f24442e009a60f2018d95c5e8cb63e5e (commit)
via 089b68192e6fd089c03331c389da1dace039c899 (commit)
via 2c68e941a15384cecb0e3b3df6def6c3e84d0a71 (commit)
via 115a5e8861ef0a46224b2cd64568b30c884908fb (commit)
via 42a0609a6e287a82ed565413c7392d40141388ae (commit)
via 71db701269dc5d2b5eb9239828a74e9c26cd7e66 (commit)
via 6714a5e7fcf4d5fde2ecd5a7f9f6504cb5ca374b (commit)
via 985ba2d04074d830fc2e8688f2d0f59a75a1c14c (commit)
via 28225aeb0336a4872bbaa3aae5d331172f1e4068 (commit)
via b70f9ce54f1f672b423999e6c07b2f0127b76666 (commit)
via 8fa57330875af11b7b61dfac6fc45080760afdf2 (commit)
via 17f521d7ffb4f3a58ca98a27395eb60d9fa34519 (commit)
via 5f42faf0818c229196b26535a0669eeb2bf27386 (commit)
via 06eb98871e82074b935d9da201118119e5719f96 (commit)
via 99887d1312315a5efb4d888251f90880846c073c (commit)
via 4ddcd079b0378d0a8556f528106ac7f60fe5cb1b (commit)
via 1e03ea4765bee6db9331a1265c8abf2c3e45caff (commit)
via a4991cffc83afea213125a24f01399d6f5707948 (commit)
via 8e31910034627dedd7259dd1e45a60768108c1e1 (commit)
via d2efca1759c9e104f67d1caf85c8e62ff06807bd (commit)
via 035b113f60302f6d9c265e6e3a63dbb3c5873153 (commit)
via 85c13201abf4a9f19dd112992a04fa65da63369c (commit)
via cefddd8efdc88c3d33e5aad4c02e96c64cbe8242 (commit)
via 0c458347e92967b5a1e6cce8267939d3ef204663 (commit)
via 54ee2b06add844efae0b0fcb97606102232974e6 (commit)
via f9347e2f053b645003d102f96a0aee49167baeef (commit)
via 5f75e2db42cea65e6f4e6b8c2b1b1dfb8e748c05 (commit)
via 29fe8719ae4f4463cf5c63ad20cf2eb1ae39cd19 (commit)
via 333a30289f47c22956bcb6a045d1cd4c797e22c3 (commit)
via 31fd95388f035bba3e70d3e8861a14511f6281e6 (commit)
via 7d7eb76d319f1ab6faeb8634f13142e7147a45ea (commit)
via 8f4b1051209f78bff358069a2a3b2f646f7a38f8 (commit)
via a518eb2b3b743259fe51b1e61d2648677e583cac (commit)
via e3483661acc3d26d4532a536bd40ed761c21d7d1 (commit)
via 1dc6c511620a852f81d5179fd2f9fbecee472e71 (commit)
via 57fcaf45fe40806dca5f001aabed9b413243183e (commit)
via afbc818e5bbc667b69e9ae9b8f080cb50ca17c53 (commit)
via ad5f111b67e2bf1c43ce9bc37e6b11b8c753f62d (commit)
via 26d20f13895fdf5c9224a7aca2b33625c511e73e (commit)
via da40bd0960806df8e2799e4fb716d41ad08b169f (commit)
via 1714431955ac7b2a1d2fd3c2cda9b806bad624cb (commit)
via da2cfffb3a3ec92c3b15841255dc704a99748fea (commit)
via df8a51cd6b8a95475dfaef08c8a56b1e1643dbfc (commit)
via 1b183730bd6ec56d51b17020e63fb2cffa2ad764 (commit)
via c9143544609d90da33eb3c2d566fc5d6a25188b2 (commit)
via d5dd5dc2838fe420c8f835975bad7052a0df55e5 (commit)
via 0809440ad583a2556c0f97fd000be9e9b7f94eb5 (commit)
via 7654764e5192fc8d34e612a6ee6608fad33a91c2 (commit)
via 0eedf70afa34167275d2135837e866b13fac4178 (commit)
via cae5f7def9dae9caf9056bc3bb77ca801cfa2229 (commit)
via e80efb5caae2acbe50ea8a1bd1151b0d2a67c1c9 (commit)
via 7b7950766b7a77cd6c2d6ecee7603dd7394c4144 (commit)
via 24af6ae1b23a4fd82c816d077c0178f0d75db2d7 (commit)
via 4869ae0f12b5a4a293c66f7304483fee1addae52 (commit)
via 7e6ac5a4967614cbe59ed5c0ec41c8be4d4cff4d (commit)
via 1965e1793598c76ceef4decd4a5a6c7755c385e3 (commit)
via 4f0c3d501d19bed5915d5d188598d3a7f1dec7f8 (commit)
via dd97299b6b3ce82b6a57e2838fbe3f81bb51f140 (commit)
via ed7698518d7eabea0f4e86acfb0660765fe31071 (commit)
via 783523246354d49b5cba8c36bb2913831e5f8e52 (commit)
via 73872ccc5bb6b80a6049b44b0113085a9c2b6934 (commit)
via 008769334163409bb933b5c94f057bb717ac6bc0 (commit)
via ec772e86990ddcfc5f26e72a0753abdfca1c0aeb (commit)
via 32568bbde8ad549452ccb57f10bb54672fda6fa6 (commit)
via 6c5fc89c06338ea829f2e4f2a7e0529548890efd (commit)
via 3108e6a627f4413819af8d2eb006016f1695b9a8 (commit)
via e1ee43ee90b7f6e6189930cb031448014d0cf638 (commit)
via 76f6b358cf612e67b586452236394feabc464373 (commit)
via d9462226315eb4a479b6bb998aa51531062635b0 (commit)
via 6814a151d680775d9804d56582a94af90cee2ab8 (commit)
via 49bb6cb300e9f6a3862c4e17112e34da1226a3c7 (commit)
via f57c43c811ee8af08e7662ad567a6d8e8c18b781 (commit)
via 5836e576fe0b78c50383cf56e1c4fb4521daeca1 (commit)
via 1bff2ab0181be31492c53351afc1c3c1e58ea05d (commit)
via 08f0ba557d67488933ddfa90030f80459efe7997 (commit)
via 2f9761bd08293de1847d3dbd0ab16482eacc414d (commit)
via 7e9b9d420b58c22c41a713b9bfbfd0f5718abb1a (commit)
via 00cca6a192eb1ab38559bf5ed9044711ed56fc4a (commit)
via 9fcb58db8d47a42e52eead600ddabf57ab7c29ed (commit)
via 18b5ad828a4149c1b768d05e477982364002a647 (commit)
via b3e79afc9f6527927c6bf6e4ee35b11e6f20185e (commit)
via bb17ad738ed9c890881361232e0dfd84e50a529d (commit)
via d8dfc75ec5c6cead3da0f3907466ce1b89373b69 (commit)
via 67c988ea479ba47da2d8dc93695188f86833436d (commit)
via 606c29cdbb9012ee99043a2da9f28b2cd302e5e4 (commit)
via c9abf73e0b57739a594aa1b4803726e612cf6313 (commit)
via 3c2266151711f532491a2369de37086bd7234dce (commit)
via 739b1b9ec3662f988ad09509bcc933ce5c23c4e8 (commit)
via 05f0008349efe56755c580e1e060115587aab352 (commit)
via 8afc85aabb9563da4de17b0b5f7d4fe574e9ad8d (commit)
via 57e511e900aa1eb175fa7f308b09516ad83492b4 (commit)
via 3950ffc9481c25262f2db2b08a0f74664c433734 (commit)
via e4ea2e1dd1cb597b02d15bd7b9323705d6342f99 (commit)
via 1275e305f6d13785687c72b8609592094a067cef (commit)
via ddac86f6bae74398f58db75e383f5945c323a099 (commit)
via b3f04be24d0d2c75808dccd9e5923ee25371cb17 (commit)
via 6e17218f2fc39a828af4d3a19bdb29243739dafd (commit)
via 708a9f8e858f0791702083e3f429fcb3eea5af15 (commit)
via a7af6b95953992a7e9554b56604cf8248d5a2bf4 (commit)
via 9a52f3d07288f8a0073736a2a273ddaeb97ddc18 (commit)
via 6853ac7e518c5c6a888c5f5bd4edb27bdb564a2c (commit)
via 2600c9f3e0cbe1072c1bc4887aa5febe8906ddec (commit)
via 7cc2336a974945d03649d924cf805fedf92bad41 (commit)
via f7029eca4ddc301167078827a6bed2219dd282a6 (commit)
via 9df819a55f00e893f2783c1e048a6d3c1cd391b2 (commit)
from b01b480adb45c3bbfcfab13e343e08c16854dedc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c4b2d43234205a0d9b1310e55b1f00dee9c20625
Merge: 3ac93fa68 94f2b4397
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Mon Jun 18 11:03:33 2018 +0200
Merge branch 'master' of git.curoverse.com:arvados into 13429-cwl-runner-storage-classes-support
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
commit 3ac93fa684e01ee2e8ababa6d9cc617cdbd3cbea
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Mon Jun 18 10:58:33 2018 +0200
Add storage_classes tests for arvados_cwl_runner
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c7ededd79..02b42fefb 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -637,6 +637,48 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
+ @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @stubs
+ def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+ def set_final_output(job_order, output_callback, **kwargs):
+ output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
+ return []
+ job.side_effect = set_final_output
+
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--local", "--storage-classes=foo",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+
+ @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
+ @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @stubs
+ def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+ def set_final_output(job_order, output_callback, **kwargs):
+ output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
+ return []
+ job.side_effect = set_final_output
+
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--local",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+
@stubs
def test_submit_container_output_ttl(self, stubs):
capture_stdout = cStringIO.StringIO()
commit 28225aeb0336a4872bbaa3aae5d331172f1e4068
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Tue Jun 12 15:10:23 2018 +0200
Update storage_classes support for arvados_cwl_runner to work correctly
when arvados_cwl_runner is run in submit mode.
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 4a6ccc0b2..b2b93bf9e 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -69,8 +69,8 @@ class ArvCwlRunner(object):
"""
def __init__(self, api_client, work_api=None, keep_client=None,
- output_name=None, output_tags=None, num_retries=4,
- thread_count=4):
+ output_name=None, output_tags=None, default_storage_classes="default",
+ num_retries=4, thread_count=4):
self.api = api_client
self.processes = {}
self.workflow_eval_lock = threading.Condition(threading.RLock())
@@ -90,6 +90,7 @@ class ArvCwlRunner(object):
self.trash_intermediate = False
self.thread_count = thread_count
self.poll_interval = 12
+ self.default_storage_classes = default_storage_classes
if keep_client is not None:
self.keep_client = keep_client
@@ -485,6 +486,7 @@ class ArvCwlRunner(object):
submit_runner_image=kwargs.get("submit_runner_image"),
intermediate_output_ttl=kwargs.get("intermediate_output_ttl"),
merged_map=merged_map,
+ default_storage_classes=self.default_storage_classes,
priority=kwargs.get("priority"),
secret_store=self.secret_store)
elif self.work_api == "jobs":
@@ -597,7 +599,7 @@ class ArvCwlRunner(object):
if self.output_tags is None:
self.output_tags = ""
- storage_classes = kwargs.get("storage_classes")
+ storage_classes = kwargs.get("storage_classes").strip().split(",")
self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
self.set_crunch_output()
@@ -720,7 +722,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
parser.add_argument("--enable-dev", action="store_true",
help="Enable loading and running development versions "
"of CWL spec.", default=False)
- parser.add_argument('--storage-classes', default="default",
+ parser.add_argument('--storage-classes', default="default", type=str,
help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
@@ -783,8 +785,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
job_order_object = None
arvargs = parser.parse_args(args)
- arvargs.storage_classes = arvargs.storage_classes.strip().split(',')
- if len(arvargs.storage_classes) > 1:
+ if len(arvargs.storage_classes.strip().split(',')) > 1:
logger.error("Multiple storage classes are not supported currently.")
return 1
@@ -817,7 +818,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client,
num_retries=4, output_name=arvargs.output_name,
- output_tags=arvargs.output_tags,
+ output_tags=arvargs.output_tags, default_storage_classes=parser.get_default("storage_classes"),
thread_count=arvargs.thread_count)
except Exception as e:
logger.error(e)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 0bec69264..05c0212fc 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -427,6 +427,9 @@ class RunnerContainer(Runner):
if kwargs.get("debug"):
command.append("--debug")
+ if kwargs.get("storage_classes") and kwargs.get("storage_classes") != self.default_storage_classes:
+ command.append("--storage-classes=" + kwargs.get("storage_classes"))
+
if self.on_error:
command.append("--on-error=" + self.on_error)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index cf91f69f8..f907d3395 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -353,8 +353,8 @@ class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
- intermediate_output_ttl=0, merged_map=None, priority=None,
- secret_store=None):
+ intermediate_output_ttl=0, merged_map=None, default_storage_classes="default",
+ priority=None, secret_store=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -376,6 +376,7 @@ class Runner(object):
self.intermediate_output_ttl = intermediate_output_ttl
self.priority = priority
self.secret_store = secret_store
+ self.default_storage_classes = default_storage_classes
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c1e9629fc..c7ededd79 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -612,6 +612,30 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_storage_classes(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=4',
+ '--enable-reuse', "--debug",
+ "--storage-classes=foo", '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
@stubs
def test_submit_container_output_ttl(self, stubs):
commit 035b113f60302f6d9c265e6e3a63dbb3c5873153
Merge: 29fe8719a 85c13201a
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Mon Jun 11 15:50:27 2018 +0200
Merge branch 'master' of git.curoverse.com:arvados into 13429-cwl-runner-storage-classes-support
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
commit 29fe8719ae4f4463cf5c63ad20cf2eb1ae39cd19
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Fri Jun 8 11:05:22 2018 +0200
Change multiple storage classes error handling in cwl_runner
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 31995ff8c..a946452b6 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -786,7 +786,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
arvargs.storage_classes = arvargs.storage_classes.strip().split(',')
if len(arvargs.storage_classes) > 1:
logger.error("Multiple storage classes are not supported currently.")
- sys.exit(1)
+ return 1
if install_sig_handlers:
arv_cmd.install_signal_handlers()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d6553aa4c..1fae1cf91 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -335,14 +335,12 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_error_when_multiple_storage_classes_specified(self, stubs):
- capture_stdout = cStringIO.StringIO()
storage_classes = "foo,bar"
-
- with self.assertRaises(SystemExit):
- arvados_cwl.main(
+ exited = arvados_cwl.main(
["--debug", "--storage-classes", storage_classes,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- capture_stdout, sys.stderr, api_client=stubs.api)
+ sys.stdin, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 1)
@mock.patch("time.sleep")
@stubs
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list