[ARVADOS] created: dabb7587d58a004a7250eab3e7e82e9916319de9
Git user
git at public.curoverse.com
Thu Nov 3 10:29:12 EDT 2016
at dabb7587d58a004a7250eab3e7e82e9916319de9 (commit)
commit dabb7587d58a004a7250eab3e7e82e9916319de9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Nov 3 10:29:07 2016 -0400
10448: Tests include enable_reuse flag. Also fix tests to not require API/keep
server to run tests.
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index e074046..759bf0b 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -321,6 +321,7 @@ class RunnerTemplate(object):
Specifically, translate CWL input specs to Arvados pipeline
format, like {"dataclass":"File","value":"xyz"}.
"""
+
spec = self.job.arvados_job_spec()
# Most of the component spec is exactly the same as the job
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 49d37eb..6f157db 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -122,7 +122,6 @@ def upload_instance(arvrunner, name, tool, job_order):
tool.tool,
tool.tool["id"],
True)
-
jobmapper = upload_dependencies(arvrunner,
os.path.basename(job_order.get("id", "#")),
tool.doc_loader,
diff --git a/sdk/cwl/tests/_rootDesc.py b/sdk/cwl/tests/_rootDesc.py
new file mode 100644
index 0000000..85e99ac
--- /dev/null
+++ b/sdk/cwl/tests/_rootDesc.py
@@ -0,0 +1,10 @@
+import json
+
+_rootDesc = None
+
+def get_rootDesc():
+ global _rootDesc
+ if not _rootDesc:
+ with open("tests/discovery-f850a12e2daa64bc0c800e6d680f8e07.json") as f:
+ _rootDesc = json.load(f)
+ return _rootDesc
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index b549997..706280f 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -53,7 +53,9 @@ class TestContainer(unittest.TestCase):
'runtime_constraints': {
'vcpus': 1,
'ram': 1073741824
- }, 'priority': 1,
+ },
+ 'use_existing': True,
+ 'priority': 1,
'mounts': {
'/var/spool/cwl': {'kind': 'tmp'}
},
@@ -118,7 +120,9 @@ class TestContainer(unittest.TestCase):
'ram': 3145728000,
'API': True,
'partition': ['blurb']
- }, 'priority': 1,
+ },
+ 'use_existing': True,
+ 'priority': 1,
'mounts': {
'/var/spool/cwl': {'kind': 'tmp'}
},
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 93b5d39..d28d8b5 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -4,17 +4,18 @@ import logging
import mock
import os
import unittest
+import copy
import arvados
import arvados_cwl
import cwltool.process
from schema_salad.ref_resolver import Loader
+from ._rootDesc import get_rootDesc
if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-
class TestJob(unittest.TestCase):
# The test passes no builder.resources
@@ -226,7 +227,8 @@ class TestWorkflow(unittest.TestCase):
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
- api._rootDesc = arvados.api('v1')._rootDesc
+ api._rootDesc = get_rootDesc()
+
runner = arvados_cwl.ArvCwlRunner(api)
self.assertEqual(runner.work_api, 'jobs')
@@ -291,7 +293,7 @@ class TestWorkflow(unittest.TestCase):
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
- api._rootDesc = arvados.api('v1')._rootDesc
+ api._rootDesc = copy.deepcopy(get_rootDesc())
del api._rootDesc.get('resources')['jobs']['methods']['create']
runner = arvados_cwl.ArvCwlRunner(api)
self.assertEqual(runner.work_api, 'containers')
diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index cd66eb1..e86f6f3 100644
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -8,11 +8,12 @@ import unittest
import arvados
import arvados_cwl
+from ._rootDesc import get_rootDesc
class TestMakeOutput(unittest.TestCase):
def setUp(self):
self.api = mock.MagicMock()
- self.api._rootDesc = arvados.api('v1')._rootDesc
+ self.api._rootDesc = get_rootDesc()
@mock.patch("arvados.collection.Collection")
@mock.patch("arvados.collection.CollectionReader")
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index 57958f7..f614465 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -12,6 +12,7 @@ import arvados.collection
import arvados_cwl
from cwltool.pathmapper import MapperEnt
+from ._rootDesc import get_rootDesc
from arvados_cwl.pathmapper import ArvPathMapper
@@ -23,7 +24,7 @@ def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPatter
class TestPathmap(unittest.TestCase):
def setUp(self):
self.api = mock.MagicMock()
- self.api._rootDesc = arvados.api('v1')._rootDesc
+ self.api._rootDesc = get_rootDesc()
def test_keepref(self):
"""Test direct keep references."""
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 7faef69..a77c34f 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -14,32 +14,38 @@ import arvados_cwl
import arvados.keep
from .matcher import JsonDiffMatcher
+from ._rootDesc import get_rootDesc
+_rootDesc = None
def stubs(func):
@functools.wraps(func)
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@mock.patch("arvados.collection.KeepClient")
+ @mock.patch("arvados.keep.KeepClient")
@mock.patch("arvados.events.subscribe")
- def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
+ def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
class Stubs:
pass
stubs = Stubs()
stubs.events = events
stubs.keepdocker = keepdocker
- stubs.keep_client = keep_client
+
def putstub(p, **kwargs):
return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- stubs.keep_client().put.side_effect = putstub
- stubs.keep_client.put.side_effect = putstub
+ keep_client1().put.side_effect = putstub
+ keep_client1.put.side_effect = putstub
+ keep_client2().put.side_effect = putstub
+ keep_client2.put.side_effect = putstub
+ stubs.keep_client = keep_client2
stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
-
stubs.api = mock.MagicMock()
- stubs.api._rootDesc = arvados.api('v1')._rootDesc
+ stubs.api._rootDesc = get_rootDesc()
+
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
}
@@ -134,7 +140,8 @@ def stubs(func):
'listing': [
{'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
]}},
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
+ 'arv:enable_reuse': True
},
'repository': 'arvados',
'script_version': arvados_cwl.__version__,
@@ -176,7 +183,7 @@ def stubs(func):
},
'state': 'Committed',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
'name': 'submit_wf.cwl',
'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
'output_path': '/var/spool/cwl',
commit 6c0e3d9ad4758b0121ffb953354d63766fd1e034
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Nov 2 17:21:56 2016 -0400
10448: Pass through enable_reuse flag to runner.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1581d20..ac1ba70 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -105,6 +105,7 @@ class ArvadosContainer(object):
container_request["mounts"] = mounts
container_request["runtime_constraints"] = runtime_constraints
+ container_request["use_existing"] = kwargs.get("enable_reuse", True)
try:
response = self.arvrunner.api.container_requests().create(
@@ -186,6 +187,12 @@ class RunnerContainer(Runner):
command = ["arvados-cwl-runner", "--local", "--api=containers"]
if self.output_name:
command.append("--output-name=" + self.output_name)
+
+ if self.enable_reuse:
+ command.append("--enable-reuse")
+ else:
+ command.append("--disable-reuse")
+
command.extend([workflowpath, jobpath])
return {
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 8228387..e074046 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -242,8 +242,12 @@ class RunnerJob(Runner):
del self.job_order["job_order"]
self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:]
+
if self.output_name:
self.job_order["arv:output_name"] = self.output_name
+
+ self.job_order["arv:enable_reuse"] = self.enable_reuse
+
return {
"script": "cwl-runner",
"script_version": __version__,
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 9b0680b..173eb93 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -63,10 +63,15 @@ def run():
adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
output_name = None
+ enable_reuse = True
if "arv:output_name" in job_order_object:
output_name = job_order_object["arv:output_name"]
del job_order_object["arv:output_name"]
+ if "arv:enable_reuse" in job_order_object:
+ enable_reuse = job_order_object["arv:enable_reuse"]
+ del job_order_object["arv:enable_reuse"]
+
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
output_name=output_name)
@@ -74,7 +79,7 @@ def run():
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
- args.enable_reuse = True
+ args.enable_reuse = enable_reuse
args.submit = False
args.debug = True
args.quiet = False
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list