[ARVADOS] updated: 5dc0047f55dc31042d0a91d3cefde9db46bb84b7
Git user
git at public.curoverse.com
Wed Apr 27 14:59:28 EDT 2016
Summary of changes:
build/package-build-dockerfiles/centos6/Dockerfile | 4 ++
build/run-tests.sh | 11 +++-
.../install-compute-node.html.textile.liquid | 4 +-
.../install-shell-server.html.textile.liquid | 4 +-
sdk/cwl/arvados_cwl/__init__.py | 39 +++++++------
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_submit.py | 19 +++++--
sdk/cwl/tests/wf/inputs_test.cwl | 7 ++-
sdk/python/arvados/events.py | 66 ++++++++++++++++++----
sdk/python/arvados/retry.py | 7 ++-
sdk/python/setup.py | 5 ++
sdk/python/tests/slow_test.py | 7 +++
sdk/python/tests/test_retry.py | 2 +-
sdk/python/tests/test_websockets.py | 22 +++++++-
services/fuse/setup.py | 5 ++
services/fuse/tests/mount_test_base.py | 14 +++--
.../performance/test_collection_performance.py | 7 +++
services/fuse/tests/slow_test.py | 1 +
services/keep-web/server_test.go | 10 +++-
tools/keep-block-check/keep-block-check.go | 13 +++--
tools/keep-block-check/keep-block-check_test.go | 15 ++++-
tools/keep-rsync/keep-rsync.go | 23 ++++----
tools/keep-rsync/keep-rsync_test.go | 16 +++++-
23 files changed, 225 insertions(+), 78 deletions(-)
create mode 100644 sdk/python/tests/slow_test.py
create mode 120000 services/fuse/tests/slow_test.py
discards 3610b230128153194c7e04cfab47d46a90489651 (commit)
discards fdb6acdccdfa1fb5410c9bb9a0709611185867be (commit)
discards f7b2c6b75413d08da0546601552cecad1ebee6e8 (commit)
discards 3ee2d4442f3643f14b9324891d566698dbe3ae36 (commit)
discards 527686ab2232bf842567b8198b120e8add8cf7c0 (commit)
discards fd71106f5be99eeda2c282611468a1262505bc78 (commit)
via 5dc0047f55dc31042d0a91d3cefde9db46bb84b7 (commit)
via 76b6838193daf06b1cef5986a318f173963717b0 (commit)
via 76eec033a5293f3e0dc7061011615fd00916df43 (commit)
via 43b014d89b9600eca06a398b1ced86d1c9cccdf9 (commit)
via e1de889290360f6dd5b5fdeab10cea997bcc6962 (commit)
via ac48d85ae411b1bcef0c194caae1a861eae9e929 (commit)
via 47a79960c81ea689445f2040b24cb76729afab06 (commit)
via 75c049d9561b08c09f207edb4ebd3bc6fb6460a6 (commit)
via 20beb96440f1bae29adb0132617aae4959e76c75 (commit)
via c1276bd9f83a7826f10e1752ac793d8a1cd3c47f (commit)
via 8cc6fc047506e3bead524f18416b78fe068c70fd (commit)
via a6401a8c59a582633de2ae197e71aad493de291a (commit)
via eebcb5e6a6cb98562212f30841a0411872ac3e89 (commit)
via 6279c22bd0e85db1bdb21ad9d95ae8816ac87a5a (commit)
via da6f122ed9c8ff0e79c7b3969c4f0960039b9bc7 (commit)
via 655e654e99a00a248639a910923e0ce66c16581c (commit)
via f5617be935a121dc339effcaafebb569109ebe5b (commit)
via 89e091b3ee1fe85a68b0a9a0a619ca7baf606d2d (commit)
via b5dc4ae38eb6da852e167952675aff59987a995c (commit)
via 8ad62b017ea0dc84613eeb43f21f32890d9ed4ed (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (3610b230128153194c7e04cfab47d46a90489651)
\
N -- N -- N (5dc0047f55dc31042d0a91d3cefde9db46bb84b7)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5dc0047f55dc31042d0a91d3cefde9db46bb84b7
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 27 14:58:11 2016 -0400
8653: Add arvados-cwl-runner --create-template flag
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8f2102c..8341624 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -4,26 +4,27 @@
import argparse
import arvados
-import arvados.events
+import arvados.collection
import arvados.commands.keepdocker
import arvados.commands.run
-import arvados.collection
+import arvados.events
import arvados.util
+import copy
+import cwltool.docker
import cwltool.draft2tool
-import cwltool.workflow
+from cwltool.errors import WorkflowException
import cwltool.main
from cwltool.process import shortname
-from cwltool.errors import WorkflowException
-import threading
-import cwltool.docker
+import cwltool.workflow
import fnmatch
-import logging
-import re
-import os
-import sys
import functools
import json
+import logging
+import os
import pkg_resources # part of setuptools
+import re
+import sys
+import threading
from cwltool.process import get_feature, adjustFiles, scandeps
from arvados.api import OrderedJsonModel
@@ -322,7 +323,13 @@ class RunnerJob(object):
for s in tool.steps:
self.upload_docker(s.embedded_tool)
- def run(self, dry_run=False, pull_image=True, **kwargs):
+ def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs):
+ """Create an Arvados job specification for this workflow.
+
+ The returned dict can be used to create a job (i.e., passed as
+ the +body+ argument to jobs().create()), or as a component in
+ a pipeline template or pipeline instance.
+ """
self.upload_docker(self.tool)
workflowfiles = set()
@@ -364,9 +371,7 @@ class RunnerJob(object):
del self.job_order["id"]
self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"])[1]
-
- response = self.arvrunner.api.jobs().create(body={
- "owner_uuid": self.arvrunner.project_uuid,
+ return {
"script": "cwl-runner",
"script_version": "master",
"repository": "arvados",
@@ -374,9 +379,19 @@ class RunnerJob(object):
"runtime_constraints": {
"docker_image": "arvados/jobs"
}
- }, find_or_create=self.enable_reuse).execute(num_retries=self.arvrunner.num_retries)
+ }
+
+ def run(self, *args, **kwargs):
+ job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
- self.arvrunner.jobs[response["uuid"]] = self
+ response = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
+
+ self.uuid = response["uuid"]
+ self.arvrunner.jobs[self.uuid] = self
logger.info("Submitted job %s", response["uuid"])
@@ -401,6 +416,99 @@ class RunnerJob(object):
finally:
del self.arvrunner.jobs[record["uuid"]]
+
+class RunnerTemplate(object):
+ """An Arvados pipeline template that invokes a CWL workflow."""
+
+ type_to_dataclass = {
+ 'boolean': 'boolean',
+ 'File': 'File',
+ 'float': 'number',
+ 'int': 'number',
+ 'string': 'text',
+ }
+
+ def __init__(self, runner, tool, job_order, enable_reuse):
+ self.runner = runner
+ self.tool = tool
+ self.job = RunnerJob(
+ runner=runner,
+ tool=tool,
+ job_order=job_order,
+ enable_reuse=enable_reuse)
+
+ def pipeline_component_spec(self):
+ """Return a component that Workbench and a-r-p-i will understand.
+
+ Specifically, translate CWL input specs to Arvados pipeline
+ format, like {"dataclass":"File","value":"xyz"}.
+ """
+ spec = self.job.arvados_job_spec()
+
+ # Most of the component spec is exactly the same as the job
+ # spec (script, script_version, etc.).
+ # spec['script_parameters'] isn't right, though. A component
+ # spec's script_parameters hash is a translation of
+ # self.tool.tool['inputs'] with defaults/overrides taken from
+ # the job order. So we move the job parameters out of the way
+ # and build a new spec['script_parameters'].
+ job_params = spec['script_parameters']
+ spec['script_parameters'] = {}
+
+ for param in self.tool.tool['inputs']:
+ param = copy.deepcopy(param)
+
+ # Data type and "required" flag...
+ types = param['type']
+ if not isinstance(types, list):
+ types = [types]
+ param['required'] = 'null' not in types
+ non_null_types = set(types) - set(['null'])
+ if len(non_null_types) == 1:
+ the_type = [c for c in non_null_types][0]
+ dataclass = self.type_to_dataclass.get(the_type)
+ if dataclass:
+ param['dataclass'] = dataclass
+ # Note: If we didn't figure out a single appropriate
+ # dataclass, we just left that attribute out. We leave
+ # the "type" attribute there in any case, which might help
+ # downstream.
+
+ # Title and description...
+ title = param.pop('label', '')
+ descr = param.pop('description', '').rstrip('\n')
+ if title:
+ param['title'] = title
+ if descr:
+ param['description'] = descr
+
+ # Fill in the value from the current job order, if any.
+ param_id = shortname(param.pop('id'))
+ value = job_params.get(param_id)
+ if value is None:
+ pass
+ elif not isinstance(value, dict):
+ param['value'] = value
+ elif param.get('dataclass') == 'File' and value.get('path'):
+ param['value'] = value['path']
+
+ spec['script_parameters'][param_id] = param
+ spec['script_parameters']['cwl:tool'] = job_params['cwl:tool']
+ return spec
+
+ def save(self):
+ job_spec = self.pipeline_component_spec()
+ response = self.runner.api.pipeline_templates().create(body={
+ "components": {
+ self.job.name: job_spec,
+ },
+ "name": self.job.name,
+ "owner_uuid": self.runner.project_uuid,
+ }).execute(num_retries=self.runner.num_retries)
+ self.uuid = response["uuid"]
+ logger.info("Created template %s", self.uuid)
+
+
class ArvPathMapper(cwltool.pathmapper.PathMapper):
"""Convert container-local paths to and from Keep collection ids."""
@@ -502,7 +610,6 @@ class ArvCwlRunner(object):
body={"state": "Failed"}).execute(num_retries=self.num_retries)
self.final_output = out
-
def on_message(self, event):
if "object_uuid" in event:
if event["object_uuid"] in self.jobs and event["event_type"] == "update":
@@ -541,11 +648,17 @@ class ArvCwlRunner(object):
self.project_uuid = args.project_uuid if args.project_uuid else useruuid
self.pipeline = None
+ if args.create_template:
+ tmpl = RunnerTemplate(self, tool, job_order, args.enable_reuse)
+ tmpl.save()
+ # cwltool.main will write our return value to stdout.
+ return tmpl.uuid
+
if args.submit:
runnerjob = RunnerJob(self, tool, job_order, args.enable_reuse)
if not args.wait:
runnerjob.run()
- return
+ return runnerjob.uuid
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
@@ -655,6 +768,7 @@ def main(args, stdout, stderr, api_client=None):
default=True, dest="submit")
exgroup.add_argument("--local", action="store_false", help="Run workflow on local host (submits jobs to Arvados).",
default=True, dest="submit")
+ exgroup.add_argument("--create-template", action="store_true", help="Create an Arvados pipeline template.")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--wait", action="store_true", help="After submitting workflow runner job, wait for completion.",
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 149c0ba..c665a00 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -30,7 +30,7 @@ setup(name='arvados-cwl-runner',
'bin/arvados-cwl-runner'
],
install_requires=[
- 'cwltool==1.0.20160421140153',
+ 'cwltool==1.0.20160427142240',
'arvados-python-client>=0.1.20160322001610'
],
test_suite='tests',
diff --git a/sdk/cwl/tests/matcher.py b/sdk/cwl/tests/matcher.py
new file mode 100644
index 0000000..d3c9316
--- /dev/null
+++ b/sdk/cwl/tests/matcher.py
@@ -0,0 +1,23 @@
+import difflib
+import json
+
+
+class JsonDiffMatcher(object):
+ """Raise AssertionError with a readable JSON diff when not __eq__().
+
+ Used with assert_called_with() so it's possible for a human to see
+ the differences between expected and actual call arguments that
+ include non-trivial data structures.
+ """
+ def __init__(self, expected):
+ self.expected = expected
+
+ def __eq__(self, actual):
+ expected_json = json.dumps(self.expected, sort_keys=True, indent=2)
+ actual_json = json.dumps(actual, sort_keys=True, indent=2)
+ if expected_json != actual_json:
+ raise AssertionError("".join(difflib.context_diff(
+ expected_json.splitlines(1),
+ actual_json.splitlines(1),
+ fromfile="Expected", tofile="Actual")))
+ return True
diff --git a/sdk/cwl/tests/order/empty_order.json b/sdk/cwl/tests/order/empty_order.json
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/sdk/cwl/tests/order/empty_order.json
@@ -0,0 +1 @@
+{}
diff --git a/sdk/cwl/tests/order/inputs_test_order.json b/sdk/cwl/tests/order/inputs_test_order.json
new file mode 100644
index 0000000..8830523
--- /dev/null
+++ b/sdk/cwl/tests/order/inputs_test_order.json
@@ -0,0 +1,9 @@
+{
+ "fileInput": {
+ "class": "File",
+ "path": "../input/blorp.txt"
+ },
+ "boolInput": true,
+ "floatInput": 1.234,
+ "optionalFloatInput": null
+}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 5734821..432aba2 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -2,12 +2,17 @@ import arvados
import arvados.keep
import arvados.collection
import arvados_cwl
+import copy
+import cStringIO
import functools
import hashlib
import mock
import sys
import unittest
+from .matcher import JsonDiffMatcher
+
+
def stubs(func):
@functools.wraps(func)
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@@ -29,7 +34,9 @@ def stubs(func):
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
stubs.api = mock.MagicMock()
- stubs.api.users().current().execute.return_value = {"uuid": stubs.fake_user_uuid}
+ stubs.api.users().current().execute.return_value = {
+ "uuid": stubs.fake_user_uuid,
+ }
stubs.api.collections().list().execute.return_value = {"items": []}
stubs.api.collections().create().execute.side_effect = ({
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
@@ -38,12 +45,16 @@ def stubs(func):
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
"portable_data_hash": "99999999999999999999999999999992+99",
})
+ stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
stubs.api.jobs().create().execute.return_value = {
- "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "uuid": stubs.expect_job_uuid,
"state": "Queued",
}
+ stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
+ stubs.api.pipeline_templates().create().execute.return_value = {
+ "uuid": stubs.expect_pipeline_template_uuid,
+ }
stubs.expect_job_spec = {
- 'owner_uuid': stubs.fake_user_uuid,
'runtime_constraints': {
'docker_image': 'arvados/jobs'
},
@@ -52,7 +63,8 @@ def stubs(func):
'path': '99999999999999999999999999999992+99/blorp.txt',
'class': 'File'
},
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ 'cwl:tool':
+ '99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
'script_version': 'master',
@@ -65,42 +77,174 @@ def stubs(func):
class TestSubmit(unittest.TestCase):
@stubs
def test_submit(self, stubs):
- arvados_cwl.main(
- ["--debug", "--submit", "--no-wait",
+ capture_stdout = cStringIO.StringIO()
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdout, sys.stderr, api_client=stubs.api)
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
stubs.api.collections().create.assert_has_calls([
mock.call(),
mock.call(body={
- 'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
+ 'manifest_text':
+ './tool 84ec4df683711de31b782505389a8843+429 '
+ '0:16:blub.txt 16:413:submit_tool.cwl\n./wf '
+ '81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
mock.call().execute(),
mock.call(body={
- 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
+ 'manifest_text':
+ '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': '#',
}, ensure_unique_name=True),
mock.call().execute()])
+ expect_job = copy.deepcopy(stubs.expect_job_spec)
+ expect_job["owner_uuid"] = stubs.fake_user_uuid
stubs.api.jobs().create.assert_called_with(
- body=stubs.expect_job_spec,
+ body=expect_job,
find_or_create=True)
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_job_uuid + '\n')
@stubs
def test_submit_with_project_uuid(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- arvados_cwl.main(
- ["--debug", "--submit", "--no-wait",
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait",
"--project-uuid", project_uuid,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
sys.stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
- expect_body = stubs.expect_job_spec.copy()
+ expect_body = copy.deepcopy(stubs.expect_job_spec)
expect_body["owner_uuid"] = project_uuid
stubs.api.jobs().create.assert_called_with(
body=expect_body,
find_or_create=True)
+
+
+class TestCreateTemplate(unittest.TestCase):
+ @stubs
+ def test_create(self, stubs):
+ project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+
+ capture_stdout = cStringIO.StringIO()
+
+ exited = arvados_cwl.main(
+ ["--create-template", "--no-wait",
+ "--project-uuid", project_uuid,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ stubs.api.pipeline_instances().create.refute_called()
+ stubs.api.jobs().create.refute_called()
+
+ expect_component = copy.deepcopy(stubs.expect_job_spec)
+ expect_component['script_parameters']['x'] = {
+ 'dataclass': 'File',
+ 'required': True,
+ 'type': 'File',
+ 'value': '99999999999999999999999999999992+99/blorp.txt',
+ }
+ expect_template = {
+ "components": {
+ "submit_wf.cwl": expect_component,
+ },
+ "name": "submit_wf.cwl",
+ "owner_uuid": project_uuid,
+ }
+ stubs.api.pipeline_templates().create.assert_called_with(
+ body=JsonDiffMatcher(expect_template))
+
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_pipeline_template_uuid + '\n')
+
+
+class TestTemplateInputs(unittest.TestCase):
+ expect_template = {
+ "components": {
+ "inputs_test.cwl": {
+ 'runtime_constraints': {
+ 'docker_image': 'arvados/jobs',
+ },
+ 'script_parameters': {
+ 'cwl:tool':
+ '99999999999999999999999999999991+99/'
+ 'wf/inputs_test.cwl',
+ 'optionalFloatInput': None,
+ 'fileInput': {
+ 'type': 'File',
+ 'dataclass': 'File',
+ 'required': True,
+ 'title': "It's a file; we expect to find some characters in it.",
+ 'description': 'If there were anything further to say, it would be said here,\nor here.'
+ },
+ 'floatInput': {
+ 'type': 'float',
+ 'dataclass': 'number',
+ 'required': True,
+ 'title': 'Floats like a duck',
+ 'default': 0.1,
+ 'value': 0.1,
+ },
+ 'optionalFloatInput': {
+ 'type': ['null', 'float'],
+ 'dataclass': 'number',
+ 'required': False,
+ },
+ 'boolInput': {
+ 'type': 'boolean',
+ 'dataclass': 'boolean',
+ 'required': True,
+ 'title': 'True or false?',
+ },
+ },
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'cwl-runner',
+ },
+ },
+ "name": "inputs_test.cwl",
+ }
+
+ @stubs
+ def test_inputs_empty(self, stubs):
+ exited = arvados_cwl.main(
+ ["--create-template", "--no-wait",
+ "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
+ cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ expect_template = copy.deepcopy(self.expect_template)
+ expect_template["owner_uuid"] = stubs.fake_user_uuid
+
+ stubs.api.pipeline_templates().create.assert_called_with(
+ body=JsonDiffMatcher(expect_template))
+
+ @stubs
+ def test_inputs(self, stubs):
+ exited = arvados_cwl.main(
+ ["--create-template", "--no-wait",
+ "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
+ cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ self.expect_template["owner_uuid"] = stubs.fake_user_uuid
+
+ expect_template = copy.deepcopy(self.expect_template)
+ expect_template["owner_uuid"] = stubs.fake_user_uuid
+ params = expect_template[
+ "components"]["inputs_test.cwl"]["script_parameters"]
+ params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
+ params["floatInput"]["value"] = 1.234
+ params["boolInput"]["value"] = True
+
+ stubs.api.pipeline_templates().create.assert_called_with(
+ body=JsonDiffMatcher(expect_template))
diff --git a/sdk/cwl/tests/wf/inputs_test.cwl b/sdk/cwl/tests/wf/inputs_test.cwl
new file mode 100644
index 0000000..91d4db0
--- /dev/null
+++ b/sdk/cwl/tests/wf/inputs_test.cwl
@@ -0,0 +1,27 @@
+# Test case for arvados-cwl-runner. Used to test propagation of
+# various input types as script_parameters in pipeline templates.
+
+class: Workflow
+inputs:
+ - id: "#fileInput"
+ type: File
+ label: It's a file; we expect to find some characters in it.
+ description: |
+ If there were anything further to say, it would be said here,
+ or here.
+ - id: "#boolInput"
+ type: boolean
+ label: True or false?
+ - id: "#floatInput"
+ type: float
+ label: Floats like a duck
+ default: 0.1
+ - id: "#optionalFloatInput"
+ type: ["null", float]
+outputs: []
+steps:
+ - id: step1
+ inputs:
+ - { id: x, source: "#x" }
+ outputs: []
+ run: ../tool/submit_tool.cwl
commit 76b6838193daf06b1cef5986a318f173963717b0
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 15:15:56 2016 -0400
8653: DRY testing code.
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 19745a0..5734821 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1,103 +1,106 @@
-import unittest
-import mock
-import arvados_cwl
-import sys
import arvados
import arvados.keep
import arvados.collection
+import arvados_cwl
+import functools
import hashlib
+import mock
+import sys
+import unittest
-class TestSubmit(unittest.TestCase):
+def stubs(func):
+ @functools.wraps(func)
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@mock.patch("arvados.collection.KeepClient")
@mock.patch("arvados.events.subscribe")
- def test_submit(self, events, keep, keepdocker):
- api = mock.MagicMock()
+ def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
+ class Stubs:
+ pass
+ stubs = Stubs()
+ stubs.events = events
+ stubs.KeepClient = KeepClient
+ stubs.keepdocker = keepdocker
+
def putstub(p, **kwargs):
return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- keep().put.side_effect = putstub
- keepdocker.return_value = True
- user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
- api.users().current().execute.return_value = {"uuid": user_uuid}
- api.collections().list().execute.return_value = {"items": []}
- api.collections().create().execute.side_effect = ({"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
- "portable_data_hash": "99999999999999999999999999999991+99"},
- {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
- "portable_data_hash": "99999999999999999999999999999992+99"})
- api.jobs().create().execute.return_value = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "state": "Queued"}
+ stubs.KeepClient().put.side_effect = putstub
+
+ stubs.keepdocker.return_value = True
+ stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+
+ stubs.api = mock.MagicMock()
+ stubs.api.users().current().execute.return_value = {"uuid": stubs.fake_user_uuid}
+ stubs.api.collections().list().execute.return_value = {"items": []}
+ stubs.api.collections().create().execute.side_effect = ({
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+ "portable_data_hash": "99999999999999999999999999999991+99",
+ }, {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "portable_data_hash": "99999999999999999999999999999992+99",
+ })
+ stubs.api.jobs().create().execute.return_value = {
+ "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "state": "Queued",
+ }
+ stubs.expect_job_spec = {
+ 'owner_uuid': stubs.fake_user_uuid,
+ 'runtime_constraints': {
+ 'docker_image': 'arvados/jobs'
+ },
+ 'script_parameters': {
+ 'x': {
+ 'path': '99999999999999999999999999999992+99/blorp.txt',
+ 'class': 'File'
+ },
+ 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ },
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'cwl-runner'
+ }
+ return func(self, stubs, *args, **kwargs)
+ return wrapped
+
- arvados_cwl.main(["--debug", "--submit", "--no-wait", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdout, sys.stderr, api_client=api)
+class TestSubmit(unittest.TestCase):
+ @stubs
+ def test_submit(self, stubs):
+ arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdout, sys.stderr, api_client=stubs.api)
- api.collections().create.assert_has_calls([
+ stubs.api.collections().create.assert_has_calls([
mock.call(),
- mock.call(body={'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
- 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'name': 'submit_wf.cwl'
- }, ensure_unique_name=True),
+ mock.call(body={
+ 'manifest_text': './tool 84ec4df683711de31b782505389a8843+429 0:16:blub.txt 16:413:submit_tool.cwl\n./wf 81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'submit_wf.cwl',
+ }, ensure_unique_name=True),
mock.call().execute(),
- mock.call(body={'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'name': '#'
- }, ensure_unique_name=True),
+ mock.call(body={
+ 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': '#',
+ }, ensure_unique_name=True),
mock.call().execute()])
- api.jobs().create.assert_called_with(
- body={
- 'owner_uuid': user_uuid,
- 'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
- },
- 'script_parameters': {
- 'x': {
- 'path': '99999999999999999999999999999992+99/blorp.txt',
- 'class': 'File'
- },
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
- },
+ stubs.api.jobs().create.assert_called_with(
+ body=stubs.expect_job_spec,
find_or_create=True)
- @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
- @mock.patch("arvados.collection.KeepClient")
- @mock.patch("arvados.events.subscribe")
- def test_submit_with_project_uuid(self, events, keep, keepdocker):
- api = mock.MagicMock()
- def putstub(p, **kwargs):
- return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- keep().put.side_effect = putstub
- keepdocker.return_value = True
- api.users().current().execute.return_value = {"uuid": "zzzzz-tpzed-zzzzzzzzzzzzzzz"}
- api.collections().list().execute.return_value = {"items": []}
- api.collections().create().execute.side_effect = ({"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
- "portable_data_hash": "99999999999999999999999999999991+99"},
- {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
- "portable_data_hash": "99999999999999999999999999999992+99"})
- api.jobs().create().execute.return_value = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "state": "Queued"}
+ @stubs
+ def test_submit_with_project_uuid(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- arvados_cwl.main(["--debug", "--submit", "--project-uuid", project_uuid,
- "--no-wait", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- sys.stdout, sys.stderr, api_client=api)
+ arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait",
+ "--project-uuid", project_uuid,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdout, sys.stderr, api_client=stubs.api)
- api.jobs().create.assert_called_with(
- body={
- 'owner_uuid': project_uuid,
- 'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
- },
- 'script_parameters': {
- 'x': {
- 'path': '99999999999999999999999999999992+99/blorp.txt',
- 'class': 'File'
- },
- 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
- },
+ expect_body = stubs.expect_job_spec.copy()
+ expect_body["owner_uuid"] = project_uuid
+ stubs.api.jobs().create.assert_called_with(
+ body=expect_body,
find_or_create=True)
commit 76eec033a5293f3e0dc7061011615fd00916df43
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 10:55:18 2016 -0400
8653: Turn off debug messages / verbose logging in test suite.
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index e1ef605..e74e000 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -1,6 +1,13 @@
-import unittest
-import mock
import arvados_cwl
+import logging
+import mock
+import unittest
+import os
+
+if not os.getenv('ARVADOS_DEBUG'):
+ logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
+ logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
+
class TestJob(unittest.TestCase):
commit 43b014d89b9600eca06a398b1ced86d1c9cccdf9
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 21 10:54:11 2016 -0400
8653: Fix whitespace.
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index bb15e60..e1ef605 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -28,9 +28,9 @@ class TestJob(unittest.TestCase):
'tasks': [{
'task.env': {'TMPDIR': '$(task.tmpdir)'},
'command': ['ls']
- }]
- },
- 'script_version': 'master',
+ }],
+ },
+ 'script_version': 'master',
'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
'repository': 'arvados',
'script': 'crunchrunner',
@@ -40,8 +40,8 @@ class TestJob(unittest.TestCase):
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
}
- },
- find_or_create=True,
+ },
+ find_or_create=True,
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
@@ -90,7 +90,7 @@ class TestJob(unittest.TestCase):
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
}
- },
+ },
find_or_create=True,
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list