[ARVADOS] updated: f7e07c3a28d4f4e1adbd957749bd825e44a2e523
Git user
git at public.curoverse.com
Wed Mar 30 14:58:50 EDT 2016
Summary of changes:
crunch_scripts/cwl-runner | 25 ++++++++-----
sdk/cwl/arvados_cwl/__init__.py | 21 ++++++++++-
sdk/cwl/tests/inp/blorp.txt | 1 +
sdk/cwl/tests/submit_test_job.json | 6 ++++
sdk/cwl/tests/test_submit.py | 58 +++++++++++++++++++++++++++++++
sdk/cwl/tests/tool/blub.txt | 1 +
sdk/cwl/tests/tool/submit_tool.cwl | 14 ++++++++
sdk/cwl/tests/wf/submit_wf.cwl | 11 ++++++
sdk/python/arvados/commands/keepdocker.py | 10 +++---
sdk/python/arvados/commands/run.py | 2 +-
10 files changed, 134 insertions(+), 15 deletions(-)
create mode 100644 sdk/cwl/tests/inp/blorp.txt
create mode 100644 sdk/cwl/tests/submit_test_job.json
create mode 100644 sdk/cwl/tests/test_submit.py
create mode 100644 sdk/cwl/tests/tool/blub.txt
create mode 100644 sdk/cwl/tests/tool/submit_tool.cwl
create mode 100644 sdk/cwl/tests/wf/submit_wf.cwl
via f7e07c3a28d4f4e1adbd957749bd825e44a2e523 (commit)
via 3fe0e22c8b6edecd2651c5ee4db105a1e19f42e8 (commit)
via 5eabedb88ce741076167efd2305216043dd9abd5 (commit)
from db1fd09c38132bbaf04e6921bdc0de3bf28abe09 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f7e07c3a28d4f4e1adbd957749bd825e44a2e523
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 30 14:58:30 2016 -0400
8654: Add missing test_submit
diff --git a/sdk/cwl/tests/inp/blorp.txt b/sdk/cwl/tests/inp/blorp.txt
new file mode 100644
index 0000000..09fc24d
--- /dev/null
+++ b/sdk/cwl/tests/inp/blorp.txt
@@ -0,0 +1 @@
+blopper blubber
diff --git a/sdk/cwl/tests/submit_test_job.json b/sdk/cwl/tests/submit_test_job.json
new file mode 100644
index 0000000..e453f42
--- /dev/null
+++ b/sdk/cwl/tests/submit_test_job.json
@@ -0,0 +1,6 @@
+{
+ "x": {
+ "class": "File",
+ "path": "inp/blorp.txt"
+ }
+}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
new file mode 100644
index 0000000..d383a48
--- /dev/null
+++ b/sdk/cwl/tests/test_submit.py
@@ -0,0 +1,58 @@
+import unittest
+import mock
+import arvados_cwl
+import sys
+import arvados
+import arvados.keep
+import arvados.collection
+import hashlib
+
+class TestSubmit(unittest.TestCase):
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ @mock.patch("arvados.collection.KeepClient")
+ @mock.patch("arvados.events.subscribe")
+ def test_submit(self, events, keep, keepdocker):
+ api = mock.MagicMock()
+ def putstub(p, **kwargs):
+ return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
+ keep().put.side_effect = putstub
+ keepdocker.return_value = True
+ api.users().current().execute.return_value = {"uuid": "zzzzz-tpzed-zzzzzzzzzzzzzzz"}
+ api.collections().list().execute.return_value = {"items": []}
+ api.collections().create().execute.side_effect = ({"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+ "portable_data_hash": "99999999999999999999999999999991+99"},
+ {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+ "portable_data_hash": "99999999999999999999999999999992+99"})
+ api.jobs().create().execute.return_value = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "state": "Queued"}
+
+ arvados_cwl.main(["--debug", "--submit", "--no-wait", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdout, sys.stderr, api_client=api)
+
+ api.collections().create.assert_has_calls([
+ mock.call(),
+ mock.call(body={'manifest_text': './tool b0fa9cffda3d37ba401bc338bb0b54a6+257 0:16:blub.txt 16:241:submit_tool.cwl\n./wf 888e237b758adf86ca66c36f7bd88a24+175 0:175:submit_wf.cwl\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'submit_wf.cwl'
+ }, ensure_unique_name=True),
+ mock.call().execute(),
+ mock.call(body={'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'submit_test_job.json'
+ }, ensure_unique_name=True),
+ mock.call().execute()])
+ api.jobs().create.assert_called_with(body={
+ 'runtime_constraints': {
+ 'docker_image': 'arvados/jobs'
+ },
+ 'script_parameters': {
+ 'x': {
+ 'path': '99999999999999999999999999999992+99/blorp.txt',
+ 'class': 'File'
+ },
+ 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
+ },
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'cwl-runner'
+ },
+ find_or_create=True)
diff --git a/sdk/cwl/tests/tool/blub.txt b/sdk/cwl/tests/tool/blub.txt
new file mode 100644
index 0000000..f12927b
--- /dev/null
+++ b/sdk/cwl/tests/tool/blub.txt
@@ -0,0 +1 @@
+blibber blubber
diff --git a/sdk/cwl/tests/tool/submit_tool.cwl b/sdk/cwl/tests/tool/submit_tool.cwl
new file mode 100644
index 0000000..4bdb235
--- /dev/null
+++ b/sdk/cwl/tests/tool/submit_tool.cwl
@@ -0,0 +1,14 @@
+class: CommandLineTool
+requirements:
+ - class: DockerRequirement
+ dockerPull: debian:8
+inputs:
+ - id: x
+ type: File
+ default:
+ class: File
+ path: blub.txt
+ inputBinding:
+ position: 1
+outputs: []
+baseCommand: cat
diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
new file mode 100644
index 0000000..104203b
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -0,0 +1,11 @@
+class: Workflow
+inputs:
+ - id: x
+ type: File
+outputs: []
+steps:
+ - id: step1
+ inputs:
+ - { id: x, source: "#x" }
+ outputs: []
+ run: ../tool/submit_tool.cwl
commit 3fe0e22c8b6edecd2651c5ee4db105a1e19f42e8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 30 14:45:12 2016 -0400
8654: Add comments
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 8071303..d628f1c 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -1,5 +1,13 @@
#!/usr/bin/env python
+# Crunch script integration for running arvados-cwl-runner (importing
+# arvados_cwl module) inside a crunch job.
+#
+# This gets the job record, transforms the script parameters into a valid CWL
+# input object, then executes the CWL runner to run the underlying workflow or
+# tool. When the workflow completes, record the output object in an output
+# collection for this runner job.
+
import arvados
import arvados_cwl
import arvados.collection
@@ -13,7 +21,8 @@ import argparse
from arvados.api import OrderedJsonModel
from cwltool.process import adjustFiles
-print cwltool.main.versionstring()
+# Print package versions
+logging.info(cwltool.main.versionstring())
api = arvados.api("v1")
@@ -32,13 +41,13 @@ try:
t = cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True)
- np = argparse.Namespace()
- np.project_uuid = arvados.current_job()["owner_uuid"]
- np.enable_reuse = True
- np.submit = False
- np.debug = True
- np.quiet = False
- outputObj = runner.arvExecutor(t, job_order_object, "", np, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
+ args = argparse.Namespace()
+ args.project_uuid = arvados.current_job()["owner_uuid"]
+ args.enable_reuse = True
+ args.submit = False
+ args.debug = True
+ args.quiet = False
+ outputObj = runner.arvExecutor(t, job_order_object, "", args, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
files = {}
def capture(path):
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3762536..5958100 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -1,5 +1,7 @@
#!/usr/bin/env python
+# Implement cwl-runner interface for submitting and running jobs on Arvados.
+
import argparse
import arvados
import arvados.events
@@ -35,6 +37,8 @@ keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.ke
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
+ """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
+
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
@@ -58,6 +62,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
class CollectionFsAccess(cwltool.process.StdFsAccess):
+ """Implement the cwltool FsAccess interface for Arvados Collections."""
+
def __init__(self, basedir):
self.collections = {}
self.basedir = basedir
@@ -114,6 +120,8 @@ class CollectionFsAccess(cwltool.process.StdFsAccess):
return os.path.exists(self._abs(fn))
class ArvadosJob(object):
+ """Submit and manage a Crunch job for executing a CWL CommandLineTool."""
+
def __init__(self, runner):
self.arvrunner = runner
self.running = False
@@ -283,6 +291,8 @@ class ArvadosJob(object):
class RunnerJob(object):
+ """Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""
+
def __init__(self, runner, tool, job_order, enable_reuse):
self.arvrunner = runner
self.tool = tool
@@ -381,6 +391,8 @@ class RunnerJob(object):
del self.arvrunner.jobs[record["uuid"]]
class ArvPathMapper(cwltool.pathmapper.PathMapper):
+ """Convert container-local paths to and from Keep collection ids."""
+
def __init__(self, arvrunner, referenced_files, basedir,
collection_pattern, file_pattern, name=None, **kwargs):
self._pathmap = arvrunner.get_uploaded()
@@ -430,6 +442,8 @@ class ArvPathMapper(cwltool.pathmapper.PathMapper):
class ArvadosCommandTool(cwltool.draft2tool.CommandLineTool):
+ """Wrap cwltool CommandLineTool to override selected methods."""
+
def __init__(self, arvrunner, toolpath_object, **kwargs):
super(ArvadosCommandTool, self).__init__(toolpath_object, **kwargs)
self.arvrunner = arvrunner
@@ -445,6 +459,9 @@ class ArvadosCommandTool(cwltool.draft2tool.CommandLineTool):
class ArvCwlRunner(object):
+ """Execute a CWL tool or workflow, submit crunch jobs, wait for them to
+ complete, and report output."""
+
def __init__(self, api_client):
self.api = api_client
self.jobs = {}
@@ -593,6 +610,8 @@ class ArvCwlRunner(object):
return self.final_output
def versionstring():
+ """Print version string of key packages for provenance and debugging."""
+
cwlpkg = pkg_resources.require("cwltool")
arvpkg = pkg_resources.require("arvados-python-client")
arvcwlpkg = pkg_resources.require("arvados-cwl-runner")
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 7a298b4..5d29c45 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -110,7 +110,7 @@ def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
def uploadfiles(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
# Find the smallest path prefix that includes all the files that need to be uploaded.
# This starts at the root and iteratively removes common parent directory prefixes
- # until all file pathes no longer have a common parent.
+ # until all file paths no longer have a common parent.
n = True
pathprefix = "/"
while n:
commit 5eabedb88ce741076167efd2305216043dd9abd5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 30 14:00:24 2016 -0400
8654: Print uuid of uploaded docker image on stderr instead of stdout.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 74f36d9..3762536 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -52,7 +52,7 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if image_tag:
args.append(image_tag)
logger.info("Uploading Docker image %s", ":".join(args[1:]))
- arvados.commands.keepdocker.main(args)
+ arvados.commands.keepdocker.main(args, stdout=sys.stderr)
return dockerRequirement["dockerImageId"]
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index e48a6d1..f665541 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -283,15 +283,15 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
return [(image['collection'], image) for image in images
if image['collection'] in existing_coll_uuids]
-def main(arguments=None):
+def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
- fmt = "{:30} {:10} {:12} {:29} {:20}"
- print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
+ fmt = "{:30} {:10} {:12} {:29} {:20}\n"
+ stdout.write(fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED"))
for i, j in list_images_in_arv(api, args.retries):
- print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
+ stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
@@ -375,7 +375,7 @@ def main(arguments=None):
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
- print(coll_uuid)
+ stdout.write(coll_uuid + "\n")
sys.exit(0)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list