[ARVADOS] updated: c7a461e97640697e390aff96bf25126972396b19
Git user
git at public.curoverse.com
Mon Oct 3 14:49:23 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 32 +++++++---
sdk/cwl/arvados_cwl/arvcontainer.py | 4 +-
sdk/cwl/arvados_cwl/arvjob.py | 9 ++-
sdk/cwl/arvados_cwl/fsaccess.py | 6 +-
sdk/cwl/arvados_cwl/pathmapper.py | 2 +
sdk/cwl/arvados_cwl/runner.py | 5 +-
sdk/cwl/tests/test_job.py | 114 +++++++++++++++++++-----------------
sdk/cwl/tests/test_make_output.py | 58 ++++++++++++++++++
sdk/cwl/tests/test_submit.py | 22 ++++---
9 files changed, 175 insertions(+), 77 deletions(-)
create mode 100644 sdk/cwl/tests/test_make_output.py
via c7a461e97640697e390aff96bf25126972396b19 (commit)
via 0e4e5d028290bc8d5a4ffe5469a6993f640027a6 (commit)
via d182d19257f6cbe71fd75c29e1a81ba59a2aa79c (commit)
from 83139cd2df6b7d86f352bee46d1ad86146bd093b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c7a461e97640697e390aff96bf25126972396b19
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Oct 3 14:48:36 2016 -0400
10165: Add test for make_output_collection
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 24f5a0d..7d5590b 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -213,7 +213,7 @@ class ArvCwlRunner(object):
adjustFileObjs(outputObj, rewrite)
with final.open("cwl.output.json", "w") as f:
- json.dump(outputObj, f, sort_keys=True, indent=4)
+ json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
final.save_new(name=name, owner_uuid=self.project_uuid, ensure_unique_name=True)
diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
new file mode 100644
index 0000000..2671297
--- /dev/null
+++ b/sdk/cwl/tests/test_make_output.py
@@ -0,0 +1,58 @@
+import logging
+import mock
+import unittest
+import os
+import functools
+import json
+import StringIO
+
+import arvados_cwl
+
+class TestMakeOutput(unittest.TestCase):
+ @mock.patch("arvados.collection.Collection")
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_make_output_collection(self, reader, col):
+ api = mock.MagicMock()
+ keep_client = mock.MagicMock()
+ runner = arvados_cwl.ArvCwlRunner(api, keep_client=keep_client)
+ runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+
+ final = mock.MagicMock()
+ col.return_value = final
+ readermock = mock.MagicMock()
+ reader.return_value = readermock
+
+ cwlout = StringIO.StringIO()
+ openmock = mock.MagicMock()
+ final.open.return_value = openmock
+ openmock.__enter__.return_value = cwlout
+
+ runner.make_output_collection("Test output", {
+ "foo": {
+ "class": "File",
+ "location": "keep:99999999999999999999999999999991+99/foo.txt",
+ "size": 3,
+ "basename": "foo.txt"
+ },
+ "bar": {
+ "class": "File",
+ "location": "keep:99999999999999999999999999999992+99/bar.txt",
+ "basename": "baz.txt"
+ }
+ })
+
+ final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
+ final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
+ final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz')])
+ self.assertEqual("""{
+ "bar": {
+ "class": "File",
+ "location": "baz.txt"
+ },
+ "foo": {
+ "class": "File",
+ "location": "foo.txt"
+ }
+}""", cwlout.getvalue())
+
+ self.assertIs(final, runner.final_output_collection)
commit 0e4e5d028290bc8d5a4ffe5469a6993f640027a6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Oct 3 12:59:51 2016 -0400
10165: Pass around keep client object instead of creating new client per
Collection.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5bdffd5..24f5a0d 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -47,7 +47,7 @@ class ArvCwlRunner(object):
"""
- def __init__(self, api_client, work_api=None):
+ def __init__(self, api_client, work_api=None, keep_client=None):
self.api = api_client
self.processes = {}
self.lock = threading.Lock()
@@ -62,6 +62,10 @@ class ArvCwlRunner(object):
self.poll_api = None
self.pipeline = None
self.final_output_collection = None
+ if keep_client is not None:
+ self.keep_client = keep_client
+ else:
+ self.keep_client = arvados.keep.KeepClient(api_client=self.api, num_retries=self.num_retries)
if self.work_api is None:
# todo: autodetect API to use.
@@ -178,14 +182,20 @@ class ArvCwlRunner(object):
generatemapper = FinalOutputPathMapper(files, "", "", separateDirs=False)
- final = arvados.collection.Collection()
+ final = arvados.collection.Collection(api_client=self.api,
+ keep_client=self.keep_client,
+ num_retries=self.num_retries)
srccollections = {}
for k,v in generatemapper.items():
sp = k.split("/")
srccollection = sp[0][5:]
if srccollection not in srccollections:
- srccollections[srccollection] = arvados.collection.CollectionReader(srccollection)
+ srccollections[srccollection] = arvados.collection.CollectionReader(
+ srccollection,
+ api_client=self.api,
+ keep_client=self.keep_client,
+ num_retries=self.num_retries)
reader = srccollections[srccollection]
try:
srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
@@ -219,7 +229,9 @@ class ArvCwlRunner(object):
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
self.pipeline = None
- make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess, api_client=self.api)
+ make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess,
+ api_client=self.api,
+ keep_client=self.keep_client)
self.fs_access = make_fs_access(kwargs["basedir"])
if kwargs.get("create_template"):
@@ -434,13 +446,14 @@ def add_arv_hints():
res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
cache["http://arvados.org/cwl"] = res.read()
res.close()
- _, cwlnames, _, _ = cwltool.process.get_schema("v1.0")
+ document_loader, cwlnames, _, _ = cwltool.process.get_schema("v1.0")
_, extnames, _, _ = schema_salad.schema.load_schema("http://arvados.org/cwl", cache=cache)
for n in extnames.names:
if not cwlnames.has_name("http://arvados.org/cwl#"+n, ""):
cwlnames.add_name("http://arvados.org/cwl#"+n, "", extnames.get_name(n, ""))
+ document_loader.idx["http://arvados.org/cwl#"+n] = {}
-def main(args, stdout, stderr, api_client=None):
+def main(args, stdout, stderr, api_client=None, keep_client=None):
parser = arg_parser()
job_order_object = None
@@ -453,7 +466,7 @@ def main(args, stdout, stderr, api_client=None):
try:
if api_client is None:
api_client=arvados.api('v1', model=OrderedJsonModel())
- runner = ArvCwlRunner(api_client, work_api=arvargs.work_api)
+ runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client)
except Exception as e:
logger.error(e)
return 1
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index aaae7d9..cd8f84e 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -161,7 +161,9 @@ class RunnerContainer(Runner):
workflowmapper = super(RunnerContainer, self).arvados_job_spec(dry_run=dry_run, pull_image=pull_image, **kwargs)
- with arvados.collection.Collection(api_client=self.arvrunner.api) as jobobj:
+ with arvados.collection.Collection(api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries) as jobobj:
with jobobj.open("cwl.input.json", "w") as f:
json.dump(self.job_order, f, sort_keys=True, indent=4)
jobobj.save_new(owner_uuid=self.arvrunner.project_uuid)
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 0818d5d..d881082 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -40,7 +40,9 @@ class ArvadosJob(object):
with Perf(metrics, "generatefiles %s" % self.name):
if self.generatefiles["listing"]:
- vwd = arvados.collection.Collection()
+ vwd = arvados.collection.Collection(api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries)
script_parameters["task.vwd"] = {}
generatemapper = InitialWorkDirPathMapper([self.generatefiles], "", "",
separateDirs=False)
@@ -169,7 +171,10 @@ class ArvadosJob(object):
try:
if record["output"]:
with Perf(metrics, "inspect log %s" % self.name):
- logc = arvados.collection.Collection(record["log"])
+ logc = arvados.collection.CollectionReader(record["log"],
+ api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries)
log = logc.open(logc.keys()[0])
tmpdir = None
outdir = None
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index e44e7a9..89a4308 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -12,9 +12,10 @@ import arvados.arvfile
class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
"""Implement the cwltool FsAccess interface for Arvados Collections."""
- def __init__(self, basedir, api_client=None):
+ def __init__(self, basedir, api_client=None, keep_client=None):
super(CollectionFsAccess, self).__init__(basedir)
self.api_client = api_client
+ self.keep_client = keep_client
self.collections = {}
def get_collection(self, path):
@@ -22,7 +23,8 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
pdh = p[0][5:]
if pdh not in self.collections:
- self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client)
+ self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
+ keep_client=self.keep_client)
return (self.collections[pdh], "/".join(p[1:]))
else:
return (None, path)
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index bd71a13..228d433 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -101,6 +101,7 @@ class ArvPathMapper(PathMapper):
if srcobj["class"] == "Directory":
if srcobj["location"] not in self._pathmap:
c = arvados.collection.Collection(api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
subdirs = []
for l in srcobj["listing"]:
@@ -119,6 +120,7 @@ class ArvPathMapper(PathMapper):
(srcobj["location"].startswith("_:") and "contents" in srcobj)):
c = arvados.collection.Collection(api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries )
subdirs = []
self.addentry(srcobj, c, ".", subdirs)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7828bfd..08e7d02 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -171,7 +171,10 @@ class Runner(object):
try:
try:
self.final_output = record["output"]
- outc = arvados.collection.Collection(self.final_output)
+ outc = arvados.collection.CollectionReader(self.final_output,
+ api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries)
with outc.open("cwl.output.json") as f:
outputs = json.load(f)
def keepify(fileobj):
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 7f31520..8317c8f 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -22,6 +22,7 @@ class TestJob(unittest.TestCase):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
+ runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
tool = {
@@ -70,6 +71,7 @@ class TestJob(unittest.TestCase):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
+ runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
tool = {
@@ -122,8 +124,8 @@ class TestJob(unittest.TestCase):
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
['docker_image_locator', 'in docker', 'arvados/jobs']])
- @mock.patch("arvados.collection.Collection")
- def test_done(self, col):
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_done(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
@@ -132,7 +134,7 @@ class TestJob(unittest.TestCase):
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
- col().open.return_value = []
+ reader().open.return_value = []
api.collections().list().execute.side_effect = ({"items": []},
{"items": [{"manifest_text": "XYZ"}]})
@@ -166,8 +168,8 @@ class TestJob(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'name': 'Output 9999999 of testjob'})
- @mock.patch("arvados.collection.Collection")
- def test_done_use_existing_collection(self, col):
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_done_use_existing_collection(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
@@ -175,7 +177,7 @@ class TestJob(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
- col().open.return_value = []
+ reader().open.return_value = []
api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
arvjob = arvados_cwl.ArvadosJob(runner)
@@ -206,58 +208,62 @@ class TestWorkflow(unittest.TestCase):
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.Collection")
def test_run(self, mockcollection):
- arvados_cwl.add_arv_hints()
+ try:
+ arvados_cwl.add_arv_hints()
- runner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
- runner.ignore_docker_for_reuse = False
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ runner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.num_retries = 0
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
- metadata["cwlVersion"] = tool["cwlVersion"]
+ tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
+ metadata["cwlVersion"] = tool["cwlVersion"]
- mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+ mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
- arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
- it.next().run()
- it.next().run()
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
+ basedir="", make_fs_access=make_fs_access, loader=document_loader,
+ makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool.formatgraph = None
+ it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
+ it.next().run()
+ it.next().run()
- with open("tests/wf/scatter2_subwf.cwl") as f:
- subwf = f.read()
+ with open("tests/wf/scatter2_subwf.cwl") as f:
+ subwf = f.read()
- mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
- runner.api.jobs().create.assert_called_with(
- body={
- 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'crunchrunner',
- 'script_parameters': {
- 'tasks': [{'task.env': {
- 'HOME': '$(task.outdir)',
- 'TMPDIR': '$(task.tmpdir)'},
- 'task.vwd': {
- 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
- 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
- },
- 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
- 'task.stdout': 'cwl.output.json'}]},
- 'runtime_constraints': {
- 'min_scratch_mb_per_node': 2048,
- 'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs',
- 'min_ram_mb_per_node': 1024
- },
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
- filters=[['repository', '=', 'arvados'],
- ['script', '=', 'crunchrunner'],
- ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']],
- find_or_create=True)
+ runner.api.jobs().create.assert_called_with(
+ body={
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'crunchrunner',
+ 'script_parameters': {
+ 'tasks': [{'task.env': {
+ 'HOME': '$(task.outdir)',
+ 'TMPDIR': '$(task.tmpdir)'},
+ 'task.vwd': {
+ 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
+ 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
+ },
+ 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
+ 'task.stdout': 'cwl.output.json'}]},
+ 'runtime_constraints': {
+ 'min_scratch_mb_per_node': 2048,
+ 'min_cores_per_node': 1,
+ 'docker_image': 'arvados/jobs',
+ 'min_ram_mb_per_node': 1024
+ },
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
+ filters=[['repository', '=', 'arvados'],
+ ['script', '=', 'crunchrunner'],
+ ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
+ ['docker_image_locator', 'in docker', 'arvados/jobs']],
+ find_or_create=True)
+ except:
+ logging.exception("")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index eb71992..6674efb 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -10,6 +10,7 @@ import mock
import sys
import unittest
import json
+import logging
from .matcher import JsonDiffMatcher
@@ -19,21 +20,23 @@ def stubs(func):
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@mock.patch("arvados.collection.KeepClient")
@mock.patch("arvados.events.subscribe")
- def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
+ def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
class Stubs:
pass
stubs = Stubs()
stubs.events = events
- stubs.KeepClient = KeepClient
stubs.keepdocker = keepdocker
+ stubs.keep_client = keep_client
def putstub(p, **kwargs):
return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
- stubs.KeepClient().put.side_effect = putstub
+ stubs.keep_client().put.side_effect = putstub
+ stubs.keep_client.put.side_effect = putstub
stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+
stubs.api = mock.MagicMock()
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
@@ -217,11 +220,14 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container(self, stubs):
capture_stdout = cStringIO.StringIO()
- exited = arvados_cwl.main(
- ["--submit", "--no-wait", "--api=containers", "--debug",
- "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
- capture_stdout, sys.stderr, api_client=stubs.api)
- self.assertEqual(exited, 0)
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
stubs.api.collections().create.assert_has_calls([
mock.call(),
commit d182d19257f6cbe71fd75c29e1a81ba59a2aa79c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Sep 30 17:35:52 2016 -0400
10165: Handle copying complete collection contents into subdirectory.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index ef4f785..5bdffd5 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -188,7 +188,8 @@ class ArvCwlRunner(object):
srccollections[srccollection] = arvados.collection.CollectionReader(srccollection)
reader = srccollections[srccollection]
try:
- final.copy("/".join(sp[1:]), v.target, source_collection=reader, overwrite=False)
+ srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
+ final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
except IOError as e:
logger.warn("While preparing output collection: %s", e)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list