[ARVADOS] updated: 014e73c96c56bc588a02bfdbdd653c10be30f80c
Git user
git at public.curoverse.com
Tue Aug 30 14:05:09 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/arvworkflow.py | 9 ++++++---
sdk/cwl/arvados_cwl/runner.py | 3 +++
sdk/cwl/tests/test_container.py | 6 ++++--
sdk/cwl/tests/test_job.py | 12 ++++++++++--
sdk/cwl/tests/test_submit.py | 6 ++----
5 files changed, 25 insertions(+), 11 deletions(-)
via 014e73c96c56bc588a02bfdbdd653c10be30f80c (commit)
from 540adbf18728558df8797bf176ec666061cea934 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 014e73c96c56bc588a02bfdbdd653c10be30f80c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 30 14:05:03 2016 -0400
9766: Fix make_workflow to not modify input object. upload_dependencies
includes starting object. Fix tests.
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index dcef67f..fb14ab4 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -1,5 +1,6 @@
import os
import json
+import copy
from cwltool.pack import pack
from cwltool.load_tool import fetch_document
@@ -11,13 +12,15 @@ def make_workflow(arvRunner, tool, job_order, project_uuid, update_uuid):
upload_docker(arvRunner, tool)
document_loader, workflowobj, uri = (tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]), tool.tool["id"])
- for inp in workflowobj["inputs"]:
+
+ packed = pack(document_loader, workflowobj, uri, tool.metadata)
+
+ main = [p for p in packed["$graph"] if p["id"] == "#main"][0]
+ for inp in main["inputs"]:
sn = shortname(inp["id"])
if sn in job_order:
inp["default"] = job_order[sn]
- packed = pack(document_loader, workflowobj, uri, tool.metadata)
-
name = os.path.basename(tool.tool["id"])
upload_dependencies(arvRunner, name, document_loader,
packed, uri, "keep:", False)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 315be0c..9f7b84a 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -51,6 +51,9 @@ def upload_dependencies(arvrunner, name, document_loader,
normalizeFilesDirs(files)
+ if "id" in workflowobj:
+ files.append({"class": "File", "location": workflowobj["id"]})
+
mapper = ArvPathMapper(arvrunner, files, "",
keepprefix+"%s",
keepprefix+"%s/%s",
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index e060037..6b5c321 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -5,6 +5,7 @@ import unittest
import os
import functools
import cwltool.process
+from schema_salad.ref_resolver import Loader
if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
@@ -35,7 +36,7 @@ class TestContainer(unittest.TestCase):
}
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access)
+ basedir="", make_fs_access=make_fs_access, loader=Loader({}))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run",
make_fs_access=make_fs_access, tmpdir="/tmp"):
@@ -88,7 +89,8 @@ class TestContainer(unittest.TestCase):
}
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
- avsc_names=avsc_names, make_fs_access=make_fs_access)
+ avsc_names=avsc_names, make_fs_access=make_fs_access,
+ loader=Loader({}))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
make_fs_access=make_fs_access, tmpdir="/tmp"):
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 21b72d1..50c4e34 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -5,6 +5,7 @@ import unittest
import os
import functools
import cwltool.process
+from schema_salad.ref_resolver import Loader
if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
@@ -28,7 +29,11 @@ class TestJob(unittest.TestCase):
"arguments": [{"valueFrom": "$(runtime.outdir)"}]
}
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access)
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs",
+ avsc_names=avsc_names, basedir="",
+ make_fs_access=make_fs_access,
+ loader=Loader({}))
+
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
j.run()
@@ -80,7 +85,10 @@ class TestJob(unittest.TestCase):
"baseCommand": "ls"
}
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
- arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, make_fs_access=make_fs_access)
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs",
+ avsc_names=avsc_names,
+ make_fs_access=make_fs_access,
+ loader=Loader({}))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
j.run()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 6ea9314..c675995 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -163,8 +163,7 @@ class TestSubmit(unittest.TestCase):
mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
'0:0:blub.txt 0:0:submit_tool.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
- 'name': 'New collection',
- 'replication_desired': None,
+ 'name': 'New collection'
}, ensure_unique_name=True),
mock.call().execute(num_retries=4),
mock.call(body={
@@ -224,7 +223,6 @@ class TestSubmit(unittest.TestCase):
'0:0:blub.txt 0:0:submit_tool.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'New collection',
- 'replication_desired': None,
}, ensure_unique_name=True),
mock.call().execute(num_retries=4),
mock.call(body={
@@ -307,7 +305,7 @@ class TestCreateWorkflow(unittest.TestCase):
"name": "submit_wf.cwl",
"description": "",
"workflow": expect_workflow
- }
+ }
}
stubs.api.workflows().create.assert_called_with(
body=JsonDiffMatcher(body))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list