[ARVADOS] updated: 670b4ba238d93910ff087794d359da0d5ac469fa
Git user
git at public.curoverse.com
Fri Oct 21 10:16:05 EDT 2016
Summary of changes:
.../test-package-python-arvados-cwl-runner.sh | 1 +
.../test-package-python-arvados-fuse.sh | 1 +
.../test-package-python-arvados-python-client.sh | 1 +
...t-package-python27-python-arvados-cwl-runner.sh | 6 ++++
...ackage-python27-python-arvados-python-client.sh | 2 +-
build/run-build-packages.sh | 4 +--
sdk/cwl/arvados_cwl/__init__.py | 3 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 11 ++-----
sdk/cwl/arvados_cwl/arvdocker.py | 14 ++++++---
sdk/cwl/arvados_cwl/arvjob.py | 28 ++++++++---------
sdk/cwl/arvados_cwl/runner.py | 8 +++++
sdk/cwl/setup.py | 15 +++++++++
sdk/cwl/tests/test_job.py | 36 ++++++++++++++--------
sdk/cwl/tests/test_submit.py | 17 +++++-----
14 files changed, 96 insertions(+), 51 deletions(-)
create mode 120000 build/package-testing/test-package-python-arvados-cwl-runner.sh
create mode 120000 build/package-testing/test-package-python-arvados-fuse.sh
create mode 120000 build/package-testing/test-package-python-arvados-python-client.sh
create mode 100755 build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
via 670b4ba238d93910ff087794d359da0d5ac469fa (commit)
via e8a76617eeec9429ddf7879048a310ba6e15e0d8 (commit)
via a70a555fbd30524568a0e019df184468f28556f0 (commit)
via e8e45c5267f216909382aad7fc976dca82008284 (commit)
via cde03921959a7edbac9f6e969b3b8b99ee3c5b86 (commit)
via 957068e7b647db0bf2188b2c9ed4bc673f0744f2 (commit)
via 162766a3f8e91024f0f40fbc3d5955b899df73a9 (commit)
via f99aa5b85a1aa3bff1f1140f6a109fd308b03d30 (commit)
via e144534106f2031b4a9bec0d80eebf094e33a285 (commit)
from 91dc5f1d7f5ad9eb2640f6089e2d0476cbf87c8e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 670b4ba238d93910ff087794d359da0d5ac469fa
Merge: 91dc5f1 e8a7661
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Oct 21 10:15:56 2016 -0400
Merge branch '10194-cwl-version-skew' closes #10194
commit e8a76617eeec9429ddf7879048a310ba6e15e0d8
Merge: a70a555 91dc5f1
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Oct 21 10:15:10 2016 -0400
Merge branch 'master' into 10194-cwl-version-skew
Conflicts:
sdk/cwl/tests/test_job.py
diff --cc sdk/cwl/arvados_cwl/arvjob.py
index c8492e5,a974531..8228387
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@@ -257,19 -256,10 +257,19 @@@ class RunnerJob(Runner)
def run(self, *args, **kwargs):
job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+ job = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
+
for k,v in job_spec["script_parameters"].items():
- if isinstance(v, dict):
+ if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
+ del job_spec["owner_uuid"]
+ job_spec["job"] = job
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
body={
"owner_uuid": self.arvrunner.project_uuid,
diff --cc sdk/cwl/tests/test_job.py
index 2b82405,7abc567..93b5d39
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@@ -220,67 -213,73 +221,77 @@@ class TestWorkflow(unittest.TestCase)
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.Collection")
- def test_run(self, mockcollection):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_run(self, list_images_in_arv, mockcollection):
- try:
- arvados_cwl.add_arv_hints()
+ arvados_cwl.add_arv_hints()
- runner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
- runner.ignore_docker_for_reuse = False
- runner.num_retries = 0
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ api = mock.MagicMock()
+ api._rootDesc = arvados.api('v1')._rootDesc
+ runner = arvados_cwl.ArvCwlRunner(api)
+ self.assertEqual(runner.work_api, 'jobs')
- list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
- runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
++ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
++ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
- tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
- metadata["cwlVersion"] = tool["cwlVersion"]
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.num_retries = 0
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+ tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
+ metadata["cwlVersion"] = tool["cwlVersion"]
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
- arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
- basedir="", make_fs_access=make_fs_access, loader=document_loader,
- makeTool=runner.arv_make_tool, metadata=metadata)
- arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
- it.next().run()
- it.next().run()
+ mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
- with open("tests/wf/scatter2_subwf.cwl") as f:
- subwf = f.read()
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
+ basedir="", make_fs_access=make_fs_access, loader=document_loader,
+ makeTool=runner.arv_make_tool, metadata=metadata)
+ arvtool.formatgraph = None
+ it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
+ it.next().run()
+ it.next().run()
- mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+ with open("tests/wf/scatter2_subwf.cwl") as f:
+ subwf = f.read()
- runner.api.jobs().create.assert_called_with(
- body={
- 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
- 'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'crunchrunner',
- 'script_parameters': {
- 'tasks': [{'task.env': {
- 'HOME': '$(task.outdir)',
- 'TMPDIR': '$(task.tmpdir)'},
- 'task.vwd': {
- 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
- 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
- },
- 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
- 'task.stdout': 'cwl.output.json'}]},
- 'runtime_constraints': {
- 'min_scratch_mb_per_node': 2048,
- 'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
- 'min_ram_mb_per_node': 1024
- },
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
- filters=[['repository', '=', 'arvados'],
- ['script', '=', 'crunchrunner'],
- ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
- find_or_create=True)
- except:
- logging.exception("")
- mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
-
+ runner.api.jobs().create.assert_called_with(
+ body={
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script_version': 'master',
+ 'script': 'crunchrunner',
+ 'script_parameters': {
+ 'tasks': [{'task.env': {
+ 'HOME': '$(task.outdir)',
+ 'TMPDIR': '$(task.tmpdir)'},
+ 'task.vwd': {
+ 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
+ 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
+ },
+ 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
+ 'task.stdout': 'cwl.output.json'}]},
+ 'runtime_constraints': {
+ 'min_scratch_mb_per_node': 2048,
+ 'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs',
++ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'min_ram_mb_per_node': 1024
+ },
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
+ filters=[['repository', '=', 'arvados'],
+ ['script', '=', 'crunchrunner'],
+ ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']],
++ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+ find_or_create=True)
+
++ mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
++ mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
++
+ def test_default_work_api(self):
+ arvados_cwl.add_arv_hints()
+
+ api = mock.MagicMock()
+ api._rootDesc = arvados.api('v1')._rootDesc
+ del api._rootDesc.get('resources')['jobs']['methods']['create']
+ runner = arvados_cwl.ArvCwlRunner(api)
+ self.assertEqual(runner.work_api, 'containers')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list