[ARVADOS] updated: 957068e7b647db0bf2188b2c9ed4bc673f0744f2
Git user
git at public.curoverse.com
Tue Oct 18 11:13:59 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 7 +------
sdk/cwl/arvados_cwl/arvdocker.py | 9 +++++----
sdk/cwl/arvados_cwl/arvjob.py | 1 +
sdk/cwl/tests/test_job.py | 30 +++++++++++++++++++++---------
sdk/cwl/tests/test_submit.py | 17 +++++++++--------
5 files changed, 37 insertions(+), 27 deletions(-)
via 957068e7b647db0bf2188b2c9ed4bc673f0744f2 (commit)
from 162766a3f8e91024f0f40fbc3d5955b899df73a9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 957068e7b647db0bf2188b2c9ed4bc673f0744f2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Oct 18 11:13:19 2016 -0400
10194: Update test
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 58b011f..56f29c5 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -179,11 +179,6 @@ class RunnerContainer(Runner):
workflowcollection = workflowcollection[5:workflowcollection.index('/')]
jobpath = "/var/lib/cwl/job/cwl.input.json"
- container_image = arv_docker_get_image(self.arvrunner.api,
- {"dockerImageId": arvados_jobs_image(self.arvrunner)},
- pull_image,
- self.arvrunner.project_uuid)
-
command = ["arvados-cwl-runner", "--local", "--api=containers"]
if self.output_name:
command.append("--output-name=" + self.output_name)
@@ -197,7 +192,7 @@ class RunnerContainer(Runner):
"cwd": "/var/spool/cwl",
"priority": 1,
"state": "Committed",
- "container_image": container_image,
+ "container_image": arvados_jobs_image(self.arvrunner),
"mounts": {
"/var/lib/cwl/workflow": {
"kind": "collection",
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index c75e783..3e5414f 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -33,11 +33,12 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
except SystemExit:
raise WorkflowException()
- images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
- image_name=image_name,
- image_tag=image_tag)
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+ image_name=image_name,
+ image_tag=image_tag)
- #return dockerRequirement["dockerImageId"]
+ if not images:
+ raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
return pdh
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index e4889b3..c8492e5 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -268,6 +268,7 @@ class RunnerJob(Runner):
if isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
+ del job_spec["owner_uuid"]
job_spec["job"] = job
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
body={
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 8d12aab..2b82405 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -18,13 +18,17 @@ class TestJob(unittest.TestCase):
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
- def test_run(self):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_run(self, list_images_in_arv):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
tool = {
"inputs": [],
"outputs": [],
@@ -52,7 +56,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_cores_per_node': 1,
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
@@ -62,18 +66,22 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']]
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
)
# The test passes some fields in builder.resources
# For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
- def test_resource_requirements(self):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_resource_requirements(self, list_images_in_arv):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
arvados_cwl.add_arv_hints()
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
@@ -116,7 +124,7 @@ class TestJob(unittest.TestCase):
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_cores_per_node': 3,
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
@@ -127,7 +135,7 @@ class TestJob(unittest.TestCase):
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']])
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
@mock.patch("arvados.collection.CollectionReader")
def test_done(self, reader):
@@ -212,7 +220,8 @@ class TestWorkflow(unittest.TestCase):
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.Collection")
- def test_run(self, mockcollection):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_run(self, list_images_in_arv, mockcollection):
try:
arvados_cwl.add_arv_hints()
@@ -222,6 +231,9 @@ class TestWorkflow(unittest.TestCase):
runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
@@ -261,14 +273,14 @@ class TestWorkflow(unittest.TestCase):
'runtime_constraints': {
'min_scratch_mb_per_node': 2048,
'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_ram_mb_per_node': 1024
},
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']],
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
find_or_create=True)
except:
logging.exception("")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d3bdf8f..f47975d 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -89,7 +89,7 @@ def stubs(func):
}
stubs.expect_job_spec = {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
},
'script_parameters': {
'x': {
@@ -115,7 +115,7 @@ def stubs(func):
'99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
- 'script_version': 'master',
+ 'script_version': arvados_cwl.__version__,
'script': 'cwl-runner'
}
stubs.pipeline_component = stubs.expect_job_spec.copy()
@@ -124,7 +124,7 @@ def stubs(func):
'state': 'RunningOnServer',
"components": {
"cwl-runner": {
- 'runtime_constraints': {'docker_image': 'arvados/jobs'},
+ 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
'script_parameters': {
'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
@@ -135,8 +135,9 @@ def stubs(func):
'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
+ 'script_version': arvados_cwl.__version__,
+ 'script': 'cwl-runner',
+ 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
}
}
}
@@ -175,7 +176,7 @@ def stubs(func):
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
'name': 'submit_wf.cwl',
- 'container_image': '99999999999999999999999999999993+99',
+ 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
'output_path': '/var/spool/cwl',
'cwd': '/var/spool/cwl',
'runtime_constraints': {
@@ -390,7 +391,7 @@ class TestTemplateInputs(unittest.TestCase):
"components": {
"inputs_test.cwl": {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
},
'script_parameters': {
'cwl:tool':
@@ -425,7 +426,7 @@ class TestTemplateInputs(unittest.TestCase):
},
},
'repository': 'arvados',
- 'script_version': 'master',
+ 'script_version': arvados_cwl.__version__,
'script': 'cwl-runner',
},
},
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list