[ARVADOS] updated: 957068e7b647db0bf2188b2c9ed4bc673f0744f2

Git user git at public.curoverse.com
Tue Oct 18 11:13:59 EDT 2016


Summary of changes:
 sdk/cwl/arvados_cwl/arvcontainer.py |  7 +------
 sdk/cwl/arvados_cwl/arvdocker.py    |  9 +++++----
 sdk/cwl/arvados_cwl/arvjob.py       |  1 +
 sdk/cwl/tests/test_job.py           | 30 +++++++++++++++++++++---------
 sdk/cwl/tests/test_submit.py        | 17 +++++++++--------
 5 files changed, 37 insertions(+), 27 deletions(-)

       via  957068e7b647db0bf2188b2c9ed4bc673f0744f2 (commit)
      from  162766a3f8e91024f0f40fbc3d5955b899df73a9 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 957068e7b647db0bf2188b2c9ed4bc673f0744f2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Oct 18 11:13:19 2016 -0400

    10194: Update test

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 58b011f..56f29c5 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -179,11 +179,6 @@ class RunnerContainer(Runner):
         workflowcollection = workflowcollection[5:workflowcollection.index('/')]
         jobpath = "/var/lib/cwl/job/cwl.input.json"
 
-        container_image = arv_docker_get_image(self.arvrunner.api,
-                                               {"dockerImageId": arvados_jobs_image(self.arvrunner)},
-                                               pull_image,
-                                               self.arvrunner.project_uuid)
-
         command = ["arvados-cwl-runner", "--local", "--api=containers"]
         if self.output_name:
             command.append("--output-name=" + self.output_name)
@@ -197,7 +192,7 @@ class RunnerContainer(Runner):
             "cwd": "/var/spool/cwl",
             "priority": 1,
             "state": "Committed",
-            "container_image": container_image,
+            "container_image": arvados_jobs_image(self.arvrunner),
             "mounts": {
                 "/var/lib/cwl/workflow": {
                     "kind": "collection",
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index c75e783..3e5414f 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -33,11 +33,12 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
         except SystemExit:
             raise WorkflowException()
 
-    images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
-                                                            image_name=image_name,
-                                                            image_tag=image_tag)
+        images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+                                                                image_name=image_name,
+                                                                image_tag=image_tag)
 
-    #return dockerRequirement["dockerImageId"]
+    if not images:
+        raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
 
     pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
     return pdh
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index e4889b3..c8492e5 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -268,6 +268,7 @@ class RunnerJob(Runner):
             if isinstance(v, dict):
                 job_spec["script_parameters"][k] = {"value": v}
 
+        del job_spec["owner_uuid"]
         job_spec["job"] = job
         self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
             body={
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 8d12aab..2b82405 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -18,13 +18,17 @@ class TestJob(unittest.TestCase):
 
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
-    def test_run(self):
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+    def test_run(self, list_images_in_arv):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.num_retries = 0
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
+        list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+        runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
         tool = {
             "inputs": [],
             "outputs": [],
@@ -52,7 +56,7 @@ class TestJob(unittest.TestCase):
                     'repository': 'arvados',
                     'script': 'crunchrunner',
                     'runtime_constraints': {
-                        'docker_image': 'arvados/jobs',
+                        'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
                         'min_cores_per_node': 1,
                         'min_ram_mb_per_node': 1024,
                         'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
@@ -62,18 +66,22 @@ class TestJob(unittest.TestCase):
                 filters=[['repository', '=', 'arvados'],
                          ['script', '=', 'crunchrunner'],
                          ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                         ['docker_image_locator', 'in docker', 'arvados/jobs']]
+                         ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
             )
 
     # The test passes some fields in builder.resources
     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
-    def test_resource_requirements(self):
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+    def test_resource_requirements(self, list_images_in_arv):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.num_retries = 0
         arvados_cwl.add_arv_hints()
 
+        list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+        runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
 
@@ -116,7 +124,7 @@ class TestJob(unittest.TestCase):
                 'repository': 'arvados',
                 'script': 'crunchrunner',
                 'runtime_constraints': {
-                    'docker_image': 'arvados/jobs',
+                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
                     'min_cores_per_node': 3,
                     'min_ram_mb_per_node': 3000,
                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
@@ -127,7 +135,7 @@ class TestJob(unittest.TestCase):
             filters=[['repository', '=', 'arvados'],
                      ['script', '=', 'crunchrunner'],
                      ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                     ['docker_image_locator', 'in docker', 'arvados/jobs']])
+                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
 
     @mock.patch("arvados.collection.CollectionReader")
     def test_done(self, reader):
@@ -212,7 +220,8 @@ class TestWorkflow(unittest.TestCase):
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.collection.Collection")
-    def test_run(self, mockcollection):
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+    def test_run(self, list_images_in_arv, mockcollection):
         try:
             arvados_cwl.add_arv_hints()
 
@@ -222,6 +231,9 @@ class TestWorkflow(unittest.TestCase):
             runner.num_retries = 0
             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
+            list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+            runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
             tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
             metadata["cwlVersion"] = tool["cwlVersion"]
 
@@ -261,14 +273,14 @@ class TestWorkflow(unittest.TestCase):
                     'runtime_constraints': {
                         'min_scratch_mb_per_node': 2048,
                         'min_cores_per_node': 1,
-                        'docker_image': 'arvados/jobs',
+                        'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
                         'min_ram_mb_per_node': 1024
                     },
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
                 filters=[['repository', '=', 'arvados'],
                          ['script', '=', 'crunchrunner'],
                          ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
-                         ['docker_image_locator', 'in docker', 'arvados/jobs']],
+                         ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
                 find_or_create=True)
         except:
             logging.exception("")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index d3bdf8f..f47975d 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -89,7 +89,7 @@ def stubs(func):
         }
         stubs.expect_job_spec = {
             'runtime_constraints': {
-                'docker_image': 'arvados/jobs'
+                'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
             },
             'script_parameters': {
                 'x': {
@@ -115,7 +115,7 @@ def stubs(func):
                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
             },
             'repository': 'arvados',
-            'script_version': 'master',
+            'script_version': arvados_cwl.__version__,
             'script': 'cwl-runner'
         }
         stubs.pipeline_component = stubs.expect_job_spec.copy()
@@ -124,7 +124,7 @@ def stubs(func):
             'state': 'RunningOnServer',
             "components": {
                 "cwl-runner": {
-                    'runtime_constraints': {'docker_image': 'arvados/jobs'},
+                    'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
                     'script_parameters': {
                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
@@ -135,8 +135,9 @@ def stubs(func):
                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
                     },
                     'repository': 'arvados',
-                    'script_version': 'master',
-                    'script': 'cwl-runner'
+                    'script_version': arvados_cwl.__version__,
+                    'script': 'cwl-runner',
+                    'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
                 }
             }
         }
@@ -175,7 +176,7 @@ def stubs(func):
             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
             'name': 'submit_wf.cwl',
-            'container_image': '99999999999999999999999999999993+99',
+            'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
             'output_path': '/var/spool/cwl',
             'cwd': '/var/spool/cwl',
             'runtime_constraints': {
@@ -390,7 +391,7 @@ class TestTemplateInputs(unittest.TestCase):
         "components": {
             "inputs_test.cwl": {
                 'runtime_constraints': {
-                    'docker_image': 'arvados/jobs',
+                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
                 },
                 'script_parameters': {
                     'cwl:tool':
@@ -425,7 +426,7 @@ class TestTemplateInputs(unittest.TestCase):
                     },
                 },
                 'repository': 'arvados',
-                'script_version': 'master',
+                'script_version': arvados_cwl.__version__,
                 'script': 'cwl-runner',
             },
         },

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list