[ARVADOS] created: 92c876d90253a0db23d40aa125cba335f7bf7e27

Git user git at public.curoverse.com
Thu Mar 31 10:21:22 EDT 2016


        at  92c876d90253a0db23d40aa125cba335f7bf7e27 (commit)


commit 92c876d90253a0db23d40aa125cba335f7bf7e27
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Mar 31 10:21:17 2016 -0400

    8857: Add --ignore-docker-for-reuse option to assist workflow development.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e3fd1fc..339b91c 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -157,16 +157,26 @@ class ArvadosJob(object):
             runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
             runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
 
+        filters = [["repository", "=", "arvados"],
+                   ["script", "=", "crunchrunner"],
+                   ["script_version", "in git", "9e5b98e8f5f4727856b53447191f9c06e3da2ba6"]]
+        if not self.arvrunner.ignore_docker_for_reuse:
+            filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]])
+
         try:
-            response = self.arvrunner.api.jobs().create(body={
-                "owner_uuid": self.arvrunner.project_uuid,
-                "script": "crunchrunner",
-                "repository": "arvados",
-                "script_version": "master",
-                "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6",
-                "script_parameters": {"tasks": [script_parameters]},
-                "runtime_constraints": runtime_constraints
-            }, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
+            response = self.arvrunner.api.jobs().create(
+                body={
+                    "owner_uuid": self.arvrunner.project_uuid,
+                    "script": "crunchrunner",
+                    "repository": "arvados",
+                    "script_version": "master",
+                    "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6",
+                    "script_parameters": {"tasks": [script_parameters]},
+                    "runtime_constraints": runtime_constraints
+                },
+                filters=filters,
+                find_or_create=kwargs.get("enable_reuse", True)
+            ).execute(num_retries=self.arvrunner.num_retries)
 
             self.arvrunner.jobs[response["uuid"]] = self
 
@@ -396,6 +406,7 @@ class ArvCwlRunner(object):
         events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
 
         self.debug = args.debug
+        self.ignore_docker_for_reuse = args.ignore_docker_for_reuse
         self.fs_access = CollectionFsAccess(input_basedir)
 
         kwargs["fs_access"] = self.fs_access
@@ -474,6 +485,9 @@ def main(args, stdout, stderr, api_client=None):
                         default=True, dest="enable_reuse",
                         help="")
     parser.add_argument("--project-uuid", type=str, help="Project that will own the workflow jobs")
+    parser.add_argument("--ignore-docker-for-reuse", action="store_true",
+                        help="Ignore Docker image version when deciding whether to reuse past jobs.",
+                        default=False)
 
     try:
         runner = ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index cb306ed..bb15e60 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -9,6 +9,8 @@ class TestJob(unittest.TestCase):
     def test_run(self):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+        runner.ignore_docker_for_reuse = False
+
         tool = {
             "inputs": [],
             "outputs": [],
@@ -18,32 +20,41 @@ class TestJob(unittest.TestCase):
         arvtool.formatgraph = None
         for j in arvtool.job({}, "", mock.MagicMock()):
             j.run()
-        runner.api.jobs().create.assert_called_with(body={
-            'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-            'runtime_constraints': {},
-            'script_parameters': {
-                'tasks': [{
-                    'task.env': {'TMPDIR': '$(task.tmpdir)'},
-                    'command': ['ls']
-                }]
+            runner.api.jobs().create.assert_called_with(
+                body={
+                    'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                    'runtime_constraints': {},
+                    'script_parameters': {
+                        'tasks': [{
+                            'task.env': {'TMPDIR': '$(task.tmpdir)'},
+                            'command': ['ls']
+                        }]
             },
             'script_version': 'master',
-            'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
-            'repository': 'arvados',
-            'script': 'crunchrunner',
-            'runtime_constraints': {
-                'docker_image': 'arvados/jobs',
-                'min_cores_per_node': 1,
-                'min_ram_mb_per_node': 1024,
-                'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
-            }
-        }, find_or_create=True)
+                    'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+                    'repository': 'arvados',
+                    'script': 'crunchrunner',
+                    'runtime_constraints': {
+                        'docker_image': 'arvados/jobs',
+                        'min_cores_per_node': 1,
+                        'min_ram_mb_per_node': 1024,
+                        'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
+                    }
+        },
+                                                    find_or_create=True,
+                filters=[['repository', '=', 'arvados'],
+                         ['script', '=', 'crunchrunner'],
+                         ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
+                         ['docker_image_locator', 'in docker', 'arvados/jobs']]
+            )
 
     # The test passes some fields in builder.resources
     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     def test_resource_requirements(self):
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+        runner.ignore_docker_for_reuse = False
+
         tool = {
             "inputs": [],
             "outputs": [],
@@ -59,26 +70,32 @@ class TestJob(unittest.TestCase):
         arvtool.formatgraph = None
         for j in arvtool.job({}, "", mock.MagicMock()):
             j.run()
-        runner.api.jobs().create.assert_called_with(body={
-            'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-            'runtime_constraints': {},
-            'script_parameters': {
-                'tasks': [{
-                    'task.env': {'TMPDIR': '$(task.tmpdir)'},
-                    'command': ['ls']
-                }]
+        runner.api.jobs().create.assert_called_with(
+            body={
+                'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                'runtime_constraints': {},
+                'script_parameters': {
+                    'tasks': [{
+                        'task.env': {'TMPDIR': '$(task.tmpdir)'},
+                        'command': ['ls']
+                    }]
             },
             'script_version': 'master',
-            'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
-            'repository': 'arvados',
-            'script': 'crunchrunner',
-            'runtime_constraints': {
-                'docker_image': 'arvados/jobs',
-                'min_cores_per_node': 3,
-                'min_ram_mb_per_node': 3000,
-                'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
-            }
-        }, find_or_create=True)
+                'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+                'repository': 'arvados',
+                'script': 'crunchrunner',
+                'runtime_constraints': {
+                    'docker_image': 'arvados/jobs',
+                    'min_cores_per_node': 3,
+                    'min_ram_mb_per_node': 3000,
+                    'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
+                }
+        },
+            find_or_create=True,
+            filters=[['repository', '=', 'arvados'],
+                     ['script', '=', 'crunchrunner'],
+                     ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
+                     ['docker_image_locator', 'in docker', 'arvados/jobs']])
 
     @mock.patch("arvados.collection.Collection")
     def test_done(self, col):
@@ -88,6 +105,7 @@ class TestJob(unittest.TestCase):
         runner.api = api
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
+        runner.ignore_docker_for_reuse = False
 
         col().open.return_value = []
         api.collections().list().execute.side_effect = ({"items": []},

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list