[ARVADOS] updated: 79b826befbc0b62a822e2d0c922e5c4ea0a3bb7c

Git user git at public.curoverse.com
Fri May 26 10:55:27 EDT 2017


Summary of changes:
 doc/user/cwl/cwl-runner.html.textile.liquid |  2 +-
 sdk/cwl/arvados_cwl/__init__.py             | 10 ++++++++--
 sdk/cwl/arvados_cwl/arvcontainer.py         | 19 +++++++++++++++++--
 sdk/cwl/tests/test_submit.py                | 23 +++++++++++++++++++++++
 4 files changed, 49 insertions(+), 5 deletions(-)

       via  79b826befbc0b62a822e2d0c922e5c4ea0a3bb7c (commit)
      from  bee4fa82c15b2188c2105fff3b52c305b38f04a6 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 79b826befbc0b62a822e2d0c922e5c4ea0a3bb7c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri May 26 10:42:47 2017 -0400

    11100: Add test for --trash-intermediate.  Add log message when intermediate
    outputs are scheduled for trash.

diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid
index 7c75c1c..f62d9e3 100644
--- a/doc/user/cwl/cwl-runner.html.textile.liquid
+++ b/doc/user/cwl/cwl-runner.html.textile.liquid
@@ -126,7 +126,7 @@ By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If yo
 
 h3. Check Docker access
 
-In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Aravdos.
+In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Arvados.
 
 You can determine if you have access to Docker by running @docker version@:
 
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index be1ec27..fbe91b9 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -346,10 +346,16 @@ class ArvCwlRunner(object):
                                                                  collection_cache=self.collection_cache)
         self.fs_access = make_fs_access(kwargs["basedir"])
 
-        self.intermediate_output_ttl = kwargs["intermediate_output_ttl"]
+
         self.trash_intermediate = kwargs["trash_intermediate"]
+        if self.trash_intermediate and self.work_api != "containers":
+            raise Exception("--trash-intermediate is only supported with --api=containers.")
+
+        self.intermediate_output_ttl = kwargs["intermediate_output_ttl"]
         if self.intermediate_output_ttl and self.work_api != "containers":
-            raise Exception("--intermediate-output-ttl is only supported when using the containers api.")
+            raise Exception("--intermediate-output-ttl is only supported with --api=containers.")
+        if self.intermediate_output_ttl < 0:
+            raise Exception("Invalid value %d for --intermediate-output-ttl, cannot be less than zero" % self.intermediate_output_ttl)
 
         if not kwargs.get("name"):
             kwargs["name"] = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index dc2d02f..b58a858 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -2,6 +2,9 @@ import logging
 import json
 import os
 import urllib
+import time
+import datetime
+import ciso8601
 
 import ruamel.yaml as yaml
 
@@ -211,8 +214,6 @@ class ArvadosContainer(object):
     def done(self, record):
         outputs = {}
         try:
-            self.arvrunner.add_intermediate_output(record["output_uuid"])
-
             container = self.arvrunner.api.containers().get(
                 uuid=record["container_uuid"]
             ).execute(num_retries=self.arvrunner.num_retries)
@@ -238,6 +239,17 @@ class ArvadosContainer(object):
                                                            num_retries=self.arvrunner.num_retries)
                 done.logtail(logc, logger, "%s error log:" % self.arvrunner.label(self))
 
+            if record["output_uuid"]:
+                if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
+                    # Compute the trash time to avoid requesting the collection record.
+                    trash_at = ciso8601.parse_datetime_unaware(record["modified_at"]) + datetime.timedelta(0, self.arvrunner.intermediate_output_ttl)
+                    aftertime = " at %s" % trash_at.strftime("%Y-%m-%d %H:%M:%S UTC") if self.arvrunner.intermediate_output_ttl else ""
+                    orpart = ", or" if self.arvrunner.trash_intermediate and self.arvrunner.intermediate_output_ttl else ""
+                    oncomplete = " upon successful completion of the workflow" if self.arvrunner.trash_intermediate else ""
+                    logger.info("%s Intermediate output %s (%s) will be trashed%s%s%s." % (
+                        self.arvrunner.label(self), record["output_uuid"], container["output"], aftertime, orpart, oncomplete))
+                self.arvrunner.add_intermediate_output(record["output_uuid"])
+
             if container["output"]:
                 outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
         except WorkflowException as e:
@@ -338,6 +350,9 @@ class RunnerContainer(Runner):
         if self.intermediate_output_ttl:
             command.append("--intermediate-output-ttl=%d" % self.intermediate_output_ttl)
 
+        if self.arvrunner.trash_intermediate:
+            command.append("--trash-intermediate")
+
         command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
 
         container_req["command"] = command
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 731b1d5..112f432 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -545,6 +545,29 @@ class TestSubmit(unittest.TestCase):
                          stubs.expect_container_request_uuid + '\n')
 
     @stubs
+    def test_submit_container_trash_intermediate(self, stubs):
+        capture_stdout = cStringIO.StringIO()
+        try:
+            exited = arvados_cwl.main(
+                ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
+                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            self.assertEqual(exited, 0)
+        except:
+            logging.exception("")
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+                                       '--enable-reuse', '--on-error=continue',
+                                       "--trash-intermediate",
+                                       '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+
+    @stubs
     def test_submit_container_output_tags(self, stubs):
         output_tags = "tag0,tag1,tag2"
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list