[ARVADOS] created: 1.1.4-177-g2074c17

Git user git at public.curoverse.com
Wed May 9 05:13:24 EDT 2018


        at  2074c175da359e754699baa2c9e9cc52468b1dd0 (commit)


commit 2074c175da359e754699baa2c9e9cc52468b1dd0
Author: Fuad Muhic <muhic.fuad at gmail.com>
Date:   Tue May 8 16:38:55 2018 +0200

    Added datetime import to arvjob.py
    
    Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index bde9700..eb6921c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -7,6 +7,7 @@ import re
 import copy
 import json
 import time
+import datetime
 
 from cwltool.process import get_feature, shortname, UnsupportedRequirement
 from cwltool.errors import WorkflowException
@@ -68,7 +69,7 @@ class ArvadosJob(object):
                 if vwd:
                     trash_time = None
                     if self.arvrunner.intermediate_output_ttl > 0:
-                        t = datetime.now() + timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                        t = datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
                         trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
 
                     current_container_uuid = None

commit f5ba876c9e637afcd90e4754346694bcea234fc7
Author: Fuad Muhic <muhic.fuad at gmail.com>
Date:   Tue May 8 16:31:42 2018 +0200

    Updated name, trash_at and properties field of intermediate collections.
    
    Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index afcf2db..72efb60 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -5,14 +5,16 @@
 import logging
 import json
 import os
+import datetime
 import urllib
 import time
-import datetime
 import ciso8601
 import uuid
 
+
 import ruamel.yaml as yaml
 
+from arvados.errors import ApiError
 from cwltool.errors import WorkflowException
 from cwltool.process import get_feature, UnsupportedRequirement, shortname
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
@@ -154,8 +156,33 @@ class ArvadosContainer(object):
 
                 keepemptydirs(vwd)
 
+                trash_time = None
+                if self.arvrunner.intermediate_output_ttl > 0:
+                    t = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                    trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                current_container_uuid = None
+                try:
+                    current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                    current_container_uuid = current_container['uuid']
+                except ApiError as e:
+                    # Status code 404 just means we're not running in a container.
+                    if e.resp.status != 404:
+                        logger.info("Getting current container: %s", e)
+                props = {"type": "Intermediate",
+                         "container": current_container_uuid}
+
                 with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                    vwd.save_new()
+                    vwd.save_new(name="Intermediate collection",
+                                 ensure_unique_name=True,
+                                 trash_at=trash_time,
+                                 properties=props)
+
+                logger.debug("Intermediate: arvcontainer")
+                logger.debug("Intermediate: uuid = %s" % vwd._api_response["uuid"])
+                logger.debug("Intermediate: name = %s" % vwd._api_response["name"])
+                logger.debug("Intermediate: trash_at = %s" % vwd._api_response["trash_at"])
+                logger.debug("Intermediate: props = " + str(vwd._api_response["properties"]))
 
                 prev = None
                 for f, p in sorteditems:
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index decd692..bde9700 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -66,8 +66,34 @@ class ArvadosJob(object):
                                 n.write(p.resolved.encode("utf-8"))
 
                 if vwd:
+                    trash_time = None
+                    if self.arvrunner.intermediate_output_ttl > 0:
+                        t = datetime.now() + timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                        trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                    current_container_uuid = None
+                    try:
+                        current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                        current_container_uuid = current_container['uuid']
+                    except ApiError as e:
+                        # Status code 404 just means we're not running in a container.
+                        if e.resp.status != 404:
+                            logger.info("Getting current container: %s", e)
+
+                    props = {"type": "Intermediate",
+                             "container": current_container_uuid}
+
                     with Perf(metrics, "generatefiles.save_new %s" % self.name):
-                        vwd.save_new()
+                        vwd.save_new(name="Intermediate collection",
+                                     ensure_unique_name=True,
+                                     trash_at=trash_time,
+                                     properties=props)
+
+                    logger.debug("Intermediate: arvjob")
+                    logger.debug("Intermediate: uuid = %s" % vwd._api_response["uuid"])
+                    logger.debug("Intermediate: name = %s" % vwd._api_response["name"])
+                    logger.debug("Intermediate: trash_at = %s" % vwd._api_response["trash_at"])
+                    logger.debug("Intermediate: props = " + str(vwd._api_response["properties"]))
 
                 for f, p in generatemapper.items():
                     if p.type == "File":
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 6fedb12..d786b9d 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -6,11 +6,14 @@ import re
 import logging
 import uuid
 import os
+import datetime
 import urllib
 
+
 import arvados.commands.run
 import arvados.collection
 
+from arvados.errors import ApiError
 from schema_salad.sourceline import SourceLine
 
 from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
@@ -164,9 +167,33 @@ class ArvPathMapper(PathMapper):
                 for l in srcobj.get("listing", []):
                     self.addentry(l, c, ".", remap)
 
-                check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
-                if not check["items"]:
-                    c.save_new(owner_uuid=self.arvrunner.project_uuid)
+                trash_time = None
+                if self.arvrunner.intermediate_output_ttl > 0:
+                    t = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                    trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                current_container_uuid = None
+                try:
+                    current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                    current_container_uuid = current_container['uuid']
+                except ApiError as e:
+                    # Status code 404 just means we're not running in a container.
+                    if e.resp.status != 404:
+                        logger.info("Getting current container: %s", e)
+
+                props = {"type": "Intermediate",
+                         "container": current_container_uuid}
+                c.save_new(name="Intermediate collection",
+                           owner_uuid=self.arvrunner.project_uuid,
+                           ensure_unique_name=True,
+                           trash_at=trash_time,
+                           properties=props)
+
+                logger.debug("Intermediate: pathmapper")
+                logger.debug("Intermediate: uuid = %s" % c._api_response["uuid"])
+                logger.debug("Intermediate: name = %s" % c._api_response["name"])
+                logger.debug("Intermediate: trash_at = %s" % c._api_response["trash_at"])
+                logger.debug("Intermediate: props = " + str(c._api_response["properties"]))
 
                 ab = self.collection_pattern % c.portable_data_hash()
                 self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
@@ -178,9 +205,33 @@ class ArvPathMapper(PathMapper):
                                                   num_retries=self.arvrunner.num_retries                                                  )
                 self.addentry(srcobj, c, ".", remap)
 
-                check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
-                if not check["items"]:
-                    c.save_new(owner_uuid=self.arvrunner.project_uuid)
+                trash_time = None
+                if self.arvrunner.intermediate_output_ttl > 0:
+                    t = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+                    trash_time = t.strftime("%Y-%m-%dT%H:%M:%S.%f000Z")
+
+                current_container_uuid = None
+                try:
+                    current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+                    current_container_uuid = current_container['uuid']
+                except ApiError as e:
+                    # Status code 404 just means we're not running in a container.
+                    if e.resp.status != 404:
+                        logger.info("Getting current container: %s", e)
+
+                props = {"type": "Intermediate",
+                         "container": current_container_uuid}
+                c.save_new(name="Intermediate collection",
+                           owner_uuid=self.arvrunner.project_uuid,
+                           ensure_unique_name=True,
+                           trash_at=trash_time,
+                           properties=props)
+
+                logger.debug("Intermediate: pathmapper")
+                logger.debug("Intermediate: uuid = %s" % c._api_response["uuid"])
+                logger.debug("Intermediate: name = %s" % c._api_response["name"])
+                logger.debug("Intermediate: trash_at = %s" % c._api_response["trash_at"])
+                logger.debug("Intermediate: props = " + str(c._api_response["properties"]))
 
                 ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
                 self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list