[ARVADOS] updated: 1.1.4-432-g4a78e8e91
Git user
git at public.curoverse.com
Mon Jul 2 12:40:26 EDT 2018
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 45 ++++++++++++++++++++-----------------
sdk/cwl/arvados_cwl/arvjob.py | 42 +++++++++++++++++++---------------
sdk/cwl/arvados_cwl/pathmapper.py | 24 +++++++++++---------
sdk/cwl/tests/test_container.py | 20 +++++++++++++++++
sdk/cwl/tests/test_job.py | 22 ++++++++++++++++++
sdk/cwl/tests/test_pathmapper.py | 24 ++++++++++++++++++++
6 files changed, 128 insertions(+), 49 deletions(-)
via 4a78e8e91fdad38e567fef0cd43aa8cb6bd33580 (commit)
from 02763280f54fd0c2a499285f8ce6afcbd8b9e082 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4a78e8e91fdad38e567fef0cd43aa8cb6bd33580
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Mon Jul 2 18:38:19 2018 +0200
Add tests for intermediate collections
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1ad276e67..014e8e58c 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -157,26 +157,11 @@ class ArvadosContainer(object):
keepemptydirs(vwd)
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
- logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
- "container": current_container_uuid}
-
- with Perf(metrics, "generatefiles.save_new %s" % self.name):
- vwd.save_new(name="Intermediate collection",
- ensure_unique_name=True,
- trash_at=trash_time,
- properties=props)
+ info = self._get_intermediate_collection_info()
+ vwd.save_new(name=info["name"],
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
+ properties=info["properties"])
prev = None
for f, p in sorteditems:
@@ -344,6 +329,26 @@ class ArvadosContainer(object):
finally:
self.output_callback(outputs, processStatus)
+ def _get_intermediate_collection_info(self):
+ trash_time = None
+ if self.arvrunner.intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+ current_container_uuid = None
+ try:
+ current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+ current_container_uuid = current_container['uuid']
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
+ logger.info("Getting current container: %s", e)
+ props = {"type": "Intermediate",
+ "container": current_container_uuid}
+
+ return {"name" : "Intermediate collection",
+ "trash_at" : trash_time,
+ "properties" : props}
+
class RunnerContainer(Runner):
"""Submit and manage a container that runs arvados-cwl-runner."""
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 2d98a53b3..ea599ea83 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -67,26 +67,12 @@ class ArvadosJob(object):
n.write(p.resolved.encode("utf-8"))
if vwd:
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
- logger.info("Getting current container: %s", e)
-
- props = {"type": "Intermediate",
- "container": current_container_uuid}
with Perf(metrics, "generatefiles.save_new %s" % self.name):
- vwd.save_new(name="Intermediate collection",
+ info = self._get_intermediate_collection_info()
+ vwd.save_new(name=info["name"],
ensure_unique_name=True,
- trash_at=trash_time,
- properties=props)
+ trash_at=info["trash_at"],
+ properties=info["properties"])
for f, p in generatemapper.items():
if p.type == "File":
@@ -284,6 +270,26 @@ class ArvadosJob(object):
finally:
self.output_callback(outputs, processStatus)
+ def _get_intermediate_collection_info(self):
+ trash_time = None
+ if self.arvrunner.intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+ current_container_uuid = None
+ try:
+ current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+ current_container_uuid = current_container['uuid']
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
+ logger.info("Getting current container: %s", e)
+ props = {"type": "Intermediate",
+ "container": current_container_uuid}
+
+ return {"name" : "Intermediate collection",
+ "trash_at" : trash_time,
+ "properties" : props}
+
class RunnerJob(Runner):
"""Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index a749b5162..b54fcdcc7 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -155,13 +155,13 @@ class ArvPathMapper(PathMapper):
for l in srcobj.get("listing", []):
self.addentry(l, c, ".", remap)
- trash_time, props = self.__get_collection_attributes()
+ info = self._get_intermediate_collection_info()
- c.save_new(name="Intermediate collection",
+ c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
- trash_at=trash_time,
- properties=props)
+ trash_at=info["trash_at"],
+ properties=info["properties"])
ab = self.collection_pattern % c.portable_data_hash()
self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
@@ -173,13 +173,13 @@ class ArvPathMapper(PathMapper):
num_retries=self.arvrunner.num_retries )
self.addentry(srcobj, c, ".", remap)
- trash_time, props = self.__get_collection_attributes()
+ info = self._get_intermediate_collection_info()
- c.save_new(name="Intermediate collection",
+ c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
- trash_at=trash_time,
- properties=props)
+ trash_at=info["trash_at"],
+ properties=info["properties"])
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),
@@ -212,7 +212,7 @@ class ArvPathMapper(PathMapper):
else:
return None
- def __get_collection_attributes(self):
+ def _get_intermediate_collection_info(self):
trash_time = None
if self.arvrunner.intermediate_output_ttl > 0:
trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
@@ -225,10 +225,12 @@ class ArvPathMapper(PathMapper):
# Status code 404 just means we're not running in a container.
if e.resp.status != 404:
logger.info("Getting current container: %s", e)
- properties = {"type": "Intermediate",
+ props = {"type": "Intermediate",
"container": current_container_uuid}
- return (trash_time, properties)
+ return {"name" : "Intermediate collection",
+ "trash_at" : trash_time,
+ "properties" : props}
class StagingPathMapper(PathMapper):
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2295e934a..a8a91efd3 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -8,6 +8,7 @@ import logging
import mock
import unittest
import os
+import datetime
import functools
import cwltool.process
import cwltool.secrets
@@ -20,6 +21,12 @@ if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def now(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
class TestContainer(unittest.TestCase):
@@ -633,3 +640,16 @@ class TestContainer(unittest.TestCase):
}
}
}))
+
+ def test_get_intermediate_collection_info(self):
+ arvrunner = mock.MagicMock()
+ arvrunner.intermediate_output_ttl = 60
+ arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ container = arvados_cwl.ArvadosContainer(arvrunner)
+
+ info = container._get_intermediate_collection_info()
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+ self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 30930dd49..1841fd3f8 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -10,6 +10,7 @@ import os
import unittest
import copy
import StringIO
+import datetime
import arvados
import arvados_cwl
@@ -24,6 +25,13 @@ if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def now(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
class TestJob(unittest.TestCase):
# The test passes no builder.resources
@@ -307,6 +315,20 @@ class TestJob(unittest.TestCase):
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+ def test_get_intermediate_collection_info(self):
+ arvrunner = mock.MagicMock()
+ arvrunner.intermediate_output_ttl = 60
+ arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ job = arvados_cwl.ArvadosJob(arvrunner)
+
+ info = job._get_intermediate_collection_info()
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+ self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+
class TestWorkflow(unittest.TestCase):
# The test passes no builder.resources
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index eaa571142..e5326147d 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -9,6 +9,7 @@ import unittest
import json
import logging
import os
+import datetime
import arvados
import arvados.keep
@@ -26,6 +27,13 @@ def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPatter
c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
c.fn = fnPattern % (pdh, os.path.basename(c.fn))
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def now(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
class TestPathmap(unittest.TestCase):
def setUp(self):
self.api = mock.MagicMock()
@@ -101,3 +109,19 @@ class TestPathmap(unittest.TestCase):
"class": "File",
"location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
+
+ def test_get_intermediate_collection_info(self):
+ self.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+ arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner.intermediate_output_ttl = 60
+
+ path_mapper = ArvPathMapper(arvrunner, [{
+ "class": "File",
+ "location": "keep:99999999999999999999999999999991+99/hw.py"
+ }], "", "/test/%s", "/test/%s/%s")
+
+ info = path_mapper._get_intermediate_collection_info()
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+ self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list