[ARVADOS] updated: 1.1.4-556-gc405f9ada
Git user
git at public.curoverse.com
Tue Jul 10 06:35:57 EDT 2018
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 4 +++-
sdk/cwl/arvados_cwl/arvjob.py | 3 ++-
sdk/cwl/arvados_cwl/pathmapper.py | 4 ++--
sdk/cwl/arvados_cwl/util.py | 9 ++++++---
sdk/cwl/tests/test_container.py | 6 ++++++
sdk/cwl/tests/test_util.py | 5 +++--
6 files changed, 22 insertions(+), 9 deletions(-)
via c405f9adab08daa3d4edef9cddd5453a79446c7b (commit)
from 9b16a843a641159867ea390c4c2384b320183b15 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c405f9adab08daa3d4edef9cddd5453a79446c7b
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Tue Jul 10 12:33:43 2018 +0200
Update get_intermediate_collection_info and add owner_uuid to save_new
method in arvcontainer.py and arvjob.py
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 2618ddeca..a2bd53c9f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -168,8 +168,9 @@ class ArvadosContainer(JobBase):
if not runtimeContext.current_container:
runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
trash_at=info["trash_at"],
properties=info["properties"])
@@ -248,6 +249,7 @@ class ArvadosContainer(JobBase):
if self.timelimit is not None:
scheduling_parameters["max_run_time"] = self.timelimit
+ container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
container_request["secret_mounts"] = secret_mounts
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 36a047213..1287fbb6e 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -79,8 +79,9 @@ class ArvadosJob(JobBase):
with Perf(metrics, "generatefiles.save_new %s" % self.name):
if not runtimeContext.current_container:
runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
+ owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
trash_at=info["trash_at"],
properties=info["properties"])
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index a3b6a4dd9..d083b78f5 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -156,7 +156,7 @@ class ArvPathMapper(PathMapper):
self.addentry(l, c, ".", remap)
container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
+ info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
@@ -175,7 +175,7 @@ class ArvPathMapper(PathMapper):
self.addentry(srcobj, c, ".", remap)
container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
+ info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index 7dafedac2..98a2a89a1 100644
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -5,8 +5,11 @@
import datetime
from arvados.errors import ApiError
-def get_intermediate_collection_info(current_container, intermediate_output_ttl):
- name = "Intermediate collection"
+def get_intermediate_collection_info(workflow_step_name, current_container, intermediate_output_ttl):
+ if workflow_step_name:
+ name = "Intermediate collection for step %s" % (workflow_step_name)
+ else:
+ name = "Intermediate collection"
trash_time = None
if intermediate_output_ttl > 0:
trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl)
@@ -25,4 +28,4 @@ def get_current_container(api, num_retries=0, logger=None):
# Status code 404 just means we're not running in a container.
if e.resp.status != 404 and logger:
logger.info("Getting current container: %s", e)
- return current_container;
+ return current_container
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index e7ce9c370..ae234414a 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -99,6 +99,7 @@ class TestContainer(unittest.TestCase):
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_run_'+str(enable_reuse),
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
@@ -185,6 +186,7 @@ class TestContainer(unittest.TestCase):
"capacity": 5242880000 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_resource_requirements',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 7200,
@@ -317,6 +319,7 @@ class TestContainer(unittest.TestCase):
}
},
'state': 'Committed',
+ 'output_name': 'Output for step test_initial_work_dir',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
@@ -404,6 +407,7 @@ class TestContainer(unittest.TestCase):
},
},
'state': 'Committed',
+ "output_name": "Output for step test_run_redirect",
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
@@ -540,6 +544,7 @@ class TestContainer(unittest.TestCase):
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_run_mounts',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
@@ -632,6 +637,7 @@ class TestContainer(unittest.TestCase):
"capacity": 1073741824 }
},
'state': 'Committed',
+ 'output_name': 'Output for step test_secrets',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
index e42197e21..2532bd596 100644
--- a/sdk/cwl/tests/test_util.py
+++ b/sdk/cwl/tests/test_util.py
@@ -19,12 +19,13 @@ datetime.datetime = MockDateTime
class TestUtil(unittest.TestCase):
def test_get_intermediate_collection_info(self):
+ name = "one"
current_container = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
intermediate_output_ttl = 120
- info = get_intermediate_collection_info(current_container, intermediate_output_ttl)
+ info = get_intermediate_collection_info(name, current_container, intermediate_output_ttl)
- self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["name"], "Intermediate collection for step one")
self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 2, 0, 0))
self.assertEqual(info["properties"], {"type" : "intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list