[ARVADOS] created: 1.1.4-722-g7eadb7fa1
Git user
git at public.curoverse.com
Fri Jul 27 12:25:12 EDT 2018
at 7eadb7fa179c18e41066709e8645a1e6eaad655c (commit)
commit 7eadb7fa179c18e41066709e8645a1e6eaad655c
Merge: 0771365c2 43175f034
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Fri Jul 27 18:24:36 2018 +0200
Merge branch 'master' of git.curoverse.com:arvados into 11942-tagging-support
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
commit 0771365c2771a913499f406a594ba1b94d1ba911
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Fri Jul 27 18:23:31 2018 +0200
Add test for output-properties
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index 590c82d20..91b1194fa 100644
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -39,7 +39,7 @@ class TestMakeOutput(unittest.TestCase):
final.open.return_value = openmock
openmock.__enter__.return_value = cwlout
- _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], "tag0,tag1,tag2", {
+ _, runner.final_output_collection = runner.make_output_collection("Test output", ["foo"], {'foo':'bar', 'baz':'quz'}, "tag0,tag1,tag2", {
"foo": {
"class": "File",
"location": "keep:99999999999999999999999999999991+99/foo.txt",
@@ -56,7 +56,7 @@ class TestMakeOutput(unittest.TestCase):
final.copy.assert_has_calls([mock.call('bar.txt', 'baz.txt', overwrite=False, source_collection=readermock)])
final.copy.assert_has_calls([mock.call('foo.txt', 'foo.txt', overwrite=False, source_collection=readermock)])
- final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'])])
+ final.save_new.assert_has_calls([mock.call(ensure_unique_name=True, name='Test output', owner_uuid='zzzzz-j7d0g-zzzzzzzzzzzzzzz', storage_classes=['foo'], properties={'foo':'bar', 'baz':'quz'})])
self.assertEqual("""{
"bar": {
"basename": "baz.txt",
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index cd4625130..c3303eea3 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -11,6 +11,7 @@ import logging
import mock
import sys
import unittest
+import tempfile
import arvados
import arvados.collection
@@ -341,6 +342,32 @@ class TestSubmit(unittest.TestCase):
sys.stdin, sys.stderr, api_client=stubs.api)
self.assertEqual(exited, 1)
+ def test_split_output_properties(self):
+ output_props = "foo:bar,baz:qux"
+ result = arvados_cwl.split_output_properties(output_props)
+
+ self.assertEqual(result, {'foo':'bar', 'baz':'qux'})
+
+ def test_parse_output_properties_yaml(self):
+ tmp = tempfile.NamedTemporaryFile()
+ with open(tmp.name, 'a') as f:
+ f.write("foo: 111\n")
+ f.write("bar: 222\n")
+ f.write("baz: 333")
+
+ result = arvados_cwl.parse_output_properties_yaml(tmp.name)
+
+ self.assertEqual(result, {'foo':111, 'bar':222, 'baz':333})
+
+ @stubs
+ def test_error_when_invalid_yaml_file_path_specified(self, stubs):
+ file_path = "foo.txt"
+ with self.assertRaises(SystemExit):
+ arvados_cwl.main(
+ ["--debug", "--output-properties-yaml", file_path,
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ sys.stdin, sys.stderr, api_client=stubs.api)
+
@mock.patch("time.sleep")
@stubs
def test_submit_on_error(self, stubs, tm):
@@ -613,6 +640,58 @@ class TestSubmit(unittest.TestCase):
stubs.expect_container_request_uuid + '\n')
@stubs
+ def test_submit_output_properties(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait", "--api=containers", "--output-properties=foo:bar,baz:qux",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=4',
+ '--enable-reuse', "--debug", '--on-error=continue',
+ "--output-properties=foo:bar,baz:qux",
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
+ @mock.patch("arvados_cwl.parse_output_properties_yaml", return_value = {'foo':'bar', 'baz':'quz'})
+ @stubs
+ def test_submit_output_properties_yaml(self, stubs, split_props):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--debug", "--submit", "--no-wait", "--api=containers", "--output-properties-yaml=config.yaml",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=4',
+ '--enable-reuse', "--debug", '--on-error=continue',
+ "--output-properties-yaml=config.yaml",
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+ @stubs
def test_submit_storage_classes(self, stubs):
capture_stdout = cStringIO.StringIO()
try:
@@ -641,7 +720,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
@stubs
- def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
+ def test_collection_content_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
def set_final_output(job_order, output_callback, runtimeContext):
output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
return []
@@ -649,14 +728,14 @@ class TestSubmit(unittest.TestCase):
try:
exited = arvados_cwl.main(
- ["--debug", "--local", "--storage-classes=foo",
+ ["--debug", "--local", "--storage-classes=foo", "--output-properties=foo:bar,baz:qux",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
self.assertEqual(exited, 0)
except:
logging.exception("")
- make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], {"foo" : "bar", "baz" : "qux"}, '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
@mock.patch("arvados_cwl.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@@ -677,7 +756,7 @@ class TestSubmit(unittest.TestCase):
except:
logging.exception("")
- make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+ make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], None, '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
@stubs
def test_submit_container_output_ttl(self, stubs):
diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
index 2532bd596..5b3b7eaca 100644
--- a/sdk/cwl/tests/test_util.py
+++ b/sdk/cwl/tests/test_util.py
@@ -43,3 +43,17 @@ class TestUtil(unittest.TestCase):
logger = mock.MagicMock()
self.assertRaises(ApiError, get_current_container(api, num_retries=0, logger=logger))
+
+ def test_merge_dict(self):
+ a = {"a":1, "b":2}
+ b = {"b":3000, "c":4000}
+ c = merge_dict(a, b)
+
+ self.assertEqual(c, {"a":1, "b":3000, "c":4000})
+
+ def test_merge_dict_none_input(self):
+ a = None
+ b = None
+ c = merge_dict(a, b)
+
+ self.assertEqual(c, None)
commit 10f0a4014ce54aa5d9c9b86d8b376f33c7b944e0
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Fri Jul 27 16:43:05 2018 +0200
fix bug when parsing output-properties from yaml file
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index cae21c678..51561d072 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -808,8 +808,13 @@ def split_output_properties(props):
def parse_output_properties_yaml(yaml_file):
if yaml_file:
- with open(yaml_file) as f:
- return yaml.safe_load(f)
+ try:
+ with open(yaml_file) as f:
+ return yaml.safe_load(f)
+ except IOError as e:
+ logger.error("Error while opening yaml file: %s" % e)
+ sys.exit(1)
+
def main(args, stdout, stderr, api_client=None, keep_client=None,
install_sig_handlers=True):
diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index f3accec45..19882f865 100644
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -33,6 +33,9 @@ def get_current_container(api, num_retries=0, logger=None):
def merge_dict(*dict_args):
result = {}
for d in dict_args:
- result.update(d)
- return result
-
+ if d:
+ result.update(d)
+ if len(result) > 0:
+ return result
+ else:
+ return None
commit b89057361dadd3e9870739a2cac64b80d92cd821
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Fri Jul 27 14:43:37 2018 +0200
Add support for tagging output collection using properties
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 131795ee2..cae21c678 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -21,6 +21,7 @@ import Queue
import time
import signal
import thread
+import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
import cwltool.main
@@ -46,6 +47,7 @@ from .perf import Perf
from .pathmapper import NoFollowPathMapper
from .task_queue import TaskQueue
from .context import ArvLoadingContext, ArvRuntimeContext
+from .util import merge_dict
from ._version import __version__
from cwltool.pack import pack
@@ -289,7 +291,7 @@ class ArvCwlRunner(object):
with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
self.check_features(v)
- def make_output_collection(self, name, storage_classes, tagsString, outputObj):
+ def make_output_collection(self, name, storage_classes, properties, tagsString, outputObj):
outputObj = copy.deepcopy(outputObj)
files = []
@@ -340,7 +342,8 @@ class ArvCwlRunner(object):
with final.open("cwl.output.json", "w") as f:
json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
- final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
+ final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes,
+ properties=properties, ensure_unique_name=True)
logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
final.api_response()["name"],
@@ -613,7 +616,8 @@ class ArvCwlRunner(object):
self.output_tags = ""
storage_classes = runtimeContext.storage_classes.strip().split(",")
- self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
+ self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes,
+ runtimeContext.output_collection_properties, self.output_tags, self.final_output)
self.set_crunch_output()
if runtimeContext.compute_checksum:
@@ -738,6 +742,12 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
parser.add_argument('--storage-classes', default="default", type=str,
help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
+ parser.add_argument('--output-properties', type=str,
+ help="Tag final output collection using properties, e.g., '--output-properties 'foo:bar,baz:qux'.")
+
+ parser.add_argument('--output-properties-yaml', type=str,
+ help="Tag final output collection using properties from YAML file.")
+
parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
help="If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).",
default=0)
@@ -791,6 +801,16 @@ def exit_signal_handler(sigcode, frame):
logger.error("Caught signal {}, exiting.".format(sigcode))
sys.exit(-sigcode)
+def split_output_properties(props):
+ if props:
+ return dict((k.strip(), v.strip()) for k,v in
+ (pairs.split(":") for pairs in props.split(",")))
+
+def parse_output_properties_yaml(yaml_file):
+ if yaml_file:
+ with open(yaml_file) as f:
+ return yaml.safe_load(f)
+
def main(args, stdout, stderr, api_client=None, keep_client=None,
install_sig_handlers=True):
parser = arg_parser()
@@ -868,6 +888,10 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
runtimeContext.make_fs_access = partial(CollectionFsAccess,
collection_cache=runner.collection_cache)
+ output_props = split_output_properties(arvargs.output_properties)
+ output_props_yaml = parse_output_properties_yaml(arvargs.output_properties_yaml)
+ runtimeContext.output_collection_properties = merge_dict(output_props_yaml, output_props)
+
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 948a9a46f..efd8411a7 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -453,6 +453,12 @@ class RunnerContainer(Runner):
if self.on_error:
command.append("--on-error=" + self.on_error)
+ if runtimeContext.output_properties:
+ command.append("--output-properties=" + runtimeContext.output_properties)
+
+ if runtimeContext.output_properties_yaml:
+ command.append("--output-properties-yaml=" + runtimeContext.output_properties_yaml)
+
if self.intermediate_output_ttl:
command.append("--intermediate-output-ttl=%d" % self.intermediate_output_ttl)
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 81e256ed5..036006e6d 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -28,6 +28,9 @@ class ArvRuntimeContext(RuntimeContext):
self.wait = True
self.cwl_runner_job = None
self.storage_classes = "default"
+ self.output_properties = None
+ self.output_properties_yaml = None
+ self.output_collection_properties = None
self.current_container = None
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index 98a2a89a1..f3accec45 100644
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -29,3 +29,10 @@ def get_current_container(api, num_retries=0, logger=None):
if e.resp.status != 404 and logger:
logger.info("Getting current container: %s", e)
return current_container
+
+def merge_dict(*dict_args):
+ result = {}
+ for d in dict_args:
+ result.update(d)
+ return result
+
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list