[ARVADOS] created: 1.1.3-207-g16feb1a
Git user
git at public.curoverse.com
Tue Mar 13 15:33:42 EDT 2018
at 16feb1a87c3e2e1955ad45caad0bd4b531f1cb26 (commit)
commit 16feb1a87c3e2e1955ad45caad0bd4b531f1cb26
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Mar 13 13:31:36 2018 -0400
13135: Fix test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 5ab452f..45f8a05 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1106,6 +1106,7 @@ class TestSubmit(unittest.TestCase):
"type": "stdout"
}
],
+ "stdout": "hashed_example.txt",
"requirements": [
{
"class": "InitialWorkDirRequirement",
commit d9b0587cbd570147f4fe7f955c73f8f1bf686cb7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Mar 13 10:05:19 2018 -0400
13135: Update docs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index 95422b6..ab4048b 100644
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -36,6 +36,8 @@ hints:
outputTTL: 3600
arv:ReuseRequirement:
enableReuse: false
+ cwltool:Secrets:
+ secrets: [input1, input2]
</pre>
The one exception to this is @arv:APIRequirement@, see note below.
@@ -101,3 +103,11 @@ Enable/disable work reuse for current process. Default true (work reuse enabled
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|enableReuse|boolean|Enable/disable work reuse for current process. Default true (work reuse enabled).|
+
+h2. cwltool:Secrets
+
+Indicate that one or more input parameters are "secret". Must be applied at the top level Workflow. Secret parameters are not stored in keep, are hidden from logs and API responses, and are wiped from the database after the workflow completes.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|secrets|array<string>|Input parameters which are considered "secret". Must be strings.|
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 9f8adc7..2ab96c9 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -27,7 +27,7 @@ $graph:
name: LoadListingEnum
symbols: [no_listing, shallow_listing, deep_listing]
-- name: Secrets
+- name: cwltool:Secrets
type: record
inVocab: false
extends: cwl:ProcessRequirement
commit 8f9afeaf64cdcbafbd26c461c91f8d19b3bff281
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Mar 12 16:45:39 2018 -0400
13135: Add secrets test to arvados-tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index d99cf6c..ea6477c 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -134,3 +134,15 @@
out: out
tool: wf/runin-with-ttl-wf.cwl
doc: "RunInSingleContainer respects outputTTL"
+
+- job: secret_test_job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "hashed_example.txt",
+ "size": 47,
+ "checksum": "sha1$f45341c7f03b4dd10646c402908d1aea0d580f5d"
+ }
+ }
+ tool: wf/secret_wf.cwl
+ doc: "Test secret input parameters"
diff --git a/sdk/cwl/tests/secret_test_job.yml b/sdk/cwl/tests/secret_test_job.yml
new file mode 100644
index 0000000..883d24e
--- /dev/null
+++ b/sdk/cwl/tests/secret_test_job.yml
@@ -0,0 +1 @@
+pw: blorp
diff --git a/sdk/cwl/tests/wf/secret_job.cwl b/sdk/cwl/tests/wf/secret_job.cwl
index aa68905..0ddeb64 100644
--- a/sdk/cwl/tests/wf/secret_job.cwl
+++ b/sdk/cwl/tests/wf/secret_job.cwl
@@ -16,4 +16,5 @@ inputs:
pw: string
outputs:
out: stdout
+stdout: hashed_example.txt
arguments: [md5sum, example.conf]
commit a3aaa7effb219c44224f61df115606bece3d294e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Mar 12 15:18:50 2018 -0400
13135: Secrets not supported with --api=jobs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5c5f550..2792b30 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -237,6 +237,8 @@ class ArvCwlRunner(object):
if not obj.get("dockerOutputDirectory").startswith('/'):
raise SourceLine(obj, "dockerOutputDirectory", validate.ValidationException).makeError(
"Option 'dockerOutputDirectory' must be an absolute path.")
+ if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
+ raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
for v in obj.itervalues():
self.check_features(v)
elif isinstance(obj, list):
commit 341f5b9ee96921018d80145712bba1200f9ea1a3
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Mar 12 14:37:56 2018 -0400
13135: Update tests for secret_mounts
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index cd555a7..522946a 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -10,6 +10,7 @@ import unittest
import os
import functools
import cwltool.process
+import cwltool.secrets
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
@@ -33,6 +34,7 @@ class TestContainer(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -85,6 +87,7 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
}))
# The test passes some fields in builder.resources
@@ -96,6 +99,8 @@ class TestContainer(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
+ runner.secret_store = cwltool.secrets.SecretStore()
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
@@ -172,7 +177,8 @@ class TestContainer(unittest.TestCase):
'scheduling_parameters': {
'partitions': ['blurb']
},
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
call_body = call_kwargs.get('body', None)
@@ -191,6 +197,8 @@ class TestContainer(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
@@ -303,7 +311,8 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
},
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
call_body = call_kwargs.get('body', None)
@@ -321,6 +330,7 @@ class TestContainer(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -388,6 +398,7 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
}))
@mock.patch("arvados.collection.Collection")
@@ -400,6 +411,7 @@ class TestContainer(unittest.TestCase):
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
runner.api.containers().get().execute.return_value = {"state":"Complete",
"output": "abc+123",
@@ -443,6 +455,7 @@ class TestContainer(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@@ -517,4 +530,102 @@ class TestContainer(unittest.TestCase):
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
+ }))
+
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_secrets(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ tool = cmap({"arguments": ["md5sum", "example.conf"],
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://commonwl.org/cwltool#Secrets",
+ "secrets": [
+ "#secret_job.cwl/pw"
+ ]
+ }
+ ],
+ "id": "#secret_job.cwl",
+ "inputs": [
+ {
+ "id": "#secret_job.cwl/pw",
+ "type": "string"
+ }
+ ],
+ "outputs": [
+ ],
+ "requirements": [
+ {
+ "class": "InitialWorkDirRequirement",
+ "listing": [
+ {
+ "entry": "username: user\npassword: $(inputs.pw)\n",
+ "entryname": "example.conf"
+ }
+ ]
+ }
+ ]})
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+ basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+ arvtool.formatgraph = None
+
+ job_order = {"pw": "blorp"}
+ runner.secret_store.store(["pw"], job_order)
+
+ for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets",
+ make_fs_access=make_fs_access, tmpdir="/tmp"):
+ j.run(enable_reuse=True, priority=500)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_secrets',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': 'arvados/jobs',
+ 'command': ['md5sum', 'example.conf'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ "secret_mounts": {
+ "/var/spool/cwl/example.conf": {
+ "content": "username: user\npassword: blorp\n",
+ "kind": "text"
+ }
+ }
}))
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c7c94fd..5ab452f 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -247,7 +247,8 @@ def stubs(func):
'ram': 1024*1024*1024
},
'use_existing': True,
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
@@ -745,7 +746,8 @@ class TestSubmit(unittest.TestCase):
'ram': 1073741824
},
'use_existing': True,
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
stubs.api.container_requests().create.assert_called_with(
@@ -863,7 +865,8 @@ class TestSubmit(unittest.TestCase):
'use_existing': True,
'properties': {
"template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
- }
+ },
+ 'secret_mounts': {}
}
stubs.api.container_requests().create.assert_called_with(
@@ -1040,13 +1043,159 @@ class TestSubmit(unittest.TestCase):
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
- "tests/wf/secret_wf.cwl", "tests/submit_test_job.json"],
+ "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
self.assertEqual(exited, 0)
except:
logging.exception("")
- expect_container = copy.deepcopy(stubs.expect_container_spec)
+
+ expect_container = {
+ "command": [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--enable-reuse",
+ "--on-error=continue",
+ "--eval-timeout=20",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ],
+ "container_image": "arvados/jobs:"+arvados_cwl.__version__,
+ "cwd": "/var/spool/cwl",
+ "mounts": {
+ "/var/lib/cwl/cwl.input.json": {
+ "content": {
+ "pw": {
+ "$include": "/secrets/s0"
+ }
+ },
+ "kind": "json"
+ },
+ "/var/lib/cwl/workflow.json": {
+ "content": {
+ "$graph": [
+ {
+ "$namespaces": {
+ "cwltool": "http://commonwl.org/cwltool#"
+ },
+ "arguments": [
+ "md5sum",
+ "example.conf"
+ ],
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://commonwl.org/cwltool#Secrets",
+ "secrets": [
+ "#secret_job.cwl/pw"
+ ]
+ }
+ ],
+ "id": "#secret_job.cwl",
+ "inputs": [
+ {
+ "id": "#secret_job.cwl/pw",
+ "type": "string"
+ }
+ ],
+ "outputs": [
+ {
+ "id": "#secret_job.cwl/out",
+ "type": "stdout"
+ }
+ ],
+ "requirements": [
+ {
+ "class": "InitialWorkDirRequirement",
+ "listing": [
+ {
+ "entry": "username: user\npassword: $(inputs.pw)\n",
+ "entryname": "example.conf"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8"
+ },
+ {
+ "class": "http://commonwl.org/cwltool#Secrets",
+ "secrets": [
+ "#main/pw"
+ ]
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "id": "#main/pw",
+ "type": "string"
+ }
+ ],
+ "outputs": [
+ {
+ "id": "#main/out",
+ "outputSource": "#main/step1/out",
+ "type": "File"
+ }
+ ],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/pw",
+ "source": "#main/pw"
+ }
+ ],
+ "out": [
+ "#main/step1/out"
+ ],
+ "run": "#secret_job.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+ },
+ "kind": "json"
+ },
+ "/var/spool/cwl": {
+ "kind": "collection",
+ "writable": True
+ },
+ "stdout": {
+ "kind": "file",
+ "path": "/var/spool/cwl/cwl.output.json"
+ }
+ },
+ "name": "secret_wf.cwl",
+ "output_path": "/var/spool/cwl",
+ "owner_uuid": None,
+ "priority": 500,
+ "properties": {},
+ "runtime_constraints": {
+ "API": True,
+ "ram": 1073741824,
+ "vcpus": 1
+ },
+ "secret_mounts": {
+ "/secrets/s0": {
+ "content": "blorp",
+ "kind": "text"
+ }
+ },
+ "state": "Committed",
+ "use_existing": True
+ }
+
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
diff --git a/sdk/cwl/tests/wf/secret_job.cwl b/sdk/cwl/tests/wf/secret_job.cwl
index 40e18e1..aa68905 100644
--- a/sdk/cwl/tests/wf/secret_job.cwl
+++ b/sdk/cwl/tests/wf/secret_job.cwl
@@ -16,4 +16,4 @@ inputs:
pw: string
outputs:
out: stdout
-arguments: [cat, example.conf]
+arguments: [md5sum, example.conf]
commit 04eca602e1bda34cfcfdd518d322050723de10d0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Mar 12 13:48:57 2018 -0400
13135: Remove testing workarounds
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index c90e79e..704bda6 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -216,9 +216,6 @@ class ArvadosContainer(object):
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
- # for testing only!
- mounts.update(secret_mounts)
-
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
container_request["secret_mounts"] = secret_mounts
@@ -366,9 +363,6 @@ class RunnerContainer(Runner):
"properties": {}
}
- # for testing
- container_req["mounts"].update(secret_mounts)
-
if self.tool.tool.get("id", "").startswith("keep:"):
sp = self.tool.tool["id"].split('/')
workflowcollection = sp[0][5:]
commit 34d4fa002c738c7a6ec1fdab4e12e748c8967f92
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 16:57:42 2018 -0500
13135: Handle secrets in file literals.
Also check for secrets leaking into command line or environment, and
fail if detected.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5b8eac5..5c5f550 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -354,7 +354,7 @@ class ArvCwlRunner(object):
make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess,
collection_cache=self.collection_cache)
self.fs_access = make_fs_access(kwargs["basedir"])
-
+ self.secret_store = kwargs.get("secret_store")
self.trash_intermediate = kwargs["trash_intermediate"]
if self.trash_intermediate and self.work_api != "containers":
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index eeadfbd..c90e79e 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -54,6 +54,12 @@ class ArvadosContainer(object):
}
runtime_constraints = {}
+ if self.arvrunner.secret_store.has_secret(self.command_line):
+ raise WorkflowException("Secret material leaked on command line, only file literals may contain secrets")
+
+ if self.arvrunner.secret_store.has_secret(self.environment):
+ raise WorkflowException("Secret material leaked in environment, only file literals may contain secrets")
+
resources = self.builder.resources
if resources is not None:
runtime_constraints["vcpus"] = resources.get("cores", 1)
@@ -69,6 +75,7 @@ class ArvadosContainer(object):
"capacity": resources.get("tmpdirSize", 0) * 2**20
}
}
+ secret_mounts = {}
scheduling_parameters = {}
rf = [self.pathmapper.mapper(f) for f in self.pathmapper.referenced_files]
@@ -119,8 +126,14 @@ class ArvadosContainer(object):
source, path = self.arvrunner.fs_access.get_collection(p.resolved)
vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
- with vwd.open(p.target, "w") as n:
- n.write(p.resolved.encode("utf-8"))
+ if self.arvrunner.secret_store.has_secret(p.resolved):
+ secret_mounts["%s/%s" % (self.outdir, p.target)] = {
+ "kind": "text",
+ "content": self.arvrunner.secret_store.retrieve(p.resolved)
+ }
+ else:
+ with vwd.open(p.target, "w") as n:
+ n.write(p.resolved.encode("utf-8"))
def keepemptydirs(p):
if isinstance(p, arvados.collection.RichCollectionBase):
@@ -136,7 +149,7 @@ class ArvadosContainer(object):
vwd.save_new()
for f, p in generatemapper.items():
- if not p.target:
+ if not p.target or self.arvrunner.secret_store.has_secret(p.resolved):
continue
mountpoint = "%s/%s" % (self.outdir, p.target)
mounts[mountpoint] = {"kind": "collection",
@@ -201,10 +214,14 @@ class ArvadosContainer(object):
self.output_ttl = self.arvrunner.intermediate_output_ttl
if self.output_ttl < 0:
- raise WorkflowError("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+ raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+
+ # for testing only!
+ mounts.update(secret_mounts)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
+ container_request["secret_mounts"] = secret_mounts
container_request["runtime_constraints"] = runtime_constraints
container_request["scheduling_parameters"] = scheduling_parameters
@@ -311,11 +328,11 @@ class RunnerContainer(Runner):
for param in sorted(self.job_order.keys()):
if self.secret_store.has_secret(self.job_order[param]):
mnt = "/secrets/s%d" % len(secret_mounts)
- self.job_order[param] = {"$include": mnt}
secret_mounts[mnt] = {
"kind": "text",
"content": self.secret_store.retrieve(self.job_order[param])
}
+ self.job_order[param] = {"$include": mnt}
container_req = {
"owner_uuid": self.arvrunner.project_uuid,
commit e73e1bc2eb23cb32eb55a5835157c4fbc0153eab
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 16:32:35 2018 -0500
13135: Secrets support WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index aee928d..5b8eac5 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -452,7 +452,8 @@ class ArvCwlRunner(object):
submit_runner_image=kwargs.get("submit_runner_image"),
intermediate_output_ttl=kwargs.get("intermediate_output_ttl"),
merged_map=merged_map,
- priority=kwargs.get("priority"))
+ priority=kwargs.get("priority"),
+ secret_store=kwargs.get("secret_store"))
elif self.work_api == "jobs":
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
self.output_name,
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 7ae2239..9f8adc7 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -27,6 +27,26 @@ $graph:
name: LoadListingEnum
symbols: [no_listing, shallow_listing, deep_listing]
+- name: Secrets
+ type: record
+ inVocab: false
+ extends: cwl:ProcessRequirement
+ fields:
+ class:
+ type: string
+ doc: "Always 'Secrets'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ secrets:
+ type: string[]
+ doc: |
+ List one or more input parameters that are sensitive (such as passwords)
+ which will be deliberately obscured from logging.
+ jsonldPredicate:
+ "_type": "@id"
+ refScope: 0
+
- name: RunInSingleContainer
type: record
extends: cwl:ProcessRequirement
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 9f83822..eeadfbd 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -9,6 +9,7 @@ import urllib
import time
import datetime
import ciso8601
+import uuid
import ruamel.yaml as yaml
@@ -306,6 +307,16 @@ class RunnerContainer(Runner):
visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
+ secret_mounts = {}
+ for param in sorted(self.job_order.keys()):
+ if self.secret_store.has_secret(self.job_order[param]):
+ mnt = "/secrets/s%d" % len(secret_mounts)
+ self.job_order[param] = {"$include": mnt}
+ secret_mounts[mnt] = {
+ "kind": "text",
+ "content": self.secret_store.retrieve(self.job_order[param])
+ }
+
container_req = {
"owner_uuid": self.arvrunner.project_uuid,
"name": self.name,
@@ -328,6 +339,7 @@ class RunnerContainer(Runner):
"writable": True
}
},
+ "secret_mounts": secret_mounts,
"runtime_constraints": {
"vcpus": 1,
"ram": 1024*1024 * self.submit_runner_ram,
@@ -337,6 +349,9 @@ class RunnerContainer(Runner):
"properties": {}
}
+ # for testing
+ container_req["mounts"].update(secret_mounts)
+
if self.tool.tool.get("id", "").startswith("keep:"):
sp = self.tool.tool["id"].split('/')
workflowcollection = sp[0][5:]
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9716ca4..053c995 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -316,7 +316,8 @@ class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
- intermediate_output_ttl=0, merged_map=None, priority=None):
+ intermediate_output_ttl=0, merged_map=None, priority=None,
+ secret_store=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -337,6 +338,7 @@ class Runner(object):
self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
self.intermediate_output_ttl = intermediate_output_ttl
self.priority = priority
+ self.secret_store = secret_store
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 5b1d737..752633a 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -41,7 +41,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180225105849',
+ 'cwltool==1.0.20180306163216',
'schema-salad==2.6.20171201034858',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 298d6aa..c7c94fd 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -231,6 +231,7 @@ def stubs(func):
'kind': 'json'
}
},
+ 'secret_mounts': {},
'state': 'Committed',
'owner_uuid': None,
'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
@@ -1033,6 +1034,25 @@ class TestSubmit(unittest.TestCase):
self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ @stubs
+ def test_submit_secrets(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/secret_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
diff --git a/sdk/cwl/tests/wf/secret_job.cwl b/sdk/cwl/tests/wf/secret_job.cwl
new file mode 100644
index 0000000..40e18e1
--- /dev/null
+++ b/sdk/cwl/tests/wf/secret_job.cwl
@@ -0,0 +1,19 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+hints:
+ "cwltool:Secrets":
+ secrets: [pw]
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: example.conf
+ entry: |
+ username: user
+ password: $(inputs.pw)
+inputs:
+ pw: string
+outputs:
+ out: stdout
+arguments: [cat, example.conf]
diff --git a/sdk/cwl/tests/wf/secret_wf.cwl b/sdk/cwl/tests/wf/secret_wf.cwl
new file mode 100644
index 0000000..17c92d6
--- /dev/null
+++ b/sdk/cwl/tests/wf/secret_wf.cwl
@@ -0,0 +1,21 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+hints:
+ "cwltool:Secrets":
+ secrets: [pw]
+ DockerRequirement:
+ dockerPull: debian:8
+inputs:
+ pw: string
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
+steps:
+ step1:
+ in:
+ pw: pw
+ out: [out]
+ run: secret_job.cwl
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list