[ARVADOS] created: 1.1.3-171-g53fb107
Git user
git at public.curoverse.com
Fri Mar 9 17:00:32 EST 2018
at 53fb107400a0e84a197aabbc558bd1a485cc5302 (commit)
commit 53fb107400a0e84a197aabbc558bd1a485cc5302
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 16:57:42 2018 -0500
13135: Handle secrets in file literals.
Also check for secrets leaking into command line or environment, and
fail if detected.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5b8eac5..5c5f550 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -354,7 +354,7 @@ class ArvCwlRunner(object):
make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess,
collection_cache=self.collection_cache)
self.fs_access = make_fs_access(kwargs["basedir"])
-
+ self.secret_store = kwargs.get("secret_store")
self.trash_intermediate = kwargs["trash_intermediate"]
if self.trash_intermediate and self.work_api != "containers":
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index eeadfbd..c90e79e 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -54,6 +54,12 @@ class ArvadosContainer(object):
}
runtime_constraints = {}
+ if self.arvrunner.secret_store.has_secret(self.command_line):
+ raise WorkflowException("Secret material leaked on command line, only file literals may contain secrets")
+
+ if self.arvrunner.secret_store.has_secret(self.environment):
+ raise WorkflowException("Secret material leaked in environment, only file literals may contain secrets")
+
resources = self.builder.resources
if resources is not None:
runtime_constraints["vcpus"] = resources.get("cores", 1)
@@ -69,6 +75,7 @@ class ArvadosContainer(object):
"capacity": resources.get("tmpdirSize", 0) * 2**20
}
}
+ secret_mounts = {}
scheduling_parameters = {}
rf = [self.pathmapper.mapper(f) for f in self.pathmapper.referenced_files]
@@ -119,8 +126,14 @@ class ArvadosContainer(object):
source, path = self.arvrunner.fs_access.get_collection(p.resolved)
vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
- with vwd.open(p.target, "w") as n:
- n.write(p.resolved.encode("utf-8"))
+ if self.arvrunner.secret_store.has_secret(p.resolved):
+ secret_mounts["%s/%s" % (self.outdir, p.target)] = {
+ "kind": "text",
+ "content": self.arvrunner.secret_store.retrieve(p.resolved)
+ }
+ else:
+ with vwd.open(p.target, "w") as n:
+ n.write(p.resolved.encode("utf-8"))
def keepemptydirs(p):
if isinstance(p, arvados.collection.RichCollectionBase):
@@ -136,7 +149,7 @@ class ArvadosContainer(object):
vwd.save_new()
for f, p in generatemapper.items():
- if not p.target:
+ if not p.target or self.arvrunner.secret_store.has_secret(p.resolved):
continue
mountpoint = "%s/%s" % (self.outdir, p.target)
mounts[mountpoint] = {"kind": "collection",
@@ -201,10 +214,14 @@ class ArvadosContainer(object):
self.output_ttl = self.arvrunner.intermediate_output_ttl
if self.output_ttl < 0:
- raise WorkflowError("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+ raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
+
+ # for testing only!
+ mounts.update(secret_mounts)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
+ container_request["secret_mounts"] = secret_mounts
container_request["runtime_constraints"] = runtime_constraints
container_request["scheduling_parameters"] = scheduling_parameters
@@ -311,11 +328,11 @@ class RunnerContainer(Runner):
for param in sorted(self.job_order.keys()):
if self.secret_store.has_secret(self.job_order[param]):
mnt = "/secrets/s%d" % len(secret_mounts)
- self.job_order[param] = {"$include": mnt}
secret_mounts[mnt] = {
"kind": "text",
"content": self.secret_store.retrieve(self.job_order[param])
}
+ self.job_order[param] = {"$include": mnt}
container_req = {
"owner_uuid": self.arvrunner.project_uuid,
commit d12d9001175d1f7fb1d45bdc0673fad312c48ab9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 16:32:35 2018 -0500
13135: Secrets support WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index aee928d..5b8eac5 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -452,7 +452,8 @@ class ArvCwlRunner(object):
submit_runner_image=kwargs.get("submit_runner_image"),
intermediate_output_ttl=kwargs.get("intermediate_output_ttl"),
merged_map=merged_map,
- priority=kwargs.get("priority"))
+ priority=kwargs.get("priority"),
+ secret_store=kwargs.get("secret_store"))
elif self.work_api == "jobs":
runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
self.output_name,
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 7ae2239..9f8adc7 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -27,6 +27,26 @@ $graph:
name: LoadListingEnum
symbols: [no_listing, shallow_listing, deep_listing]
+- name: Secrets
+ type: record
+ inVocab: false
+ extends: cwl:ProcessRequirement
+ fields:
+ class:
+ type: string
+ doc: "Always 'Secrets'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ secrets:
+ type: string[]
+ doc: |
+ List one or more input parameters that are sensitive (such as passwords)
+ which will be deliberately obscured from logging.
+ jsonldPredicate:
+ "_type": "@id"
+ refScope: 0
+
- name: RunInSingleContainer
type: record
extends: cwl:ProcessRequirement
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 9f83822..eeadfbd 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -9,6 +9,7 @@ import urllib
import time
import datetime
import ciso8601
+import uuid
import ruamel.yaml as yaml
@@ -306,6 +307,16 @@ class RunnerContainer(Runner):
visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
+ secret_mounts = {}
+ for param in sorted(self.job_order.keys()):
+ if self.secret_store.has_secret(self.job_order[param]):
+ mnt = "/secrets/s%d" % len(secret_mounts)
+ self.job_order[param] = {"$include": mnt}
+ secret_mounts[mnt] = {
+ "kind": "text",
+ "content": self.secret_store.retrieve(self.job_order[param])
+ }
+
container_req = {
"owner_uuid": self.arvrunner.project_uuid,
"name": self.name,
@@ -328,6 +339,7 @@ class RunnerContainer(Runner):
"writable": True
}
},
+ "secret_mounts": secret_mounts,
"runtime_constraints": {
"vcpus": 1,
"ram": 1024*1024 * self.submit_runner_ram,
@@ -337,6 +349,9 @@ class RunnerContainer(Runner):
"properties": {}
}
+ # for testing
+ container_req["mounts"].update(secret_mounts)
+
if self.tool.tool.get("id", "").startswith("keep:"):
sp = self.tool.tool["id"].split('/')
workflowcollection = sp[0][5:]
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9716ca4..053c995 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -316,7 +316,8 @@ class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
- intermediate_output_ttl=0, merged_map=None, priority=None):
+ intermediate_output_ttl=0, merged_map=None, priority=None,
+ secret_store=None):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
@@ -337,6 +338,7 @@ class Runner(object):
self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
self.intermediate_output_ttl = intermediate_output_ttl
self.priority = priority
+ self.secret_store = secret_store
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 5b1d737..752633a 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -41,7 +41,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180225105849',
+ 'cwltool==1.0.20180306163216',
'schema-salad==2.6.20171201034858',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 298d6aa..c7c94fd 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -231,6 +231,7 @@ def stubs(func):
'kind': 'json'
}
},
+ 'secret_mounts': {},
'state': 'Committed',
'owner_uuid': None,
'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
@@ -1033,6 +1034,25 @@ class TestSubmit(unittest.TestCase):
self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ @stubs
+ def test_submit_secrets(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/secret_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
diff --git a/sdk/cwl/tests/wf/secret_job.cwl b/sdk/cwl/tests/wf/secret_job.cwl
new file mode 100644
index 0000000..40e18e1
--- /dev/null
+++ b/sdk/cwl/tests/wf/secret_job.cwl
@@ -0,0 +1,19 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+hints:
+ "cwltool:Secrets":
+ secrets: [pw]
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: example.conf
+ entry: |
+ username: user
+ password: $(inputs.pw)
+inputs:
+ pw: string
+outputs:
+ out: stdout
+arguments: [cat, example.conf]
diff --git a/sdk/cwl/tests/wf/secret_wf.cwl b/sdk/cwl/tests/wf/secret_wf.cwl
new file mode 100644
index 0000000..17c92d6
--- /dev/null
+++ b/sdk/cwl/tests/wf/secret_wf.cwl
@@ -0,0 +1,21 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+hints:
+ "cwltool:Secrets":
+ secrets: [pw]
+ DockerRequirement:
+ dockerPull: debian:8
+inputs:
+ pw: string
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
+steps:
+ step1:
+ in:
+ pw: pw
+ out: [out]
+ run: secret_job.cwl
commit a75460e1e432c24a1938d985f6aa3d3714873203
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 15:29:22 2018 -0500
Fix packaging for python-cwltest. refs #13140
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/build.list b/build/build.list
index a216e67..da7c030 100644
--- a/build/build.list
+++ b/build/build.list
@@ -14,7 +14,7 @@ debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|uritemplate|3.0.0|2|pyt
debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|httplib2|0.9.2|3|python|all
debian8,debian9,ubuntu1204,centos7|ws4py|0.3.5|2|python|all
debian8,debian9,ubuntu1204,centos7|pykka|1.2.1|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404|six|1.10.0|2|python|all
+debian8,debian9,ubuntu1204,ubuntu1404,centos7|six|1.10.0|2|python|all
debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|ciso8601|1.0.3|3|python|amd64
debian8,debian9,ubuntu1204,centos7|pycrypto|2.6.1|3|python|amd64
debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604|backports.ssl_match_hostname|3.5.0.1|2|python|all
@@ -41,8 +41,9 @@ centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
+debian8,ubuntu1404,centos7|subprocess32|3.2.7|2|python|all
all|ruamel.yaml|0.13.7|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
-all|cwltest|1.0.20180209171722|3|python|all|--depends 'python-futures >= 3.0.5'
+all|cwltest|1.0.20180209171722|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32'
all|junit-xml|1.7|3|python|all
all|rdflib-jsonld|0.4.0|2|python|all
all|futures|3.0.5|2|python|all
diff --git a/build/package-testing/test-package-python-cwltest.sh b/build/package-testing/test-package-python-cwltest.sh
new file mode 120000
index 0000000..9b6545b
--- /dev/null
+++ b/build/package-testing/test-package-python-cwltest.sh
@@ -0,0 +1 @@
+test-package-python27-python-cwltest.sh
\ No newline at end of file
diff --git a/build/package-testing/test-package-python27-python-cwltest.sh b/build/package-testing/test-package-python27-python-cwltest.sh
new file mode 100755
index 0000000..395cefc
--- /dev/null
+++ b/build/package-testing/test-package-python27-python-cwltest.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec python <<EOF
+import cwltest
+EOF
diff --git a/build/package-testing/test-packages-debian9.sh b/build/package-testing/test-packages-debian9.sh
deleted file mode 100755
index b4ea35c..0000000
--- a/build/package-testing/test-packages-debian9.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -eu
-
-# Multiple .deb based distros symlink to this script, so extract the target
-# from the invocation path.
-target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
-
-export ARV_PACKAGES_DIR="/arvados/packages/$target"
-
-dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
-
-apt-get -qq update
-apt-get --assume-yes --allow-unauthenticated install "$1"
-
-dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
-
-set +e
-diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" > "$ARV_PACKAGES_DIR/$1.diff"
-set -e
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-export ARV_PACKAGES_DIR="/arvados/packages/$target"
-
-dpkg-deb -x $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) .
-
-while read so && [ -n "$so" ]; do
- echo
- echo "== Packages dependencies for $so =="
- ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
-done <<EOF
-$(find -name '*.so')
-EOF
-
-exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/build/package-testing/test-packages-debian9.sh b/build/package-testing/test-packages-debian9.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/build/package-testing/test-packages-debian9.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
commit 60a95ea63c000ea4d0fcc3f95af801433b386cf8
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Feb 20 16:16:09 2018 -0300
12737: Update API server to rails 4.2.10
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/api/Gemfile b/services/api/Gemfile
index 4cb5671..7d4d4bb 100644
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -4,7 +4,7 @@
source 'https://rubygems.org'
-gem 'rails', '~> 4.0'
+gem 'rails', '~> 4.2'
gem 'responders', '~> 2.0'
gem 'protected_attributes'
@@ -21,8 +21,14 @@ group :test, :development do
gem 'mocha', require: false
end
+# We'll need to update related code prior to Rails 5.
+# See: https://github.com/rails/activerecord-deprecated_finders
+gem 'activerecord-deprecated_finders', require: 'active_record/deprecated_finders'
+
# pg is the only supported database driver.
-gem 'pg'
+# Note: Rails 4.2 is not compatible with pg 1.0
+# (See: https://github.com/rails/rails/pull/31671)
+gem 'pg', '~> 0.18'
gem 'multi_json'
gem 'oj'
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index b2de3f5..30cd8a5 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -8,57 +8,57 @@ GIT
GEM
remote: https://rubygems.org/
specs:
- actionmailer (4.2.5.2)
- actionpack (= 4.2.5.2)
- actionview (= 4.2.5.2)
- activejob (= 4.2.5.2)
+ actionmailer (4.2.10)
+ actionpack (= 4.2.10)
+ actionview (= 4.2.10)
+ activejob (= 4.2.10)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 1.0, >= 1.0.5)
- actionpack (4.2.5.2)
- actionview (= 4.2.5.2)
- activesupport (= 4.2.5.2)
+ actionpack (4.2.10)
+ actionview (= 4.2.10)
+ activesupport (= 4.2.10)
rack (~> 1.6)
rack-test (~> 0.6.2)
rails-dom-testing (~> 1.0, >= 1.0.5)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (4.2.5.2)
- activesupport (= 4.2.5.2)
+ actionview (4.2.10)
+ activesupport (= 4.2.10)
builder (~> 3.1)
erubis (~> 2.7.0)
rails-dom-testing (~> 1.0, >= 1.0.5)
- rails-html-sanitizer (~> 1.0, >= 1.0.2)
- activejob (4.2.5.2)
- activesupport (= 4.2.5.2)
+ rails-html-sanitizer (~> 1.0, >= 1.0.3)
+ activejob (4.2.10)
+ activesupport (= 4.2.10)
globalid (>= 0.3.0)
- activemodel (4.2.5.2)
- activesupport (= 4.2.5.2)
+ activemodel (4.2.10)
+ activesupport (= 4.2.10)
builder (~> 3.1)
- activerecord (4.2.5.2)
- activemodel (= 4.2.5.2)
- activesupport (= 4.2.5.2)
+ activerecord (4.2.10)
+ activemodel (= 4.2.10)
+ activesupport (= 4.2.10)
arel (~> 6.0)
- activesupport (4.2.5.2)
+ activerecord-deprecated_finders (1.0.4)
+ activesupport (4.2.10)
i18n (~> 0.7)
- json (~> 1.7, >= 1.7.7)
minitest (~> 5.1)
thread_safe (~> 0.3, >= 0.3.4)
tzinfo (~> 1.1)
- acts_as_api (1.0.0)
+ acts_as_api (1.0.1)
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
- addressable (2.5.1)
- public_suffix (~> 2.0, >= 2.0.2)
+ addressable (2.5.2)
+ public_suffix (>= 2.0.2, < 4.0)
andand (1.3.3)
arel (6.0.4)
- arvados (0.1.20170629115132)
- activesupport (>= 3, < 4.2.6)
+ arvados (0.1.20180302192246)
+ activesupport (>= 3)
andand (~> 1.3, >= 1.3.3)
google-api-client (>= 0.7, < 0.8.9)
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-cli (0.1.20170817171636)
+ arvados-cli (0.1.20171211220040)
activesupport (>= 3.2.13, < 5)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1, >= 0.1.20150128223554)
@@ -78,33 +78,33 @@ GEM
net-sftp (>= 2.0.0)
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
- coffee-rails (4.2.1)
+ coffee-rails (4.2.2)
coffee-script (>= 2.2.0)
- railties (>= 4.0.0, < 5.2.x)
+ railties (>= 4.0.0)
coffee-script (2.4.1)
coffee-script-source
execjs
coffee-script-source (1.12.2)
concurrent-ruby (1.0.5)
- crass (1.0.2)
- curb (0.9.3)
- database_cleaner (1.5.3)
+ crass (1.0.3)
+ curb (0.9.4)
+ database_cleaner (1.6.2)
erubis (2.7.0)
- eventmachine (1.2.3)
+ eventmachine (1.2.5)
execjs (2.7.0)
extlib (0.9.16)
- factory_girl (4.8.0)
+ factory_girl (4.9.0)
activesupport (>= 3.0.0)
- factory_girl_rails (4.8.0)
- factory_girl (~> 4.8.0)
+ factory_girl_rails (4.9.0)
+ factory_girl (~> 4.9.0)
railties (>= 3.0.0)
- faraday (0.11.0)
+ faraday (0.12.2)
multipart-post (>= 1.2, < 3)
faye-websocket (0.10.7)
eventmachine (>= 0.12.0)
websocket-driver (>= 0.5.1)
- globalid (0.3.7)
- activesupport (>= 4.1.0)
+ globalid (0.4.1)
+ activesupport (>= 4.2.0)
google-api-client (0.8.7)
activesupport (>= 3.2, < 5.0)
addressable (~> 2.3)
@@ -116,25 +116,25 @@ GEM
multi_json (~> 1.10)
retriable (~> 1.4)
signet (~> 0.6)
- googleauth (0.5.1)
- faraday (~> 0.9)
- jwt (~> 1.4)
+ googleauth (0.6.2)
+ faraday (~> 0.12)
+ jwt (>= 1.4, < 3.0)
logging (~> 2.0)
memoist (~> 0.12)
multi_json (~> 1.11)
os (~> 0.9)
signet (~> 0.7)
- hashie (3.5.5)
- highline (1.7.8)
+ hashie (3.5.7)
+ highline (1.7.10)
hike (1.2.3)
httpclient (2.8.3)
- i18n (0.9.0)
+ i18n (0.9.5)
concurrent-ruby (~> 1.0)
- jquery-rails (4.2.2)
+ jquery-rails (4.3.1)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
- json (1.8.6)
+ json (2.1.0)
jwt (1.5.6)
launchy (2.4.3)
addressable (~> 2.3)
@@ -143,97 +143,97 @@ GEM
logging (2.2.2)
little-plugger (~> 1.1)
multi_json (~> 1.10)
- lograge (0.7.1)
- actionpack (>= 4, < 5.2)
- activesupport (>= 4, < 5.2)
- railties (>= 4, < 5.2)
+ lograge (0.9.0)
+ actionpack (>= 4)
+ activesupport (>= 4)
+ railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.1.1)
+ loofah (2.2.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
- mail (2.6.4)
- mime-types (>= 1.16, < 4)
+ mail (2.7.0)
+ mini_mime (>= 0.1.1)
memoist (0.16.0)
metaclass (0.0.4)
- mime-types (3.1)
- mime-types-data (~> 3.2015)
- mime-types-data (3.2016.0521)
+ mini_mime (1.0.0)
mini_portile2 (2.3.0)
- minitest (5.10.3)
- mocha (1.2.1)
+ minitest (5.11.3)
+ mocha (1.3.0)
metaclass (~> 0.0.1)
- multi_json (1.12.1)
+ multi_json (1.13.1)
multi_xml (0.6.0)
multipart-post (2.0.0)
net-scp (1.2.1)
net-ssh (>= 2.6.5)
net-sftp (2.1.2)
net-ssh (>= 2.6.5)
- net-ssh (4.1.0)
+ net-ssh (4.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
- nokogiri (1.8.1)
+ nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
- oauth2 (1.3.1)
- faraday (>= 0.8, < 0.12)
+ oauth2 (1.4.0)
+ faraday (>= 0.8, < 0.13)
jwt (~> 1.0)
multi_json (~> 1.3)
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
oj (2.18.5)
oj_mimic_json (1.0.1)
- omniauth (1.4.2)
+ omniauth (1.4.3)
hashie (>= 1.2, < 4)
- rack (>= 1.0, < 3)
- omniauth-oauth2 (1.4.0)
- oauth2 (~> 1.0)
+ rack (>= 1.6.2, < 3)
+ omniauth-oauth2 (1.5.0)
+ oauth2 (~> 1.1)
omniauth (~> 1.2)
os (0.9.6)
- passenger (5.1.2)
+ passenger (5.2.1)
rack
rake (>= 0.8.1)
- pg (0.20.0)
- power_assert (1.0.1)
- protected_attributes (1.1.3)
+ pg (0.21.0)
+ power_assert (1.1.1)
+ protected_attributes (1.1.4)
activemodel (>= 4.0.1, < 5.0)
- public_suffix (2.0.5)
- rack (1.6.8)
+ public_suffix (3.0.2)
+ rack (1.6.9)
rack-test (0.6.3)
rack (>= 1.0)
- rails (4.2.5.2)
- actionmailer (= 4.2.5.2)
- actionpack (= 4.2.5.2)
- actionview (= 4.2.5.2)
- activejob (= 4.2.5.2)
- activemodel (= 4.2.5.2)
- activerecord (= 4.2.5.2)
- activesupport (= 4.2.5.2)
+ rails (4.2.10)
+ actionmailer (= 4.2.10)
+ actionpack (= 4.2.10)
+ actionview (= 4.2.10)
+ activejob (= 4.2.10)
+ activemodel (= 4.2.10)
+ activerecord (= 4.2.10)
+ activesupport (= 4.2.10)
bundler (>= 1.3.0, < 2.0)
- railties (= 4.2.5.2)
+ railties (= 4.2.10)
sprockets-rails
rails-deprecated_sanitizer (1.0.3)
activesupport (>= 4.2.0.alpha)
- rails-dom-testing (1.0.8)
- activesupport (>= 4.2.0.beta, < 5.0)
+ rails-dom-testing (1.0.9)
+ activesupport (>= 4.2.0, < 5.0)
nokogiri (~> 1.6)
rails-deprecated_sanitizer (>= 1.0.1)
rails-html-sanitizer (1.0.3)
loofah (~> 2.0)
- rails-observers (0.1.2)
- activemodel (~> 4.0)
- railties (4.2.5.2)
- actionpack (= 4.2.5.2)
- activesupport (= 4.2.5.2)
+ rails-observers (0.1.5)
+ activemodel (>= 4.0)
+ railties (4.2.10)
+ actionpack (= 4.2.10)
+ activesupport (= 4.2.10)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (12.2.1)
+ rake (12.3.0)
ref (2.0.0)
- request_store (1.3.2)
- responders (2.3.0)
- railties (>= 4.2.0, < 5.1)
+ request_store (1.4.0)
+ rack (>= 1.4)
+ responders (2.4.0)
+ actionpack (>= 4.2.0, < 5.3)
+ railties (>= 4.2.0, < 5.3)
retriable (1.4.1)
- ruby-prof (0.16.2)
+ ruby-prof (0.17.0)
rvm-capistrano (1.5.6)
capistrano (~> 2.15.4)
safe_yaml (1.0.4)
@@ -243,10 +243,10 @@ GEM
sass (~> 3.2.2)
sprockets (~> 2.8, < 3.0)
sprockets-rails (~> 2.0)
- signet (0.7.3)
+ signet (0.8.1)
addressable (~> 2.3)
faraday (~> 0.9)
- jwt (~> 1.5)
+ jwt (>= 1.5, < 3.0)
multi_json (~> 1.10)
simplecov (0.7.1)
multi_json (~> 1.0)
@@ -264,7 +264,7 @@ GEM
activesupport (>= 3.0)
sprockets (>= 2.8, < 4.0)
sshkey (1.9.0)
- test-unit (3.2.3)
+ test-unit (3.2.7)
power_assert
test_after_commit (1.1.0)
activerecord (>= 3.2)
@@ -275,19 +275,20 @@ GEM
thread_safe (0.3.6)
tilt (1.4.1)
trollop (2.1.2)
- tzinfo (1.2.4)
+ tzinfo (1.2.5)
thread_safe (~> 0.1)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.6.5)
+ websocket-driver (0.7.0)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.2)
+ websocket-extensions (0.1.3)
PLATFORMS
ruby
DEPENDENCIES
+ activerecord-deprecated_finders
acts_as_api
andand
arvados (>= 0.1.20150615153458)
@@ -307,9 +308,9 @@ DEPENDENCIES
omniauth (~> 1.4.0)
omniauth-oauth2 (~> 1.1)
passenger
- pg
+ pg (~> 0.18)
protected_attributes
- rails (~> 4.0)
+ rails (~> 4.2)
rails-observers
responders (~> 2.0)
ruby-prof
@@ -327,4 +328,4 @@ DEPENDENCIES
uglifier (~> 2.0)
BUNDLED WITH
- 1.16.0
+ 1.16.1
commit ab71c039f0c9ebd269f96b80cf969d8a87c48e98
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 9 14:11:28 2018 -0500
13134: Support for secret mounts in crunch-run.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index daafc49..e0e8c77 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -16,6 +16,7 @@ type Container struct {
Environment map[string]string `json:"environment"`
LockedByUUID string `json:"locked_by_uuid"`
Mounts map[string]Mount `json:"mounts"`
+ SecretMounts map[string]Mount `json:"secret_mounts"`
Output string `json:"output"`
OutputPath string `json:"output_path"`
Priority int `json:"priority"`
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 653e0b4..31af0bd 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -396,10 +396,21 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
for bind := range runner.Container.Mounts {
binds = append(binds, bind)
}
+ for bind := range runner.Container.SecretMounts {
+ if runner.Container.SecretMounts[bind].Kind != "json" &&
+ runner.Container.SecretMounts[bind].Kind != "text" {
+ return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.",
+ bind, runner.Container.SecretMounts[bind].Kind)
+ }
+ binds = append(binds, bind)
+ }
sort.Strings(binds)
for _, bind := range binds {
- mnt := runner.Container.Mounts[bind]
+ mnt, ok := runner.Container.Mounts[bind]
+ if !ok {
+ mnt = runner.Container.SecretMounts[bind]
+ }
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
@@ -428,8 +439,8 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
- if mnt.Kind != "collection" {
- return fmt.Errorf("Only mount points of kind 'collection' are supported underneath the output_path: %v", bind)
+ if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
+ return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
@@ -503,26 +514,35 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
runner.HostOutputDir = tmpdir
}
- case mnt.Kind == "json":
- jsondata, err := json.Marshal(mnt.Content)
- if err != nil {
- return fmt.Errorf("encoding json data: %v", err)
+ case mnt.Kind == "json" || mnt.Kind == "text":
+ var filedata []byte
+ if mnt.Kind == "json" {
+ filedata, err = json.Marshal(mnt.Content)
+ if err != nil {
+ return fmt.Errorf("encoding json data: %v", err)
+ }
+ } else {
+ text, ok := mnt.Content.(string)
+ if !ok {
+ return fmt.Errorf("content for mount %q must be a string", bind)
+ }
+ filedata = []byte(text)
}
- // Create a tempdir with a single file
- // (instead of just a tempfile): this way we
- // can ensure the file is world-readable
- // inside the container, without having to
- // make it world-readable on the docker host.
- tmpdir, err := runner.MkTempDir(runner.parentTemp, "json")
+
+ tmpdir, err := runner.MkTempDir(runner.parentTemp, mnt.Kind)
if err != nil {
return fmt.Errorf("creating temp dir: %v", err)
}
- tmpfn := filepath.Join(tmpdir, "mountdata.json")
- err = ioutil.WriteFile(tmpfn, jsondata, 0644)
+ tmpfn := filepath.Join(tmpdir, "mountdata."+mnt.Kind)
+ err = ioutil.WriteFile(tmpfn, filedata, 0444)
if err != nil {
return fmt.Errorf("writing temp file: %v", err)
}
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+ if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
+ copyFiles = append(copyFiles, copyFile{tmpfn, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
+ } else {
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+ }
case mnt.Kind == "git_tree":
tmpdir, err := runner.MkTempDir(runner.parentTemp, "git_tree")
@@ -1259,6 +1279,16 @@ func (runner *ContainerRunner) CaptureOutput() error {
}
sort.Strings(binds)
+ // Delete secret mounts so they don't get saved to the output collection.
+ for bind := range runner.Container.SecretMounts {
+ if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
+ err = os.Remove(runner.HostOutputDir + bind[len(runner.Container.OutputPath):])
+ if err != nil {
+ return fmt.Errorf("Unable to remove secret mount: %v", err)
+ }
+ }
+ }
+
var manifestText string
collectionMetafile := fmt.Sprintf("%s/.arvados#collection", runner.HostOutputDir)
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 94b7133..487e95e 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1211,6 +1211,35 @@ func (s *TestSuite) TestSetupMounts(c *C) {
checkEmpty()
}
+ for _, test := range []struct {
+ in interface{}
+ out string
+ }{
+ {in: "foo", out: `foo`},
+ {in: nil, out: "error"},
+ {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
+ } {
+ i = 0
+ cr.ArvMountPoint = ""
+ cr.Container.Mounts = map[string]arvados.Mount{
+ "/mnt/test.txt": {Kind: "text", Content: test.in},
+ }
+ err := cr.SetupMounts()
+ if test.out == "error" {
+ c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
+ } else {
+ c.Check(err, IsNil)
+ sort.StringSlice(cr.Binds).Sort()
+ c.Check(cr.Binds, DeepEquals, []string{realTemp + "/text2/mountdata.text:/mnt/test.txt:ro"})
+ content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
+ c.Check(err, IsNil)
+ c.Check(content, DeepEquals, []byte(test.out))
+ }
+ os.RemoveAll(cr.ArvMountPoint)
+ cr.CleanupDirs()
+ checkEmpty()
+ }
+
// Read-only mount points are allowed underneath output_dir mount point
{
i = 0
@@ -1277,13 +1306,13 @@ func (s *TestSuite) TestSetupMounts(c *C) {
cr.Container.Mounts = make(map[string]arvados.Mount)
cr.Container.Mounts = map[string]arvados.Mount{
"/tmp": {Kind: "tmp"},
- "/tmp/foo": {Kind: "json"},
+ "/tmp/foo": {Kind: "tmp"},
}
cr.OutputPath = "/tmp"
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
+ c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
@@ -2035,3 +2064,55 @@ func (s *TestSuite) TestBadCommand3(c *C) {
c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
}
+
+func (s *TestSuite) TestSecretTextMountPoint(c *C) {
+ // under normal mounts, gets captured in output, oops
+ helperRecord := `{
+ "command": ["true"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "mounts": {
+ "/tmp": {"kind": "tmp"},
+ "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
+ },
+ "secret_mounts": {
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
+ c.Check(api.CalledWith("collection.manifest_text", ""), IsNil)
+
+ // under secret mounts, not captured in output
+ helperRecord = `{
+ "command": ["true"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "mounts": {
+ "/tmp": {"kind": "tmp"}
+ },
+ "secret_mounts": {
+ "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ api, _, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
+ c.Check(api.CalledWith("collection.manifest_text", ""), NotNil)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list