[ARVADOS] updated: 9cb3552d964d162d6eacff40435650440bc6f692
Git user
git at public.curoverse.com
Wed Sep 28 16:19:20 EDT 2016
Summary of changes:
.../postinst | 11 +++
.../prerm | 8 ++
build/package-build-dockerfiles/Makefile | 2 +-
build/package-build-dockerfiles/centos6/Dockerfile | 2 +-
build/package-build-dockerfiles/centos7/Dockerfile | 2 +-
build/package-build-dockerfiles/debian7/Dockerfile | 2 +-
build/package-build-dockerfiles/debian8/Dockerfile | 2 +-
.../ubuntu1204/Dockerfile | 2 +-
.../ubuntu1404/Dockerfile | 2 +-
build/run-library.sh | 14 ++-
sdk/pam/setup.py | 7 ++
services/arv-git-httpd/arv-git-httpd.service | 3 +-
services/arv-git-httpd/main.go | 2 +-
.../crunch-dispatch-slurm.service | 1 +
services/dockercleaner/MANIFEST.in | 1 +
.../arvados-docker-cleaner.service} | 5 +-
services/dockercleaner/arvados_docker/cleaner.py | 107 ++++++++++++++++-----
services/dockercleaner/setup.py | 17 ++--
services/dockercleaner/tests/test_cleaner.py | 85 ++++++++++++----
services/keep-balance/keep-balance.service | 1 +
services/keep-web/keep-web.service | 1 +
services/keepproxy/keepproxy.service | 1 +
22 files changed, 215 insertions(+), 63 deletions(-)
rename build/{go-package-scripts => go-python-package-scripts}/postinst (74%)
rename build/{go-package-scripts => go-python-package-scripts}/prerm (61%)
copy services/{keep-balance/keep-balance.service => dockercleaner/arvados-docker-cleaner.service} (50%)
via 9cb3552d964d162d6eacff40435650440bc6f692 (commit)
via 2a4cf07eb109934ade1f77202da6002db9458663 (commit)
via 0bcf6869ed21baf852ed9278413a93df6b002b43 (commit)
via 68d57ec546640255f78f3aae5abf8514df0a003a (commit)
via d6fa7140440d1dd852ca90780abcf74dd99345f3 (commit)
via c213cdeae6dc9487a7eded068adfbd8d88c9757e (commit)
via 64c8efd05e7d8bed255a083694e0b8125ecbc372 (commit)
via ced114dfe4f8fda480a0d37fae9c0107c90f5cf7 (commit)
via 586e959c9cfbe256682836be98eb451dddaf3ae9 (commit)
via ae50ec2fec20b96a208619421a97a7b41c7a6ac5 (commit)
via 86593d23ec8bbe3f18e3f2d5d76cc0ffa486d084 (commit)
via e1f0da5e2657adceb6bfec870f96ac7e604341a8 (commit)
via ba6d65ca676c8517828c53682cd948b1450e281b (commit)
via 14c3753c7b8128b02f9cdbeaccd338a646506851 (commit)
from eec17149459fb71f8be88ec2f8ac61bc8f41f3cb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9cb3552d964d162d6eacff40435650440bc6f692
Merge: eec1714 2a4cf07
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 16:19:03 2016 -0400
Merge branch '9953-dockercleaner-config' closes #9953
commit 2a4cf07eb109934ade1f77202da6002db9458663
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 16:18:00 2016 -0400
9953: Add arvados-docker-cleaner.service to MANIFEST.in.
Fixes error during `setup.py install`:
Command "/tmp/tmp.lpB0zPHYWs/VENV3DIR/bin/python3 -u -c "import setuptools, tokenize;__file__=/tmp/pip-6gqvw70n-build/setup.py;exec(compile(getattr(tokenize, open, open)(__file__).read().replace(rn, n), __file__, exec))" install --record /tmp/pip-ljhdcczu-record/install-record.txt --single-version-externally-managed --compile --install-headers /tmp/tmp.lpB0zPHYWs/VENV3DIR/include/site/python3.4/arvados-docker-cleaner" failed with error code 1 in /tmp/pip-6gqvw70n-build/
diff --git a/services/dockercleaner/MANIFEST.in b/services/dockercleaner/MANIFEST.in
index 4db8152..5430662 100644
--- a/services/dockercleaner/MANIFEST.in
+++ b/services/dockercleaner/MANIFEST.in
@@ -1 +1,2 @@
include agpl-3.0.txt
+include arvados-docker-cleaner.service
commit 0bcf6869ed21baf852ed9278413a93df6b002b43
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 11:13:40 2016 -0400
9950: 9953: 9954: 9955: 9957: Add AssertPathExists= to unit file to avoid startup until config file exists.
diff --git a/services/arv-git-httpd/arv-git-httpd.service b/services/arv-git-httpd/arv-git-httpd.service
index 8aaf674..f71c2ff 100644
--- a/services/arv-git-httpd/arv-git-httpd.service
+++ b/services/arv-git-httpd/arv-git-httpd.service
@@ -2,6 +2,7 @@
Description=Arvados git server
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/arvados-git-httpd/arvados-git-httpd.yml
[Service]
Type=notify
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
index 5b37e3c..34ba80b 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
@@ -2,6 +2,7 @@
Description=Arvados Crunch Dispatcher for SLURM
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml
[Service]
Type=notify
diff --git a/services/dockercleaner/arvados-docker-cleaner.service b/services/dockercleaner/arvados-docker-cleaner.service
index 43e4ab8..28653ae 100644
--- a/services/dockercleaner/arvados-docker-cleaner.service
+++ b/services/dockercleaner/arvados-docker-cleaner.service
@@ -2,6 +2,7 @@
Description=Arvados Docker Image Cleaner
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/docker-cleaner/docker-cleaner.json
[Service]
Type=simple
diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 634b44b..157e42c 100644
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -2,6 +2,7 @@
Description=Arvados Keep Balance
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
[Service]
Type=simple
diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
index da56212..24be771 100644
--- a/services/keep-web/keep-web.service
+++ b/services/keep-web/keep-web.service
@@ -2,6 +2,7 @@
Description=Arvados Keep web gateway
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/keep-web/keep-web.yml
[Service]
Type=notify
diff --git a/services/keepproxy/keepproxy.service b/services/keepproxy/keepproxy.service
index 5bd3036..c340fab 100644
--- a/services/keepproxy/keepproxy.service
+++ b/services/keepproxy/keepproxy.service
@@ -2,6 +2,7 @@
Description=Arvados Keep Proxy
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/keepproxy/keepproxy.yml
[Service]
Type=notify
commit 68d57ec546640255f78f3aae5abf8514df0a003a
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 11:12:11 2016 -0400
9950: Fix arvados-git-httpd executable path. Update config path to /etc/arvados/git-httpd/.
diff --git a/services/arv-git-httpd/arv-git-httpd.service b/services/arv-git-httpd/arv-git-httpd.service
index 1182a0e..8aaf674 100644
--- a/services/arv-git-httpd/arv-git-httpd.service
+++ b/services/arv-git-httpd/arv-git-httpd.service
@@ -5,7 +5,7 @@ After=network.target
[Service]
Type=notify
-ExecStart=/usr/bin/arv-git-httpd
+ExecStart=/usr/bin/arvados-git-httpd
Restart=always
[Install]
diff --git a/services/arv-git-httpd/main.go b/services/arv-git-httpd/main.go
index 26d2914..dd28136 100644
--- a/services/arv-git-httpd/main.go
+++ b/services/arv-git-httpd/main.go
@@ -35,7 +35,7 @@ func defaultConfig() *Config {
}
func init() {
- const defaultCfgPath = "/etc/arvados/arv-git-httpd/arv-git-httpd.yml"
+ const defaultCfgPath = "/etc/arvados/git-httpd/git-httpd.yml"
const deprecated = " (DEPRECATED -- use config file instead)"
flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
"Address to listen on, \"host:port\" or \":port\"."+deprecated)
commit d6fa7140440d1dd852ca90780abcf74dd99345f3
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 10:47:36 2016 -0400
9953: Update default config path.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 8b8c772..5ac8100 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -294,7 +294,7 @@ def parse_arguments(arguments):
formatter_class=Formatter,
)
parser.add_argument(
- '--config', action='store', type=str, default='/etc/arvados/docker-cleaner/config.json',
+ '--config', action='store', type=str, default='/etc/arvados/docker-cleaner/docker-cleaner.json',
help="configuration file")
deprecated = " (DEPRECATED -- use config file instead)"
commit c213cdeae6dc9487a7eded068adfbd8d88c9757e
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 28 09:44:07 2016 -0400
9953: If config cannot be read/parsed, show just the error without the stack trace.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 9dd7b12..8b8c772 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -257,8 +257,12 @@ def load_config(arguments):
args = parse_arguments(arguments)
config = default_config()
- with open(args.config, 'r') as f:
- config.update(json.load(f))
+ try:
+ with open(args.config, 'r') as f:
+ c = json.load(f)
+ config.update(c)
+ except (FileNotFoundError, IOError, ValueError) as error:
+ sys.exit('error reading config file {}: {}'.format(args.config, error))
configargs = vars(args).copy()
configargs.pop('config')
commit 64c8efd05e7d8bed255a083694e0b8125ecbc372
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:47:29 2016 -0400
9953: Install/remove systemd unit files wherever systemd exists, even if it is not running.
diff --git a/build/go-python-package-scripts/postinst b/build/go-python-package-scripts/postinst
index 1e10333..051c8bd 100755
--- a/build/go-python-package-scripts/postinst
+++ b/build/go-python-package-scripts/postinst
@@ -11,7 +11,8 @@ systemd_unit="${pkg}.service"
case "${1}" in
configure)
- if [ -e /run/systemd/system ]; then
+ if [ -d /lib/systemd/system ]
+ then
# Python packages put all data files in /usr, so we copy
# them to /lib at install time.
py_unit="/usr/share/doc/${pkg}/${pkg}.service"
@@ -19,7 +20,9 @@ case "${1}" in
then
cp "${py_unit}" /lib/systemd/system/
fi
+ fi
+ if [ -e /run/systemd/system ]; then
eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
case "${UnitFileState}" in
disabled)
diff --git a/build/go-python-package-scripts/prerm b/build/go-python-package-scripts/prerm
index 7f4b843..c6ec18c 100755
--- a/build/go-python-package-scripts/prerm
+++ b/build/go-python-package-scripts/prerm
@@ -14,14 +14,14 @@ case "${1}" in
if [ -e /run/systemd/system ]; then
systemctl stop "${systemd_unit}" || true
systemctl disable "${systemd_unit}" || true
+ fi
- # Unit files from Python packages get installed by
- # postinst so we have to remove them explicitly here.
- py_unit="/usr/share/doc/${pkg}/${pkg}.service"
- if [ -e "${py_unit}" ]
- then
- rm "/lib/systemd/system/${pkg}.service" || true
- fi
+ # Unit files from Python packages get installed by postinst so
+ # we have to remove them explicitly here.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ rm "/lib/systemd/system/${pkg}.service" || true
fi
;;
esac
commit ced114dfe4f8fda480a0d37fae9c0107c90f5cf7
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:41:01 2016 -0400
9953: Update build scripts to Go 1.7.1.
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 2180b87..18694ed 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -24,7 +24,7 @@ ubuntu1404/generated: common-generated-all
test -d ubuntu1404/generated || mkdir ubuntu1404/generated
cp -rlt ubuntu1404/generated common-generated/*
-GOTARBALL=go1.6.2.linux-amd64.tar.gz
+GOTARBALL=go1.7.1.linux-amd64.tar.gz
common-generated-all: common-generated/$(GOTARBALL)
diff --git a/build/package-build-dockerfiles/centos6/Dockerfile b/build/package-build-dockerfiles/centos6/Dockerfile
index 8d969e6..8ea81f4 100644
--- a/build/package-build-dockerfiles/centos6/Dockerfile
+++ b/build/package-build-dockerfiles/centos6/Dockerfile
@@ -5,7 +5,7 @@ MAINTAINER Brett Smith <brett at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install RVM
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 311aaa2..4fcd640 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -5,7 +5,7 @@ MAINTAINER Brett Smith <brett at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install RVM
diff --git a/build/package-build-dockerfiles/debian7/Dockerfile b/build/package-build-dockerfiles/debian7/Dockerfile
index ddad542..7632c94 100644
--- a/build/package-build-dockerfiles/debian7/Dockerfile
+++ b/build/package-build-dockerfiles/debian7/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 80f06a2..977cd24 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1204/Dockerfile b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
index 2f628b0..b0dd906 100644
--- a/build/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index b9c003a..91c5e5b 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
commit 586e959c9cfbe256682836be98eb451dddaf3ae9
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:40:37 2016 -0400
9953: De-duplicate Go and Python postinst/prerm scripts.
diff --git a/build/go-package-scripts/postinst b/build/go-python-package-scripts/postinst
similarity index 77%
rename from build/go-package-scripts/postinst
rename to build/go-python-package-scripts/postinst
index 0663465..1e10333 100755
--- a/build/go-package-scripts/postinst
+++ b/build/go-python-package-scripts/postinst
@@ -12,6 +12,14 @@ systemd_unit="${pkg}.service"
case "${1}" in
configure)
if [ -e /run/systemd/system ]; then
+ # Python packages put all data files in /usr, so we copy
+ # them to /lib at install time.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ cp "${py_unit}" /lib/systemd/system/
+ fi
+
eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
case "${UnitFileState}" in
disabled)
diff --git a/build/go-package-scripts/prerm b/build/go-python-package-scripts/prerm
similarity index 59%
rename from build/go-package-scripts/prerm
rename to build/go-python-package-scripts/prerm
index 02772e4..7f4b843 100755
--- a/build/go-package-scripts/prerm
+++ b/build/go-python-package-scripts/prerm
@@ -14,6 +14,14 @@ case "${1}" in
if [ -e /run/systemd/system ]; then
systemctl stop "${systemd_unit}" || true
systemctl disable "${systemd_unit}" || true
+
+ # Unit files from Python packages get installed by
+ # postinst so we have to remove them explicitly here.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ rm "/lib/systemd/system/${pkg}.service" || true
+ fi
fi
;;
esac
diff --git a/build/python-package-scripts/postinst b/build/python-package-scripts/postinst
deleted file mode 100755
index 27e4601..0000000
--- a/build/python-package-scripts/postinst
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# NOTE: This package name detection will only work on Debian.
-# If this postinst script ever starts doing work on Red Hat,
-# we'll need to adapt this code accordingly.
-script="$(basename "${0}")"
-pkg="${script%.postinst}"
-systemd_unit="${pkg}.service"
-
-case "${1}" in
- configure)
- if [ -e /run/systemd/system ]; then
- cp "/usr/share/doc/${pkg}/${pkg}.service" /lib/systemd/system/
- eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
- case "${UnitFileState}" in
- disabled)
- # Failing to enable or start the service is not a
- # package error, so don't let errors here
- # propagate up.
- systemctl enable "${systemd_unit}" || true
- systemctl start "${systemd_unit}" || true
- ;;
- enabled)
- systemctl daemon-reload || true
- systemctl reload-or-try-restart "${systemd_unit}" || true
- ;;
- esac
- fi
- ;;
-esac
diff --git a/build/python-package-scripts/prerm b/build/python-package-scripts/prerm
deleted file mode 100755
index 6d75f81..0000000
--- a/build/python-package-scripts/prerm
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# NOTE: This package name detection will only work on Debian.
-# If this prerm script ever starts doing work on Red Hat,
-# we'll need to adapt this code accordingly.
-script="$(basename "${0}")"
-pkg="${script%.prerm}"
-systemd_unit="${pkg}.service"
-
-case "${1}" in
- remove)
- if [ -e /run/systemd/system ]; then
- systemctl stop "${systemd_unit}" || true
- systemctl disable "${systemd_unit}" || true
- rm "/lib/systemd/system/${pkg}.service"
- fi
- ;;
-esac
diff --git a/build/run-library.sh b/build/run-library.sh
index e445541..73a99da 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -105,8 +105,8 @@ package_go_binary() {
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
- --after-install "${WORKSPACE}/build/go-package-scripts/postinst"
- --before-remove "${WORKSPACE}/build/go-package-scripts/prerm"
+ --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
"${systemd_unit}=/lib/systemd/system/${prog}.service")
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
@@ -268,8 +268,8 @@ fpm_build () {
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
COMMAND_ARR+=(
- --after-install "${WORKSPACE}/build/python-package-scripts/postinst"
- --before-remove "${WORKSPACE}/build/python-package-scripts/prerm"
+ --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
)
fi
commit ae50ec2fec20b96a208619421a97a7b41c7a6ac5
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:08:37 2016 -0400
9953: Add systemd unit file for arvados-docker-cleaner.
diff --git a/build/python-package-scripts/postinst b/build/python-package-scripts/postinst
new file mode 100755
index 0000000..27e4601
--- /dev/null
+++ b/build/python-package-scripts/postinst
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this postinst script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.postinst}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ configure)
+ if [ -e /run/systemd/system ]; then
+ cp "/usr/share/doc/${pkg}/${pkg}.service" /lib/systemd/system/
+ eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
+ case "${UnitFileState}" in
+ disabled)
+ # Failing to enable or start the service is not a
+ # package error, so don't let errors here
+ # propagate up.
+ systemctl enable "${systemd_unit}" || true
+ systemctl start "${systemd_unit}" || true
+ ;;
+ enabled)
+ systemctl daemon-reload || true
+ systemctl reload-or-try-restart "${systemd_unit}" || true
+ ;;
+ esac
+ fi
+ ;;
+esac
diff --git a/build/python-package-scripts/prerm b/build/python-package-scripts/prerm
new file mode 100755
index 0000000..6d75f81
--- /dev/null
+++ b/build/python-package-scripts/prerm
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this prerm script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.prerm}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ remove)
+ if [ -e /run/systemd/system ]; then
+ systemctl stop "${systemd_unit}" || true
+ systemctl disable "${systemd_unit}" || true
+ rm "/lib/systemd/system/${pkg}.service"
+ fi
+ ;;
+esac
diff --git a/build/run-library.sh b/build/run-library.sh
index 067285d..e445541 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -265,6 +265,14 @@ fpm_build () {
# that will take precedence, as desired.
COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
+ then
+ COMMAND_ARR+=(
+ --after-install "${WORKSPACE}/build/python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/python-package-scripts/prerm"
+ )
+ fi
+
# Append --depends X and other arguments specified by fpm-info.sh in
# the package source dir. These are added last so they can override
# the arguments added by this script.
diff --git a/services/dockercleaner/arvados-docker-cleaner.service b/services/dockercleaner/arvados-docker-cleaner.service
new file mode 100644
index 0000000..43e4ab8
--- /dev/null
+++ b/services/dockercleaner/arvados-docker-cleaner.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Arvados Docker Image Cleaner
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/env arvados-docker-cleaner
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 15b9b4e..535650e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -25,7 +25,7 @@ setup(name="arvados-docker-cleaner",
'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
},
data_files=[
- ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
+ ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt', 'arvados-docker-cleaner.service']),
],
install_requires=[
'docker-py==1.7.2',
commit 86593d23ec8bbe3f18e3f2d5d76cc0ffa486d084
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:09:03 2016 -0400
9953: Install python data files to /usr/share/... not /usr/data/share/...
diff --git a/build/run-library.sh b/build/run-library.sh
index 4495a4e..067285d 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -213,6 +213,7 @@ fpm_build () {
--python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
--prefix "$PYTHON2_PREFIX" \
--python-install-lib "$PYTHON2_INSTALL_LIB" \
+ --python-install-data . \
--exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
--depends "$PYTHON2_PACKAGE"
# Fix --iteration for #9242.
@@ -229,6 +230,7 @@ fpm_build () {
--python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
--prefix "$PYTHON3_PREFIX" \
--python-install-lib "$PYTHON3_INSTALL_LIB" \
+ --python-install-data . \
--exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
--depends "$PYTHON3_PACKAGE"
# Fix --iteration for #9242.
diff --git a/sdk/pam/setup.py b/sdk/pam/setup.py
index c194013..e1046ac 100755
--- a/sdk/pam/setup.py
+++ b/sdk/pam/setup.py
@@ -37,6 +37,13 @@ setup(name='arvados-pam',
('share/pam-configs', ['pam-configs/arvados']),
('share/doc/arvados-pam', ['LICENSE-2.0.txt', 'README.rst']),
('share/doc/arvados-pam/examples', glob.glob('examples/*')),
+
+ # The arvados build scripts used to install data files to
+ # "/usr/data/*" but now install them to "/usr/*". Here, we
+ # install an extra copy in the old location so existing pam
+ # configs can still work. When old systems have had a chance
+ # to update to the new paths, this line can be removed.
+ ('data/lib/security', ['lib/libpam_arvados.py']),
],
install_requires=[
'arvados-python-client>=0.1.20150801000000',
commit e1f0da5e2657adceb6bfec870f96ac7e604341a8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 10:26:10 2016 -0400
9953: PEP-8
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 5531bf5..9dd7b12 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -21,6 +21,7 @@ SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
logger = logging.getLogger('arvados_docker.cleaner')
+
def return_when_docker_not_found(result=None):
# If the decorated function raises a 404 error from Docker, return
# `result` instead.
@@ -36,7 +37,9 @@ def return_when_docker_not_found(result=None):
return docker_not_found_wrapper
return docker_not_found_decorator
+
class DockerImage:
+
def __init__(self, image_hash):
self.docker_id = image_hash['Id']
self.size = image_hash['VirtualSize']
@@ -47,6 +50,7 @@ class DockerImage:
class DockerImages:
+
def __init__(self, target_size):
self.target_size = target_size
self.images = {}
@@ -118,6 +122,7 @@ class DockerImages:
class DockerEventHandlers:
# This class maps Docker event types to the names of methods that should
# receive those events.
+
def __init__(self):
self.handler_names = collections.defaultdict(list)
@@ -221,7 +226,8 @@ class DockerImageCleaner(DockerImageUseRecorder):
try:
self.docker_client.remove_image(image_id)
except docker.errors.APIError as error:
- logger.warning("Failed to remove image %s: %s", image_id, error)
+ logger.warning(
+ "Failed to remove image %s: %s", image_id, error)
else:
logger.info("Removed image %s", image_id)
self.images.del_image(image_id)
@@ -231,8 +237,9 @@ class DockerImageCleaner(DockerImageUseRecorder):
unknown_ids = {image['Id'] for image in self.docker_client.images()
if not self.images.has_image(image['Id'])}
for image_id in (unknown_ids - self.logged_unknown):
- logger.info("Image %s is loaded but unused, so it won't be cleaned",
- image_id)
+ logger.info(
+ "Image %s is loaded but unused, so it won't be cleaned",
+ image_id)
self.logged_unknown = unknown_ids
@@ -245,6 +252,7 @@ def human_size(size_str):
size_str = size_str[:-1]
return int(size_str) * multiplier
+
def load_config(arguments):
args = parse_arguments(arguments)
@@ -261,6 +269,7 @@ def load_config(arguments):
return config
+
def default_config():
return {
'Quota': '1G',
@@ -268,6 +277,7 @@ def default_config():
'Verbose': 0,
}
+
def parse_arguments(arguments):
class Formatter(argparse.ArgumentDefaultsHelpFormatter,
argparse.RawDescriptionHelpFormatter):
@@ -280,7 +290,7 @@ def parse_arguments(arguments):
formatter_class=Formatter,
)
parser.add_argument(
- '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+ '--config', action='store', type=str, default='/etc/arvados/docker-cleaner/config.json',
help="configuration file")
deprecated = " (DEPRECATED -- use config file instead)"
@@ -299,14 +309,16 @@ def parse_arguments(arguments):
return parser.parse_args(arguments)
+
def setup_logging(config):
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
- '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
- '%Y-%m-%d %H:%M:%S'))
+ '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
+ '%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
logger.setLevel(logging.ERROR - (10 * config['Verbose']))
+
def run(config, docker_client):
start_time = int(time.time())
logger.debug("Loading Docker activity through present")
@@ -324,6 +336,7 @@ def run(config, docker_client):
logger.info("Listening for docker events")
cleaner.run()
+
def main(arguments=sys.argv[1:]):
config = load_config(arguments)
setup_logging(config)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index cabafe9..9fbd3e3 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -15,13 +15,16 @@ from arvados_docker import cleaner
MAX_DOCKER_ID = (16 ** 64) - 1
+
def MockDockerId():
return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID))
+
def MockContainer(image_hash):
return {'Id': MockDockerId(),
'Image': image_hash['Id']}
+
def MockImage(*, size=0, vsize=None, tags=[]):
if vsize is None:
vsize = random.randint(100, 2000000)
@@ -31,6 +34,7 @@ def MockImage(*, size=0, vsize=None, tags=[]):
'Size': size,
'VirtualSize': vsize}
+
class MockEvent(dict):
ENCODING = 'utf-8'
event_seq = itertools.count(1)
@@ -48,6 +52,7 @@ class MockEvent(dict):
class MockException(docker.errors.APIError):
+
def __init__(self, status_code):
response = mock.Mock(name='response')
response.status_code = status_code
@@ -55,6 +60,7 @@ class MockException(docker.errors.APIError):
class DockerImageTestCase(unittest.TestCase):
+
def test_used_at_sets_last_used(self):
image = cleaner.DockerImage(MockImage())
image.used_at(5)
@@ -74,6 +80,7 @@ class DockerImageTestCase(unittest.TestCase):
class DockerImagesTestCase(unittest.TestCase):
+
def setUp(self):
self.mock_images = []
@@ -336,6 +343,7 @@ class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase):
class HumanSizeTestCase(unittest.TestCase):
+
def check(self, human_str, count, exp):
self.assertEqual(count * (1024 ** exp),
cleaner.human_size(human_str))
@@ -362,6 +370,7 @@ class HumanSizeTestCase(unittest.TestCase):
class RunTestCase(unittest.TestCase):
+
def setUp(self):
self.config = cleaner.default_config()
self.config['Quota'] = 1000000
@@ -384,6 +393,7 @@ class RunTestCase(unittest.TestCase):
@mock.patch('docker.Client', name='docker_client')
@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
class MainTestCase(unittest.TestCase):
+
def test_client_api_version(self, run_mock, docker_client):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
cf.write('{"Quota":"1000T"}')
@@ -392,7 +402,8 @@ class MainTestCase(unittest.TestCase):
self.assertEqual(1, docker_client.call_count)
# 1.14 is the first version that's well defined, going back to
# Docker 1.2, and still supported up to at least Docker 1.9.
- # See <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
+ # See
+ # <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
self.assertEqual('1.14',
docker_client.call_args[1].get('version'))
self.assertEqual(1, run_mock.call_count)
@@ -400,18 +411,21 @@ class MainTestCase(unittest.TestCase):
class ConfigTestCase(unittest.TestCase):
+
def test_load_config(self):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
- cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
cf.flush()
config = cleaner.load_config(['--config', cf.name])
- self.assertEqual(1000<<40, config['Quota'])
+ self.assertEqual(1000 << 40, config['Quota'])
self.assertEqual("always", config['RemoveStoppedContainers'])
self.assertEqual(2, config['Verbose'])
def test_args_override_config(self):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
- cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
cf.flush()
config = cleaner.load_config([
'--config', cf.name,
@@ -419,7 +433,7 @@ class ConfigTestCase(unittest.TestCase):
'--remove-stopped-containers', 'never',
'--verbose',
])
- self.assertEqual(1<<30, config['Quota'])
+ self.assertEqual(1 << 30, config['Quota'])
self.assertEqual('never', config['RemoveStoppedContainers'])
self.assertEqual(1, config['Verbose'])
@@ -448,13 +462,16 @@ class ContainerRemovalTestCase(unittest.TestCase):
def test_remove_onexit(self):
self.config['RemoveStoppedContainers'] = 'onexit'
cleaner.run(self.config, self.docker_client)
- self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
+ self.docker_client.remove_container.assert_called_once_with(
+ self.newCID, v=True)
def test_remove_always(self):
self.config['RemoveStoppedContainers'] = 'always'
cleaner.run(self.config, self.docker_client)
- self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
- self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
+ self.docker_client.remove_container.assert_any_call(
+ self.existingCID, v=True)
+ self.docker_client.remove_container.assert_any_call(
+ self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
@@ -472,7 +489,8 @@ class ContainerRemovalTestCase(unittest.TestCase):
# exited containers?
self.docker_client.assert_has_calls([
mock.call.events(since=mock.ANY),
- mock.call.containers(filters={'status':'exited'})])
+ mock.call.containers(filters={'status': 'exited'})])
# Asked to delete the container twice?
- self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID, v=True)] * 2)
+ self.docker_client.remove_container.assert_has_calls(
+ [mock.call(self.existingCID, v=True)] * 2)
self.assertEqual(2, self.docker_client.remove_container.call_count)
commit ba6d65ca676c8517828c53682cd948b1450e281b
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 10:17:59 2016 -0400
9953: Load config from file.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index cdb6602..5531bf5 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -15,6 +15,7 @@ import sys
import time
import docker
+import json
SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
@@ -244,53 +245,92 @@ def human_size(size_str):
size_str = size_str[:-1]
return int(size_str) * multiplier
+def load_config(arguments):
+ args = parse_arguments(arguments)
+
+ config = default_config()
+ with open(args.config, 'r') as f:
+ config.update(json.load(f))
+
+ configargs = vars(args).copy()
+ configargs.pop('config')
+ config.update({k: v for k, v in configargs.items() if v})
+
+ if isinstance(config['Quota'], str):
+ config['Quota'] = human_size(config['Quota'])
+
+ return config
+
+def default_config():
+ return {
+ 'Quota': '1G',
+ 'RemoveStoppedContainers': 'always',
+ 'Verbose': 0,
+ }
+
def parse_arguments(arguments):
+ class Formatter(argparse.ArgumentDefaultsHelpFormatter,
+ argparse.RawDescriptionHelpFormatter):
+ pass
parser = argparse.ArgumentParser(
prog="arvados_docker.cleaner",
- description="clean old Docker images from Arvados compute nodes")
+ description="clean old Docker images from Arvados compute nodes",
+ epilog="Example config file:\n\n{}".format(
+ json.dumps(default_config(), indent=4)),
+ formatter_class=Formatter,
+ )
+ parser.add_argument(
+ '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+ help="configuration file")
+
+ deprecated = " (DEPRECATED -- use config file instead)"
parser.add_argument(
- '--quota', action='store', type=human_size, required=True,
- help="space allowance for Docker images, suffixed with K/M/G/T")
+ '--quota', action='store', type=human_size, dest='Quota',
+ help="space allowance for Docker images, suffixed with K/M/G/T" + deprecated)
parser.add_argument(
- '--remove-stopped-containers', type=str, default='always',
+ '--remove-stopped-containers', type=str, default='always', dest='RemoveStoppedContainers',
choices=['never', 'onexit', 'always'],
help="""when to remove stopped containers (default: always, i.e., remove
stopped containers found at startup, and remove containers as
- soon as they exit)""")
+ soon as they exit)""" + deprecated)
parser.add_argument(
- '--verbose', '-v', action='count', default=0,
- help="log more information")
+ '--verbose', '-v', action='count', default=0, dest='Verbose',
+ help="log more information" + deprecated)
+
return parser.parse_args(arguments)
-def setup_logging(args):
+def setup_logging(config):
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
- logger.setLevel(logging.ERROR - (10 * args.verbose))
+ logger.setLevel(logging.ERROR - (10 * config['Verbose']))
-def run(args, docker_client):
+def run(config, docker_client):
start_time = int(time.time())
logger.debug("Loading Docker activity through present")
- images = DockerImages.from_daemon(args.quota, docker_client)
+ images = DockerImages.from_daemon(config['Quota'], docker_client)
use_recorder = DockerImageUseRecorder(
images, docker_client, docker_client.events(since=1, until=start_time))
use_recorder.run()
cleaner = DockerImageCleaner(
images, docker_client, docker_client.events(since=start_time),
- remove_containers_onexit=args.remove_stopped_containers != 'never')
+ remove_containers_onexit=config['RemoveStoppedContainers'] != 'never')
cleaner.check_stopped_containers(
- remove=args.remove_stopped_containers == 'always')
+ remove=config['RemoveStoppedContainers'] == 'always')
logger.info("Checking image quota at startup")
cleaner.clean_images()
logger.info("Listening for docker events")
cleaner.run()
-def main(arguments):
- args = parse_arguments(arguments)
- setup_logging(args)
- run(args, docker.Client(version='1.14'))
+def main(arguments=sys.argv[1:]):
+ config = load_config(arguments)
+ setup_logging(config)
+ try:
+ run(config, docker.Client(version='1.14'))
+ except KeyboardInterrupt:
+ sys.exit(1)
if __name__ == '__main__':
- main(sys.argv[1:])
+ main()
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 3ca9714..15b9b4e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -21,17 +21,20 @@ setup(name="arvados-docker-cleaner",
download_url="https://github.com/curoverse/arvados.git",
license="GNU Affero General Public License version 3.0",
packages=find_packages(),
+ entry_points={
+ 'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
+ },
data_files=[
('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
],
install_requires=[
- 'docker-py==1.7.2',
- ],
+ 'docker-py==1.7.2',
+ ],
tests_require=[
- 'pbr<1.7.0',
- 'mock',
- ],
+ 'pbr<1.7.0',
+ 'mock',
+ ],
test_suite='tests',
zip_safe=False,
cmdclass={'egg_info': tagger},
- )
+)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index 3cb172e..cabafe9 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -4,6 +4,7 @@ import collections
import itertools
import json
import random
+import tempfile
import time
import unittest
@@ -362,14 +363,14 @@ class HumanSizeTestCase(unittest.TestCase):
class RunTestCase(unittest.TestCase):
def setUp(self):
- self.args = mock.MagicMock(name='args')
- self.args.quota = 1000000
+ self.config = cleaner.default_config()
+ self.config['Quota'] = 1000000
self.docker_client = mock.MagicMock(name='docker_client')
def test_run(self):
test_start_time = int(time.time())
self.docker_client.events.return_value = []
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(2, self.docker_client.events.call_count)
event_kwargs = [args[1] for args in
self.docker_client.events.call_args_list]
@@ -384,7 +385,10 @@ class RunTestCase(unittest.TestCase):
@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
class MainTestCase(unittest.TestCase):
def test_client_api_version(self, run_mock, docker_client):
- cleaner.main(['--quota', '1000T'])
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T"}')
+ cf.flush()
+ cleaner.main(['--config', cf.name])
self.assertEqual(1, docker_client.call_count)
# 1.14 is the first version that's well defined, going back to
# Docker 1.2, and still supported up to at least Docker 1.9.
@@ -395,11 +399,36 @@ class MainTestCase(unittest.TestCase):
self.assertIs(run_mock.call_args[0][1], docker_client())
+class ConfigTestCase(unittest.TestCase):
+ def test_load_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config(['--config', cf.name])
+ self.assertEqual(1000<<40, config['Quota'])
+ self.assertEqual("always", config['RemoveStoppedContainers'])
+ self.assertEqual(2, config['Verbose'])
+
+ def test_args_override_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config([
+ '--config', cf.name,
+ '--quota', '1G',
+ '--remove-stopped-containers', 'never',
+ '--verbose',
+ ])
+ self.assertEqual(1<<30, config['Quota'])
+ self.assertEqual('never', config['RemoveStoppedContainers'])
+ self.assertEqual(1, config['Verbose'])
+
+
class ContainerRemovalTestCase(unittest.TestCase):
LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
def setUp(self):
- self.args = mock.MagicMock(name='args')
+ self.config = cleaner.default_config()
self.docker_client = mock.MagicMock(name='docker_client')
self.existingCID = MockDockerId()
self.docker_client.containers.return_value = [{
@@ -417,28 +446,28 @@ class ContainerRemovalTestCase(unittest.TestCase):
for e in self.LIFECYCLE]
def test_remove_onexit(self):
- self.args.remove_stopped_containers = 'onexit'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'onexit'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
def test_remove_always(self):
- self.args.remove_stopped_containers = 'always'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'always'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
- self.args.remove_stopped_containers = 'never'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'never'
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(0, self.docker_client.remove_container.call_count)
def test_container_exited_between_subscribe_events_and_check_existing(self):
- self.args.remove_stopped_containers = 'always'
+ self.config['RemoveStoppedContainers'] = 'always'
self.docker_client.events.return_value = [
MockEvent(e, docker_id=self.existingCID).encoded()
for e in ['die', 'destroy']]
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
# Subscribed to events before getting the list of existing
# exited containers?
self.docker_client.assert_has_calls([
commit 14c3753c7b8128b02f9cdbeaccd338a646506851
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 22 23:58:30 2016 -0400
9953: Ignore non-container events (volume, network) and events with no status, instead of crashing.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 88b8a4b..cdb6602 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -148,7 +148,9 @@ class DockerEventListener:
def run(self):
for event in self.events:
event = json.loads(event.decode(self.ENCODING))
- for method_name in self.event_handlers.for_event(event['status']):
+ if event.get('Type', 'container') != 'container':
+ continue
+ for method_name in self.event_handlers.for_event(event.get('status')):
getattr(self, method_name)(event)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list