[ARVADOS] created: 4d5de0e464c1de110de46588f3193c4677ac719c
Git user
git at public.curoverse.com
Fri Sep 23 17:41:10 EDT 2016
at 4d5de0e464c1de110de46588f3193c4677ac719c (commit)
commit 4d5de0e464c1de110de46588f3193c4677ac719c
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:47:29 2016 -0400
9953: Install/remove systemd unit files wherever systemd exists, even if it is not running.
diff --git a/build/go-python-package-scripts/postinst b/build/go-python-package-scripts/postinst
index 1e10333..051c8bd 100755
--- a/build/go-python-package-scripts/postinst
+++ b/build/go-python-package-scripts/postinst
@@ -11,7 +11,8 @@ systemd_unit="${pkg}.service"
case "${1}" in
configure)
- if [ -e /run/systemd/system ]; then
+ if [ -d /lib/systemd/system ]
+ then
# Python packages put all data files in /usr, so we copy
# them to /lib at install time.
py_unit="/usr/share/doc/${pkg}/${pkg}.service"
@@ -19,7 +20,9 @@ case "${1}" in
then
cp "${py_unit}" /lib/systemd/system/
fi
+ fi
+ if [ -e /run/systemd/system ]; then
eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
case "${UnitFileState}" in
disabled)
diff --git a/build/go-python-package-scripts/prerm b/build/go-python-package-scripts/prerm
index 7f4b843..c6ec18c 100755
--- a/build/go-python-package-scripts/prerm
+++ b/build/go-python-package-scripts/prerm
@@ -14,14 +14,14 @@ case "${1}" in
if [ -e /run/systemd/system ]; then
systemctl stop "${systemd_unit}" || true
systemctl disable "${systemd_unit}" || true
+ fi
- # Unit files from Python packages get installed by
- # postinst so we have to remove them explicitly here.
- py_unit="/usr/share/doc/${pkg}/${pkg}.service"
- if [ -e "${py_unit}" ]
- then
- rm "/lib/systemd/system/${pkg}.service" || true
- fi
+ # Unit files from Python packages get installed by postinst so
+ # we have to remove them explicitly here.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ rm "/lib/systemd/system/${pkg}.service" || true
fi
;;
esac
commit 90b70c212d63149a1205d40bb5d63eb56c8883cb
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:41:01 2016 -0400
9953: Update build scripts to Go 1.7.1.
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 2180b87..18694ed 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -24,7 +24,7 @@ ubuntu1404/generated: common-generated-all
test -d ubuntu1404/generated || mkdir ubuntu1404/generated
cp -rlt ubuntu1404/generated common-generated/*
-GOTARBALL=go1.6.2.linux-amd64.tar.gz
+GOTARBALL=go1.7.1.linux-amd64.tar.gz
common-generated-all: common-generated/$(GOTARBALL)
diff --git a/build/package-build-dockerfiles/centos6/Dockerfile b/build/package-build-dockerfiles/centos6/Dockerfile
index 8d969e6..8ea81f4 100644
--- a/build/package-build-dockerfiles/centos6/Dockerfile
+++ b/build/package-build-dockerfiles/centos6/Dockerfile
@@ -5,7 +5,7 @@ MAINTAINER Brett Smith <brett at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install RVM
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 311aaa2..4fcd640 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -5,7 +5,7 @@ MAINTAINER Brett Smith <brett at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install RVM
diff --git a/build/package-build-dockerfiles/debian7/Dockerfile b/build/package-build-dockerfiles/debian7/Dockerfile
index ddad542..7632c94 100644
--- a/build/package-build-dockerfiles/debian7/Dockerfile
+++ b/build/package-build-dockerfiles/debian7/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 80f06a2..977cd24 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1204/Dockerfile b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
index 2f628b0..b0dd906 100644
--- a/build/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index b9c003a..91c5e5b 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
commit f59617da8da5b154a7909fccaf2a83588d665da5
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:40:37 2016 -0400
9953: De-duplicate Go and Python postinst/prerm scripts.
diff --git a/build/go-package-scripts/postinst b/build/go-python-package-scripts/postinst
similarity index 77%
rename from build/go-package-scripts/postinst
rename to build/go-python-package-scripts/postinst
index 0663465..1e10333 100755
--- a/build/go-package-scripts/postinst
+++ b/build/go-python-package-scripts/postinst
@@ -12,6 +12,14 @@ systemd_unit="${pkg}.service"
case "${1}" in
configure)
if [ -e /run/systemd/system ]; then
+ # Python packages put all data files in /usr, so we copy
+ # them to /lib at install time.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ cp "${py_unit}" /lib/systemd/system/
+ fi
+
eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
case "${UnitFileState}" in
disabled)
diff --git a/build/go-package-scripts/prerm b/build/go-python-package-scripts/prerm
similarity index 59%
rename from build/go-package-scripts/prerm
rename to build/go-python-package-scripts/prerm
index 02772e4..7f4b843 100755
--- a/build/go-package-scripts/prerm
+++ b/build/go-python-package-scripts/prerm
@@ -14,6 +14,14 @@ case "${1}" in
if [ -e /run/systemd/system ]; then
systemctl stop "${systemd_unit}" || true
systemctl disable "${systemd_unit}" || true
+
+ # Unit files from Python packages get installed by
+ # postinst so we have to remove them explicitly here.
+ py_unit="/usr/share/doc/${pkg}/${pkg}.service"
+ if [ -e "${py_unit}" ]
+ then
+ rm "/lib/systemd/system/${pkg}.service" || true
+ fi
fi
;;
esac
diff --git a/build/python-package-scripts/postinst b/build/python-package-scripts/postinst
deleted file mode 100755
index 27e4601..0000000
--- a/build/python-package-scripts/postinst
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# NOTE: This package name detection will only work on Debian.
-# If this postinst script ever starts doing work on Red Hat,
-# we'll need to adapt this code accordingly.
-script="$(basename "${0}")"
-pkg="${script%.postinst}"
-systemd_unit="${pkg}.service"
-
-case "${1}" in
- configure)
- if [ -e /run/systemd/system ]; then
- cp "/usr/share/doc/${pkg}/${pkg}.service" /lib/systemd/system/
- eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
- case "${UnitFileState}" in
- disabled)
- # Failing to enable or start the service is not a
- # package error, so don't let errors here
- # propagate up.
- systemctl enable "${systemd_unit}" || true
- systemctl start "${systemd_unit}" || true
- ;;
- enabled)
- systemctl daemon-reload || true
- systemctl reload-or-try-restart "${systemd_unit}" || true
- ;;
- esac
- fi
- ;;
-esac
diff --git a/build/python-package-scripts/prerm b/build/python-package-scripts/prerm
deleted file mode 100755
index 6d75f81..0000000
--- a/build/python-package-scripts/prerm
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# NOTE: This package name detection will only work on Debian.
-# If this prerm script ever starts doing work on Red Hat,
-# we'll need to adapt this code accordingly.
-script="$(basename "${0}")"
-pkg="${script%.prerm}"
-systemd_unit="${pkg}.service"
-
-case "${1}" in
- remove)
- if [ -e /run/systemd/system ]; then
- systemctl stop "${systemd_unit}" || true
- systemctl disable "${systemd_unit}" || true
- rm "/lib/systemd/system/${pkg}.service"
- fi
- ;;
-esac
diff --git a/build/run-library.sh b/build/run-library.sh
index e445541..73a99da 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -105,8 +105,8 @@ package_go_binary() {
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
- --after-install "${WORKSPACE}/build/go-package-scripts/postinst"
- --before-remove "${WORKSPACE}/build/go-package-scripts/prerm"
+ --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
"${systemd_unit}=/lib/systemd/system/${prog}.service")
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
@@ -268,8 +268,8 @@ fpm_build () {
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
COMMAND_ARR+=(
- --after-install "${WORKSPACE}/build/python-package-scripts/postinst"
- --before-remove "${WORKSPACE}/build/python-package-scripts/prerm"
+ --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
)
fi
commit 2e05df2ccfd6ddf4143278a4249107e3cf110d89
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:08:37 2016 -0400
9953: Add systemd unit file for arvados-docker-cleaner.
diff --git a/build/python-package-scripts/postinst b/build/python-package-scripts/postinst
new file mode 100755
index 0000000..27e4601
--- /dev/null
+++ b/build/python-package-scripts/postinst
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this postinst script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.postinst}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ configure)
+ if [ -e /run/systemd/system ]; then
+ cp "/usr/share/doc/${pkg}/${pkg}.service" /lib/systemd/system/
+ eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
+ case "${UnitFileState}" in
+ disabled)
+ # Failing to enable or start the service is not a
+ # package error, so don't let errors here
+ # propagate up.
+ systemctl enable "${systemd_unit}" || true
+ systemctl start "${systemd_unit}" || true
+ ;;
+ enabled)
+ systemctl daemon-reload || true
+ systemctl reload-or-try-restart "${systemd_unit}" || true
+ ;;
+ esac
+ fi
+ ;;
+esac
diff --git a/build/python-package-scripts/prerm b/build/python-package-scripts/prerm
new file mode 100755
index 0000000..6d75f81
--- /dev/null
+++ b/build/python-package-scripts/prerm
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this prerm script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.prerm}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ remove)
+ if [ -e /run/systemd/system ]; then
+ systemctl stop "${systemd_unit}" || true
+ systemctl disable "${systemd_unit}" || true
+ rm "/lib/systemd/system/${pkg}.service"
+ fi
+ ;;
+esac
diff --git a/build/run-library.sh b/build/run-library.sh
index 067285d..e445541 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -265,6 +265,14 @@ fpm_build () {
# that will take precedence, as desired.
COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
+ then
+ COMMAND_ARR+=(
+ --after-install "${WORKSPACE}/build/python-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/python-package-scripts/prerm"
+ )
+ fi
+
# Append --depends X and other arguments specified by fpm-info.sh in
# the package source dir. These are added last so they can override
# the arguments added by this script.
diff --git a/services/dockercleaner/arvados-docker-cleaner.service b/services/dockercleaner/arvados-docker-cleaner.service
new file mode 100644
index 0000000..43e4ab8
--- /dev/null
+++ b/services/dockercleaner/arvados-docker-cleaner.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Arvados Docker Image Cleaner
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/env arvados-docker-cleaner
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 15b9b4e..535650e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -25,7 +25,7 @@ setup(name="arvados-docker-cleaner",
'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
},
data_files=[
- ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
+ ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt', 'arvados-docker-cleaner.service']),
],
install_requires=[
'docker-py==1.7.2',
commit 9b537477dbe8dd49c15ea14676d47f58c0b8f371
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 16:09:03 2016 -0400
9953: Install python data files to /usr/share/... not /usr/data/share/...
diff --git a/build/run-library.sh b/build/run-library.sh
index 4495a4e..067285d 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -213,6 +213,7 @@ fpm_build () {
--python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
--prefix "$PYTHON2_PREFIX" \
--python-install-lib "$PYTHON2_INSTALL_LIB" \
+ --python-install-data . \
--exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
--depends "$PYTHON2_PACKAGE"
# Fix --iteration for #9242.
@@ -229,6 +230,7 @@ fpm_build () {
--python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
--prefix "$PYTHON3_PREFIX" \
--python-install-lib "$PYTHON3_INSTALL_LIB" \
+ --python-install-data . \
--exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
--depends "$PYTHON3_PACKAGE"
# Fix --iteration for #9242.
diff --git a/sdk/pam/setup.py b/sdk/pam/setup.py
index c194013..e1046ac 100755
--- a/sdk/pam/setup.py
+++ b/sdk/pam/setup.py
@@ -37,6 +37,13 @@ setup(name='arvados-pam',
('share/pam-configs', ['pam-configs/arvados']),
('share/doc/arvados-pam', ['LICENSE-2.0.txt', 'README.rst']),
('share/doc/arvados-pam/examples', glob.glob('examples/*')),
+
+ # The arvados build scripts used to install data files to
+ # "/usr/data/*" but now install them to "/usr/*". Here, we
+ # install an extra copy in the old location so existing pam
+ # configs can still work. When old systems have had a chance
+ # to update to the new paths, this line can be removed.
+ ('data/lib/security', ['lib/libpam_arvados.py']),
],
install_requires=[
'arvados-python-client>=0.1.20150801000000',
commit 8aec6b1c9836bc7062339969b2c38eed68e574d8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 10:26:10 2016 -0400
9953: PEP-8
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 5531bf5..9dd7b12 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -21,6 +21,7 @@ SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
logger = logging.getLogger('arvados_docker.cleaner')
+
def return_when_docker_not_found(result=None):
# If the decorated function raises a 404 error from Docker, return
# `result` instead.
@@ -36,7 +37,9 @@ def return_when_docker_not_found(result=None):
return docker_not_found_wrapper
return docker_not_found_decorator
+
class DockerImage:
+
def __init__(self, image_hash):
self.docker_id = image_hash['Id']
self.size = image_hash['VirtualSize']
@@ -47,6 +50,7 @@ class DockerImage:
class DockerImages:
+
def __init__(self, target_size):
self.target_size = target_size
self.images = {}
@@ -118,6 +122,7 @@ class DockerImages:
class DockerEventHandlers:
# This class maps Docker event types to the names of methods that should
# receive those events.
+
def __init__(self):
self.handler_names = collections.defaultdict(list)
@@ -221,7 +226,8 @@ class DockerImageCleaner(DockerImageUseRecorder):
try:
self.docker_client.remove_image(image_id)
except docker.errors.APIError as error:
- logger.warning("Failed to remove image %s: %s", image_id, error)
+ logger.warning(
+ "Failed to remove image %s: %s", image_id, error)
else:
logger.info("Removed image %s", image_id)
self.images.del_image(image_id)
@@ -231,8 +237,9 @@ class DockerImageCleaner(DockerImageUseRecorder):
unknown_ids = {image['Id'] for image in self.docker_client.images()
if not self.images.has_image(image['Id'])}
for image_id in (unknown_ids - self.logged_unknown):
- logger.info("Image %s is loaded but unused, so it won't be cleaned",
- image_id)
+ logger.info(
+ "Image %s is loaded but unused, so it won't be cleaned",
+ image_id)
self.logged_unknown = unknown_ids
@@ -245,6 +252,7 @@ def human_size(size_str):
size_str = size_str[:-1]
return int(size_str) * multiplier
+
def load_config(arguments):
args = parse_arguments(arguments)
@@ -261,6 +269,7 @@ def load_config(arguments):
return config
+
def default_config():
return {
'Quota': '1G',
@@ -268,6 +277,7 @@ def default_config():
'Verbose': 0,
}
+
def parse_arguments(arguments):
class Formatter(argparse.ArgumentDefaultsHelpFormatter,
argparse.RawDescriptionHelpFormatter):
@@ -280,7 +290,7 @@ def parse_arguments(arguments):
formatter_class=Formatter,
)
parser.add_argument(
- '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+ '--config', action='store', type=str, default='/etc/arvados/docker-cleaner/config.json',
help="configuration file")
deprecated = " (DEPRECATED -- use config file instead)"
@@ -299,14 +309,16 @@ def parse_arguments(arguments):
return parser.parse_args(arguments)
+
def setup_logging(config):
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
- '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
- '%Y-%m-%d %H:%M:%S'))
+ '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
+ '%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
logger.setLevel(logging.ERROR - (10 * config['Verbose']))
+
def run(config, docker_client):
start_time = int(time.time())
logger.debug("Loading Docker activity through present")
@@ -324,6 +336,7 @@ def run(config, docker_client):
logger.info("Listening for docker events")
cleaner.run()
+
def main(arguments=sys.argv[1:]):
config = load_config(arguments)
setup_logging(config)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index cabafe9..9fbd3e3 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -15,13 +15,16 @@ from arvados_docker import cleaner
MAX_DOCKER_ID = (16 ** 64) - 1
+
def MockDockerId():
return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID))
+
def MockContainer(image_hash):
return {'Id': MockDockerId(),
'Image': image_hash['Id']}
+
def MockImage(*, size=0, vsize=None, tags=[]):
if vsize is None:
vsize = random.randint(100, 2000000)
@@ -31,6 +34,7 @@ def MockImage(*, size=0, vsize=None, tags=[]):
'Size': size,
'VirtualSize': vsize}
+
class MockEvent(dict):
ENCODING = 'utf-8'
event_seq = itertools.count(1)
@@ -48,6 +52,7 @@ class MockEvent(dict):
class MockException(docker.errors.APIError):
+
def __init__(self, status_code):
response = mock.Mock(name='response')
response.status_code = status_code
@@ -55,6 +60,7 @@ class MockException(docker.errors.APIError):
class DockerImageTestCase(unittest.TestCase):
+
def test_used_at_sets_last_used(self):
image = cleaner.DockerImage(MockImage())
image.used_at(5)
@@ -74,6 +80,7 @@ class DockerImageTestCase(unittest.TestCase):
class DockerImagesTestCase(unittest.TestCase):
+
def setUp(self):
self.mock_images = []
@@ -336,6 +343,7 @@ class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase):
class HumanSizeTestCase(unittest.TestCase):
+
def check(self, human_str, count, exp):
self.assertEqual(count * (1024 ** exp),
cleaner.human_size(human_str))
@@ -362,6 +370,7 @@ class HumanSizeTestCase(unittest.TestCase):
class RunTestCase(unittest.TestCase):
+
def setUp(self):
self.config = cleaner.default_config()
self.config['Quota'] = 1000000
@@ -384,6 +393,7 @@ class RunTestCase(unittest.TestCase):
@mock.patch('docker.Client', name='docker_client')
@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
class MainTestCase(unittest.TestCase):
+
def test_client_api_version(self, run_mock, docker_client):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
cf.write('{"Quota":"1000T"}')
@@ -392,7 +402,8 @@ class MainTestCase(unittest.TestCase):
self.assertEqual(1, docker_client.call_count)
# 1.14 is the first version that's well defined, going back to
# Docker 1.2, and still supported up to at least Docker 1.9.
- # See <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
+ # See
+ # <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
self.assertEqual('1.14',
docker_client.call_args[1].get('version'))
self.assertEqual(1, run_mock.call_count)
@@ -400,18 +411,21 @@ class MainTestCase(unittest.TestCase):
class ConfigTestCase(unittest.TestCase):
+
def test_load_config(self):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
- cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
cf.flush()
config = cleaner.load_config(['--config', cf.name])
- self.assertEqual(1000<<40, config['Quota'])
+ self.assertEqual(1000 << 40, config['Quota'])
self.assertEqual("always", config['RemoveStoppedContainers'])
self.assertEqual(2, config['Verbose'])
def test_args_override_config(self):
with tempfile.NamedTemporaryFile(mode='wt') as cf:
- cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
cf.flush()
config = cleaner.load_config([
'--config', cf.name,
@@ -419,7 +433,7 @@ class ConfigTestCase(unittest.TestCase):
'--remove-stopped-containers', 'never',
'--verbose',
])
- self.assertEqual(1<<30, config['Quota'])
+ self.assertEqual(1 << 30, config['Quota'])
self.assertEqual('never', config['RemoveStoppedContainers'])
self.assertEqual(1, config['Verbose'])
@@ -448,13 +462,16 @@ class ContainerRemovalTestCase(unittest.TestCase):
def test_remove_onexit(self):
self.config['RemoveStoppedContainers'] = 'onexit'
cleaner.run(self.config, self.docker_client)
- self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
+ self.docker_client.remove_container.assert_called_once_with(
+ self.newCID, v=True)
def test_remove_always(self):
self.config['RemoveStoppedContainers'] = 'always'
cleaner.run(self.config, self.docker_client)
- self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
- self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
+ self.docker_client.remove_container.assert_any_call(
+ self.existingCID, v=True)
+ self.docker_client.remove_container.assert_any_call(
+ self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
@@ -472,7 +489,8 @@ class ContainerRemovalTestCase(unittest.TestCase):
# exited containers?
self.docker_client.assert_has_calls([
mock.call.events(since=mock.ANY),
- mock.call.containers(filters={'status':'exited'})])
+ mock.call.containers(filters={'status': 'exited'})])
# Asked to delete the container twice?
- self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID, v=True)] * 2)
+ self.docker_client.remove_container.assert_has_calls(
+ [mock.call(self.existingCID, v=True)] * 2)
self.assertEqual(2, self.docker_client.remove_container.call_count)
commit a02b6825d8209560f9132add66303d8a97af58b8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Sep 23 10:17:59 2016 -0400
9953: Load config from file.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index cdb6602..5531bf5 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -15,6 +15,7 @@ import sys
import time
import docker
+import json
SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
@@ -244,53 +245,92 @@ def human_size(size_str):
size_str = size_str[:-1]
return int(size_str) * multiplier
+def load_config(arguments):
+ args = parse_arguments(arguments)
+
+ config = default_config()
+ with open(args.config, 'r') as f:
+ config.update(json.load(f))
+
+ configargs = vars(args).copy()
+ configargs.pop('config')
+ config.update({k: v for k, v in configargs.items() if v})
+
+ if isinstance(config['Quota'], str):
+ config['Quota'] = human_size(config['Quota'])
+
+ return config
+
+def default_config():
+ return {
+ 'Quota': '1G',
+ 'RemoveStoppedContainers': 'always',
+ 'Verbose': 0,
+ }
+
def parse_arguments(arguments):
+ class Formatter(argparse.ArgumentDefaultsHelpFormatter,
+ argparse.RawDescriptionHelpFormatter):
+ pass
parser = argparse.ArgumentParser(
prog="arvados_docker.cleaner",
- description="clean old Docker images from Arvados compute nodes")
+ description="clean old Docker images from Arvados compute nodes",
+ epilog="Example config file:\n\n{}".format(
+ json.dumps(default_config(), indent=4)),
+ formatter_class=Formatter,
+ )
+ parser.add_argument(
+ '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+ help="configuration file")
+
+ deprecated = " (DEPRECATED -- use config file instead)"
parser.add_argument(
- '--quota', action='store', type=human_size, required=True,
- help="space allowance for Docker images, suffixed with K/M/G/T")
+ '--quota', action='store', type=human_size, dest='Quota',
+ help="space allowance for Docker images, suffixed with K/M/G/T" + deprecated)
parser.add_argument(
- '--remove-stopped-containers', type=str, default='always',
+ '--remove-stopped-containers', type=str, default='always', dest='RemoveStoppedContainers',
choices=['never', 'onexit', 'always'],
help="""when to remove stopped containers (default: always, i.e., remove
stopped containers found at startup, and remove containers as
- soon as they exit)""")
+ soon as they exit)""" + deprecated)
parser.add_argument(
- '--verbose', '-v', action='count', default=0,
- help="log more information")
+ '--verbose', '-v', action='count', default=0, dest='Verbose',
+ help="log more information" + deprecated)
+
return parser.parse_args(arguments)
-def setup_logging(args):
+def setup_logging(config):
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
- logger.setLevel(logging.ERROR - (10 * args.verbose))
+ logger.setLevel(logging.ERROR - (10 * config['Verbose']))
-def run(args, docker_client):
+def run(config, docker_client):
start_time = int(time.time())
logger.debug("Loading Docker activity through present")
- images = DockerImages.from_daemon(args.quota, docker_client)
+ images = DockerImages.from_daemon(config['Quota'], docker_client)
use_recorder = DockerImageUseRecorder(
images, docker_client, docker_client.events(since=1, until=start_time))
use_recorder.run()
cleaner = DockerImageCleaner(
images, docker_client, docker_client.events(since=start_time),
- remove_containers_onexit=args.remove_stopped_containers != 'never')
+ remove_containers_onexit=config['RemoveStoppedContainers'] != 'never')
cleaner.check_stopped_containers(
- remove=args.remove_stopped_containers == 'always')
+ remove=config['RemoveStoppedContainers'] == 'always')
logger.info("Checking image quota at startup")
cleaner.clean_images()
logger.info("Listening for docker events")
cleaner.run()
-def main(arguments):
- args = parse_arguments(arguments)
- setup_logging(args)
- run(args, docker.Client(version='1.14'))
+def main(arguments=sys.argv[1:]):
+ config = load_config(arguments)
+ setup_logging(config)
+ try:
+ run(config, docker.Client(version='1.14'))
+ except KeyboardInterrupt:
+ sys.exit(1)
if __name__ == '__main__':
- main(sys.argv[1:])
+ main()
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 3ca9714..15b9b4e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -21,17 +21,20 @@ setup(name="arvados-docker-cleaner",
download_url="https://github.com/curoverse/arvados.git",
license="GNU Affero General Public License version 3.0",
packages=find_packages(),
+ entry_points={
+ 'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
+ },
data_files=[
('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
],
install_requires=[
- 'docker-py==1.7.2',
- ],
+ 'docker-py==1.7.2',
+ ],
tests_require=[
- 'pbr<1.7.0',
- 'mock',
- ],
+ 'pbr<1.7.0',
+ 'mock',
+ ],
test_suite='tests',
zip_safe=False,
cmdclass={'egg_info': tagger},
- )
+)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index 3cb172e..cabafe9 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -4,6 +4,7 @@ import collections
import itertools
import json
import random
+import tempfile
import time
import unittest
@@ -362,14 +363,14 @@ class HumanSizeTestCase(unittest.TestCase):
class RunTestCase(unittest.TestCase):
def setUp(self):
- self.args = mock.MagicMock(name='args')
- self.args.quota = 1000000
+ self.config = cleaner.default_config()
+ self.config['Quota'] = 1000000
self.docker_client = mock.MagicMock(name='docker_client')
def test_run(self):
test_start_time = int(time.time())
self.docker_client.events.return_value = []
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(2, self.docker_client.events.call_count)
event_kwargs = [args[1] for args in
self.docker_client.events.call_args_list]
@@ -384,7 +385,10 @@ class RunTestCase(unittest.TestCase):
@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
class MainTestCase(unittest.TestCase):
def test_client_api_version(self, run_mock, docker_client):
- cleaner.main(['--quota', '1000T'])
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T"}')
+ cf.flush()
+ cleaner.main(['--config', cf.name])
self.assertEqual(1, docker_client.call_count)
# 1.14 is the first version that's well defined, going back to
# Docker 1.2, and still supported up to at least Docker 1.9.
@@ -395,11 +399,36 @@ class MainTestCase(unittest.TestCase):
self.assertIs(run_mock.call_args[0][1], docker_client())
+class ConfigTestCase(unittest.TestCase):
+ def test_load_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config(['--config', cf.name])
+ self.assertEqual(1000<<40, config['Quota'])
+ self.assertEqual("always", config['RemoveStoppedContainers'])
+ self.assertEqual(2, config['Verbose'])
+
+ def test_args_override_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config([
+ '--config', cf.name,
+ '--quota', '1G',
+ '--remove-stopped-containers', 'never',
+ '--verbose',
+ ])
+ self.assertEqual(1<<30, config['Quota'])
+ self.assertEqual('never', config['RemoveStoppedContainers'])
+ self.assertEqual(1, config['Verbose'])
+
+
class ContainerRemovalTestCase(unittest.TestCase):
LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
def setUp(self):
- self.args = mock.MagicMock(name='args')
+ self.config = cleaner.default_config()
self.docker_client = mock.MagicMock(name='docker_client')
self.existingCID = MockDockerId()
self.docker_client.containers.return_value = [{
@@ -417,28 +446,28 @@ class ContainerRemovalTestCase(unittest.TestCase):
for e in self.LIFECYCLE]
def test_remove_onexit(self):
- self.args.remove_stopped_containers = 'onexit'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'onexit'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
def test_remove_always(self):
- self.args.remove_stopped_containers = 'always'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'always'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
- self.args.remove_stopped_containers = 'never'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'never'
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(0, self.docker_client.remove_container.call_count)
def test_container_exited_between_subscribe_events_and_check_existing(self):
- self.args.remove_stopped_containers = 'always'
+ self.config['RemoveStoppedContainers'] = 'always'
self.docker_client.events.return_value = [
MockEvent(e, docker_id=self.existingCID).encoded()
for e in ['die', 'destroy']]
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
# Subscribed to events before getting the list of existing
# exited containers?
self.docker_client.assert_has_calls([
commit d602a345a5dd7a4ebd6423892acd6e96bfd74969
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 22 23:58:30 2016 -0400
9953: Ignore non-container events (volume, network) and events with no status, instead of crashing.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 88b8a4b..cdb6602 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -148,7 +148,9 @@ class DockerEventListener:
def run(self):
for event in self.events:
event = json.loads(event.decode(self.ENCODING))
- for method_name in self.event_handlers.for_event(event['status']):
+ if event.get('Type', 'container') != 'container':
+ continue
+ for method_name in self.event_handlers.for_event(event.get('status')):
getattr(self, method_name)(event)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list