[ARVADOS] created: 246413f02ca8711f117a8575fccfec82e5dbfe23
Git user
git at public.curoverse.com
Fri Oct 28 02:58:31 EDT 2016
at 246413f02ca8711f117a8575fccfec82e5dbfe23 (commit)
commit 246413f02ca8711f117a8575fccfec82e5dbfe23
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 28 02:57:47 2016 -0400
10343: Update arvados-docker-cleaner install docs to use packaged systemd unit file.
diff --git a/doc/_includes/_install_docker_cleaner.liquid b/doc/_includes/_install_docker_cleaner.liquid
index 5671a54..6db5e4a 100644
--- a/doc/_includes/_install_docker_cleaner.liquid
+++ b/doc/_includes/_install_docker_cleaner.liquid
@@ -3,39 +3,53 @@ h2. Configure the Docker cleaner
The arvados-docker-cleaner program removes least recently used Docker images as needed to keep disk usage below a configured limit.
{% include 'notebox_begin' %}
-This also removes all containers as soon as they exit, as if they were run with @docker run --rm at . If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or run it with @--remove-stopped-containers never at .
+This also removes all containers as soon as they exit, as if they were run with @docker run --rm at . If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or configure it with @"RemoveStoppedContainers":"never"@.
{% include 'notebox_end' %}
-Create a file @/etc/systemd/system/arvados-docker-cleaner.service@ in an editor. Include the text below as its contents. Make sure to edit the @ExecStart@ line appropriately for your compute node.
+Create a file @/etc/arvados/docker-cleaner/docker-cleaner.json@ in an editor, with the following contents.
<notextile>
-<pre><code>[Service]
-# Most deployments will want a quota that's at least 10G. From there,
-# a larger quota can help reduce compute overhead by preventing reloading
-# the same Docker image repeatedly, but will leave less space for other
-# files on the same storage (usually Docker volumes). Make sure the quota
-# is less than the total space available for Docker images.
-# If your deployment uses a Python 3 Software Collection, uncomment the
-# ExecStart line below, and delete the following one:
-# ExecStart=scl enable python33 "python3 -m arvados_docker.cleaner --quota <span class="userinput">20G</span>"
-ExecStart=python3 -m arvados_docker.cleaner --quota <span class="userinput">20G</span>
-Restart=always
-RestartPreventExitStatus=2
-
-[Install]
-WantedBy=default.target
-
-[Unit]
-After=docker.service
+<pre><code>{
+ "Quota": "<span class="userinput">10G</span>",
+ "RemoveStoppedContainers": "always"
+}
</code></pre>
</notextile>
-Then enable and start the service:
+*Choosing a quota:* Most deployments will want a quota that's at least 10G. From there, a larger quota can help reduce compute overhead by preventing reloading the same Docker image repeatedly, but will leave less space for other files on the same storage (usually Docker volumes). Make sure the quota is less than the total space available for Docker images.
+
+*On Debian-based systems,* the @arvados-docker-cleaner@ service is enabled automatically when installed. Restart the service after updating the configuration file:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-docker-cleaner</span>
+</code></pre>
+</notextile>
+
+*On Red Hat-based systems,* first install the systemd unit file included in the distribution.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo cp /usr/share/doc/arvados-docker-cleaner/arvados-docker-cleaner.service /lib/systemd/system/</span>
+</code></pre>
+</notextile>
+
+{% include 'notebox_begin' %}
+
+If your deployment uses a Python 3 Software Collection, the package files will be installed to a different location, and you must modify the ExecStart line as described in the comments in the unit file.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo cp /opt/rh/python33/root/usr/share/doc/arvados-docker-cleaner/arvados-docker-cleaner.service /lib/systemd/system/</span>
+~$ <span class="userinput">sudo nano /lib/systemd/system/arvados-docker-cleaner.service</span>
+</code></pre>
+</notextile>
+
+{% include 'notebox_end' %}
+
+Next, use @systemctl@ to enable and start the service.
<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl enable arvados-docker-cleaner.service</span>
-~$ <span class="userinput">sudo systemctl start arvados-docker-cleaner.service</span>
+<pre><code>~$ <span class="userinput">sudo systemctl enable arvados-docker-cleaner</span>
+~$ <span class="userinput">sudo systemctl start arvados-docker-cleaner</span>
</code></pre>
</notextile>
-If you are using a different daemon supervisor, or if you want to test the daemon in a terminal window, use the command on the @ExecStart@ line above.
+*If you are using a different daemon supervisor,* or if you want to test the daemon in a terminal window, run @arvados-docker-cleaner at . Run @arvados-docker-cleaner --help@ for more configuration options.
diff --git a/services/dockercleaner/arvados-docker-cleaner.service b/services/dockercleaner/arvados-docker-cleaner.service
index 28653ae..416ea44 100644
--- a/services/dockercleaner/arvados-docker-cleaner.service
+++ b/services/dockercleaner/arvados-docker-cleaner.service
@@ -6,9 +6,14 @@ AssertPathExists=/etc/arvados/docker-cleaner/docker-cleaner.json
[Service]
Type=simple
-ExecStart=/usr/bin/env arvados-docker-cleaner
Restart=always
RestartSec=10s
+RestartPreventExitStatus=2
+#
+# If your deployment uses a Python 3 Software Collection, uncomment the
+# ExecStart line below, and delete the following one:
+#ExecStart=/usr/bin/env scl enable python33 arvados-docker-cleaner
+ExecStart=/usr/bin/env arvados-docker-cleaner
[Install]
WantedBy=multi-user.target
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 535650e..7b7a471 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -29,6 +29,7 @@ setup(name="arvados-docker-cleaner",
],
install_requires=[
'docker-py==1.7.2',
+ 'setuptools',
],
tests_require=[
'pbr<1.7.0',
commit d966c816e478d1747c6e0e39d3430526120c655f
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 27 12:55:35 2016 -0400
10343: Allow running without config file, if -config arg is not given.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 5ac8100..9c03c91 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -17,6 +17,8 @@ import time
import docker
import json
+DEFAULT_CONFIG_FILE = '/etc/arvados/docker-cleaner/docker-cleaner.json'
+
SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
logger = logging.getLogger('arvados_docker.cleaner')
@@ -262,7 +264,13 @@ def load_config(arguments):
c = json.load(f)
config.update(c)
except (FileNotFoundError, IOError, ValueError) as error:
- sys.exit('error reading config file {}: {}'.format(args.config, error))
+ if (isinstance(error, FileNotFoundError) and
+ args.config == DEFAULT_CONFIG_FILE):
+ logger.warning("DEPRECATED: default config file not found; "
+ "relying on command line configuration")
+ else:
+ sys.exit('error reading config file {}: {}'.format(
+ args.config, error))
configargs = vars(args).copy()
configargs.pop('config')
@@ -294,7 +302,7 @@ def parse_arguments(arguments):
formatter_class=Formatter,
)
parser.add_argument(
- '--config', action='store', type=str, default='/etc/arvados/docker-cleaner/docker-cleaner.json',
+ '--config', action='store', type=str, default=DEFAULT_CONFIG_FILE,
help="configuration file")
deprecated = " (DEPRECATED -- use config file instead)"
@@ -314,12 +322,15 @@ def parse_arguments(arguments):
return parser.parse_args(arguments)
-def setup_logging(config):
+def setup_logging():
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
+
+
+def configure_logging(config):
logger.setLevel(logging.ERROR - (10 * config['Verbose']))
@@ -342,8 +353,9 @@ def run(config, docker_client):
def main(arguments=sys.argv[1:]):
+ setup_logging()
config = load_config(arguments)
- setup_logging(config)
+ configure_logging(config)
try:
run(config, docker.Client(version='1.14'))
except KeyboardInterrupt:
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index 9fbd3e3..37c1d76 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -3,6 +3,7 @@
import collections
import itertools
import json
+import os
import random
import tempfile
import time
@@ -437,6 +438,11 @@ class ConfigTestCase(unittest.TestCase):
self.assertEqual('never', config['RemoveStoppedContainers'])
self.assertEqual(1, config['Verbose'])
+ def test_args_no_config(self):
+ self.assertEqual(False, os.path.exists(cleaner.DEFAULT_CONFIG_FILE))
+ config = cleaner.load_config(['--quota', '1G'])
+ self.assertEqual(1 << 30, config['Quota'])
+
class ContainerRemovalTestCase(unittest.TestCase):
LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list