[ARVADOS] created: 2018340fc4048f64c1c2fecd074686eb0c8034df
Git user
git at public.curoverse.com
Thu Sep 22 23:58:54 EDT 2016
at 2018340fc4048f64c1c2fecd074686eb0c8034df (commit)
commit 2018340fc4048f64c1c2fecd074686eb0c8034df
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 22 23:58:44 2016 -0400
9960: Load config from file.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index cdb6602..a7d3d26 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -15,6 +15,7 @@ import sys
import time
import docker
+import json
SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
@@ -244,53 +245,91 @@ def human_size(size_str):
size_str = size_str[:-1]
return int(size_str) * multiplier
+def load_config(arguments):
+ args = parse_arguments(arguments)
+
+ config = default_config()
+ with open(args.config, 'r') as f:
+ config.update(json.load(f))
+
+ configargs = vars(args).copy()
+ configargs.pop('config')
+ configargs.pop('dump_config')
+ config.update({k:v for k,v in configargs.items() if v})
+
+ if args.dump_config:
+ print(json.dump(config))
+ sys.exit(0)
+
+ if isinstance(config['Quota'], str):
+ config['Quota'] = human_size(config['Quota'])
+
+ return config
+
+def default_config():
+ return {
+ 'Quota': '1G',
+ 'RemoveStoppedContainers': 'always',
+ 'Verbose': 0,
+ }
+
def parse_arguments(arguments):
+ deprecated = " (DEPRECATED -- use config file instead)"
parser = argparse.ArgumentParser(
prog="arvados_docker.cleaner",
description="clean old Docker images from Arvados compute nodes")
parser.add_argument(
- '--quota', action='store', type=human_size, required=True,
- help="space allowance for Docker images, suffixed with K/M/G/T")
+ '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+ help="configuration file")
parser.add_argument(
- '--remove-stopped-containers', type=str, default='always',
+ '--dump-config', action='store_true',
+ help="display current configuration")
+ parser.add_argument(
+ '--quota', action='store', type=human_size, dest='Quota',
+ help="space allowance for Docker images, suffixed with K/M/G/T"+deprecated)
+ parser.add_argument(
+ '--remove-stopped-containers', type=str, default='always', dest='RemoveStoppedContainers',
choices=['never', 'onexit', 'always'],
help="""when to remove stopped containers (default: always, i.e., remove
stopped containers found at startup, and remove containers as
- soon as they exit)""")
+ soon as they exit)"""+deprecated)
parser.add_argument(
- '--verbose', '-v', action='count', default=0,
- help="log more information")
+ '--verbose', '-v', action='count', default=0, dest='Verbose',
+ help="log more information"+deprecated)
return parser.parse_args(arguments)
-def setup_logging(args):
+def setup_logging(config):
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
logger.addHandler(log_handler)
- logger.setLevel(logging.ERROR - (10 * args.verbose))
+ logger.setLevel(logging.ERROR - (10 * config['Verbose']))
-def run(args, docker_client):
+def run(config, docker_client):
start_time = int(time.time())
logger.debug("Loading Docker activity through present")
- images = DockerImages.from_daemon(args.quota, docker_client)
+ images = DockerImages.from_daemon(config['Quota'], docker_client)
use_recorder = DockerImageUseRecorder(
images, docker_client, docker_client.events(since=1, until=start_time))
use_recorder.run()
cleaner = DockerImageCleaner(
images, docker_client, docker_client.events(since=start_time),
- remove_containers_onexit=args.remove_stopped_containers != 'never')
+ remove_containers_onexit=config['RemoveStoppedContainers'] != 'never')
cleaner.check_stopped_containers(
- remove=args.remove_stopped_containers == 'always')
+ remove=config['RemoveStoppedContainers'] == 'always')
logger.info("Checking image quota at startup")
cleaner.clean_images()
logger.info("Listening for docker events")
cleaner.run()
-def main(arguments):
- args = parse_arguments(arguments)
- setup_logging(args)
- run(args, docker.Client(version='1.14'))
+def main(arguments=sys.argv[1:]):
+ config = load_config(arguments)
+ setup_logging(config)
+ try:
+ run(config, docker.Client(version='1.14'))
+ except KeyboardInterrupt:
+ sys.exit(1)
if __name__ == '__main__':
- main(sys.argv[1:])
+ main()
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 3ca9714..15b9b4e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -21,17 +21,20 @@ setup(name="arvados-docker-cleaner",
download_url="https://github.com/curoverse/arvados.git",
license="GNU Affero General Public License version 3.0",
packages=find_packages(),
+ entry_points={
+ 'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
+ },
data_files=[
('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
],
install_requires=[
- 'docker-py==1.7.2',
- ],
+ 'docker-py==1.7.2',
+ ],
tests_require=[
- 'pbr<1.7.0',
- 'mock',
- ],
+ 'pbr<1.7.0',
+ 'mock',
+ ],
test_suite='tests',
zip_safe=False,
cmdclass={'egg_info': tagger},
- )
+)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index 3cb172e..cabafe9 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -4,6 +4,7 @@ import collections
import itertools
import json
import random
+import tempfile
import time
import unittest
@@ -362,14 +363,14 @@ class HumanSizeTestCase(unittest.TestCase):
class RunTestCase(unittest.TestCase):
def setUp(self):
- self.args = mock.MagicMock(name='args')
- self.args.quota = 1000000
+ self.config = cleaner.default_config()
+ self.config['Quota'] = 1000000
self.docker_client = mock.MagicMock(name='docker_client')
def test_run(self):
test_start_time = int(time.time())
self.docker_client.events.return_value = []
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(2, self.docker_client.events.call_count)
event_kwargs = [args[1] for args in
self.docker_client.events.call_args_list]
@@ -384,7 +385,10 @@ class RunTestCase(unittest.TestCase):
@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
class MainTestCase(unittest.TestCase):
def test_client_api_version(self, run_mock, docker_client):
- cleaner.main(['--quota', '1000T'])
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T"}')
+ cf.flush()
+ cleaner.main(['--config', cf.name])
self.assertEqual(1, docker_client.call_count)
# 1.14 is the first version that's well defined, going back to
# Docker 1.2, and still supported up to at least Docker 1.9.
@@ -395,11 +399,36 @@ class MainTestCase(unittest.TestCase):
self.assertIs(run_mock.call_args[0][1], docker_client())
+class ConfigTestCase(unittest.TestCase):
+ def test_load_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config(['--config', cf.name])
+ self.assertEqual(1000<<40, config['Quota'])
+ self.assertEqual("always", config['RemoveStoppedContainers'])
+ self.assertEqual(2, config['Verbose'])
+
+ def test_args_override_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config([
+ '--config', cf.name,
+ '--quota', '1G',
+ '--remove-stopped-containers', 'never',
+ '--verbose',
+ ])
+ self.assertEqual(1<<30, config['Quota'])
+ self.assertEqual('never', config['RemoveStoppedContainers'])
+ self.assertEqual(1, config['Verbose'])
+
+
class ContainerRemovalTestCase(unittest.TestCase):
LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
def setUp(self):
- self.args = mock.MagicMock(name='args')
+ self.config = cleaner.default_config()
self.docker_client = mock.MagicMock(name='docker_client')
self.existingCID = MockDockerId()
self.docker_client.containers.return_value = [{
@@ -417,28 +446,28 @@ class ContainerRemovalTestCase(unittest.TestCase):
for e in self.LIFECYCLE]
def test_remove_onexit(self):
- self.args.remove_stopped_containers = 'onexit'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'onexit'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
def test_remove_always(self):
- self.args.remove_stopped_containers = 'always'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'always'
+ cleaner.run(self.config, self.docker_client)
self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
- self.args.remove_stopped_containers = 'never'
- cleaner.run(self.args, self.docker_client)
+ self.config['RemoveStoppedContainers'] = 'never'
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(0, self.docker_client.remove_container.call_count)
def test_container_exited_between_subscribe_events_and_check_existing(self):
- self.args.remove_stopped_containers = 'always'
+ self.config['RemoveStoppedContainers'] = 'always'
self.docker_client.events.return_value = [
MockEvent(e, docker_id=self.existingCID).encoded()
for e in ['die', 'destroy']]
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
# Subscribed to events before getting the list of existing
# exited containers?
self.docker_client.assert_has_calls([
commit 61e666cce1727b223ccaae656879889ee2e7d99b
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 22 23:58:30 2016 -0400
9960: Ignore non-container events (volume, network) and events with no status, instead of crashing.
diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 88b8a4b..cdb6602 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -148,7 +148,9 @@ class DockerEventListener:
def run(self):
for event in self.events:
event = json.loads(event.decode(self.ENCODING))
- for method_name in self.event_handlers.for_event(event['status']):
+ if event.get('Type', 'container') != 'container':
+ continue
+ for method_name in self.event_handlers.for_event(event.get('status')):
getattr(self, method_name)(event)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list