[ARVADOS] updated: 2b1104dbf59f5b69f3068e9155cf955278c54ace

Git user git at public.curoverse.com
Fri Sep 23 10:25:02 EDT 2016


Summary of changes:
 services/dockercleaner/arvados_docker/cleaner.py | 29 ++++++++++++------------
 1 file changed, 15 insertions(+), 14 deletions(-)

  discards  2018340fc4048f64c1c2fecd074686eb0c8034df (commit)
       via  2b1104dbf59f5b69f3068e9155cf955278c54ace (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (2018340fc4048f64c1c2fecd074686eb0c8034df)
            \
             N -- N -- N (2b1104dbf59f5b69f3068e9155cf955278c54ace)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 2b1104dbf59f5b69f3068e9155cf955278c54ace
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Sep 23 10:17:59 2016 -0400

    9960: Load config from file.

diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index cdb6602..5531bf5 100755
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -15,6 +15,7 @@ import sys
 import time
 
 import docker
+import json
 
 SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
 
@@ -244,53 +245,92 @@ def human_size(size_str):
         size_str = size_str[:-1]
     return int(size_str) * multiplier
 
+def load_config(arguments):
+    args = parse_arguments(arguments)
+
+    config = default_config()
+    with open(args.config, 'r') as f:
+        config.update(json.load(f))
+
+    configargs = vars(args).copy()
+    configargs.pop('config')
+    config.update({k: v for k, v in configargs.items() if v})
+
+    if isinstance(config['Quota'], str):
+        config['Quota'] = human_size(config['Quota'])
+
+    return config
+
+def default_config():
+    return {
+        'Quota': '1G',
+        'RemoveStoppedContainers': 'always',
+        'Verbose': 0,
+    }
+
 def parse_arguments(arguments):
+    class Formatter(argparse.ArgumentDefaultsHelpFormatter,
+                    argparse.RawDescriptionHelpFormatter):
+        pass
     parser = argparse.ArgumentParser(
         prog="arvados_docker.cleaner",
-        description="clean old Docker images from Arvados compute nodes")
+        description="clean old Docker images from Arvados compute nodes",
+        epilog="Example config file:\n\n{}".format(
+            json.dumps(default_config(), indent=4)),
+        formatter_class=Formatter,
+    )
+    parser.add_argument(
+        '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+        help="configuration file")
+
+    deprecated = " (DEPRECATED -- use config file instead)"
     parser.add_argument(
-        '--quota', action='store', type=human_size, required=True,
-        help="space allowance for Docker images, suffixed with K/M/G/T")
+        '--quota', action='store', type=human_size, dest='Quota',
+        help="space allowance for Docker images, suffixed with K/M/G/T" + deprecated)
     parser.add_argument(
-        '--remove-stopped-containers', type=str, default='always',
+        '--remove-stopped-containers', type=str, default='always', dest='RemoveStoppedContainers',
         choices=['never', 'onexit', 'always'],
         help="""when to remove stopped containers (default: always, i.e., remove
         stopped containers found at startup, and remove containers as
-        soon as they exit)""")
+        soon as they exit)""" + deprecated)
     parser.add_argument(
-        '--verbose', '-v', action='count', default=0,
-        help="log more information")
+        '--verbose', '-v', action='count', default=0, dest='Verbose',
+        help="log more information" + deprecated)
+
     return parser.parse_args(arguments)
 
-def setup_logging(args):
+def setup_logging(config):
     log_handler = logging.StreamHandler()
     log_handler.setFormatter(logging.Formatter(
             '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
             '%Y-%m-%d %H:%M:%S'))
     logger.addHandler(log_handler)
-    logger.setLevel(logging.ERROR - (10 * args.verbose))
+    logger.setLevel(logging.ERROR - (10 * config['Verbose']))
 
-def run(args, docker_client):
+def run(config, docker_client):
     start_time = int(time.time())
     logger.debug("Loading Docker activity through present")
-    images = DockerImages.from_daemon(args.quota, docker_client)
+    images = DockerImages.from_daemon(config['Quota'], docker_client)
     use_recorder = DockerImageUseRecorder(
         images, docker_client, docker_client.events(since=1, until=start_time))
     use_recorder.run()
     cleaner = DockerImageCleaner(
         images, docker_client, docker_client.events(since=start_time),
-        remove_containers_onexit=args.remove_stopped_containers != 'never')
+        remove_containers_onexit=config['RemoveStoppedContainers'] != 'never')
     cleaner.check_stopped_containers(
-        remove=args.remove_stopped_containers == 'always')
+        remove=config['RemoveStoppedContainers'] == 'always')
     logger.info("Checking image quota at startup")
     cleaner.clean_images()
     logger.info("Listening for docker events")
     cleaner.run()
 
-def main(arguments):
-    args = parse_arguments(arguments)
-    setup_logging(args)
-    run(args, docker.Client(version='1.14'))
+def main(arguments=sys.argv[1:]):
+    config = load_config(arguments)
+    setup_logging(config)
+    try:
+        run(config, docker.Client(version='1.14'))
+    except KeyboardInterrupt:
+        sys.exit(1)
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main()
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 3ca9714..15b9b4e 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -21,17 +21,20 @@ setup(name="arvados-docker-cleaner",
       download_url="https://github.com/curoverse/arvados.git",
       license="GNU Affero General Public License version 3.0",
       packages=find_packages(),
+      entry_points={
+          'console_scripts': ['arvados-docker-cleaner=arvados_docker.cleaner:main'],
+      },
       data_files=[
           ('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt']),
       ],
       install_requires=[
-        'docker-py==1.7.2',
-        ],
+          'docker-py==1.7.2',
+      ],
       tests_require=[
-        'pbr<1.7.0',
-        'mock',
-        ],
+          'pbr<1.7.0',
+          'mock',
+      ],
       test_suite='tests',
       zip_safe=False,
       cmdclass={'egg_info': tagger},
-      )
+)
diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py
index 3cb172e..cabafe9 100644
--- a/services/dockercleaner/tests/test_cleaner.py
+++ b/services/dockercleaner/tests/test_cleaner.py
@@ -4,6 +4,7 @@ import collections
 import itertools
 import json
 import random
+import tempfile
 import time
 import unittest
 
@@ -362,14 +363,14 @@ class HumanSizeTestCase(unittest.TestCase):
 
 class RunTestCase(unittest.TestCase):
     def setUp(self):
-        self.args = mock.MagicMock(name='args')
-        self.args.quota = 1000000
+        self.config = cleaner.default_config()
+        self.config['Quota'] = 1000000
         self.docker_client = mock.MagicMock(name='docker_client')
 
     def test_run(self):
         test_start_time = int(time.time())
         self.docker_client.events.return_value = []
-        cleaner.run(self.args, self.docker_client)
+        cleaner.run(self.config, self.docker_client)
         self.assertEqual(2, self.docker_client.events.call_count)
         event_kwargs = [args[1] for args in
                         self.docker_client.events.call_args_list]
@@ -384,7 +385,10 @@ class RunTestCase(unittest.TestCase):
 @mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
 class MainTestCase(unittest.TestCase):
     def test_client_api_version(self, run_mock, docker_client):
-        cleaner.main(['--quota', '1000T'])
+        with tempfile.NamedTemporaryFile(mode='wt') as cf:
+            cf.write('{"Quota":"1000T"}')
+            cf.flush()
+            cleaner.main(['--config', cf.name])
         self.assertEqual(1, docker_client.call_count)
         # 1.14 is the first version that's well defined, going back to
         # Docker 1.2, and still supported up to at least Docker 1.9.
@@ -395,11 +399,36 @@ class MainTestCase(unittest.TestCase):
         self.assertIs(run_mock.call_args[0][1], docker_client())
 
 
+class ConfigTestCase(unittest.TestCase):
+    def test_load_config(self):
+        with tempfile.NamedTemporaryFile(mode='wt') as cf:
+            cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+            cf.flush()
+            config = cleaner.load_config(['--config', cf.name])
+        self.assertEqual(1000<<40, config['Quota'])
+        self.assertEqual("always", config['RemoveStoppedContainers'])
+        self.assertEqual(2, config['Verbose'])
+
+    def test_args_override_config(self):
+        with tempfile.NamedTemporaryFile(mode='wt') as cf:
+            cf.write('{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+            cf.flush()
+            config = cleaner.load_config([
+                '--config', cf.name,
+                '--quota', '1G',
+                '--remove-stopped-containers', 'never',
+                '--verbose',
+            ])
+        self.assertEqual(1<<30, config['Quota'])
+        self.assertEqual('never', config['RemoveStoppedContainers'])
+        self.assertEqual(1, config['Verbose'])
+
+
 class ContainerRemovalTestCase(unittest.TestCase):
     LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
 
     def setUp(self):
-        self.args = mock.MagicMock(name='args')
+        self.config = cleaner.default_config()
         self.docker_client = mock.MagicMock(name='docker_client')
         self.existingCID = MockDockerId()
         self.docker_client.containers.return_value = [{
@@ -417,28 +446,28 @@ class ContainerRemovalTestCase(unittest.TestCase):
             for e in self.LIFECYCLE]
 
     def test_remove_onexit(self):
-        self.args.remove_stopped_containers = 'onexit'
-        cleaner.run(self.args, self.docker_client)
+        self.config['RemoveStoppedContainers'] = 'onexit'
+        cleaner.run(self.config, self.docker_client)
         self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
 
     def test_remove_always(self):
-        self.args.remove_stopped_containers = 'always'
-        cleaner.run(self.args, self.docker_client)
+        self.config['RemoveStoppedContainers'] = 'always'
+        cleaner.run(self.config, self.docker_client)
         self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
         self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
         self.assertEqual(2, self.docker_client.remove_container.call_count)
 
     def test_remove_never(self):
-        self.args.remove_stopped_containers = 'never'
-        cleaner.run(self.args, self.docker_client)
+        self.config['RemoveStoppedContainers'] = 'never'
+        cleaner.run(self.config, self.docker_client)
         self.assertEqual(0, self.docker_client.remove_container.call_count)
 
     def test_container_exited_between_subscribe_events_and_check_existing(self):
-        self.args.remove_stopped_containers = 'always'
+        self.config['RemoveStoppedContainers'] = 'always'
         self.docker_client.events.return_value = [
             MockEvent(e, docker_id=self.existingCID).encoded()
             for e in ['die', 'destroy']]
-        cleaner.run(self.args, self.docker_client)
+        cleaner.run(self.config, self.docker_client)
         # Subscribed to events before getting the list of existing
         # exited containers?
         self.docker_client.assert_has_calls([

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list