[ARVADOS] created: b6423b5e5261f1ddaa88db2548e2190f82de21e8

Git user git at public.curoverse.com
Mon Mar 6 17:10:17 EST 2017


        at  b6423b5e5261f1ddaa88db2548e2190f82de21e8 (commit)


commit b6423b5e5261f1ddaa88db2548e2190f82de21e8
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Mon Mar 6 19:04:35 2017 -0300

    11139: Added default config parameter 'node_mem_scaling' to be applied to node's ram sticker values. Its default is 95%.
    Added test checking the default config and that the value is actually applied to the server calculations.

diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index b54461c..aa9b3e2 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -45,7 +45,8 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
                        'max_total_price': '0',
                        'boot_fail_after': str(sys.maxint),
                        'node_stale_after': str(60 * 60 * 2),
-                       'watchdog': '600'},
+                       'watchdog': '600',
+                       'node_mem_scaling': '0.95'},
             'Logging': {'file': '/dev/stderr',
                         'level': 'WARNING'},
         }.iteritems():
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index abe8852..f6e9249 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -19,13 +19,14 @@ class ServerCalculator(object):
     """
 
     class CloudSizeWrapper(object):
-        def __init__(self, real_size, **kwargs):
+        def __init__(self, real_size, node_mem_scaling, **kwargs):
             self.real = real_size
             for name in ['id', 'name', 'ram', 'disk', 'bandwidth', 'price',
                          'extra']:
                 setattr(self, name, getattr(self.real, name))
             self.cores = kwargs.pop('cores')
             self.scratch = self.disk
+            self.ram = int(self.ram * node_mem_scaling)
             for name, override in kwargs.iteritems():
                 if not hasattr(self, name):
                     raise ValueError("unrecognized size field '%s'" % (name,))
@@ -42,8 +43,9 @@ class ServerCalculator(object):
             return True
 
 
-    def __init__(self, server_list, max_nodes=None, max_price=None):
-        self.cloud_sizes = [self.CloudSizeWrapper(s, **kws)
+    def __init__(self, server_list, max_nodes=None, max_price=None,
+                 node_mem_scaling=0.95):
+        self.cloud_sizes = [self.CloudSizeWrapper(s, node_mem_scaling, **kws)
                             for s, kws in server_list]
         self.cloud_sizes.sort(key=lambda s: s.price)
         self.max_nodes = max_nodes or float('inf')
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 9c45318..93f6cbd 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -71,7 +71,8 @@ def build_server_calculator(config):
         abort("No valid node sizes configured")
     return ServerCalculator(cloud_size_list,
                             config.getint('Daemon', 'max_nodes'),
-                            config.getfloat('Daemon', 'max_total_price'))
+                            config.getfloat('Daemon', 'max_total_price'),
+                            config.getfloat('Daemon', 'node_mem_scaling'))
 
 def launch_pollers(config, server_calculator):
     poll_time = config.getint('Daemon', 'poll_time')
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index e566928..3a17d38 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -58,6 +58,10 @@ testlogger = INFO
         self.assertEqual(1, kwargs['cores'])
         self.assertEqual(0.8, kwargs['price'])
 
+    def test_default_node_mem_scaling(self):
+        config = self.load_config()
+        self.assertEqual(0.95, config.getfloat('Daemon', 'node_mem_scaling'))
+
     def test_shutdown_windows(self):
         config = self.load_config()
         self.assertEqual([52, 6, 2], config.shutdown_windows())
diff --git a/services/nodemanager/tests/test_jobqueue.py b/services/nodemanager/tests/test_jobqueue.py
index b1bf3f6..2c43052 100644
--- a/services/nodemanager/tests/test_jobqueue.py
+++ b/services/nodemanager/tests/test_jobqueue.py
@@ -28,6 +28,13 @@ class ServerCalculatorTestCase(unittest.TestCase):
         servlist = self.calculate(servcalc, {'min_nodes': 3})
         self.assertEqual(3, len(servlist))
 
+    def test_default_5pct_ram_value_decrease(self):
+        servcalc = self.make_calculator([1])
+        servlist = self.calculate(servcalc, {'min_ram_mb_per_node': 128})
+        self.assertEqual(0, len(servlist))
+        servlist = self.calculate(servcalc, {'min_ram_mb_per_node': 121})
+        self.assertEqual(1, len(servlist))
+
     def test_implicit_server_count(self):
         servcalc = self.make_calculator([1])
         servlist = self.calculate(servcalc, {}, {'min_nodes': 3})

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list