[ARVADOS] created: b6423b5e5261f1ddaa88db2548e2190f82de21e8
Git user
git at public.curoverse.com
Mon Mar 6 17:10:17 EST 2017
at b6423b5e5261f1ddaa88db2548e2190f82de21e8 (commit)
commit b6423b5e5261f1ddaa88db2548e2190f82de21e8
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Mar 6 19:04:35 2017 -0300
11139: Added default config parameter 'node_mem_scaling' to be applied to node's ram sticker values. Its default is 95%.
Added test checking the default config and that the value is actually applied to the server calculations.
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index b54461c..aa9b3e2 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -45,7 +45,8 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
'max_total_price': '0',
'boot_fail_after': str(sys.maxint),
'node_stale_after': str(60 * 60 * 2),
- 'watchdog': '600'},
+ 'watchdog': '600',
+ 'node_mem_scaling': '0.95'},
'Logging': {'file': '/dev/stderr',
'level': 'WARNING'},
}.iteritems():
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index abe8852..f6e9249 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -19,13 +19,14 @@ class ServerCalculator(object):
"""
class CloudSizeWrapper(object):
- def __init__(self, real_size, **kwargs):
+ def __init__(self, real_size, node_mem_scaling, **kwargs):
self.real = real_size
for name in ['id', 'name', 'ram', 'disk', 'bandwidth', 'price',
'extra']:
setattr(self, name, getattr(self.real, name))
self.cores = kwargs.pop('cores')
self.scratch = self.disk
+ self.ram = int(self.ram * node_mem_scaling)
for name, override in kwargs.iteritems():
if not hasattr(self, name):
raise ValueError("unrecognized size field '%s'" % (name,))
@@ -42,8 +43,9 @@ class ServerCalculator(object):
return True
- def __init__(self, server_list, max_nodes=None, max_price=None):
- self.cloud_sizes = [self.CloudSizeWrapper(s, **kws)
+ def __init__(self, server_list, max_nodes=None, max_price=None,
+ node_mem_scaling=0.95):
+ self.cloud_sizes = [self.CloudSizeWrapper(s, node_mem_scaling, **kws)
for s, kws in server_list]
self.cloud_sizes.sort(key=lambda s: s.price)
self.max_nodes = max_nodes or float('inf')
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 9c45318..93f6cbd 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -71,7 +71,8 @@ def build_server_calculator(config):
abort("No valid node sizes configured")
return ServerCalculator(cloud_size_list,
config.getint('Daemon', 'max_nodes'),
- config.getfloat('Daemon', 'max_total_price'))
+ config.getfloat('Daemon', 'max_total_price'),
+ config.getfloat('Daemon', 'node_mem_scaling'))
def launch_pollers(config, server_calculator):
poll_time = config.getint('Daemon', 'poll_time')
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index e566928..3a17d38 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -58,6 +58,10 @@ testlogger = INFO
self.assertEqual(1, kwargs['cores'])
self.assertEqual(0.8, kwargs['price'])
+ def test_default_node_mem_scaling(self):
+ config = self.load_config()
+ self.assertEqual(0.95, config.getfloat('Daemon', 'node_mem_scaling'))
+
def test_shutdown_windows(self):
config = self.load_config()
self.assertEqual([52, 6, 2], config.shutdown_windows())
diff --git a/services/nodemanager/tests/test_jobqueue.py b/services/nodemanager/tests/test_jobqueue.py
index b1bf3f6..2c43052 100644
--- a/services/nodemanager/tests/test_jobqueue.py
+++ b/services/nodemanager/tests/test_jobqueue.py
@@ -28,6 +28,13 @@ class ServerCalculatorTestCase(unittest.TestCase):
servlist = self.calculate(servcalc, {'min_nodes': 3})
self.assertEqual(3, len(servlist))
+ def test_default_5pct_ram_value_decrease(self):
+ servcalc = self.make_calculator([1])
+ servlist = self.calculate(servcalc, {'min_ram_mb_per_node': 128})
+ self.assertEqual(0, len(servlist))
+ servlist = self.calculate(servcalc, {'min_ram_mb_per_node': 121})
+ self.assertEqual(1, len(servlist))
+
def test_implicit_server_count(self):
servcalc = self.make_calculator([1])
servlist = self.calculate(servcalc, {}, {'min_nodes': 3})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list