[ARVADOS] updated: 1.1.4-327-g9fcb58d
Git user
git at public.curoverse.com
Wed May 30 17:09:27 EDT 2018
Summary of changes:
services/nodemanager/arvnodeman/computenode/driver/azure.py | 6 ++++++
services/nodemanager/arvnodeman/computenode/driver/ec2.py | 5 +++++
services/nodemanager/arvnodeman/computenode/driver/gce.py | 5 +++++
services/nodemanager/arvnodeman/config.py | 7 ++++---
services/nodemanager/arvnodeman/jobqueue.py | 2 +-
services/nodemanager/arvnodeman/launcher.py | 2 +-
services/nodemanager/arvnodeman/test/fake_driver.py | 2 +-
services/nodemanager/tests/test_computenode_driver.py | 2 +-
services/nodemanager/tests/test_computenode_driver_ec2.py | 8 +++++---
services/nodemanager/tests/test_config.py | 3 +--
services/nodemanager/tests/testutil.py | 2 ++
11 files changed, 32 insertions(+), 12 deletions(-)
via 9fcb58db8d47a42e52eead600ddabf57ab7c29ed (commit)
via 18b5ad828a4149c1b768d05e477982364002a647 (commit)
via b3e79afc9f6527927c6bf6e4ee35b11e6f20185e (commit)
via bb17ad738ed9c890881361232e0dfd84e50a529d (commit)
from 05f0008349efe56755c580e1e060115587aab352 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9fcb58db8d47a42e52eead600ddabf57ab7c29ed
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed May 30 18:08:34 2018 -0300
7478: Test fixes.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 5d03308..978c6aa 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -49,7 +49,7 @@ class FakeDriver(NodeDriver):
ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0]
if ex_userdata:
ping_url = ex_userdata
- if ex_metadata:
+ elif ex_metadata:
ping_url = ex_metadata["arv-ping-url"]
ping_url += "&instance_id=" + nodeid
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
diff --git a/services/nodemanager/tests/test_computenode_driver.py b/services/nodemanager/tests/test_computenode_driver.py
index 128a29e..4bf4c39 100644
--- a/services/nodemanager/tests/test_computenode_driver.py
+++ b/services/nodemanager/tests/test_computenode_driver.py
@@ -80,7 +80,7 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
for an_error, is_cloud_error in errors:
self.driver_mock().create_node.side_effect = an_error
with self.assertRaises(an_error):
- driver.create_node('1', 'id_1')
+ driver.create_node(testutil.MockSize(1), 'id_1')
if is_cloud_error:
error_count += 1
self.assertEqual(error_count, status.tracker.get('create_node_errors'))
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index 297eac0..ed0e0bd 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -56,9 +56,11 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
driver.create_node(testutil.MockSize(1), arv_node)
create_method = self.driver_mock().create_node
self.assertTrue(create_method.called)
- self.assertEqual(
- {'test':'testvalue'},
- create_method.call_args[1].get('ex_metadata', {'arg': 'missing'})
+ self.assertIn(
+ ('test', 'testvalue'),
+ create_method.call_args[1].get(
+ 'ex_metadata',
+ {'arg': 'missing'}).items()
)
def test_hostname_from_arvados_node(self):
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index 921281b..15716d5 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -53,8 +53,7 @@ testlogger = INFO
def test_list_sizes(self):
config = self.load_config()
- client = config.new_cloud_client()
- sizes = config.node_sizes(client.list_sizes())
+ sizes = config.node_sizes()
self.assertEqual(1, len(sizes))
size, kwargs = sizes[0]
self.assertEqual('Small', size.name)
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 555144c..857e09f 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -87,6 +87,8 @@ class MockSize(object):
self.bandwidth = 16 * factor
self.price = float(factor)
self.extra = {}
+ self.real = self
+ self.preemptable = False
def __eq__(self, other):
return self.id == other.id
commit 18b5ad828a4149c1b768d05e477982364002a647
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed May 30 18:02:30 2018 -0300
7478: Set arvados-node-size-id tag on node creation.
This tag has the Arvados assigned node size id, not the cloud size.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/azure.py b/services/nodemanager/arvnodeman/computenode/driver/azure.py
index e0f260a..32f6b6c 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/azure.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/azure.py
@@ -78,6 +78,12 @@ echo %s > /var/tmp/arv-node-data/meta-data/instance-type
def _init_image(self, urn):
return "image", self.get_image(urn)
+ def create_node(self, size, arvados_node):
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ self.create_kwargs.setdefault('ex_tags', {})
+ self.create_kwargs['ex_tags'].update({'arvados-node-size-id': size.id})
+ return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
def list_nodes(self):
# Azure only supports filtering node lists by resource group.
# Do our own filtering based on tag.
diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 1442a12..ae2eb85 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -100,6 +100,11 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
self.real.ex_create_tags(cloud_node,
{'Name': arvados_node_fqdn(arvados_node)})
+ def create_node(self, size, arvados_node):
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ self.create_kwargs['ex_metadata'].update({'arvados-node-size-id': size.id})
+ return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
def list_nodes(self):
# Need to populate Node.size
nodes = super(ComputeNodeDriver, self).list_nodes()
diff --git a/services/nodemanager/arvnodeman/computenode/driver/gce.py b/services/nodemanager/arvnodeman/computenode/driver/gce.py
index 3f1d575..8ab991b 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/gce.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/gce.py
@@ -109,6 +109,11 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
return result
+ def create_node(self, size, arvados_node):
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ self.create_kwargs['ex_metadata'].update({'arvados-node-size-id': size.id})
+ return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
def list_nodes(self):
# The GCE libcloud driver only supports filtering node lists by zone.
# Do our own filtering based on tag list.
commit b3e79afc9f6527927c6bf6e4ee35b11e6f20185e
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed May 30 18:00:29 2018 -0300
7478: Override CloudSizeWrapper id with config Size name.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index 8655046..19d92ad 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -139,13 +139,13 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
self.get_section('Cloud Create'),
driver_class=driver_class)
- def node_sizes(self, all_sizes):
+ def node_sizes(self):
"""Finds all acceptable NodeSizes for our installation.
Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
returned by libcloud that matches a size listed in our config file.
"""
-
+ all_sizes = self.new_cloud_client().list_sizes()
size_kwargs = {}
section_types = {
'price': float,
@@ -159,8 +159,9 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
if 'preemptable' not in size_spec:
size_spec['preemptable'] = False
if 'instance_type' not in size_spec:
- # Assume instance type is Size name is missing
+ # Assume instance type is Size name if missing
size_spec['instance_type'] = sec_words[1]
+ size_spec['id'] = sec_words[1]
size_kwargs[sec_words[1]] = size_spec
# EC2 node sizes are identified by id. GCE sizes are identified by name.
matching_sizes = []
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 888abf5..a1e2303 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -71,7 +71,7 @@ def setup_logging(path, level, **sublevels):
return root_logger
def build_server_calculator(config):
- cloud_size_list = config.node_sizes(config.new_cloud_client().list_sizes())
+ cloud_size_list = config.node_sizes()
if not cloud_size_list:
abort("No valid node sizes configured")
return ServerCalculator(cloud_size_list,
commit bb17ad738ed9c890881361232e0dfd84e50a529d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed May 30 17:55:59 2018 -0300
7478: Ignore redundant instance_type field on CloudSizeWrapper
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index 6ca28e6..f3fdc8e 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -39,8 +39,8 @@ class ServerCalculator(object):
self.scratch = self.disk * 1000
self.ram = int(self.ram * node_mem_scaling)
self.preemptable = False
- self.instance_type = None
for name, override in kwargs.iteritems():
+ if name == 'instance_type': continue
if not hasattr(self, name):
raise ValueError("unrecognized size field '%s'" % (name,))
setattr(self, name, override)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list