[ARVADOS] updated: 37916fadb45b240c0731a328ddc153b723959ad3
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 17 10:38:26 EST 2015
Summary of changes:
.../nodemanager/arvnodeman/computenode/__init__.py | 9 +++-
.../arvnodeman/computenode/driver/gce.py | 50 ++++++++++++++--------
.../tests/test_computenode_driver_gce.py | 50 ++++++++++------------
3 files changed, 61 insertions(+), 48 deletions(-)
via 37916fadb45b240c0731a328ddc153b723959ad3 (commit)
from 39e2e62f726916ef78b9bd4da004ae4ea2ce0817 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 37916fadb45b240c0731a328ddc153b723959ad3
Author: Brett Smith <brett at curoverse.com>
Date: Tue Feb 17 10:38:16 2015 -0500
4138: Fixup GCE boot time calculations.
Store boot time in node metadata when we create a node. Assume 0 if
that's not available.
diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index 4955992..30fe516 100644
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -5,13 +5,18 @@ from __future__ import absolute_import, print_function
import itertools
import time
+ARVADOS_TIMEFMT = '%Y-%m-%dT%H:%M:%SZ'
+
def arvados_node_fqdn(arvados_node, default_hostname='dynamic.compute'):
hostname = arvados_node.get('hostname') or default_hostname
return '{}.{}'.format(hostname, arvados_node['domain'])
def arvados_node_mtime(node):
- return time.mktime(time.strptime(node['modified_at'] + 'UTC',
- '%Y-%m-%dT%H:%M:%SZ%Z')) - time.timezone
+ return arvados_timestamp(node['modified_at'])
+
+def arvados_timestamp(timestr):
+ return time.mktime(time.strptime(timestr + 'UTC',
+ ARVADOS_TIMEFMT + '%Z')) - time.timezone
def timestamp_fresh(timestamp, fresh_time):
return (time.time() - timestamp) < fresh_time
diff --git a/services/nodemanager/arvnodeman/computenode/driver/gce.py b/services/nodemanager/arvnodeman/computenode/driver/gce.py
index 6921839..d6ea2b2 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/gce.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/gce.py
@@ -10,7 +10,7 @@ import libcloud.compute.providers as cloud_provider
import libcloud.compute.types as cloud_types
from . import BaseComputeNodeDriver
-from .. import arvados_node_fqdn
+from .. import arvados_node_fqdn, arvados_timestamp, ARVADOS_TIMEFMT
class ComputeNodeDriver(BaseComputeNodeDriver):
"""Compute node driver wrapper for GCE
@@ -19,7 +19,6 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
"""
DEFAULT_DRIVER = cloud_provider.get_driver(cloud_types.Provider.GCE)
SEARCH_CACHE = {}
- node_start_times = {}
def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
driver_class=DEFAULT_DRIVER):
@@ -71,6 +70,8 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
result = {'name': 'compute-{}-{}'.format(node_id, cluster_id),
'ex_metadata': self.create_kwargs['ex_metadata'].copy(),
'ex_tags': list(self.node_tags)}
+ result['ex_metadata']['booted_at'] = time.strftime(ARVADOS_TIMEFMT,
+ time.gmtime())
result['ex_metadata']['hostname'] = arvados_node_fqdn(arvados_node)
result['ex_metadata']['user-data'] = self._make_ping_url(arvados_node)
return result
@@ -82,15 +83,33 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
super(ComputeNodeDriver, self).list_nodes()
if self.node_tags.issubset(node.extra.get('tags', []))]
+ @classmethod
+ def _find_metadata(cls, metadata_items, key):
+ # Given a list of two-item metadata dictonaries, return the one with
+ # the named key. Raise KeyError if not found.
+ try:
+ return next(data_dict for data_dict in metadata_items
+ if data_dict.get('key') == key)
+ except StopIteration:
+ raise KeyError(key)
+
+ @classmethod
+ def _get_metadata(cls, metadata_items, key, *default):
+ try:
+ return cls._find_metadata(metadata_items, key)['value']
+ except KeyError:
+ if default:
+ return default[0]
+ raise
+
def sync_node(self, cloud_node, arvados_node):
hostname = arvados_node_fqdn(arvados_node)
metadata_req = cloud_node.extra['metadata'].copy()
- for data_dict in metadata_req.setdefault('items', []):
- if data_dict['key'] == 'hostname':
- data_dict['value'] = hostname
- break
- else:
- metadata_req['items'].append({'key': 'hostname', 'value': hostname})
+ metadata_items = metadata_req.setdefault('items', [])
+ try:
+ self._find_metadata(metadata_items, 'hostname')['value'] = hostname
+ except KeyError:
+ metadata_items.append({'key': 'hostname', 'value': hostname})
response = self.real.connection.async_request(
'/zones/{}/instances/{}/setMetadata'.format(
cloud_node.extra['zone'].name, cloud_node.name),
@@ -98,15 +117,10 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
if not response.success():
raise Exception("setMetadata error: {}".format(response.error))
- def destroy_node(self, node):
- success = super(ComputeNodeDriver, self).destroy_node(node)
- if success:
- self.node_start_times.pop(node.id, None)
- return success
-
@classmethod
def node_start_time(cls, node):
- # Launch time isn't available on GCE node records. Thankfully that's
- # not too big a deal because they have by-minute billing.
- # Fake an answer based on the first time we see it.
- return cls.node_start_times.setdefault(node.id, time.time())
+ try:
+ return arvados_timestamp(cls._get_metadata(
+ node.extra['metadata']['items'], 'booted_at'))
+ except KeyError:
+ return 0
diff --git a/services/nodemanager/tests/test_computenode_driver_gce.py b/services/nodemanager/tests/test_computenode_driver_gce.py
index 26f8ca7..f995a8d 100644
--- a/services/nodemanager/tests/test_computenode_driver_gce.py
+++ b/services/nodemanager/tests/test_computenode_driver_gce.py
@@ -73,13 +73,17 @@ class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
driver = self.new_driver(list_kwargs={'tags': 'good, great'})
self.assertItemsEqual(['5', '6'], [n.id for n in driver.list_nodes()])
- def check_sync_node_updates_hostname_tag(self, plain_metadata):
- start_metadata = {
+ def build_gce_metadata(self, metadata_dict):
+ # Convert a plain metadata dictionary to the GCE data structure.
+ return {
'kind': 'compute#metadata',
'fingerprint': 'testprint',
- 'items': [{'key': key, 'value': plain_metadata[key]}
- for key in plain_metadata],
+ 'items': [{'key': key, 'value': metadata_dict[key]}
+ for key in metadata_dict],
}
+
+ def check_sync_node_updates_hostname_tag(self, plain_metadata):
+ start_metadata = self.build_gce_metadata(plain_metadata)
arv_node = testutil.arvados_node_mock(1)
cloud_node = testutil.cloud_node_mock(
2, metadata=start_metadata.copy(),
@@ -115,32 +119,22 @@ class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
self.assertIs(err_check.exception.__class__, Exception)
self.assertIn('sync error test', str(err_check.exception))
- def test_node_create_time_static(self):
- node = testutil.cloud_node_mock()
- with mock.patch('time.time', return_value=1) as time_mock:
- result1 = gce.ComputeNodeDriver.node_start_time(node)
- time_mock.return_value += 1
- self.assertEqual(result1,
- gce.ComputeNodeDriver.node_start_time(node))
-
- def test_node_create_time_varies_by_node(self):
- node1 = testutil.cloud_node_mock(1)
- node2 = testutil.cloud_node_mock(2)
- with mock.patch('time.time', return_value=1) as time_mock:
- start_time1 = gce.ComputeNodeDriver.node_start_time(node1)
- time_mock.return_value += 1
- self.assertNotEqual(start_time1,
- gce.ComputeNodeDriver.node_start_time(node2))
-
- def test_node_create_time_not_remembered_after_delete(self):
+ def test_node_create_time_zero_for_unknown_nodes(self):
node = testutil.cloud_node_mock()
+ self.assertEqual(0, gce.ComputeNodeDriver.node_start_time(node))
+
+ def test_node_create_time_for_known_node(self):
+ node = testutil.cloud_node_mock(metadata=self.build_gce_metadata(
+ {'booted_at': '1970-01-01T00:01:05Z'}))
+ self.assertEqual(65, gce.ComputeNodeDriver.node_start_time(node))
+
+ def test_node_create_time_recorded_when_node_boots(self):
+ start_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+ arv_node = testutil.arvados_node_mock()
driver = self.new_driver()
- with mock.patch('time.time', return_value=1) as time_mock:
- result1 = gce.ComputeNodeDriver.node_start_time(node)
- driver.destroy_node(node)
- time_mock.return_value += 1
- self.assertNotEqual(result1,
- gce.ComputeNodeDriver.node_start_time(node))
+ driver.create_node(testutil.MockSize(1), arv_node)
+ metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
+ self.assertLessEqual(start_time, metadata.get('booted_at'))
def test_deliver_ssh_key_in_metadata(self):
test_ssh_key = 'ssh-rsa-foo'
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list