[ARVADOS] updated: 1.1.4-327-g9fcb58d

Git user git at public.curoverse.com
Wed May 30 17:09:27 EDT 2018


Summary of changes:
 services/nodemanager/arvnodeman/computenode/driver/azure.py | 6 ++++++
 services/nodemanager/arvnodeman/computenode/driver/ec2.py   | 5 +++++
 services/nodemanager/arvnodeman/computenode/driver/gce.py   | 5 +++++
 services/nodemanager/arvnodeman/config.py                   | 7 ++++---
 services/nodemanager/arvnodeman/jobqueue.py                 | 2 +-
 services/nodemanager/arvnodeman/launcher.py                 | 2 +-
 services/nodemanager/arvnodeman/test/fake_driver.py         | 2 +-
 services/nodemanager/tests/test_computenode_driver.py       | 2 +-
 services/nodemanager/tests/test_computenode_driver_ec2.py   | 8 +++++---
 services/nodemanager/tests/test_config.py                   | 3 +--
 services/nodemanager/tests/testutil.py                      | 2 ++
 11 files changed, 32 insertions(+), 12 deletions(-)

       via  9fcb58db8d47a42e52eead600ddabf57ab7c29ed (commit)
       via  18b5ad828a4149c1b768d05e477982364002a647 (commit)
       via  b3e79afc9f6527927c6bf6e4ee35b11e6f20185e (commit)
       via  bb17ad738ed9c890881361232e0dfd84e50a529d (commit)
      from  05f0008349efe56755c580e1e060115587aab352 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 9fcb58db8d47a42e52eead600ddabf57ab7c29ed
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Wed May 30 18:08:34 2018 -0300

    7478: Test fixes.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 5d03308..978c6aa 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -49,7 +49,7 @@ class FakeDriver(NodeDriver):
             ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0]
         if ex_userdata:
             ping_url = ex_userdata
-        if ex_metadata:
+        elif ex_metadata:
             ping_url = ex_metadata["arv-ping-url"]
         ping_url += "&instance_id=" + nodeid
         ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
diff --git a/services/nodemanager/tests/test_computenode_driver.py b/services/nodemanager/tests/test_computenode_driver.py
index 128a29e..4bf4c39 100644
--- a/services/nodemanager/tests/test_computenode_driver.py
+++ b/services/nodemanager/tests/test_computenode_driver.py
@@ -80,7 +80,7 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
         for an_error, is_cloud_error in errors:
             self.driver_mock().create_node.side_effect = an_error
             with self.assertRaises(an_error):
-                driver.create_node('1', 'id_1')
+                driver.create_node(testutil.MockSize(1), 'id_1')
             if is_cloud_error:
                 error_count += 1
             self.assertEqual(error_count, status.tracker.get('create_node_errors'))
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index 297eac0..ed0e0bd 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -56,9 +56,11 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
         driver.create_node(testutil.MockSize(1), arv_node)
         create_method = self.driver_mock().create_node
         self.assertTrue(create_method.called)
-        self.assertEqual(
-            {'test':'testvalue'},
-            create_method.call_args[1].get('ex_metadata', {'arg': 'missing'})
+        self.assertIn(
+            ('test', 'testvalue'),
+            create_method.call_args[1].get(
+                'ex_metadata',
+                {'arg': 'missing'}).items()
         )
 
     def test_hostname_from_arvados_node(self):
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index 921281b..15716d5 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -53,8 +53,7 @@ testlogger = INFO
 
     def test_list_sizes(self):
         config = self.load_config()
-        client = config.new_cloud_client()
-        sizes = config.node_sizes(client.list_sizes())
+        sizes = config.node_sizes()
         self.assertEqual(1, len(sizes))
         size, kwargs = sizes[0]
         self.assertEqual('Small', size.name)
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 555144c..857e09f 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -87,6 +87,8 @@ class MockSize(object):
         self.bandwidth = 16 * factor
         self.price = float(factor)
         self.extra = {}
+        self.real = self
+        self.preemptable = False
 
     def __eq__(self, other):
         return self.id == other.id

commit 18b5ad828a4149c1b768d05e477982364002a647
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Wed May 30 18:02:30 2018 -0300

    7478: Set arvados-node-size-id tag on node creation.
    
    This tag has the Arvados assigned node size id, not the cloud size.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/azure.py b/services/nodemanager/arvnodeman/computenode/driver/azure.py
index e0f260a..32f6b6c 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/azure.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/azure.py
@@ -78,6 +78,12 @@ echo %s > /var/tmp/arv-node-data/meta-data/instance-type
     def _init_image(self, urn):
         return "image", self.get_image(urn)
 
+    def create_node(self, size, arvados_node):
+        # Set up tag indicating the Arvados assigned Cloud Size id.
+        self.create_kwargs.setdefault('ex_tags', {})
+        self.create_kwargs['ex_tags'].update({'arvados-node-size-id': size.id})
+        return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
     def list_nodes(self):
         # Azure only supports filtering node lists by resource group.
         # Do our own filtering based on tag.
diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 1442a12..ae2eb85 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -100,6 +100,11 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         self.real.ex_create_tags(cloud_node,
                                  {'Name': arvados_node_fqdn(arvados_node)})
 
+    def create_node(self, size, arvados_node):
+        # Set up tag indicating the Arvados assigned Cloud Size id.
+        self.create_kwargs['ex_metadata'].update({'arvados-node-size-id': size.id})
+        return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
     def list_nodes(self):
         # Need to populate Node.size
         nodes = super(ComputeNodeDriver, self).list_nodes()
diff --git a/services/nodemanager/arvnodeman/computenode/driver/gce.py b/services/nodemanager/arvnodeman/computenode/driver/gce.py
index 3f1d575..8ab991b 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/gce.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/gce.py
@@ -109,6 +109,11 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         return result
 
 
+    def create_node(self, size, arvados_node):
+        # Set up tag indicating the Arvados assigned Cloud Size id.
+        self.create_kwargs['ex_metadata'].update({'arvados-node-size-id': size.id})
+        return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
     def list_nodes(self):
         # The GCE libcloud driver only supports filtering node lists by zone.
         # Do our own filtering based on tag list.

commit b3e79afc9f6527927c6bf6e4ee35b11e6f20185e
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Wed May 30 18:00:29 2018 -0300

    7478: Override CloudSizeWrapper id with config Size name.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index 8655046..19d92ad 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -139,13 +139,13 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
                                         self.get_section('Cloud Create'),
                                         driver_class=driver_class)
 
-    def node_sizes(self, all_sizes):
+    def node_sizes(self):
         """Finds all acceptable NodeSizes for our installation.
 
         Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
         returned by libcloud that matches a size listed in our config file.
         """
-
+        all_sizes = self.new_cloud_client().list_sizes()
         size_kwargs = {}
         section_types = {
             'price': float,
@@ -159,8 +159,9 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
             if 'preemptable' not in size_spec:
                 size_spec['preemptable'] = False
             if 'instance_type' not in size_spec:
-                # Assume instance type is Size name is missing
+                # Assume instance type is Size name if missing
                 size_spec['instance_type'] = sec_words[1]
+            size_spec['id'] = sec_words[1]
             size_kwargs[sec_words[1]] = size_spec
         # EC2 node sizes are identified by id. GCE sizes are identified by name.
         matching_sizes = []
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 888abf5..a1e2303 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -71,7 +71,7 @@ def setup_logging(path, level, **sublevels):
     return root_logger
 
 def build_server_calculator(config):
-    cloud_size_list = config.node_sizes(config.new_cloud_client().list_sizes())
+    cloud_size_list = config.node_sizes()
     if not cloud_size_list:
         abort("No valid node sizes configured")
     return ServerCalculator(cloud_size_list,

commit bb17ad738ed9c890881361232e0dfd84e50a529d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Wed May 30 17:55:59 2018 -0300

    7478: Ignore redundant instance_type field on CloudSizeWrapper
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index 6ca28e6..f3fdc8e 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -39,8 +39,8 @@ class ServerCalculator(object):
             self.scratch = self.disk * 1000
             self.ram = int(self.ram * node_mem_scaling)
             self.preemptable = False
-            self.instance_type = None
             for name, override in kwargs.iteritems():
+                if name == 'instance_type': continue
                 if not hasattr(self, name):
                     raise ValueError("unrecognized size field '%s'" % (name,))
                 setattr(self, name, override)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list