[ARVADOS] created: 58a7e4f0854e392de979c531ad397bb508a77779

Git user git at public.curoverse.com
Tue Jun 20 10:33:31 EDT 2017


        at  58a7e4f0854e392de979c531ad397bb508a77779 (commit)


commit 58a7e4f0854e392de979c531ad397bb508a77779
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Jun 20 10:25:09 2017 -0400

    8186: Round up ec2 ebs volume sizes.  Ensure size is in valid range.  Add test for creating ec2 instance with ebs storage.

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index b256eca..9670c9c 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -70,13 +70,18 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         kw = {'name': self.create_cloud_name(arvados_node),
                 'ex_userdata': self._make_ping_url(arvados_node)}
         # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
-        scratch = size.scratch / 1000
+        scratch = int(size.scratch / 1000) + 1
         if scratch > size.disk:
+            volsize = scratch - size.disk
+            if volsize < 1:
+                volsize = 1
+            if volsize > 16384:
+                volsize = 16384
             kw["ex_blockdevicemappings"] = [{
                 "DeviceName": "/dev/xvdt",
                 "Ebs": {
                     "DeleteOnTermination": True,
-                    "VolumeSize": scratch - size.disk,
+                    "VolumeSize": volsize,
                     "VolumeType": "gp2"
                 }}]
         return kw
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index 14df360..e7a36eb 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -96,3 +96,34 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
         node = testutil.cloud_node_mock()
         node.name = name
         self.assertEqual(name, ec2.ComputeNodeDriver.node_fqdn(node))
+
+    def test_create_ebs_volume(self):
+        arv_node = testutil.arvados_node_mock()
+        driver = self.new_driver()
+        # libcloud/ec2 "disk" sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        size = testutil.MockSize(1)
+        size.disk=5
+        size.scratch=20000
+        driver.create_node(size, arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertTrue(create_method.called)
+        self.assertEqual([{
+            "DeviceName": "/dev/xvdt",
+            "Ebs": {
+                "DeleteOnTermination": True,
+                "VolumeSize": 16,
+                "VolumeType": "gp2"
+            }}],
+                         create_method.call_args[1].get('ex_blockdevicemappings'))
+
+    def test_ebs_volume_not_needed(self):
+        arv_node = testutil.arvados_node_mock()
+        driver = self.new_driver()
+        # libcloud/ec2 "disk" sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        size = testutil.MockSize(1)
+        size.disk=80
+        size.scratch=20000
+        driver.create_node(size, arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertTrue(create_method.called)
+        self.assertIsNone(create_method.call_args[1].get('ex_blockdevicemappings'))
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 41f4ed1..1ee7690 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -79,7 +79,8 @@ class MockSize(object):
         self.id = 'z{}.test'.format(factor)
         self.name = self.id
         self.ram = 128 * factor
-        self.disk = 100 * factor
+        self.disk = factor   # GB
+        self.scratch = 1000 * factor # MB
         self.bandwidth = 16 * factor
         self.price = float(factor)
         self.extra = {}

commit 495c902b59cebb965c610b7bec7fca64831c6e3a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Jun 12 19:43:19 2017 +0000

    8186: Set virtual device path.  Require libcloud 0.20 because 0.18 has mistakes in the ec2 instance size table.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 6a9f473..b256eca 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -73,6 +73,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         scratch = size.scratch / 1000
         if scratch > size.disk:
             kw["ex_blockdevicemappings"] = [{
+                "DeviceName": "/dev/xvdt",
                 "Ebs": {
                     "DeleteOnTermination": True,
                     "VolumeSize": scratch - size.disk,
diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py
index 5eb923e..9343191 100644
--- a/services/nodemanager/setup.py
+++ b/services/nodemanager/setup.py
@@ -29,7 +29,7 @@ setup(name='arvados-node-manager',
           ('share/doc/arvados-node-manager', ['agpl-3.0.txt', 'README.rst']),
       ],
       install_requires=[
-          'apache-libcloud>=0.16',
+          'apache-libcloud>=0.20',
           'arvados-python-client>=0.1.20150206225333',
           'future',
           'pykka',
@@ -37,14 +37,14 @@ setup(name='arvados-node-manager',
           'setuptools'
       ],
       dependency_links=[
-          "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev4.zip"
+          "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.20.2.dev3.zip"
       ],
       test_suite='tests',
       tests_require=[
           'requests',
           'pbr<1.7.0',
           'mock>=1.0',
-          'apache-libcloud==0.18.1.dev4',
+          'apache-libcloud==0.20.2.dev3',
       ],
       zip_safe=False,
       cmdclass={'egg_info': tagger},

commit 319c1aa1ffcb065492eb1878687c993d7638e94e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jun 14 09:28:27 2017 -0400

    8186: Add EBS storage if necessary so there is sufficient scratch space.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 8deabbd..6a9f473 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -67,8 +67,18 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
     create_cloud_name = staticmethod(arvados_node_fqdn)
 
     def arvados_create_kwargs(self, size, arvados_node):
-        return {'name': self.create_cloud_name(arvados_node),
+        kw = {'name': self.create_cloud_name(arvados_node),
                 'ex_userdata': self._make_ping_url(arvados_node)}
+        # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        scratch = size.scratch / 1000
+        if scratch > size.disk:
+            kw["ex_blockdevicemappings"] = [{
+                "Ebs": {
+                    "DeleteOnTermination": True,
+                    "VolumeSize": scratch - size.disk,
+                    "VolumeType": "gp2"
+                }}]
+        return kw
 
     def post_create_node(self, cloud_node):
         self.real.ex_create_tags(cloud_node, self.tags)
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 7e63c78..5fc5673 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -387,7 +387,7 @@ class NodeManagerDaemonActor(actor_class):
             arvados_client=self._new_arvados(),
             arvados_node=arvados_node,
             cloud_client=self._new_cloud(),
-            cloud_size=cloud_size).proxy()
+            cloud_size=self.server_calculator.find_size(cloud_size.id)).proxy()
         self.booting[new_setup.actor_ref.actor_urn] = new_setup
         self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
 
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 8e7cf2f..1e15002 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -1,6 +1,9 @@
 import re
 import urllib
 import ssl
+import time
+
+from arvnodeman.computenode import ARVADOS_TIMEFMT
 
 from libcloud.compute.base import NodeSize, Node, NodeDriver, NodeState
 from libcloud.common.exceptions import BaseHTTPError
@@ -29,12 +32,16 @@ class FakeDriver(NodeDriver):
                     ex_resource_group=None,
                     ex_user_name=None,
                     ex_tags=None,
-                    ex_network=None):
+                    ex_network=None,
+                    ex_userdata=None):
         global all_nodes, create_calls
         create_calls += 1
         n = Node(name, name, NodeState.RUNNING, [], [], self, size=size, extra={"tags": ex_tags})
         all_nodes.append(n)
-        ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_customdata:
+            ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_userdata:
+            ping_url = ex_userdata
         ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         ctx.verify_mode = ssl.CERT_NONE
         f = urllib.urlopen(ping_url, "", context=ctx)
@@ -125,3 +132,24 @@ class RetryDriver(FakeDriver):
                     ex_user_name=ex_user_name,
                     ex_tags=ex_tags,
                     ex_network=ex_network)
+
+class FakeAwsDriver(FakeDriver):
+
+    def create_node(self, name=None,
+                    size=None,
+                    image=None,
+                    auth=None,
+                    ex_userdata=None,
+                    ex_blockdevicemappings=None):
+        n = super(FakeAwsDriver, self).create_node(name=name,
+                                                      size=size,
+                                                      image=image,
+                                                      auth=auth,
+                                                      ex_userdata=ex_userdata)
+        n.extra = {"launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1]}
+        return n
+
+    def list_sizes(self, **kwargs):
+        return [NodeSize("m3.xlarge", "Extra Large Instance", 3500, 80, 0, 0, self),
+                NodeSize("m4.xlarge", "Extra Large Instance", 3500, 0, 0, 0, self),
+                NodeSize("m4.2xlarge", "Double Extra Large Instance", 7000, 0, 0, 0, self)]

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list