[ARVADOS] created: 495c902b59cebb965c610b7bec7fca64831c6e3a

Git user git at public.curoverse.com
Wed Jun 14 09:29:13 EDT 2017


        at  495c902b59cebb965c610b7bec7fca64831c6e3a (commit)


commit 495c902b59cebb965c610b7bec7fca64831c6e3a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Jun 12 19:43:19 2017 +0000

    8186: Set virtual device path.  Require libcloud 0.20 because 0.18 has mistakes in the ec2 instance size table.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 6a9f473..b256eca 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -73,6 +73,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         scratch = size.scratch / 1000
         if scratch > size.disk:
             kw["ex_blockdevicemappings"] = [{
+                "DeviceName": "/dev/xvdt",
                 "Ebs": {
                     "DeleteOnTermination": True,
                     "VolumeSize": scratch - size.disk,
diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py
index 5eb923e..9343191 100644
--- a/services/nodemanager/setup.py
+++ b/services/nodemanager/setup.py
@@ -29,7 +29,7 @@ setup(name='arvados-node-manager',
           ('share/doc/arvados-node-manager', ['agpl-3.0.txt', 'README.rst']),
       ],
       install_requires=[
-          'apache-libcloud>=0.16',
+          'apache-libcloud>=0.20',
           'arvados-python-client>=0.1.20150206225333',
           'future',
           'pykka',
@@ -37,14 +37,14 @@ setup(name='arvados-node-manager',
           'setuptools'
       ],
       dependency_links=[
-          "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev4.zip"
+          "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.20.2.dev3.zip"
       ],
       test_suite='tests',
       tests_require=[
           'requests',
           'pbr<1.7.0',
           'mock>=1.0',
-          'apache-libcloud==0.18.1.dev4',
+          'apache-libcloud==0.20.2.dev3',
       ],
       zip_safe=False,
       cmdclass={'egg_info': tagger},

commit 319c1aa1ffcb065492eb1878687c993d7638e94e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jun 14 09:28:27 2017 -0400

    8186: Add EBS storage if necessary so there is sufficient scratch space.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 8deabbd..6a9f473 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -67,8 +67,18 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
     create_cloud_name = staticmethod(arvados_node_fqdn)
 
     def arvados_create_kwargs(self, size, arvados_node):
-        return {'name': self.create_cloud_name(arvados_node),
+        kw = {'name': self.create_cloud_name(arvados_node),
                 'ex_userdata': self._make_ping_url(arvados_node)}
+        # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        scratch = size.scratch / 1000
+        if scratch > size.disk:
+            kw["ex_blockdevicemappings"] = [{
+                "Ebs": {
+                    "DeleteOnTermination": True,
+                    "VolumeSize": scratch - size.disk,
+                    "VolumeType": "gp2"
+                }}]
+        return kw
 
     def post_create_node(self, cloud_node):
         self.real.ex_create_tags(cloud_node, self.tags)
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 7e63c78..5fc5673 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -387,7 +387,7 @@ class NodeManagerDaemonActor(actor_class):
             arvados_client=self._new_arvados(),
             arvados_node=arvados_node,
             cloud_client=self._new_cloud(),
-            cloud_size=cloud_size).proxy()
+            cloud_size=self.server_calculator.find_size(cloud_size.id)).proxy()
         self.booting[new_setup.actor_ref.actor_urn] = new_setup
         self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
 
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 8e7cf2f..1e15002 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -1,6 +1,9 @@
 import re
 import urllib
 import ssl
+import time
+
+from arvnodeman.computenode import ARVADOS_TIMEFMT
 
 from libcloud.compute.base import NodeSize, Node, NodeDriver, NodeState
 from libcloud.common.exceptions import BaseHTTPError
@@ -29,12 +32,16 @@ class FakeDriver(NodeDriver):
                     ex_resource_group=None,
                     ex_user_name=None,
                     ex_tags=None,
-                    ex_network=None):
+                    ex_network=None,
+                    ex_userdata=None):
         global all_nodes, create_calls
         create_calls += 1
         n = Node(name, name, NodeState.RUNNING, [], [], self, size=size, extra={"tags": ex_tags})
         all_nodes.append(n)
-        ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_customdata:
+            ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_userdata:
+            ping_url = ex_userdata
         ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         ctx.verify_mode = ssl.CERT_NONE
         f = urllib.urlopen(ping_url, "", context=ctx)
@@ -125,3 +132,24 @@ class RetryDriver(FakeDriver):
                     ex_user_name=ex_user_name,
                     ex_tags=ex_tags,
                     ex_network=ex_network)
+
+class FakeAwsDriver(FakeDriver):
+
+    def create_node(self, name=None,
+                    size=None,
+                    image=None,
+                    auth=None,
+                    ex_userdata=None,
+                    ex_blockdevicemappings=None):
+        n = super(FakeAwsDriver, self).create_node(name=name,
+                                                      size=size,
+                                                      image=image,
+                                                      auth=auth,
+                                                      ex_userdata=ex_userdata)
+        n.extra = {"launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1]}
+        return n
+
+    def list_sizes(self, **kwargs):
+        return [NodeSize("m3.xlarge", "Extra Large Instance", 3500, 80, 0, 0, self),
+                NodeSize("m4.xlarge", "Extra Large Instance", 3500, 0, 0, 0, self),
+                NodeSize("m4.2xlarge", "Double Extra Large Instance", 7000, 0, 0, 0, self)]

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list