[ARVADOS] created: e098acbe8e934fd2f416864c8bf1d33314cfd08d

[Git user]

        at  e098acbe8e934fd2f416864c8bf1d33314cfd08d (commit)


commit e098acbe8e934fd2f416864c8bf1d33314cfd08d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Jun 9 11:22:14 2017 -0400

    8186: Add EBS storage if necessary so there is sufficient scratch space.

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 8deabbd..6a9f473 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -67,8 +67,18 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
     create_cloud_name = staticmethod(arvados_node_fqdn)
 
     def arvados_create_kwargs(self, size, arvados_node):
-        return {'name': self.create_cloud_name(arvados_node),
+        kw = {'name': self.create_cloud_name(arvados_node),
                 'ex_userdata': self._make_ping_url(arvados_node)}
+        # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        scratch = size.scratch / 1000
+        if scratch > size.disk:
+            kw["ex_blockdevicemappings"] = [{
+                "Ebs": {
+                    "DeleteOnTermination": True,
+                    "VolumeSize": scratch - size.disk,
+                    "VolumeType": "gp2"
+                }}]
+        return kw
 
     def post_create_node(self, cloud_node):
         self.real.ex_create_tags(cloud_node, self.tags)
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index f3b9765..4cf9c0b 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -398,7 +398,7 @@ class NodeManagerDaemonActor(actor_class):
             arvados_client=self._new_arvados(),
             arvados_node=arvados_node,
             cloud_client=self._new_cloud(),
-            cloud_size=cloud_size,
+            cloud_size=self.server_calculator.find_size(cloud_size.id),
             assigned_hostname=assigned_hostname).proxy()
         self.booting[new_setup.actor_ref.actor_urn] = new_setup
         self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 8e7cf2f..1e15002 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -1,6 +1,9 @@
 import re
 import urllib
 import ssl
+import time
+
+from arvnodeman.computenode import ARVADOS_TIMEFMT
 
 from libcloud.compute.base import NodeSize, Node, NodeDriver, NodeState
 from libcloud.common.exceptions import BaseHTTPError
@@ -29,12 +32,16 @@ class FakeDriver(NodeDriver):
                     ex_resource_group=None,
                     ex_user_name=None,
                     ex_tags=None,
-                    ex_network=None):
+                    ex_network=None,
+                    ex_userdata=None):
         global all_nodes, create_calls
         create_calls += 1
         n = Node(name, name, NodeState.RUNNING, [], [], self, size=size, extra={"tags": ex_tags})
         all_nodes.append(n)
-        ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_customdata:
+            ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+        if ex_userdata:
+            ping_url = ex_userdata
         ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         ctx.verify_mode = ssl.CERT_NONE
         f = urllib.urlopen(ping_url, "", context=ctx)
@@ -125,3 +132,24 @@ class RetryDriver(FakeDriver):
                     ex_user_name=ex_user_name,
                     ex_tags=ex_tags,
                     ex_network=ex_network)
+
+class FakeAwsDriver(FakeDriver):
+
+    def create_node(self, name=None,
+                    size=None,
+                    image=None,
+                    auth=None,
+                    ex_userdata=None,
+                    ex_blockdevicemappings=None):
+        n = super(FakeAwsDriver, self).create_node(name=name,
+                                                      size=size,
+                                                      image=image,
+                                                      auth=auth,
+                                                      ex_userdata=ex_userdata)
+        n.extra = {"launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1]}
+        return n
+
+    def list_sizes(self, **kwargs):
+        return [NodeSize("m3.xlarge", "Extra Large Instance", 3500, 80, 0, 0, self),
+                NodeSize("m4.xlarge", "Extra Large Instance", 3500, 0, 0, 0, self),
+                NodeSize("m4.2xlarge", "Double Extra Large Instance", 7000, 0, 0, 0, self)]

-----------------------------------------------------------------------


hooks/post-receive
--