[ARVADOS] created: 495c902b59cebb965c610b7bec7fca64831c6e3a
Git user
git at public.curoverse.com
Wed Jun 14 09:29:13 EDT 2017
at 495c902b59cebb965c610b7bec7fca64831c6e3a (commit)
commit 495c902b59cebb965c610b7bec7fca64831c6e3a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Jun 12 19:43:19 2017 +0000
8186: Set virtual device path. Require libcloud 0.20 because 0.18 has mistakes in the ec2 instance size table.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 6a9f473..b256eca 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -73,6 +73,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
scratch = size.scratch / 1000
if scratch > size.disk:
kw["ex_blockdevicemappings"] = [{
+ "DeviceName": "/dev/xvdt",
"Ebs": {
"DeleteOnTermination": True,
"VolumeSize": scratch - size.disk,
diff --git a/services/nodemanager/setup.py b/services/nodemanager/setup.py
index 5eb923e..9343191 100644
--- a/services/nodemanager/setup.py
+++ b/services/nodemanager/setup.py
@@ -29,7 +29,7 @@ setup(name='arvados-node-manager',
('share/doc/arvados-node-manager', ['agpl-3.0.txt', 'README.rst']),
],
install_requires=[
- 'apache-libcloud>=0.16',
+ 'apache-libcloud>=0.20',
'arvados-python-client>=0.1.20150206225333',
'future',
'pykka',
@@ -37,14 +37,14 @@ setup(name='arvados-node-manager',
'setuptools'
],
dependency_links=[
- "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev4.zip"
+ "https://github.com/curoverse/libcloud/archive/apache-libcloud-0.20.2.dev3.zip"
],
test_suite='tests',
tests_require=[
'requests',
'pbr<1.7.0',
'mock>=1.0',
- 'apache-libcloud==0.18.1.dev4',
+ 'apache-libcloud==0.20.2.dev3',
],
zip_safe=False,
cmdclass={'egg_info': tagger},
commit 319c1aa1ffcb065492eb1878687c993d7638e94e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jun 14 09:28:27 2017 -0400
8186: Add EBS storage if necessary so there is sufficient scratch space.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 8deabbd..6a9f473 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -67,8 +67,18 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
create_cloud_name = staticmethod(arvados_node_fqdn)
def arvados_create_kwargs(self, size, arvados_node):
- return {'name': self.create_cloud_name(arvados_node),
+ kw = {'name': self.create_cloud_name(arvados_node),
'ex_userdata': self._make_ping_url(arvados_node)}
+ # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
+ scratch = size.scratch / 1000
+ if scratch > size.disk:
+ kw["ex_blockdevicemappings"] = [{
+ "Ebs": {
+ "DeleteOnTermination": True,
+ "VolumeSize": scratch - size.disk,
+ "VolumeType": "gp2"
+ }}]
+ return kw
def post_create_node(self, cloud_node):
self.real.ex_create_tags(cloud_node, self.tags)
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 7e63c78..5fc5673 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -387,7 +387,7 @@ class NodeManagerDaemonActor(actor_class):
arvados_client=self._new_arvados(),
arvados_node=arvados_node,
cloud_client=self._new_cloud(),
- cloud_size=cloud_size).proxy()
+ cloud_size=self.server_calculator.find_size(cloud_size.id)).proxy()
self.booting[new_setup.actor_ref.actor_urn] = new_setup
self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 8e7cf2f..1e15002 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -1,6 +1,9 @@
import re
import urllib
import ssl
+import time
+
+from arvnodeman.computenode import ARVADOS_TIMEFMT
from libcloud.compute.base import NodeSize, Node, NodeDriver, NodeState
from libcloud.common.exceptions import BaseHTTPError
@@ -29,12 +32,16 @@ class FakeDriver(NodeDriver):
ex_resource_group=None,
ex_user_name=None,
ex_tags=None,
- ex_network=None):
+ ex_network=None,
+ ex_userdata=None):
global all_nodes, create_calls
create_calls += 1
n = Node(name, name, NodeState.RUNNING, [], [], self, size=size, extra={"tags": ex_tags})
all_nodes.append(n)
- ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+ if ex_customdata:
+ ping_url = re.search(r"echo '(.*)' > /var/tmp/arv-node-data/arv-ping-url", ex_customdata).groups(1)[0] + "&instance_id=" + name
+ if ex_userdata:
+ ping_url = ex_userdata
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ctx.verify_mode = ssl.CERT_NONE
f = urllib.urlopen(ping_url, "", context=ctx)
@@ -125,3 +132,24 @@ class RetryDriver(FakeDriver):
ex_user_name=ex_user_name,
ex_tags=ex_tags,
ex_network=ex_network)
+
+class FakeAwsDriver(FakeDriver):
+
+ def create_node(self, name=None,
+ size=None,
+ image=None,
+ auth=None,
+ ex_userdata=None,
+ ex_blockdevicemappings=None):
+ n = super(FakeAwsDriver, self).create_node(name=name,
+ size=size,
+ image=image,
+ auth=auth,
+ ex_userdata=ex_userdata)
+ n.extra = {"launch_time": time.strftime(ARVADOS_TIMEFMT, time.gmtime())[:-1]}
+ return n
+
+ def list_sizes(self, **kwargs):
+ return [NodeSize("m3.xlarge", "Extra Large Instance", 3500, 80, 0, 0, self),
+ NodeSize("m4.xlarge", "Extra Large Instance", 3500, 0, 0, 0, self),
+ NodeSize("m4.2xlarge", "Double Extra Large Instance", 7000, 0, 0, 0, self)]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list