[ARVADOS] updated: 8552d32092e45a1f6ee1424e92882ec84b51cb8a
git at public.curoverse.com
git at public.curoverse.com
Wed Aug 19 11:12:17 EDT 2015
Summary of changes:
.../arvnodeman/computenode/driver/azure.py | 8 ++-
services/nodemanager/doc/azure.example.cfg | 54 +++++++++++++++----
...ver_ec2.py => test_computenode_driver_azure.py} | 63 ++++++++--------------
3 files changed, 70 insertions(+), 55 deletions(-)
copy services/nodemanager/tests/{test_computenode_driver_ec2.py => test_computenode_driver_azure.py} (52%)
via 8552d32092e45a1f6ee1424e92882ec84b51cb8a (commit)
via 0fde046a6c68909ae25af809557fcd64eb7264d7 (commit)
via 45a172b5f59ea7464b7241212464bf9113a18f36 (commit)
from c46ab7c622127315b4e90c98e859588a3403267a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8552d32092e45a1f6ee1424e92882ec84b51cb8a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Aug 19 11:13:34 2015 -0400
6507: Paste in command line for creating service principals instead of just
referencing generic documentation.
diff --git a/services/nodemanager/doc/azure.example.cfg b/services/nodemanager/doc/azure.example.cfg
index 6bfd28b..e92431f 100644
--- a/services/nodemanager/doc/azure.example.cfg
+++ b/services/nodemanager/doc/azure.example.cfg
@@ -88,18 +88,18 @@ provider = azure
shutdown_windows = 20, 999999
[Cloud Credentials]
-# Get these using "azure account list" with the azure CLI.
+# Use "azure account list" with the azure CLI to get these values.
tenant_id = 00000000-0000-0000-0000-000000000000
subscription_id = 00000000-0000-0000-0000-000000000000
-# Follow the directions on the following page to create an Active Directory
-# "service principal" that NodeManager will use to authenticate. Note that the
-# role must be "Owner" to manipulate resources.
-#
+# The following directions are based on
# https://azure.microsoft.com/en-us/documentation/articles/resource-group-authenticate-service-principal/
#
-# The key is the the "Application Id" and the secret is the password provided
-# when creating the service principal.
+# azure ad app create --name "<Your Application Display Name>" --home-page "<https://YourApplicationHomePage>" --identifier-uris "<https://YouApplicationUri>" --password <Your_Password>
+# azure ad sp create "<Application_Id>"
+# azure role assignment create --objectId "<Object_Id>" -o Owner -c /subscriptions/{subscriptionId}/
+#
+# Use <Application_Id> for "key" and the <Your_Password> for "secret"
#
key = 00000000-0000-0000-0000-000000000000
secret = PASSWORD
commit 0fde046a6c68909ae25af809557fcd64eb7264d7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Aug 19 11:07:53 2015 -0400
6507: Added notes to sample azure configuration file
diff --git a/services/nodemanager/doc/azure.example.cfg b/services/nodemanager/doc/azure.example.cfg
index 8fad85d..6bfd28b 100644
--- a/services/nodemanager/doc/azure.example.cfg
+++ b/services/nodemanager/doc/azure.example.cfg
@@ -88,24 +88,56 @@ provider = azure
shutdown_windows = 20, 999999
[Cloud Credentials]
-subscription_id = SUBSCRIPTION_ID
-key_file = PATH_TO_PEM_FILE
+# Get these using "azure account list" with the azure CLI.
+tenant_id = 00000000-0000-0000-0000-000000000000
+subscription_id = 00000000-0000-0000-0000-000000000000
+
+# Follow the directions on the following page to create an Active Directory
+# "service principal" that NodeManager will use to authenticate. Note that the
+# role must be "Owner" to manipulate resources.
+#
+# https://azure.microsoft.com/en-us/documentation/articles/resource-group-authenticate-service-principal/
+#
+# The key is the the "Application Id" and the secret is the password provided
+# when creating the service principal.
+#
+key = 00000000-0000-0000-0000-000000000000
+secret = PASSWORD
timeout = 60
+region = East US
[Cloud List]
-# This section defines filters that find compute nodes.
-# Tags that you specify here will automatically be added to nodes you create.
-# Replace colons in Microsoft filters with underscores
-# (e.g., write "tag:mytag" as "tag_mytag").
-instance-state-name = running
+# The resource group in which the compute node virtual machines will be created
+# and listed.
+ex_resource_group = ArvadosResourceGroup
+
+[Cloud Create]
+# The image id, in the form "Publisher:Offer:SKU:Version"
+image = Canonical:UbuntuServer:14.04.3-LTS:14.04.201508050
+
+# Path to a local ssh key file that will be used to provision new nodes.
+ssh_key = /home/arvadosuser/.ssh/id_rsa.pub
+
+# The account name for the admin user that will be provisioned on new nodes.
+ex_user_name = arvadosuser
+
+# The Azure storage account that will be used to store the node OS disk images.
+ex_storage_account = arvadosstorage
+
+# The virtual network the VMs will be associated with.
+ex_network = ArvadosNetwork
+
+# Optional subnet of the virtual network.
+#ex_subnet = default
+
+# Node tags
tag_arvados-class = dynamic-compute
tag_cluster = zyxwv
-[Cloud Create]
-image: ???
-ex_cloud_service_name: ???
+# the API server to ping
+ping_host = hostname:port
-[Size A3]
+[Size Standard_A2]
# You can define any number of Size sections to list Azure sizes you're
# willing to use. The Node Manager should boot the cheapest size(s) that
# can run jobs in the queue (N.B.: defining more than one size has not been
commit 45a172b5f59ea7464b7241212464bf9113a18f36
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Aug 19 10:36:42 2015 -0400
6507: Add basic azure driver tests.
diff --git a/services/nodemanager/arvnodeman/computenode/driver/azure.py b/services/nodemanager/arvnodeman/computenode/driver/azure.py
index ecab75c..725020e 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/azure.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/azure.py
@@ -19,7 +19,11 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
driver_class=DEFAULT_DRIVER):
- list_kwargs["ex_resource_group"] = create_kwargs["ex_resource_group"]
+
+ if not list_kwargs.get("ex_resource_group"):
+ raise Exception("Must include ex_resource_group in Cloud List configuration (list_kwargs)")
+
+ create_kwargs["ex_resource_group"] = list_kwargs["ex_resource_group"]
self.tags = {key[4:]: value
for key, value in create_kwargs.iteritems()
@@ -43,7 +47,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
def sync_node(self, cloud_node, arvados_node):
hostname = arvados_node_fqdn(arvados_node)
- self.real.ex_create_tags(cloud_node.id, {"hostname": hostname})
+ self.real.ex_create_tags(cloud_node, {"hostname": hostname})
def _init_image(self, urn):
return "image", self.list_images(ex_urn=urn)[0]
diff --git a/services/nodemanager/tests/test_computenode_driver_azure.py b/services/nodemanager/tests/test_computenode_driver_azure.py
new file mode 100644
index 0000000..049e708
--- /dev/null
+++ b/services/nodemanager/tests/test_computenode_driver_azure.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import, print_function
+
+import ssl
+import time
+import unittest
+
+import libcloud.common.types as cloud_types
+import mock
+
+import arvnodeman.computenode.driver.azure as azure
+from . import testutil
+
+class AzureComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
+ TEST_CLASS = azure.ComputeNodeDriver
+
+ def new_driver(self, auth_kwargs={}, list_kwargs={}, create_kwargs={}):
+ list_kwargs.setdefault("ex_resource_group", "TestResourceGroup")
+ return super(AzureComputeNodeDriverTestCase, self).new_driver(auth_kwargs, list_kwargs, create_kwargs)
+
+ def test_driver_instantiation(self):
+ kwargs = {'key': 'testkey'}
+ driver = self.new_driver(auth_kwargs=kwargs)
+ self.assertTrue(self.driver_mock.called)
+ self.assertEqual(kwargs, self.driver_mock.call_args[1])
+
+ def test_create_image_loaded_at_initialization(self):
+ list_method = self.driver_mock().list_images
+ list_method.return_value = [testutil.cloud_object_mock(c)
+ for c in 'abc']
+ driver = self.new_driver(create_kwargs={'image': 'id_b'})
+ self.assertEqual(1, list_method.call_count)
+
+ def test_create_includes_ping_url(self):
+ arv_node = testutil.arvados_node_mock(info={'ping_secret': 'ssshh'})
+ driver = self.new_driver()
+ driver.create_node(testutil.MockSize(1), arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ print(create_method.call_args[1])
+ self.assertIn('ping_secret=ssshh',
+ create_method.call_args[1].get('ex_tags', {}).get('arv-ping-url', ""))
+
+ def test_name_from_new_arvados_node(self):
+ arv_node = testutil.arvados_node_mock(hostname=None)
+ driver = self.new_driver()
+ self.assertEqual('compute-000000000000063-zzzzz',
+ driver.arvados_create_kwargs(arv_node)['name'])
+
+ def check_node_tagged(self, cloud_node, expected_tags):
+ tag_mock = self.driver_mock().ex_create_tags
+ self.assertTrue(tag_mock.called)
+ self.assertIs(cloud_node, tag_mock.call_args[0][0])
+ self.assertEqual(expected_tags, tag_mock.call_args[0][1])
+
+ def test_sync_node(self):
+ arv_node = testutil.arvados_node_mock(1)
+ cloud_node = testutil.cloud_node_mock(2)
+ driver = self.new_driver()
+ driver.sync_node(cloud_node, arv_node)
+ self.check_node_tagged(cloud_node,
+ {'hostname': 'compute1.zzzzz.arvadosapi.com'})
+
+ def test_node_create_time(self):
+ refsecs = int(time.time())
+ reftuple = time.gmtime(refsecs)
+ node = testutil.cloud_node_mock()
+ node.extra = {'tags': {'booted_at': time.strftime('%Y-%m-%dT%H:%M:%S.000Z',
+ reftuple)}}
+ self.assertEqual(refsecs, azure.ComputeNodeDriver.node_start_time(node))
+
+ def test_node_fqdn(self):
+ name = 'fqdntest.zzzzz.arvadosapi.com'
+ node = testutil.cloud_node_mock()
+ node.extra = {'tags': {"hostname": name}}
+ self.assertEqual(name, azure.ComputeNodeDriver.node_fqdn(node))
+
+ def test_cloud_exceptions(self):
+ for error in [Exception("test exception"),
+ IOError("test exception"),
+ ssl.SSLError("test exception"),
+ cloud_types.LibcloudError("test exception")]:
+ self.assertTrue(azure.ComputeNodeDriver.is_cloud_exception(error),
+ "{} not flagged as cloud exception".format(error))
+
+ def test_noncloud_exceptions(self):
+ self.assertFalse(
+ azure.ComputeNodeDriver.is_cloud_exception(ValueError("test error")),
+ "ValueError flagged as cloud exception")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list