[ARVADOS] updated: 1.2.0-38-g10c41c20f
Git user
git at public.curoverse.com
Thu Aug 23 16:22:33 EDT 2018
Summary of changes:
lib/dispatchcloud/azure.go | 88 +++++++++++++++++++++++++++--------------
lib/dispatchcloud/azure_test.go | 34 ++++++++++++++--
lib/dispatchcloud/provider.go | 8 ++--
3 files changed, 93 insertions(+), 37 deletions(-)
via 10c41c20f05537e5249983ddb5ebc8fd2441b1c3 (commit)
from 441b77d7a3fe7568b696fd0e88109e98ca51bbd4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 10c41c20f05537e5249983ddb5ebc8fd2441b1c3
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Aug 23 16:22:12 2018 -0400
13964: VM names are namespaced. Can set/get tags.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 7f7e34fb1..dcfcf0564 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -11,6 +11,7 @@ import (
"net/http"
"regexp"
"strconv"
+ "strings"
"sync"
"time"
@@ -191,18 +192,20 @@ type AzureProvider struct {
storageAcctClient storageacct.AccountsClient
azureEnv azure.Environment
interfaces map[string]network.Interface
+ dispatcherID string
+ namePrefix string
}
-func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (prv Provider, err error) {
+func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (prv Provider, err error) {
ap := AzureProvider{}
- err = ap.setup(azcfg, arvcfg)
+ err = ap.setup(azcfg, arvcfg, dispatcherID)
if err != nil {
return nil, err
}
return &ap, nil
}
-func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (err error) {
+func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (err error) {
az.azconfig = azcfg
az.arvconfig = arvcfg
vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
@@ -233,31 +236,36 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster
az.netClient = &InterfacesClientImpl{netClient}
az.storageAcctClient = storageAcctClient
+ az.dispatcherID = dispatcherID
+ az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
+
return nil
}
func (az *AzureProvider) Create(ctx context.Context,
instanceType arvados.InstanceType,
imageId ImageID,
- instanceTag []InstanceTag) (Instance, error) {
+ newTags map[string]string) (Instance, error) {
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
return nil, err
}
- name = "compute-" + name
+ name = az.namePrefix + name
log.Printf("name is %v", name)
timestamp := time.Now().Format(time.RFC3339Nano)
+ tags := make(map[string]*string)
+ tags["created-at"] = ×tamp
+ for k, v := range newTags {
+ tags["dispatch-"+k] = &v
+ }
+
nicParameters := network.Interface{
Location: &az.azconfig.Location,
- Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
- },
+ Tags: tags,
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
IPConfigurations: &[]network.InterfaceIPConfiguration{
network.InterfaceIPConfiguration{
@@ -292,14 +300,11 @@ func (az *AzureProvider) Create(ctx context.Context,
log.Printf("URI instance vhd %v", instance_vhd)
+ tags["arvados-instance-type"] = &instanceType.Name
+
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
- Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-instance-type": &instanceType.Name,
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
- },
+ Tags: tags,
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
@@ -307,7 +312,7 @@ func (az *AzureProvider) Create(ctx context.Context,
StorageProfile: &compute.StorageProfile{
OsDisk: &compute.OSDisk{
OsType: compute.Linux,
- Name: to.StringPtr(fmt.Sprintf("%v-os", name)),
+ Name: to.StringPtr(name + "-os"),
CreateOption: compute.FromImage,
Image: &compute.VirtualHardDisk{
URI: to.StringPtr(string(imageId)),
@@ -376,9 +381,8 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
if err != nil {
return nil, WrapAzureError(err)
}
- if result.Value().Tags["arvados-class"] != nil &&
- result.Value().Tags["arvados-instance-type"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
+ if strings.HasPrefix(*result.Value().Name, az.namePrefix) &&
+ result.Value().Tags["arvados-instance-type"] != nil {
instances = append(instances, &AzureInstance{
provider: az,
vm: result.Value(),
@@ -425,15 +429,12 @@ func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Int
for ; result.NotDone(); err = result.Next() {
if err != nil {
log.Printf("Error listing nics: %v", err)
- return interfaces, WrapAzureError(nil)
+ return interfaces, nil
}
- if result.Value().Tags["arvados-class"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
-
+ if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
if result.Value().VirtualMachine != nil {
interfaces[*result.Value().ID] = result.Value()
} else {
-
if result.Value().Tags["created-at"] != nil {
created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
if err == nil {
@@ -493,7 +494,7 @@ func (az *AzureProvider) ManageBlobs(ctx context.Context) {
}()
}
- page := storage.ListBlobsParameters{Prefix: "compute-"}
+ page := storage.ListBlobsParameters{Prefix: az.namePrefix}
for {
response, err := blobcont.ListBlobs(page)
@@ -536,12 +537,41 @@ func (ai *AzureInstance) InstanceType() arvados.InstanceType {
return ai.instanceType
}
-func (ai *AzureInstance) SetTags([]InstanceTag) error {
+func (ai *AzureInstance) SetTags(ctx context.Context, newTags map[string]string) error {
+ tags := make(map[string]*string)
+
+ for k, v := range ai.vm.Tags {
+ if !strings.HasPrefix(k, "dispatch-") {
+ tags[k] = v
+ }
+ }
+ for k, v := range newTags {
+ tags["dispatch-"+k] = &v
+ }
+
+ vmParameters := compute.VirtualMachine{
+ Location: &ai.provider.azconfig.Location,
+ Tags: tags,
+ }
+ vm, err := ai.provider.vmClient.CreateOrUpdate(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
+ if err != nil {
+ return WrapAzureError(err)
+ }
+ ai.vm = vm
+
return nil
}
-func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
- return nil, nil
+func (ai *AzureInstance) GetTags(ctx context.Context) (map[string]string, error) {
+ tags := make(map[string]string)
+
+ for k, v := range ai.vm.Tags {
+ if strings.HasPrefix(k, "dispatch-") {
+ tags[k[9:]] = *v
+ }
+ }
+
+ return tags, nil
}
func (ai *AzureInstance) Destroy(ctx context.Context) error {
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index f10121984..a5a173bee 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -83,14 +83,16 @@ func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
if err != nil {
return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, cluster)
+ ap, err := NewAzureProvider(cfg, cluster, "test123")
return ap, ImageID(cfg.Image), cluster, err
} else {
ap := AzureProvider{
azconfig: AzureProviderConfig{
BlobContainer: "vhds",
},
- arvconfig: cluster,
+ arvconfig: cluster,
+ dispatcherID: "test123",
+ namePrefix: "compute-test123-",
}
ap.vmClient = &VirtualMachinesClientStub{}
ap.netClient = &InterfacesClientStub{}
@@ -106,7 +108,7 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
inst, err := ap.Create(context.Background(),
cluster.InstanceTypes["tiny"],
- img, []InstanceTag{"tag1"})
+ img, map[string]string{"tag1": "bleep"})
c.Assert(err, check.IsNil)
@@ -124,7 +126,8 @@ func (*AzureProviderSuite) TestListInstances(c *check.C) {
c.Assert(err, check.IsNil)
for _, i := range l {
- log.Printf("%v %v %v", i.String(), i.Address(), i.InstanceType())
+ tg, _ := i.GetTags(context.Background())
+ log.Printf("%v %v %v %v", i.String(), i.Address(), i.InstanceType(), tg)
}
}
@@ -208,3 +211,26 @@ func (*AzureProviderSuite) TestWrapError(c *check.C) {
_, ok = wrapped.(QuotaError)
c.Check(ok, check.Equals, true)
}
+
+func (*AzureProviderSuite) TestSetTags(c *check.C) {
+ ap, _, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+ l, err := ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ if len(l) > 0 {
+ err = l[0].SetTags(context.Background(), map[string]string{"foo": "bar"})
+ if err != nil {
+ c.Fatal("Error setting tags", err)
+ }
+ }
+ l, err = ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ if len(l) > 0 {
+ tg, _ := l[0].GetTags(context.Background())
+ log.Printf("tags are %v", tg)
+ }
+}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index d8b1ad6c0..efaf2a6c3 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -43,16 +43,16 @@ type Instance interface {
// Configured Arvados instance type
InstanceType() arvados.InstanceType
// Get tags
- GetTags() ([]InstanceTag, error)
+ GetTags(context.Context) (map[string]string, error)
// Replace tags with the given tags
- SetTags([]InstanceTag) error
+ SetTags(context.Context, map[string]string) error
// Shut down the node
- Destroy(ctx context.Context) error
+ Destroy(context.Context) error
// SSH server hostname or IP address, or empty string if unknown pending creation.
Address() string
}
type Provider interface {
- Create(context.Context, arvados.InstanceType, ImageID, []InstanceTag) (Instance, error)
+ Create(context.Context, arvados.InstanceType, ImageID, map[string]string) (Instance, error)
Instances(context.Context) ([]Instance, error)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list