[ARVADOS] updated: 1.2.0-35-g71efe2082
Git user
git at public.curoverse.com
Wed Aug 22 22:42:39 EDT 2018
Summary of changes:
lib/dispatchcloud/azure.go | 127 ++++++++++++++++++++++----------
lib/dispatchcloud/azure_test.go | 158 ++++++++++++++++++++++++++++++++++++++++
lib/dispatchcloud/provider.go | 5 +-
3 files changed, 250 insertions(+), 40 deletions(-)
create mode 100644 lib/dispatchcloud/azure_test.go
via 71efe208264f60c03cbb1994b8e9e44dadddff0e (commit)
via 13b9e9f1121574c5d46a71165430997a25e0fc5b (commit)
from 11d96a31330468540bff69d11a3e6e20c49ee676 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 71efe208264f60c03cbb1994b8e9e44dadddff0e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Aug 22 22:42:26 2018 -0400
13964: Return Arvados instance type
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 29631e6dc..195bd8182 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -230,9 +230,10 @@ func (az *AzureProvider) Create(ctx context.Context,
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
+ "arvados-class": to.StringPtr("crunch-dynamic-compute"),
+ "arvados-instance-type": &instanceType.Name,
+ "arvados-cluster": &az.arvconfig.ClusterID,
+ "created-at": ×tamp,
},
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
@@ -311,11 +312,13 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
return nil, err
}
if result.Value().Tags["arvados-class"] != nil &&
+ result.Value().Tags["arvados-instance-type"] != nil &&
(*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
instances = append(instances, &AzureInstance{
- provider: az,
- vm: result.Value(),
- nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
+ provider: az,
+ vm: result.Value(),
+ nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID],
+ instanceType: az.arvconfig.InstanceTypes[(*result.Value().Tags["arvados-instance-type"])]})
}
}
return instances, nil
@@ -464,10 +467,6 @@ func (ai *AzureInstance) String() string {
return *ai.vm.Name
}
-func (ai *AzureInstance) ProviderType() string {
- return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
-}
-
func (ai *AzureInstance) InstanceType() arvados.InstanceType {
return ai.instanceType
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index 2791ea6bf..bcba51bfd 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -61,45 +61,49 @@ func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName
var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
-func GetProvider() (Provider, ImageID, error) {
+func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
+ cluster := arvados.Cluster{
+ InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
+ "tiny": arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "Standard_D1_v2",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ })}
if *live != "" {
cfg := AzureProviderConfig{}
err := config.LoadFile(&cfg, *live)
if err != nil {
- return nil, ImageID(""), err
+ return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, arvados.Cluster{})
- return ap, ImageID(cfg.Image), err
+ ap, err := NewAzureProvider(cfg, cluster)
+ return ap, ImageID(cfg.Image), cluster, err
} else {
ap := AzureProvider{
azconfig: AzureProviderConfig{
BlobContainer: "vhds",
},
+ arvconfig: cluster,
}
ap.vmClient = &VirtualMachinesClientStub{}
ap.netClient = &InterfacesClientStub{}
- return &ap, ImageID("blob"), nil
+ return &ap, ImageID("blob"), cluster, nil
}
}
func (*AzureProviderSuite) TestCreate(c *check.C) {
- ap, img, err := GetProvider()
+ ap, img, cluster, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
inst, err := ap.Create(context.Background(),
- arvados.InstanceType{
- Name: "tiny",
- ProviderType: "Standard_D1_v2",
- VCPUs: 1,
- RAM: 4000000000,
- Scratch: 10000000000,
- Price: .02,
- Preemptible: false,
- },
- img,
- []InstanceTag{"tag1"})
+ cluster.InstanceTypes["tiny"],
+ img, []InstanceTag{"tag1"})
c.Assert(err, check.IsNil)
@@ -107,7 +111,7 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
}
func (*AzureProviderSuite) TestListInstances(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -117,12 +121,12 @@ func (*AzureProviderSuite) TestListInstances(c *check.C) {
c.Assert(err, check.IsNil)
for _, i := range l {
- log.Printf("%v %v", i.String(), i.Address())
+ log.Printf("%v %v %v", i.String(), i.Address(), i.InstanceType())
}
}
func (*AzureProviderSuite) TestManageNics(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -131,7 +135,7 @@ func (*AzureProviderSuite) TestManageNics(c *check.C) {
}
func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -140,7 +144,7 @@ func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
}
func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index e896ac650..c5411128a 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -18,9 +18,8 @@ type ImageID string
type Instance interface {
// String typically returns the cloud-provided instance ID.
String() string
- // Cloud provider's "instance type" ID. Matches a key in
- // configured arvados.InstanceTypeMap.
- ProviderType() string
+ // Configured Arvados instance type
+ InstanceType() arvados.InstanceType
// Get tags
GetTags() ([]InstanceTag, error)
// Replace tags with the given tags
commit 13b9e9f1121574c5d46a71165430997a25e0fc5b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Aug 22 22:13:23 2018 -0400
13964: Full lifecycle for NICs/VHDs/VMs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 6f769bfaa..29631e6dc 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -125,6 +125,7 @@ type AzureProvider struct {
netClient InterfacesClientWrapper
storageAcctClient storageacct.AccountsClient
azureEnv azure.Environment
+ interfaces map[string]network.Interface
}
func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (prv Provider, err error) {
@@ -279,7 +280,7 @@ func (az *AzureProvider) Create(ctx context.Context,
},
}
- vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name+"-compute", vmParameters)
+ vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name, vmParameters)
if err != nil {
return nil, err
}
@@ -293,6 +294,11 @@ func (az *AzureProvider) Create(ctx context.Context,
}
func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
+ interfaces, err := az.ManageNics(ctx)
+ if err != nil {
+ return nil, err
+ }
+
result, err := az.vmClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
return nil, err
@@ -304,59 +310,80 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
if err != nil {
return nil, err
}
- log.Printf("%v", *result.Value().Name)
if result.Value().Tags["arvados-class"] != nil &&
(*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
instances = append(instances, &AzureInstance{
provider: az,
- vm: result.Value()})
+ vm: result.Value(),
+ nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
}
}
return instances, nil
}
-func (az *AzureProvider) DeleteDanglingNics(ctx context.Context) {
+func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
- return
+ return nil, err
}
+ interfaces := make(map[string]network.Interface)
+
timestamp := time.Now()
wg := sync.WaitGroup{}
- defer wg.Wait()
+ deletechannel := make(chan string, 20)
+ defer func() {
+ wg.Wait()
+ close(deletechannel)
+ }()
+ for i := 0; i < 4; i += 1 {
+ go func() {
+ for {
+ nicname, ok := <-deletechannel
+ if !ok {
+ return
+ }
+ _, delerr := az.netClient.Delete(context.Background(), az.azconfig.ResourceGroup, nicname)
+ if delerr != nil {
+ log.Printf("Error deleting %v: %v", nicname, delerr)
+ } else {
+ log.Printf("Deleted %v", nicname)
+ }
+ wg.Done()
+ }
+ }()
+ }
+
for ; result.NotDone(); err = result.Next() {
if err != nil {
log.Printf("Error listing nics: %v", err)
- return
- }
- if !result.NotDone() {
- return
+ return interfaces, nil
}
if result.Value().Tags["arvados-class"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" &&
- result.Value().VirtualMachine == nil {
-
- if result.Value().Tags["created-at"] != nil {
- created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
- if err == nil {
- log.Printf("found dangling NIC %v created %v seconds ago", *result.Value().Name, timestamp.Sub(created_at).Seconds())
- if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
- log.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
- wg.Add(1)
- go func(nicname string) {
- r, delerr := az.netClient.Delete(context.Background(), az.azconfig.ResourceGroup, nicname)
- log.Printf("%v %v", r, delerr)
- wg.Done()
- }(*result.Value().Name)
+ (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
+
+ if result.Value().VirtualMachine != nil {
+ interfaces[*result.Value().ID] = result.Value()
+ } else {
+
+ if result.Value().Tags["created-at"] != nil {
+ created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
+ if err == nil {
+ //log.Printf("found dangling NIC %v created %v seconds ago", *result.Value().Name, timestamp.Sub(created_at).Seconds())
+ if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ log.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
+ wg.Add(1)
+ deletechannel <- *result.Value().Name
+ }
}
}
}
}
}
-
+ return interfaces, nil
}
-func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
+func (az *AzureProvider) ManageBlobs(ctx context.Context) {
result, err := az.storageAcctClient.ListKeys(ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
if err != nil {
log.Printf("Couldn't get account keys %v", err)
@@ -374,6 +401,30 @@ func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
blobcont := blobsvc.GetContainerReference(az.azconfig.BlobContainer)
timestamp := time.Now()
+ wg := sync.WaitGroup{}
+ deletechannel := make(chan storage.Blob, 20)
+ defer func() {
+ wg.Wait()
+ close(deletechannel)
+ }()
+ for i := 0; i < 4; i += 1 {
+ go func() {
+ for {
+ blob, ok := <-deletechannel
+ if !ok {
+ return
+ }
+ err := blob.Delete(nil)
+ if err != nil {
+ log.Printf("error deleting %v: %v", blob.Name, err)
+ } else {
+ log.Printf("Deleted blob %v", blob.Name)
+ }
+ wg.Done()
+ }
+ }()
+ }
+
page := storage.ListBlobsParameters{Prefix: "compute-"}
for {
@@ -388,7 +439,10 @@ func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
b.Properties.LeaseState == "available" &&
b.Properties.LeaseStatus == "unlocked" &&
age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+
log.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
+ wg.Add(1)
+ deletechannel <- b
}
}
if response.NextMarker != "" {
@@ -407,7 +461,7 @@ type AzureInstance struct {
}
func (ai *AzureInstance) String() string {
- return *ai.vm.ID
+ return *ai.vm.Name
}
func (ai *AzureInstance) ProviderType() string {
@@ -428,7 +482,7 @@ func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
func (ai *AzureInstance) Destroy(ctx context.Context) error {
_, err := ai.provider.vmClient.Delete(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
- // check response code
+ // check response code?
return err
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
new file mode 100644
index 000000000..2791ea6bf
--- /dev/null
+++ b/lib/dispatchcloud/azure_test.go
@@ -0,0 +1,154 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package dispatchcloud
+
+import (
+ "context"
+ "flag"
+ "log"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
+ "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
+ "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
+ "github.com/Azure/go-autorest/autorest/to"
+ check "gopkg.in/check.v1"
+)
+
+type AzureProviderSuite struct{}
+
+var _ = check.Suite(&AzureProviderSuite{})
+
+type VirtualMachinesClientStub struct{}
+
+func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ VMName string,
+ parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
+ parameters.ID = &VMName
+ return parameters, nil
+}
+
+func (*VirtualMachinesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ return nil, nil
+}
+
+func (*VirtualMachinesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
+ return compute.VirtualMachineListResultIterator{}, nil
+}
+
+type InterfacesClientStub struct{}
+
+func (*InterfacesClientStub) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ nicName string,
+ parameters network.Interface) (result network.Interface, err error) {
+ parameters.ID = to.StringPtr(nicName)
+ (*parameters.IPConfigurations)[0].PrivateIPAddress = to.StringPtr("192.168.5.5")
+ return parameters, nil
+}
+
+func (*InterfacesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ return nil, nil
+}
+
+func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
+ return network.InterfaceListResultIterator{}, nil
+}
+
+var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
+
+func GetProvider() (Provider, ImageID, error) {
+ if *live != "" {
+ cfg := AzureProviderConfig{}
+ err := config.LoadFile(&cfg, *live)
+ if err != nil {
+ return nil, ImageID(""), err
+ }
+ ap, err := NewAzureProvider(cfg, arvados.Cluster{})
+ return ap, ImageID(cfg.Image), err
+ } else {
+ ap := AzureProvider{
+ azconfig: AzureProviderConfig{
+ BlobContainer: "vhds",
+ },
+ }
+ ap.vmClient = &VirtualMachinesClientStub{}
+ ap.netClient = &InterfacesClientStub{}
+ return &ap, ImageID("blob"), nil
+ }
+}
+
+func (*AzureProviderSuite) TestCreate(c *check.C) {
+ ap, img, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ inst, err := ap.Create(context.Background(),
+ arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "Standard_D1_v2",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ img,
+ []InstanceTag{"tag1"})
+
+ c.Assert(err, check.IsNil)
+
+ log.Printf("Result %v %v", inst.String(), inst.Address())
+}
+
+func (*AzureProviderSuite) TestListInstances(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(context.Background())
+
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ log.Printf("%v %v", i.String(), i.Address())
+ }
+}
+
+func (*AzureProviderSuite) TestManageNics(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ ap.(*AzureProvider).ManageNics(context.Background())
+}
+
+func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ ap.(*AzureProvider).ManageBlobs(context.Background())
+}
+
+func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ c.Check(i.Destroy(context.Background()), check.IsNil)
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list