[ARVADOS] created: 1.3.0-163-g1b2a4c982
Git user
git at public.curoverse.com
Wed Jan 9 16:28:22 EST 2019
at 1b2a4c98249e3a46242f7d10c00a70feeeb2a843 (commit)
commit 1b2a4c98249e3a46242f7d10c00a70feeeb2a843
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 9 16:13:56 2019 -0500
14324: Azure driver for crunch-dispatch-cloud
* Adds context parameter to methods of InstanceSet interface that may block
* Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index cb4437256..0a26e8513 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -76,6 +76,7 @@ lib/cli
lib/cmd
lib/controller
lib/crunchstat
+lib/cloud
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
@@ -929,6 +930,7 @@ gostuff=(
lib/cmd
lib/controller
lib/crunchstat
+ lib/cloud
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
diff --git a/lib/cloud/azure.go b/lib/cloud/azure.go
index 734be7bef..2ee668372 100644
--- a/lib/cloud/azure.go
+++ b/lib/cloud/azure.go
@@ -26,6 +26,7 @@ import (
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/Azure/go-autorest/autorest/to"
"github.com/jmcvetta/randutil"
+ "github.com/mitchellh/mapstructure"
"golang.org/x/crypto/ssh"
)
@@ -196,14 +197,13 @@ type AzureInstanceSet struct {
namePrefix string
}
-func NewAzureInstanceSet(config map[string]interface{}, dispatcherID string) (prv InstanceProvider, err error) {
+func NewAzureInstanceSet(config map[string]interface{}, dispatcherID InstanceSetID) (prv InstanceSet, err error) {
azcfg := AzureInstanceSetConfig{}
- err = mapstructure.Decode(config, &azcfg)
- if err != nil {
+ if err = mapstructure.Decode(config, &azcfg); err != nil {
return nil, err
}
ap := AzureInstanceSet{}
- err = ap.setup(azcfg, dispatcherID)
+ err = ap.setup(azcfg, string(dispatcherID))
if err != nil {
return nil, err
}
@@ -376,7 +376,7 @@ echo '%s-%s' > /home/crunch/node-token`, name, newTags["node-token"])))
}, nil
}
-func (az *AzureInstanceSet) Instances(ctx context.Context) ([]Instance, error) {
+func (az *AzureInstanceSet) Instances(ctx context.Context, _ InstanceTags) ([]Instance, error) {
interfaces, err := az.ManageNics(ctx)
if err != nil {
return nil, err
@@ -549,6 +549,10 @@ func (ai *AzureInstance) String() string {
return *ai.vm.Name
}
+func (ai *AzureInstance) ProviderType() string {
+ return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
+}
+
func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) error {
tags := make(map[string]*string)
@@ -575,7 +579,7 @@ func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) erro
return nil
}
-func (ai *AzureInstance) Tags(ctx context.Context) (InstanceTags, error) {
+func (ai *AzureInstance) Tags() InstanceTags {
tags := make(map[string]string)
for k, v := range ai.vm.Tags {
@@ -584,7 +588,7 @@ func (ai *AzureInstance) Tags(ctx context.Context) (InstanceTags, error) {
}
}
- return tags, nil
+ return tags
}
func (ai *AzureInstance) Destroy(ctx context.Context) error {
@@ -596,10 +600,10 @@ func (ai *AzureInstance) Address() string {
return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
}
-func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.PublicKey, client *ssh.Client) error {
+func (ai *AzureInstance) VerifyHostKey(ctx context.Context, receivedKey ssh.PublicKey, client *ssh.Client) error {
remoteFingerprint := ssh.FingerprintSHA256(receivedKey)
- tags, _ := ai.Tags(ctx)
+ tags := ai.Tags()
tg := tags["ssh-pubkey-fingerprint"]
if tg != "" {
diff --git a/lib/cloud/azure_test.go b/lib/cloud/azure_test.go
index aaa3d2846..a123bc4bb 100644
--- a/lib/cloud/azure_test.go
+++ b/lib/cloud/azure_test.go
@@ -27,9 +27,9 @@ import (
check "gopkg.in/check.v1"
)
-type AzureProviderSuite struct{}
+type AzureInstanceSetSuite struct{}
-var _ = check.Suite(&AzureProviderSuite{})
+var _ = check.Suite(&AzureInstanceSetSuite{})
type VirtualMachinesClientStub struct{}
@@ -71,7 +71,7 @@ func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName
var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
-func GetProvider() (InstanceProvider, ImageID, arvados.Cluster, error) {
+func GetInstanceSet() (InstanceSet, ImageID, arvados.Cluster, error) {
cluster := arvados.Cluster{
InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
"tiny": arvados.InstanceType{
@@ -85,16 +85,16 @@ func GetProvider() (InstanceProvider, ImageID, arvados.Cluster, error) {
},
})}
if *live != "" {
- cfg := AzureProviderConfig{}
+ cfg := make(map[string]interface{})
err := config.LoadFile(&cfg, *live)
if err != nil {
return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, "test123")
- return ap, ImageID(cfg.Image), cluster, err
+ ap, err := NewAzureInstanceSet(cfg, "test123")
+ return ap, ImageID(cfg["image"].(string)), cluster, err
} else {
- ap := AzureProvider{
- azconfig: AzureProviderConfig{
+ ap := AzureInstanceSet{
+ azconfig: AzureInstanceSetConfig{
BlobContainer: "vhds",
},
dispatcherID: "test123",
@@ -106,8 +106,8 @@ func GetProvider() (InstanceProvider, ImageID, arvados.Cluster, error) {
}
}
-func (*AzureProviderSuite) TestCreate(c *check.C) {
- ap, img, cluster, err := GetProvider()
+func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
+ ap, img, cluster, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -132,52 +132,52 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
c.Assert(err, check.IsNil)
- tg, _ := inst.Tags(context.Background())
+ tg := inst.Tags()
log.Printf("Result %v %v %v", inst.String(), inst.Address(), tg)
}
-func (*AzureProviderSuite) TestListInstances(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestListInstances(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- l, err := ap.Instances(context.Background())
+ l, err := ap.Instances(context.Background(), nil)
c.Assert(err, check.IsNil)
for _, i := range l {
- tg, _ := i.Tags(context.Background())
+ tg := i.Tags()
log.Printf("%v %v %v", i.String(), i.Address(), tg)
}
}
-func (*AzureProviderSuite) TestManageNics(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestManageNics(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- ap.(*AzureProvider).ManageNics(context.Background())
+ ap.(*AzureInstanceSet).ManageNics(context.Background())
}
-func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestManageBlobs(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- ap.(*AzureProvider).ManageBlobs(context.Background())
+ ap.(*AzureInstanceSet).ManageBlobs(context.Background())
}
-func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestDestroyInstances(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- l, err := ap.Instances(context.Background())
+ l, err := ap.Instances(context.Background(), nil)
c.Assert(err, check.IsNil)
for _, i := range l {
@@ -185,13 +185,13 @@ func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
}
}
-func (*AzureProviderSuite) TestDeleteFake(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestDeleteFake(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- _, err = ap.(*AzureProvider).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
+ _, err = ap.(*AzureInstanceSet).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
de, ok := err.(autorest.DetailedError)
if ok {
@@ -201,7 +201,7 @@ func (*AzureProviderSuite) TestDeleteFake(c *check.C) {
}
}
-func (*AzureProviderSuite) TestWrapError(c *check.C) {
+func (*AzureInstanceSetSuite) TestWrapError(c *check.C) {
retryError := autorest.DetailedError{
Original: &azure.RequestError{
DetailedError: autorest.DetailedError{
@@ -234,12 +234,12 @@ func (*AzureProviderSuite) TestWrapError(c *check.C) {
c.Check(ok, check.Equals, true)
}
-func (*AzureProviderSuite) TestSetTags(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestSetTags(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- l, err := ap.Instances(context.Background())
+ l, err := ap.Instances(context.Background(), nil)
c.Assert(err, check.IsNil)
if len(l) > 0 {
@@ -248,21 +248,21 @@ func (*AzureProviderSuite) TestSetTags(c *check.C) {
c.Fatal("Error setting tags", err)
}
}
- l, err = ap.Instances(context.Background())
+ l, err = ap.Instances(context.Background(), nil)
c.Assert(err, check.IsNil)
if len(l) > 0 {
- tg, _ := l[0].Tags(context.Background())
+ tg := l[0].Tags()
log.Printf("tags are %v", tg)
}
}
-func (*AzureProviderSuite) TestSSH(c *check.C) {
- ap, _, _, err := GetProvider()
+func (*AzureInstanceSetSuite) TestSSH(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- l, err := ap.Instances(context.Background())
+ l, err := ap.Instances(context.Background(), nil)
c.Assert(err, check.IsNil)
if len(l) > 0 {
@@ -316,7 +316,7 @@ func SetupSSHClient(c *check.C, inst Instance) (*ssh.Client, error) {
return nil, errors.New("BUG: key was never provided to HostKeyCallback")
}
- err = inst.VerifyPublicKey(context.Background(), receivedKey, client)
+ err = inst.VerifyHostKey(context.Background(), receivedKey, client)
c.Assert(err, check.IsNil)
return client, nil
diff --git a/lib/cloud/interfaces.go b/lib/cloud/interfaces.go
index e3a072582..c2b1acbab 100644
--- a/lib/cloud/interfaces.go
+++ b/lib/cloud/interfaces.go
@@ -5,6 +5,7 @@
package cloud
import (
+ "context"
"io"
"time"
@@ -66,7 +67,7 @@ type ExecutorTarget interface {
// VerifyHostKey can use it to make outgoing network
// connections from the instance -- e.g., to use the cloud's
// "this instance's metadata" API.
- VerifyHostKey(ssh.PublicKey, *ssh.Client) error
+ VerifyHostKey(context.Context, ssh.PublicKey, *ssh.Client) error
}
// Instance is implemented by the provider-specific instance types.
@@ -88,10 +89,10 @@ type Instance interface {
Tags() InstanceTags
// Replace tags with the given tags
- SetTags(InstanceTags) error
+ SetTags(context.Context, InstanceTags) error
// Shut down the node
- Destroy() error
+ Destroy(context.Context) error
}
// An InstanceSet manages a set of VM instances created by an elastic
@@ -105,7 +106,7 @@ type InstanceSet interface {
//
// The returned error should implement RateLimitError and
// QuotaError where applicable.
- Create(arvados.InstanceType, ImageID, InstanceTags, ssh.PublicKey) (Instance, error)
+ Create(context.Context, arvados.InstanceType, ImageID, InstanceTags, ssh.PublicKey) (Instance, error)
// Return all instances, including ones that are booting or
// shutting down. Optionally, filter out nodes that don't have
@@ -117,7 +118,7 @@ type InstanceSet interface {
// Instance object each time. Thus, the caller is responsible
// for de-duplicating the returned instances by comparing the
// InstanceIDs returned by the instances' ID() methods.
- Instances(InstanceTags) ([]Instance, error)
+ Instances(context.Context, InstanceTags) ([]Instance, error)
// Stop any background tasks and release other resources.
Stop()
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 33823a828..87122a85c 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -5,6 +5,7 @@
package dispatchcloud
import (
+ "context"
"encoding/json"
"io/ioutil"
"math/rand"
@@ -165,7 +166,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
deadline := time.Now().Add(time.Second)
for range time.NewTicker(10 * time.Millisecond).C {
- insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
+ insts, err := s.stubDriver.InstanceSets()[0].Instances(context.TODO(), nil)
c.Check(err, check.IsNil)
queue.Update()
ents, _ := queue.Entries()
diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index fe5362437..97c1bc773 100644
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -12,7 +12,7 @@ import (
)
var drivers = map[string]cloud.Driver{
- "azure": "",
+ "azure": cloud.DriverFunc(cloud.NewAzureInstanceSet),
}
func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID) (cloud.InstanceSet, error) {
diff --git a/lib/dispatchcloud/instance_set_proxy.go b/lib/dispatchcloud/instance_set_proxy.go
index e728b67cd..80c510458 100644
--- a/lib/dispatchcloud/instance_set_proxy.go
+++ b/lib/dispatchcloud/instance_set_proxy.go
@@ -5,6 +5,8 @@
package dispatchcloud
import (
+ "context"
+
"git.curoverse.com/arvados.git/lib/cloud"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"golang.org/x/crypto/ssh"
@@ -14,12 +16,12 @@ type instanceSetProxy struct {
cloud.InstanceSet
}
-func (is *instanceSetProxy) Create(it arvados.InstanceType, id cloud.ImageID, tags cloud.InstanceTags, pk ssh.PublicKey) (cloud.Instance, error) {
+func (is *instanceSetProxy) Create(ctx context.Context, it arvados.InstanceType, id cloud.ImageID, tags cloud.InstanceTags, pk ssh.PublicKey) (cloud.Instance, error) {
// TODO: return if Create failed recently with a RateLimitError or QuotaError
- return is.InstanceSet.Create(it, id, tags, pk)
+ return is.InstanceSet.Create(ctx, it, id, tags, pk)
}
-func (is *instanceSetProxy) Instances(tags cloud.InstanceTags) ([]cloud.Instance, error) {
+func (is *instanceSetProxy) Instances(ctx context.Context, tags cloud.InstanceTags) ([]cloud.Instance, error) {
// TODO: return if Instances failed recently with a RateLimitError
- return is.InstanceSet.Instances(tags)
+ return is.InstanceSet.Instances(ctx, tags)
}
diff --git a/lib/dispatchcloud/ssh_executor/executor.go b/lib/dispatchcloud/ssh_executor/executor.go
index b5dba9870..c9b0101a7 100644
--- a/lib/dispatchcloud/ssh_executor/executor.go
+++ b/lib/dispatchcloud/ssh_executor/executor.go
@@ -8,6 +8,7 @@ package ssh_executor
import (
"bytes"
+ "context"
"errors"
"io"
"net"
@@ -180,7 +181,7 @@ func (exr *Executor) setupSSHClient() (*ssh.Client, error) {
}
if exr.hostKey == nil || !bytes.Equal(exr.hostKey.Marshal(), receivedKey.Marshal()) {
- err = target.VerifyHostKey(receivedKey, client)
+ err = target.VerifyHostKey(context.TODO(), receivedKey, client)
if err != nil {
return nil, err
}
diff --git a/lib/dispatchcloud/test/lame_instance_set.go b/lib/dispatchcloud/test/lame_instance_set.go
index baab407a7..c52f7c645 100644
--- a/lib/dispatchcloud/test/lame_instance_set.go
+++ b/lib/dispatchcloud/test/lame_instance_set.go
@@ -5,6 +5,7 @@
package test
import (
+ "context"
"fmt"
"math/rand"
"sync"
@@ -24,13 +25,13 @@ type LameInstanceSet struct {
}
// Create returns a new instance.
-func (p *LameInstanceSet) Create(instType arvados.InstanceType, imageID cloud.ImageID, tags cloud.InstanceTags, pubkey ssh.PublicKey) (cloud.Instance, error) {
+func (p *LameInstanceSet) Create(_ context.Context, instType arvados.InstanceType, imageID cloud.ImageID, tags cloud.InstanceTags, pubkey ssh.PublicKey) (cloud.Instance, error) {
inst := &lameInstance{
p: p,
id: cloud.InstanceID(fmt.Sprintf("lame-%x", rand.Uint64())),
providerType: instType.ProviderType,
}
- inst.SetTags(tags)
+ inst.SetTags(context.TODO(), tags)
if p.Hold != nil {
p.Hold <- true
}
@@ -44,7 +45,7 @@ func (p *LameInstanceSet) Create(instType arvados.InstanceType, imageID cloud.Im
}
// Instances returns the instances that haven't been destroyed.
-func (p *LameInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
+func (p *LameInstanceSet) Instances(context.Context, cloud.InstanceTags) ([]cloud.Instance, error) {
p.mtx.Lock()
defer p.mtx.Unlock()
var instances []cloud.Instance
@@ -89,7 +90,7 @@ func (inst *lameInstance) Address() string {
return "0.0.0.0:1234"
}
-func (inst *lameInstance) SetTags(tags cloud.InstanceTags) error {
+func (inst *lameInstance) SetTags(_ context.Context, tags cloud.InstanceTags) error {
inst.p.mtx.Lock()
defer inst.p.mtx.Unlock()
inst.tags = cloud.InstanceTags{}
@@ -99,7 +100,7 @@ func (inst *lameInstance) SetTags(tags cloud.InstanceTags) error {
return nil
}
-func (inst *lameInstance) Destroy() error {
+func (inst *lameInstance) Destroy(context.Context) error {
if inst.p.Hold != nil {
inst.p.Hold <- true
}
@@ -113,6 +114,6 @@ func (inst *lameInstance) Tags() cloud.InstanceTags {
return inst.tags
}
-func (inst *lameInstance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
+func (inst *lameInstance) VerifyHostKey(context.Context, ssh.PublicKey, *ssh.Client) error {
return nil
}
diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go
index 8bdfaa947..8f40b85eb 100644
--- a/lib/dispatchcloud/test/stub_driver.go
+++ b/lib/dispatchcloud/test/stub_driver.go
@@ -5,6 +5,7 @@
package test
import (
+ "context"
"crypto/rand"
"errors"
"fmt"
@@ -68,7 +69,7 @@ type StubInstanceSet struct {
stopped bool
}
-func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, tags cloud.InstanceTags, authKey ssh.PublicKey) (cloud.Instance, error) {
+func (sis *StubInstanceSet) Create(_ context.Context, it arvados.InstanceType, image cloud.ImageID, tags cloud.InstanceTags, authKey ssh.PublicKey) (cloud.Instance, error) {
sis.mtx.Lock()
defer sis.mtx.Unlock()
if sis.stopped {
@@ -96,7 +97,7 @@ func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID,
return svm.Instance(), nil
}
-func (sis *StubInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
+func (sis *StubInstanceSet) Instances(context.Context, cloud.InstanceTags) ([]cloud.Instance, error) {
sis.mtx.RLock()
defer sis.mtx.RUnlock()
var r []cloud.Instance
@@ -261,7 +262,7 @@ func (si stubInstance) Address() string {
return si.addr
}
-func (si stubInstance) Destroy() error {
+func (si stubInstance) Destroy(_ context.Context) error {
if math_rand.Float64() < si.svm.sis.driver.ErrorRateDestroy {
return errors.New("instance could not be destroyed")
}
@@ -277,7 +278,7 @@ func (si stubInstance) ProviderType() string {
return si.svm.providerType
}
-func (si stubInstance) SetTags(tags cloud.InstanceTags) error {
+func (si stubInstance) SetTags(_ context.Context, tags cloud.InstanceTags) error {
tags = copyTags(tags)
svm := si.svm
go func() {
@@ -296,7 +297,7 @@ func (si stubInstance) String() string {
return string(si.svm.id)
}
-func (si stubInstance) VerifyHostKey(key ssh.PublicKey, client *ssh.Client) error {
+func (si stubInstance) VerifyHostKey(_ context.Context, key ssh.PublicKey, client *ssh.Client) error {
buf := make([]byte, 512)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index ff5f762c1..30a1312d3 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -5,6 +5,7 @@
package worker
import (
+ "context"
"io"
"sort"
"strings"
@@ -225,7 +226,7 @@ func (wp *Pool) Create(it arvados.InstanceType) error {
wp.creating[it] = append(wp.creating[it], now)
go func() {
defer wp.notify()
- inst, err := wp.instanceSet.Create(it, wp.imageID, tags, nil)
+ inst, err := wp.instanceSet.Create(context.TODO(), it, wp.imageID, tags, nil)
wp.mtx.Lock()
defer wp.mtx.Unlock()
// Remove our timestamp marker from wp.creating
@@ -626,7 +627,7 @@ func (wp *Pool) getInstancesAndSync() error {
wp.setupOnce.Do(wp.setup)
wp.logger.Debug("getting instance list")
threshold := time.Now()
- instances, err := wp.instanceSet.Instances(cloud.InstanceTags{})
+ instances, err := wp.instanceSet.Instances(context.TODO(), cloud.InstanceTags{})
if err != nil {
return err
}
diff --git a/lib/dispatchcloud/worker/worker.go b/lib/dispatchcloud/worker/worker.go
index c26186309..a4578b26d 100644
--- a/lib/dispatchcloud/worker/worker.go
+++ b/lib/dispatchcloud/worker/worker.go
@@ -6,6 +6,7 @@ package worker
import (
"bytes"
+ "context"
"strings"
"sync"
"time"
@@ -311,7 +312,7 @@ func (wkr *worker) shutdown() {
wkr.state = StateShutdown
go wkr.wp.notify()
go func() {
- err := wkr.instance.Destroy()
+ err := wkr.instance.Destroy(context.TODO())
if err != nil {
wkr.logger.WithError(err).Warn("shutdown failed")
return
commit 7f5b859e82a65ac27c403ff2416c471f9c770ac1
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 9 11:35:55 2019 -0500
14324: Azure driver WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/cloud/azure.go
similarity index 95%
rename from lib/dispatchcloud/azure.go
rename to lib/cloud/azure.go
index 92970d1de..734be7bef 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/cloud/azure.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-package dispatchcloud
+package cloud
import (
"context"
@@ -29,7 +29,7 @@ import (
"golang.org/x/crypto/ssh"
)
-type AzureProviderConfig struct {
+type AzureInstanceSetConfig struct {
SubscriptionID string `json:"subscription_id"`
ClientID string `json:"key"`
ClientSecret string `json:"secret"`
@@ -185,8 +185,8 @@ func WrapAzureError(err error) error {
return err
}
-type AzureProvider struct {
- azconfig AzureProviderConfig
+type AzureInstanceSet struct {
+ azconfig AzureInstanceSetConfig
vmClient VirtualMachinesClientWrapper
netClient InterfacesClientWrapper
storageAcctClient storageacct.AccountsClient
@@ -196,8 +196,13 @@ type AzureProvider struct {
namePrefix string
}
-func NewAzureProvider(azcfg AzureProviderConfig, dispatcherID string) (prv InstanceProvider, err error) {
- ap := AzureProvider{}
+func NewAzureInstanceSet(config map[string]interface{}, dispatcherID string) (prv InstanceProvider, err error) {
+ azcfg := AzureInstanceSetConfig{}
+ err = mapstructure.Decode(config, &azcfg)
+ if err != nil {
+ return nil, err
+ }
+ ap := AzureInstanceSet{}
err = ap.setup(azcfg, dispatcherID)
if err != nil {
return nil, err
@@ -205,7 +210,7 @@ func NewAzureProvider(azcfg AzureProviderConfig, dispatcherID string) (prv Insta
return &ap, nil
}
-func (az *AzureProvider) setup(azcfg AzureProviderConfig, dispatcherID string) (err error) {
+func (az *AzureInstanceSet) setup(azcfg AzureInstanceSetConfig, dispatcherID string) (err error) {
az.azconfig = azcfg
vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
@@ -241,7 +246,7 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, dispatcherID string) (
return nil
}
-func (az *AzureProvider) Create(ctx context.Context,
+func (az *AzureInstanceSet) Create(ctx context.Context,
instanceType arvados.InstanceType,
imageId ImageID,
newTags InstanceTags,
@@ -371,7 +376,7 @@ echo '%s-%s' > /home/crunch/node-token`, name, newTags["node-token"])))
}, nil
}
-func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
+func (az *AzureInstanceSet) Instances(ctx context.Context) ([]Instance, error) {
interfaces, err := az.ManageNics(ctx)
if err != nil {
return nil, err
@@ -398,7 +403,7 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
return instances, nil
}
-func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
+func (az *AzureInstanceSet) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
return nil, WrapAzureError(err)
@@ -457,7 +462,7 @@ func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Int
return interfaces, nil
}
-func (az *AzureProvider) ManageBlobs(ctx context.Context) {
+func (az *AzureInstanceSet) ManageBlobs(ctx context.Context) {
result, err := az.storageAcctClient.ListKeys(ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
if err != nil {
log.Printf("Couldn't get account keys %v", err)
@@ -527,11 +532,11 @@ func (az *AzureProvider) ManageBlobs(ctx context.Context) {
}
}
-func (az *AzureProvider) Stop() {
+func (az *AzureInstanceSet) Stop() {
}
type AzureInstance struct {
- provider *AzureProvider
+ provider *AzureInstanceSet
nic network.Interface
vm compute.VirtualMachine
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/cloud/azure_test.go
similarity index 99%
rename from lib/dispatchcloud/azure_test.go
rename to lib/cloud/azure_test.go
index 568a403a5..aaa3d2846 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/cloud/azure_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-package dispatchcloud
+package cloud
import (
"context"
diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index 295fd6105..fe5362437 100644
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -11,7 +11,9 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
)
-var drivers = map[string]cloud.Driver{}
+var drivers = map[string]cloud.Driver{
+ "azure": "",
+}
func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID) (cloud.InstanceSet, error) {
driver, ok := drivers[cluster.CloudVMs.Driver]
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
deleted file mode 100644
index 9e3af0767..000000000
--- a/lib/dispatchcloud/provider.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package dispatchcloud
-
-import (
- "context"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "golang.org/x/crypto/ssh"
-)
-
-// A RateLimitError should be returned by a Provider when the cloud
-// service indicates it is rejecting all API calls for some time
-// interval.
-type RateLimitError interface {
- // Time before which the caller should expect requests to
- // fail.
- EarliestRetry() time.Time
- error
-}
-
-// A QuotaError should be returned by a Provider when the cloud
-// service indicates the account cannot create more VMs than already
-// exist.
-type QuotaError interface {
- // If true, don't create more instances until some existing
- // instances are destroyed. If false, don't handle the error
- // as a quota error.
- IsQuotaError() bool
- error
-}
-
-type InstanceTags map[string]string
-type InstanceID string
-type ImageID string
-
-// instance is implemented by the provider-specific instance types.
-type Instance interface {
- // ID returns the provider's instance ID. It must be stable
- // for the life of the instance.
- ID() InstanceID
-
- // String typically returns the cloud-provided instance ID.
- String() string
-
- // Get tags
- Tags(context.Context) (InstanceTags, error)
-
- // Replace tags with the given tags
- SetTags(context.Context, InstanceTags) error
-
- // Shut down the node
- Destroy(context.Context) error
-
- // SSH server hostname or IP address, or empty string if unknown pending creation.
- Address() string
-
- // Return nil if the given public key matches the instance's
- // SSH server key. If the provided ssh client is not nil,
- // VerifyPublicKey can use it to make outgoing network
- // connections from the instance -- e.g., to use the cloud's
- // "this instance's metadata" API.
- VerifyPublicKey(context.Context, ssh.PublicKey, *ssh.Client) error
-}
-
-type InstanceProvider interface {
- // Create a new instance. If supported by the driver, add the
- // provided public key to /root/.ssh/authorized_keys.
- //
- // The returned error should implement RateLimitError and
- // QuotaError where applicable.
- Create(context.Context, arvados.InstanceType, ImageID, InstanceTags, ssh.PublicKey) (Instance, error)
-
- // Return all instances, including ones that are booting or
- // shutting down.
- //
- // An instance returned by successive calls to Instances() may
- // -- but does not need to -- be represented by the same
- // Instance object each time. Thus, the caller is responsible
- // for de-duplicating the returned instances by comparing the
- // InstanceIDs returned by the instances' ID() methods.
- Instances(context.Context) ([]Instance, error)
-
- // Stop any background tasks and release other resources.
- Stop()
-}
commit dc19260dea6ee0b7186a50245ad31c2cbe5e6b2a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 16:41:07 2018 -0400
13964: Fail on missing node-token tag
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 32b81321a..92970d1de 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -605,6 +605,11 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
}
}
+ nodetokenTag := tags["node-token"]
+ if nodetokenTag == "" {
+ return fmt.Errorf("Missing node token tag")
+ }
+
sess, err := client.NewSession()
if err != nil {
return err
@@ -617,7 +622,7 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
nodetoken := strings.TrimSpace(string(nodetokenbytes))
- expectedToken := fmt.Sprintf("%s-%s", *ai.vm.Name, tags["node-token"])
+ expectedToken := fmt.Sprintf("%s-%s", *ai.vm.Name, nodetokenTag)
log.Printf("%q %q", nodetoken, expectedToken)
if strings.TrimSpace(nodetoken) != expectedToken {
commit 5dd601ec0484d193d5d71975dc1d6b85ea901f88
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 16:27:08 2018 -0400
13964: Forget the node token once the key fingerprint is known
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 6cc4e9908..32b81321a 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -643,6 +643,7 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
}
tags["ssh-pubkey-fingerprint"] = sp[1]
+ delete(tags, "node-token")
ai.SetTags(ctx, tags)
return nil
}
commit 351505f3a2b2b4460a4214097d63f53ab00ec42f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 16:15:55 2018 -0400
13964: Fix string pointer capture
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 28efe4f5c..6cc4e9908 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -264,7 +264,8 @@ func (az *AzureProvider) Create(ctx context.Context,
tags := make(map[string]*string)
tags["created-at"] = ×tamp
for k, v := range newTags {
- tags["dispatch-"+k] = &v
+ newstr := v
+ tags["dispatch-"+k] = &newstr
}
tags["dispatch-instance-type"] = &instanceType.Name
@@ -552,7 +553,8 @@ func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) erro
}
}
for k, v := range newTags {
- tags["dispatch-"+k] = &v
+ newstr := v
+ tags["dispatch-"+k] = &newstr
}
vmParameters := compute.VirtualMachine{
@@ -599,7 +601,7 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
if remoteFingerprint == tg {
return nil
} else {
- return fmt.Errorf("Key fingerprint did not match")
+ return fmt.Errorf("Key fingerprint did not match, expected %q got %q", tg, remoteFingerprint)
}
}
@@ -619,7 +621,7 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
log.Printf("%q %q", nodetoken, expectedToken)
if strings.TrimSpace(nodetoken) != expectedToken {
- return fmt.Errorf("Node token did not match")
+ return fmt.Errorf("Node token did not match, expected %q got %q", expectedToken, nodetoken)
}
sess, err = client.NewSession()
@@ -637,7 +639,7 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
log.Printf("%q %q", remoteFingerprint, sp[1])
if remoteFingerprint != sp[1] {
- return fmt.Errorf("Key fingerprint did not match")
+ return fmt.Errorf("Key fingerprint did not match, expected %q got %q", sp[1], remoteFingerprint)
}
tags["ssh-pubkey-fingerprint"] = sp[1]
commit 925644cc7c7e6761566a4059ac9ed9987653931e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 16:11:12 2018 -0400
13964: Trim space
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 378e8bea3..28efe4f5c 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -608,35 +608,39 @@ func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.Pu
return err
}
- nodetoken, err := sess.Output("cat /home/crunch/node-token")
+ nodetokenbytes, err := sess.Output("cat /home/crunch/node-token")
if err != nil {
return err
}
+ nodetoken := strings.TrimSpace(string(nodetokenbytes))
+
expectedToken := fmt.Sprintf("%s-%s", *ai.vm.Name, tags["node-token"])
- log.Printf("%q %q", string(nodetoken), expectedToken)
+ log.Printf("%q %q", nodetoken, expectedToken)
- if string(nodetoken) == expectedToken {
- sess, err := client.NewSession()
- if err != nil {
- return err
- }
+ if strings.TrimSpace(nodetoken) != expectedToken {
+ return fmt.Errorf("Node token did not match")
+ }
- keyfingerprintbytes, err := sess.Output("ssh-keygen -E sha256 -l -f /etc/ssh/ssh_host_rsa_key.pub")
- if err != nil {
- return err
- }
+ sess, err = client.NewSession()
+ if err != nil {
+ return err
+ }
+
+ keyfingerprintbytes, err := sess.Output("ssh-keygen -E sha256 -l -f /etc/ssh/ssh_host_rsa_key.pub")
+ if err != nil {
+ return err
+ }
- sp := strings.Split(string(keyfingerprintbytes), " ")
+ sp := strings.Split(string(keyfingerprintbytes), " ")
- log.Printf("%q %q", remoteFingerprint, sp[1])
+ log.Printf("%q %q", remoteFingerprint, sp[1])
- if remoteFingerprint == sp[1] {
- tags["ssh-pubkey-fingerprint"] = sp[1]
- ai.SetTags(ctx, tags)
- return nil
- }
+ if remoteFingerprint != sp[1] {
+ return fmt.Errorf("Key fingerprint did not match")
}
- return fmt.Errorf("Key fingerprint did not match")
+ tags["ssh-pubkey-fingerprint"] = sp[1]
+ ai.SetTags(ctx, tags)
+ return nil
}
commit e3dc6df1694586eca13f2f20eb064b8b16d115e5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 15:55:43 2018 -0400
13964: ssh key checking
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 563246ece..378e8bea3 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -588,3 +588,55 @@ func (ai *AzureInstance) Destroy(ctx context.Context) error {
func (ai *AzureInstance) Address() string {
return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
}
+
+func (ai *AzureInstance) VerifyPublicKey(ctx context.Context, receivedKey ssh.PublicKey, client *ssh.Client) error {
+ remoteFingerprint := ssh.FingerprintSHA256(receivedKey)
+
+ tags, _ := ai.Tags(ctx)
+
+ tg := tags["ssh-pubkey-fingerprint"]
+ if tg != "" {
+ if remoteFingerprint == tg {
+ return nil
+ } else {
+ return fmt.Errorf("Key fingerprint did not match")
+ }
+ }
+
+ sess, err := client.NewSession()
+ if err != nil {
+ return err
+ }
+
+ nodetoken, err := sess.Output("cat /home/crunch/node-token")
+ if err != nil {
+ return err
+ }
+
+ expectedToken := fmt.Sprintf("%s-%s", *ai.vm.Name, tags["node-token"])
+ log.Printf("%q %q", string(nodetoken), expectedToken)
+
+ if string(nodetoken) == expectedToken {
+ sess, err := client.NewSession()
+ if err != nil {
+ return err
+ }
+
+ keyfingerprintbytes, err := sess.Output("ssh-keygen -E sha256 -l -f /etc/ssh/ssh_host_rsa_key.pub")
+ if err != nil {
+ return err
+ }
+
+ sp := strings.Split(string(keyfingerprintbytes), " ")
+
+ log.Printf("%q %q", remoteFingerprint, sp[1])
+
+ if remoteFingerprint == sp[1] {
+ tags["ssh-pubkey-fingerprint"] = sp[1]
+ ai.SetTags(ctx, tags)
+ return nil
+ }
+ }
+
+ return fmt.Errorf("Key fingerprint did not match")
+}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index 29356f1f7..568a403a5 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -316,19 +316,8 @@ func SetupSSHClient(c *check.C, inst Instance) (*ssh.Client, error) {
return nil, errors.New("BUG: key was never provided to HostKeyCallback")
}
- log.Printf("receivedKey %v", receivedKey)
- log.Printf("fingerprint %v", ssh.FingerprintSHA256(receivedKey))
- tags, err := inst.Tags(context.Background())
+ err = inst.VerifyPublicKey(context.Background(), receivedKey, client)
c.Assert(err, check.IsNil)
- log.Printf("ssh-pubkey %q", tags["ssh-pubkey"])
-
- /*if wkr.publicKey == nil || !bytes.Equal(wkr.publicKey.Marshal(), receivedKey.Marshal()) {
- err = wkr.instance.VerifyPublicKey(receivedKey, client)
- if err != nil {
- return nil, err
- }
- wkr.publicKey = receivedKey
- }*/
return client, nil
}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index ed5eb8fe2..9e3af0767 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -57,6 +57,13 @@ type Instance interface {
// SSH server hostname or IP address, or empty string if unknown pending creation.
Address() string
+
+ // Return nil if the given public key matches the instance's
+ // SSH server key. If the provided ssh client is not nil,
+ // VerifyPublicKey can use it to make outgoing network
+ // connections from the instance -- e.g., to use the cloud's
+ // "this instance's metadata" API.
+ VerifyPublicKey(context.Context, ssh.PublicKey, *ssh.Client) error
}
type InstanceProvider interface {
commit 40e5a87a683bf463bac2f6a41beb8b2ee96bab27
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 15:11:58 2018 -0400
13964: ssh key verification wip
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index c68a41809..563246ece 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -261,14 +261,14 @@ func (az *AzureProvider) Create(ctx context.Context,
timestamp := time.Now().Format(time.RFC3339Nano)
- newTags["instance-type"] = instanceType.Name
-
tags := make(map[string]*string)
tags["created-at"] = ×tamp
for k, v := range newTags {
tags["dispatch-"+k] = &v
}
+ tags["dispatch-instance-type"] = &instanceType.Name
+
nicParameters := network.Interface{
Location: &az.azconfig.Location,
Tags: tags,
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index 2a8cc186c..29356f1f7 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -126,13 +126,14 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
inst, err := ap.Create(context.Background(),
cluster.InstanceTypes["tiny"],
- img, map[string]string{"instance-type": "tiny",
+ img, map[string]string{
"node-token": nodetoken},
pk)
c.Assert(err, check.IsNil)
- log.Printf("Result %v %v", inst.String(), inst.Address())
+ tg, _ := inst.Tags(context.Background())
+ log.Printf("Result %v %v %v", inst.String(), inst.Address(), tg)
}
@@ -266,7 +267,7 @@ func (*AzureProviderSuite) TestSSH(c *check.C) {
if len(l) > 0 {
- sshclient, err := SetupSSHClient(c, l[0].Address()+":2222")
+ sshclient, err := SetupSSHClient(c, l[0])
c.Assert(err, check.IsNil)
sess, err := sshclient.NewSession()
@@ -281,7 +282,8 @@ func (*AzureProviderSuite) TestSSH(c *check.C) {
}
}
-func SetupSSHClient(c *check.C, addr string) (*ssh.Client, error) {
+func SetupSSHClient(c *check.C, inst Instance) (*ssh.Client, error) {
+ addr := inst.Address() + ":2222"
if addr == "" {
return nil, errors.New("instance has no address")
}
@@ -314,6 +316,13 @@ func SetupSSHClient(c *check.C, addr string) (*ssh.Client, error) {
return nil, errors.New("BUG: key was never provided to HostKeyCallback")
}
+ log.Printf("receivedKey %v", receivedKey)
+ log.Printf("fingerprint %v", ssh.FingerprintSHA256(receivedKey))
+ tags, err := inst.Tags(context.Background())
+ c.Assert(err, check.IsNil)
+
+ log.Printf("ssh-pubkey %q", tags["ssh-pubkey"])
+
/*if wkr.publicKey == nil || !bytes.Equal(wkr.publicKey.Marshal(), receivedKey.Marshal()) {
err = wkr.instance.VerifyPublicKey(receivedKey, client)
if err != nil {
commit 146257d644f6149d5504aa0652286ddef6a2049e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 14:47:28 2018 -0400
13964: Set and check node-token WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index da2e3eb8f..c68a41809 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -6,6 +6,7 @@ package dispatchcloud
import (
"context"
+ "encoding/base64"
"fmt"
"log"
"net/http"
@@ -246,6 +247,10 @@ func (az *AzureProvider) Create(ctx context.Context,
newTags InstanceTags,
publicKey ssh.PublicKey) (Instance, error) {
+ if len(newTags["node-token"]) == 0 {
+ return nil, fmt.Errorf("Must provide tag 'node-token'")
+ }
+
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
return nil, err
@@ -256,6 +261,8 @@ func (az *AzureProvider) Create(ctx context.Context,
timestamp := time.Now().Format(time.RFC3339Nano)
+ newTags["instance-type"] = instanceType.Name
+
tags := make(map[string]*string)
tags["created-at"] = ×tamp
for k, v := range newTags {
@@ -299,6 +306,9 @@ func (az *AzureProvider) Create(ctx context.Context,
log.Printf("URI instance vhd %v", instance_vhd)
+ customData := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`#!/bin/sh
+echo '%s-%s' > /home/crunch/node-token`, name, newTags["node-token"])))
+
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
Tags: tags,
@@ -343,7 +353,7 @@ func (az *AzureProvider) Create(ctx context.Context,
},
},
},
- //CustomData: to.StringPtr(""),
+ CustomData: &customData,
},
},
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index d5998aae6..2a8cc186c 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -22,6 +22,7 @@ import (
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/to"
+ "github.com/jmcvetta/randutil"
"golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
)
@@ -120,9 +121,13 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
pk, _, _, _, err := ssh.ParseAuthorizedKey(keybytes)
c.Assert(err, check.IsNil)
+ nodetoken, err := randutil.String(40, "abcdefghijklmnopqrstuvwxyz0123456789")
+ c.Assert(err, check.IsNil)
+
inst, err := ap.Create(context.Background(),
cluster.InstanceTypes["tiny"],
- img, map[string]string{"tag1": "bleep"},
+ img, map[string]string{"instance-type": "tiny",
+ "node-token": nodetoken},
pk)
c.Assert(err, check.IsNil)
@@ -267,7 +272,7 @@ func (*AzureProviderSuite) TestSSH(c *check.C) {
sess, err := sshclient.NewSession()
c.Assert(err, check.IsNil)
- out, err := sess.Output("ls /")
+ out, err := sess.Output("cat /home/crunch/node-token")
c.Assert(err, check.IsNil)
log.Printf("%v", string(out))
commit ce01f835b561911f87e0d264ac72dbabc552281f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 14:24:28 2018 -0400
13964: Tweak test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index c23360b32..d5998aae6 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -270,7 +270,7 @@ func (*AzureProviderSuite) TestSSH(c *check.C) {
out, err := sess.Output("ls /")
c.Assert(err, check.IsNil)
- log.Printf("%v", out)
+ log.Printf("%v", string(out))
sshclient.Conn.Close()
}
commit e7dfae5f99f9291789be40a6505738307137360f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Aug 31 11:42:01 2018 -0400
13964: SSH access WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 21e8d28d8..da2e3eb8f 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -25,6 +25,7 @@ import (
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/Azure/go-autorest/autorest/to"
"github.com/jmcvetta/randutil"
+ "golang.org/x/crypto/ssh"
)
type AzureProviderConfig struct {
@@ -40,7 +41,6 @@ type AzureProviderConfig struct {
StorageAccount string `json:"storage_account"`
BlobContainer string `json:"blob_container"`
Image string `json:"image"`
- AuthorizedKey string `json:"authorized_key"`
DeleteDanglingResourcesAfter float64 `json:"delete_dangling_resources_after"`
}
@@ -186,7 +186,6 @@ func WrapAzureError(err error) error {
type AzureProvider struct {
azconfig AzureProviderConfig
- arvconfig arvados.Cluster
vmClient VirtualMachinesClientWrapper
netClient InterfacesClientWrapper
storageAcctClient storageacct.AccountsClient
@@ -196,18 +195,17 @@ type AzureProvider struct {
namePrefix string
}
-func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (prv Provider, err error) {
+func NewAzureProvider(azcfg AzureProviderConfig, dispatcherID string) (prv InstanceProvider, err error) {
ap := AzureProvider{}
- err = ap.setup(azcfg, arvcfg, dispatcherID)
+ err = ap.setup(azcfg, dispatcherID)
if err != nil {
return nil, err
}
return &ap, nil
}
-func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (err error) {
+func (az *AzureProvider) setup(azcfg AzureProviderConfig, dispatcherID string) (err error) {
az.azconfig = azcfg
- az.arvconfig = arvcfg
vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
@@ -245,7 +243,8 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster
func (az *AzureProvider) Create(ctx context.Context,
instanceType arvados.InstanceType,
imageId ImageID,
- newTags InstanceTags) (Instance, error) {
+ newTags InstanceTags,
+ publicKey ssh.PublicKey) (Instance, error) {
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
@@ -300,8 +299,6 @@ func (az *AzureProvider) Create(ctx context.Context,
log.Printf("URI instance vhd %v", instance_vhd)
- tags["arvados-instance-type"] = &instanceType.Name
-
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
Tags: tags,
@@ -334,14 +331,14 @@ func (az *AzureProvider) Create(ctx context.Context,
},
OsProfile: &compute.OSProfile{
ComputerName: &name,
- AdminUsername: to.StringPtr("arvados"),
+ AdminUsername: to.StringPtr("crunch"),
LinuxConfiguration: &compute.LinuxConfiguration{
DisablePasswordAuthentication: to.BoolPtr(true),
SSH: &compute.SSHConfiguration{
PublicKeys: &[]compute.SSHPublicKey{
compute.SSHPublicKey{
- Path: to.StringPtr("/home/arvados/.ssh/authorized_keys"),
- KeyData: to.StringPtr(az.azconfig.AuthorizedKey),
+ Path: to.StringPtr("/home/crunch/.ssh/authorized_keys"),
+ KeyData: to.StringPtr(string(ssh.MarshalAuthorizedKey(publicKey))),
},
},
},
@@ -357,10 +354,9 @@ func (az *AzureProvider) Create(ctx context.Context,
}
return &AzureInstance{
- instanceType: instanceType,
- provider: az,
- nic: nic,
- vm: vm,
+ provider: az,
+ nic: nic,
+ vm: vm,
}, nil
}
@@ -381,13 +377,11 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
if err != nil {
return nil, WrapAzureError(err)
}
- if strings.HasPrefix(*result.Value().Name, az.namePrefix) &&
- result.Value().Tags["arvados-instance-type"] != nil {
+ if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
instances = append(instances, &AzureInstance{
- provider: az,
- vm: result.Value(),
- nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID],
- instanceType: az.arvconfig.InstanceTypes[(*result.Value().Tags["arvados-instance-type"])]})
+ provider: az,
+ vm: result.Value(),
+ nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
}
}
return instances, nil
@@ -522,19 +516,21 @@ func (az *AzureProvider) ManageBlobs(ctx context.Context) {
}
}
+func (az *AzureProvider) Stop() {
+}
+
type AzureInstance struct {
- instanceType arvados.InstanceType
- provider *AzureProvider
- nic network.Interface
- vm compute.VirtualMachine
+ provider *AzureProvider
+ nic network.Interface
+ vm compute.VirtualMachine
}
-func (ai *AzureInstance) String() string {
- return *ai.vm.Name
+func (ai *AzureInstance) ID() InstanceID {
+ return InstanceID(*ai.vm.ID)
}
-func (ai *AzureInstance) InstanceType() arvados.InstanceType {
- return ai.instanceType
+func (ai *AzureInstance) String() string {
+ return *ai.vm.Name
}
func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) error {
@@ -562,7 +558,7 @@ func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) erro
return nil
}
-func (ai *AzureInstance) GetTags(ctx context.Context) (InstanceTags, error) {
+func (ai *AzureInstance) Tags(ctx context.Context) (InstanceTags, error) {
tags := make(map[string]string)
for k, v := range ai.vm.Tags {
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index a5a173bee..c23360b32 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -6,9 +6,14 @@ package dispatchcloud
import (
"context"
+ "errors"
"flag"
+ "io/ioutil"
"log"
+ "net"
"net/http"
+ "os"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/config"
@@ -17,6 +22,7 @@ import (
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/to"
+ "golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
)
@@ -64,7 +70,7 @@ func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName
var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
-func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
+func GetProvider() (InstanceProvider, ImageID, arvados.Cluster, error) {
cluster := arvados.Cluster{
InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
"tiny": arvados.InstanceType{
@@ -83,14 +89,13 @@ func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
if err != nil {
return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, cluster, "test123")
+ ap, err := NewAzureProvider(cfg, "test123")
return ap, ImageID(cfg.Image), cluster, err
} else {
ap := AzureProvider{
azconfig: AzureProviderConfig{
BlobContainer: "vhds",
},
- arvconfig: cluster,
dispatcherID: "test123",
namePrefix: "compute-test123-",
}
@@ -106,13 +111,24 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
c.Fatal("Error making provider", err)
}
+ f, err := os.Open("azconfig_sshkey.pub")
+ c.Assert(err, check.IsNil)
+
+ keybytes, err := ioutil.ReadAll(f)
+ c.Assert(err, check.IsNil)
+
+ pk, _, _, _, err := ssh.ParseAuthorizedKey(keybytes)
+ c.Assert(err, check.IsNil)
+
inst, err := ap.Create(context.Background(),
cluster.InstanceTypes["tiny"],
- img, map[string]string{"tag1": "bleep"})
+ img, map[string]string{"tag1": "bleep"},
+ pk)
c.Assert(err, check.IsNil)
log.Printf("Result %v %v", inst.String(), inst.Address())
+
}
func (*AzureProviderSuite) TestListInstances(c *check.C) {
@@ -126,8 +142,8 @@ func (*AzureProviderSuite) TestListInstances(c *check.C) {
c.Assert(err, check.IsNil)
for _, i := range l {
- tg, _ := i.GetTags(context.Background())
- log.Printf("%v %v %v %v", i.String(), i.Address(), i.InstanceType(), tg)
+ tg, _ := i.Tags(context.Background())
+ log.Printf("%v %v %v", i.String(), i.Address(), tg)
}
}
@@ -230,7 +246,75 @@ func (*AzureProviderSuite) TestSetTags(c *check.C) {
c.Assert(err, check.IsNil)
if len(l) > 0 {
- tg, _ := l[0].GetTags(context.Background())
+ tg, _ := l[0].Tags(context.Background())
log.Printf("tags are %v", tg)
}
}
+
+func (*AzureProviderSuite) TestSSH(c *check.C) {
+ ap, _, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+ l, err := ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ if len(l) > 0 {
+
+ sshclient, err := SetupSSHClient(c, l[0].Address()+":2222")
+ c.Assert(err, check.IsNil)
+
+ sess, err := sshclient.NewSession()
+ c.Assert(err, check.IsNil)
+
+ out, err := sess.Output("ls /")
+ c.Assert(err, check.IsNil)
+
+ log.Printf("%v", out)
+
+ sshclient.Conn.Close()
+ }
+}
+
+func SetupSSHClient(c *check.C, addr string) (*ssh.Client, error) {
+ if addr == "" {
+ return nil, errors.New("instance has no address")
+ }
+
+ f, err := os.Open("azconfig_sshkey")
+ c.Assert(err, check.IsNil)
+
+ keybytes, err := ioutil.ReadAll(f)
+ c.Assert(err, check.IsNil)
+
+ priv, err := ssh.ParsePrivateKey(keybytes)
+ c.Assert(err, check.IsNil)
+
+ var receivedKey ssh.PublicKey
+ client, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{
+ User: "crunch",
+ Auth: []ssh.AuthMethod{
+ ssh.PublicKeys(priv),
+ },
+ HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+ receivedKey = key
+ return nil
+ },
+ Timeout: time.Minute,
+ })
+
+ if err != nil {
+ return nil, err
+ } else if receivedKey == nil {
+ return nil, errors.New("BUG: key was never provided to HostKeyCallback")
+ }
+
+ /*if wkr.publicKey == nil || !bytes.Equal(wkr.publicKey.Marshal(), receivedKey.Marshal()) {
+ err = wkr.instance.VerifyPublicKey(receivedKey, client)
+ if err != nil {
+ return nil, err
+ }
+ wkr.publicKey = receivedKey
+ }*/
+ return client, nil
+}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index 3322575dc..ed5eb8fe2 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -9,6 +9,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "golang.org/x/crypto/ssh"
)
// A RateLimitError should be returned by a Provider when the cloud
@@ -38,21 +39,44 @@ type ImageID string
// instance is implemented by the provider-specific instance types.
type Instance interface {
+ // ID returns the provider's instance ID. It must be stable
+ // for the life of the instance.
+ ID() InstanceID
+
// String typically returns the cloud-provided instance ID.
String() string
- // Configured Arvados instance type
- InstanceType() arvados.InstanceType
+
// Get tags
- GetTags(context.Context) (InstanceTags, error)
+ Tags(context.Context) (InstanceTags, error)
+
// Replace tags with the given tags
SetTags(context.Context, InstanceTags) error
+
// Shut down the node
Destroy(context.Context) error
+
// SSH server hostname or IP address, or empty string if unknown pending creation.
Address() string
}
-type Provider interface {
- Create(context.Context, arvados.InstanceType, ImageID, InstanceTags) (Instance, error)
+type InstanceProvider interface {
+ // Create a new instance. If supported by the driver, add the
+ // provided public key to /root/.ssh/authorized_keys.
+ //
+ // The returned error should implement RateLimitError and
+ // QuotaError where applicable.
+ Create(context.Context, arvados.InstanceType, ImageID, InstanceTags, ssh.PublicKey) (Instance, error)
+
+ // Return all instances, including ones that are booting or
+ // shutting down.
+ //
+ // An instance returned by successive calls to Instances() may
+ // -- but does not need to -- be represented by the same
+ // Instance object each time. Thus, the caller is responsible
+ // for de-duplicating the returned instances by comparing the
+ // InstanceIDs returned by the instances' ID() methods.
Instances(context.Context) ([]Instance, error)
+
+ // Stop any background tasks and release other resources.
+ Stop()
}
commit 137f46c8040d22e70e4b17a495e4171c59a8033f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Aug 23 16:38:28 2018 -0400
13964: InstanceTags type
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index dcfcf0564..21e8d28d8 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -245,7 +245,7 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster
func (az *AzureProvider) Create(ctx context.Context,
instanceType arvados.InstanceType,
imageId ImageID,
- newTags map[string]string) (Instance, error) {
+ newTags InstanceTags) (Instance, error) {
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
@@ -537,7 +537,7 @@ func (ai *AzureInstance) InstanceType() arvados.InstanceType {
return ai.instanceType
}
-func (ai *AzureInstance) SetTags(ctx context.Context, newTags map[string]string) error {
+func (ai *AzureInstance) SetTags(ctx context.Context, newTags InstanceTags) error {
tags := make(map[string]*string)
for k, v := range ai.vm.Tags {
@@ -562,7 +562,7 @@ func (ai *AzureInstance) SetTags(ctx context.Context, newTags map[string]string)
return nil
}
-func (ai *AzureInstance) GetTags(ctx context.Context) (map[string]string, error) {
+func (ai *AzureInstance) GetTags(ctx context.Context) (InstanceTags, error) {
tags := make(map[string]string)
for k, v := range ai.vm.Tags {
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index efaf2a6c3..3322575dc 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -32,7 +32,7 @@ type QuotaError interface {
error
}
-type InstanceTag string
+type InstanceTags map[string]string
type InstanceID string
type ImageID string
@@ -43,9 +43,9 @@ type Instance interface {
// Configured Arvados instance type
InstanceType() arvados.InstanceType
// Get tags
- GetTags(context.Context) (map[string]string, error)
+ GetTags(context.Context) (InstanceTags, error)
// Replace tags with the given tags
- SetTags(context.Context, map[string]string) error
+ SetTags(context.Context, InstanceTags) error
// Shut down the node
Destroy(context.Context) error
// SSH server hostname or IP address, or empty string if unknown pending creation.
@@ -53,6 +53,6 @@ type Instance interface {
}
type Provider interface {
- Create(context.Context, arvados.InstanceType, ImageID, map[string]string) (Instance, error)
+ Create(context.Context, arvados.InstanceType, ImageID, InstanceTags) (Instance, error)
Instances(context.Context) ([]Instance, error)
}
commit ca801f2402482c9192f3c75461134c10834f2fcc
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Aug 23 16:22:12 2018 -0400
13964: VM names are namespaced. Can set/get tags.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 7f7e34fb1..dcfcf0564 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -11,6 +11,7 @@ import (
"net/http"
"regexp"
"strconv"
+ "strings"
"sync"
"time"
@@ -191,18 +192,20 @@ type AzureProvider struct {
storageAcctClient storageacct.AccountsClient
azureEnv azure.Environment
interfaces map[string]network.Interface
+ dispatcherID string
+ namePrefix string
}
-func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (prv Provider, err error) {
+func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (prv Provider, err error) {
ap := AzureProvider{}
- err = ap.setup(azcfg, arvcfg)
+ err = ap.setup(azcfg, arvcfg, dispatcherID)
if err != nil {
return nil, err
}
return &ap, nil
}
-func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (err error) {
+func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster, dispatcherID string) (err error) {
az.azconfig = azcfg
az.arvconfig = arvcfg
vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
@@ -233,31 +236,36 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster
az.netClient = &InterfacesClientImpl{netClient}
az.storageAcctClient = storageAcctClient
+ az.dispatcherID = dispatcherID
+ az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
+
return nil
}
func (az *AzureProvider) Create(ctx context.Context,
instanceType arvados.InstanceType,
imageId ImageID,
- instanceTag []InstanceTag) (Instance, error) {
+ newTags map[string]string) (Instance, error) {
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
return nil, err
}
- name = "compute-" + name
+ name = az.namePrefix + name
log.Printf("name is %v", name)
timestamp := time.Now().Format(time.RFC3339Nano)
+ tags := make(map[string]*string)
+ tags["created-at"] = ×tamp
+ for k, v := range newTags {
+ tags["dispatch-"+k] = &v
+ }
+
nicParameters := network.Interface{
Location: &az.azconfig.Location,
- Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
- },
+ Tags: tags,
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
IPConfigurations: &[]network.InterfaceIPConfiguration{
network.InterfaceIPConfiguration{
@@ -292,14 +300,11 @@ func (az *AzureProvider) Create(ctx context.Context,
log.Printf("URI instance vhd %v", instance_vhd)
+ tags["arvados-instance-type"] = &instanceType.Name
+
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
- Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-instance-type": &instanceType.Name,
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
- },
+ Tags: tags,
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
@@ -307,7 +312,7 @@ func (az *AzureProvider) Create(ctx context.Context,
StorageProfile: &compute.StorageProfile{
OsDisk: &compute.OSDisk{
OsType: compute.Linux,
- Name: to.StringPtr(fmt.Sprintf("%v-os", name)),
+ Name: to.StringPtr(name + "-os"),
CreateOption: compute.FromImage,
Image: &compute.VirtualHardDisk{
URI: to.StringPtr(string(imageId)),
@@ -376,9 +381,8 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
if err != nil {
return nil, WrapAzureError(err)
}
- if result.Value().Tags["arvados-class"] != nil &&
- result.Value().Tags["arvados-instance-type"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
+ if strings.HasPrefix(*result.Value().Name, az.namePrefix) &&
+ result.Value().Tags["arvados-instance-type"] != nil {
instances = append(instances, &AzureInstance{
provider: az,
vm: result.Value(),
@@ -425,15 +429,12 @@ func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Int
for ; result.NotDone(); err = result.Next() {
if err != nil {
log.Printf("Error listing nics: %v", err)
- return interfaces, WrapAzureError(nil)
+ return interfaces, nil
}
- if result.Value().Tags["arvados-class"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
-
+ if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
if result.Value().VirtualMachine != nil {
interfaces[*result.Value().ID] = result.Value()
} else {
-
if result.Value().Tags["created-at"] != nil {
created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
if err == nil {
@@ -493,7 +494,7 @@ func (az *AzureProvider) ManageBlobs(ctx context.Context) {
}()
}
- page := storage.ListBlobsParameters{Prefix: "compute-"}
+ page := storage.ListBlobsParameters{Prefix: az.namePrefix}
for {
response, err := blobcont.ListBlobs(page)
@@ -536,12 +537,41 @@ func (ai *AzureInstance) InstanceType() arvados.InstanceType {
return ai.instanceType
}
-func (ai *AzureInstance) SetTags([]InstanceTag) error {
+func (ai *AzureInstance) SetTags(ctx context.Context, newTags map[string]string) error {
+ tags := make(map[string]*string)
+
+ for k, v := range ai.vm.Tags {
+ if !strings.HasPrefix(k, "dispatch-") {
+ tags[k] = v
+ }
+ }
+ for k, v := range newTags {
+ tags["dispatch-"+k] = &v
+ }
+
+ vmParameters := compute.VirtualMachine{
+ Location: &ai.provider.azconfig.Location,
+ Tags: tags,
+ }
+ vm, err := ai.provider.vmClient.CreateOrUpdate(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
+ if err != nil {
+ return WrapAzureError(err)
+ }
+ ai.vm = vm
+
return nil
}
-func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
- return nil, nil
+func (ai *AzureInstance) GetTags(ctx context.Context) (map[string]string, error) {
+ tags := make(map[string]string)
+
+ for k, v := range ai.vm.Tags {
+ if strings.HasPrefix(k, "dispatch-") {
+ tags[k[9:]] = *v
+ }
+ }
+
+ return tags, nil
}
func (ai *AzureInstance) Destroy(ctx context.Context) error {
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index f10121984..a5a173bee 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -83,14 +83,16 @@ func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
if err != nil {
return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, cluster)
+ ap, err := NewAzureProvider(cfg, cluster, "test123")
return ap, ImageID(cfg.Image), cluster, err
} else {
ap := AzureProvider{
azconfig: AzureProviderConfig{
BlobContainer: "vhds",
},
- arvconfig: cluster,
+ arvconfig: cluster,
+ dispatcherID: "test123",
+ namePrefix: "compute-test123-",
}
ap.vmClient = &VirtualMachinesClientStub{}
ap.netClient = &InterfacesClientStub{}
@@ -106,7 +108,7 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
inst, err := ap.Create(context.Background(),
cluster.InstanceTypes["tiny"],
- img, []InstanceTag{"tag1"})
+ img, map[string]string{"tag1": "bleep"})
c.Assert(err, check.IsNil)
@@ -124,7 +126,8 @@ func (*AzureProviderSuite) TestListInstances(c *check.C) {
c.Assert(err, check.IsNil)
for _, i := range l {
- log.Printf("%v %v %v", i.String(), i.Address(), i.InstanceType())
+ tg, _ := i.GetTags(context.Background())
+ log.Printf("%v %v %v %v", i.String(), i.Address(), i.InstanceType(), tg)
}
}
@@ -208,3 +211,26 @@ func (*AzureProviderSuite) TestWrapError(c *check.C) {
_, ok = wrapped.(QuotaError)
c.Check(ok, check.Equals, true)
}
+
+func (*AzureProviderSuite) TestSetTags(c *check.C) {
+ ap, _, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+ l, err := ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ if len(l) > 0 {
+ err = l[0].SetTags(context.Background(), map[string]string{"foo": "bar"})
+ if err != nil {
+ c.Fatal("Error setting tags", err)
+ }
+ }
+ l, err = ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ if len(l) > 0 {
+ tg, _ := l[0].GetTags(context.Background())
+ log.Printf("tags are %v", tg)
+ }
+}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index d8b1ad6c0..efaf2a6c3 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -43,16 +43,16 @@ type Instance interface {
// Configured Arvados instance type
InstanceType() arvados.InstanceType
// Get tags
- GetTags() ([]InstanceTag, error)
+ GetTags(context.Context) (map[string]string, error)
// Replace tags with the given tags
- SetTags([]InstanceTag) error
+ SetTags(context.Context, map[string]string) error
// Shut down the node
- Destroy(ctx context.Context) error
+ Destroy(context.Context) error
// SSH server hostname or IP address, or empty string if unknown pending creation.
Address() string
}
type Provider interface {
- Create(context.Context, arvados.InstanceType, ImageID, []InstanceTag) (Instance, error)
+ Create(context.Context, arvados.InstanceType, ImageID, map[string]string) (Instance, error)
Instances(context.Context) ([]Instance, error)
}
commit 95e21426a5a27c24f7aa561df53d88aee6e0360f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Aug 23 15:21:05 2018 -0400
13964: Don't panic running tests on stubs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index 0c8c8d534..f10121984 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -31,6 +31,7 @@ func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
VMName string,
parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
parameters.ID = &VMName
+ parameters.Name = &VMName
return parameters, nil
}
@@ -167,9 +168,12 @@ func (*AzureProviderSuite) TestDeleteFake(c *check.C) {
_, err = ap.(*AzureProvider).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
- rq := err.(autorest.DetailedError).Original.(*azure.RequestError)
+ de, ok := err.(autorest.DetailedError)
+ if ok {
+ rq := de.Original.(*azure.RequestError)
- log.Printf("%v %q %q", rq.Response.StatusCode, rq.ServiceError.Code, rq.ServiceError.Message)
+ log.Printf("%v %q %q", rq.Response.StatusCode, rq.ServiceError.Code, rq.ServiceError.Message)
+ }
}
func (*AzureProviderSuite) TestWrapError(c *check.C) {
commit abecf37b0d4028782f44d04540402a4857352735
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Aug 23 15:15:42 2018 -0400
13964: Detect and report rate limit and quota exceeded errors
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 195bd8182..7f7e34fb1 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -9,6 +9,8 @@ import (
"fmt"
"log"
"net/http"
+ "regexp"
+ "strconv"
"sync"
"time"
@@ -17,6 +19,7 @@ import (
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
"github.com/Azure/azure-sdk-for-go/storage"
+ "github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/Azure/go-autorest/autorest/to"
@@ -60,23 +63,25 @@ func (cl *VirtualMachinesClientImpl) CreateOrUpdate(ctx context.Context,
future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
if err != nil {
- return compute.VirtualMachine{}, err
+ return compute.VirtualMachine{}, WrapAzureError(err)
}
future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.Result(cl.inner)
+ r, err := future.Result(cl.inner)
+ return r, WrapAzureError(err)
}
func (cl *VirtualMachinesClientImpl) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
err = future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.Response(), err
+ return future.Response(), WrapAzureError(err)
}
func (cl *VirtualMachinesClientImpl) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
- return cl.inner.ListComplete(ctx, resourceGroupName)
+ r, err := cl.inner.ListComplete(ctx, resourceGroupName)
+ return r, WrapAzureError(err)
}
type InterfacesClientWrapper interface {
@@ -95,10 +100,10 @@ type InterfacesClientImpl struct {
func (cl *InterfacesClientImpl) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
err = future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.Response(), err
+ return future.Response(), WrapAzureError(err)
}
func (cl *InterfacesClientImpl) CreateOrUpdate(ctx context.Context,
@@ -108,14 +113,74 @@ func (cl *InterfacesClientImpl) CreateOrUpdate(ctx context.Context,
future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
if err != nil {
- return network.Interface{}, err
+ return network.Interface{}, WrapAzureError(err)
}
future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.Result(cl.inner)
+ r, err := future.Result(cl.inner)
+ return r, WrapAzureError(err)
}
func (cl *InterfacesClientImpl) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
- return cl.inner.ListComplete(ctx, resourceGroupName)
+ r, err := cl.inner.ListComplete(ctx, resourceGroupName)
+ return r, WrapAzureError(err)
+}
+
+var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`)
+
+type AzureRateLimitError struct {
+ azure.RequestError
+ earliestRetry time.Time
+}
+
+func (ar *AzureRateLimitError) EarliestRetry() time.Time {
+ return ar.earliestRetry
+}
+
+type AzureQuotaError struct {
+ azure.RequestError
+}
+
+func (ar *AzureQuotaError) IsQuotaError() bool {
+ return true
+}
+
+func WrapAzureError(err error) error {
+ de, ok := err.(autorest.DetailedError)
+ if !ok {
+ return err
+ }
+ rq, ok := de.Original.(*azure.RequestError)
+ if !ok {
+ return err
+ }
+ if rq.Response == nil {
+ return err
+ }
+ if rq.Response.StatusCode == 429 || len(rq.Response.Header["Retry-After"]) >= 1 {
+ // API throttling
+ ra := rq.Response.Header["Retry-After"][0]
+ earliestRetry, parseErr := http.ParseTime(ra)
+ if parseErr != nil {
+ // Could not parse as a timestamp, must be number of seconds
+ dur, parseErr := strconv.ParseInt(ra, 10, 64)
+ if parseErr != nil {
+ earliestRetry = time.Now().Add(time.Duration(dur) * time.Second)
+ }
+ }
+ if parseErr != nil {
+ // Couldn't make sense of retry-after,
+ // so set retry to 20 seconds
+ earliestRetry = time.Now().Add(20 * time.Second)
+ }
+ return &AzureRateLimitError{*rq, earliestRetry}
+ }
+ if rq.ServiceError == nil {
+ return err
+ }
+ if quotaRe.FindString(rq.ServiceError.Code) != "" || quotaRe.FindString(rq.ServiceError.Message) != "" {
+ return &AzureQuotaError{*rq}
+ }
+ return err
}
type AzureProvider struct {
@@ -214,7 +279,7 @@ func (az *AzureProvider) Create(ctx context.Context,
}
nic, err := az.netClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
log.Printf("Created NIC %v", *nic.ID)
@@ -283,7 +348,7 @@ func (az *AzureProvider) Create(ctx context.Context,
vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name, vmParameters)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
return &AzureInstance{
@@ -302,14 +367,14 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
result, err := az.vmClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
instances := make([]Instance, 0)
for ; result.NotDone(); err = result.Next() {
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
if result.Value().Tags["arvados-class"] != nil &&
result.Value().Tags["arvados-instance-type"] != nil &&
@@ -327,7 +392,7 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
- return nil, err
+ return nil, WrapAzureError(err)
}
interfaces := make(map[string]network.Interface)
@@ -360,7 +425,7 @@ func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Int
for ; result.NotDone(); err = result.Next() {
if err != nil {
log.Printf("Error listing nics: %v", err)
- return interfaces, nil
+ return interfaces, WrapAzureError(nil)
}
if result.Value().Tags["arvados-class"] != nil &&
(*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
@@ -481,8 +546,7 @@ func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
func (ai *AzureInstance) Destroy(ctx context.Context) error {
_, err := ai.provider.vmClient.Delete(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
- // check response code?
- return err
+ return WrapAzureError(err)
}
func (ai *AzureInstance) Address() string {
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index bcba51bfd..0c8c8d534 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -14,6 +14,8 @@ import (
"git.curoverse.com/arvados.git/sdk/go/config"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
+ "github.com/Azure/go-autorest/autorest"
+ "github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/to"
check "gopkg.in/check.v1"
)
@@ -156,3 +158,49 @@ func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
c.Check(i.Destroy(context.Background()), check.IsNil)
}
}
+
+func (*AzureProviderSuite) TestDeleteFake(c *check.C) {
+ ap, _, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ _, err = ap.(*AzureProvider).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
+
+ rq := err.(autorest.DetailedError).Original.(*azure.RequestError)
+
+ log.Printf("%v %q %q", rq.Response.StatusCode, rq.ServiceError.Code, rq.ServiceError.Message)
+}
+
+func (*AzureProviderSuite) TestWrapError(c *check.C) {
+ retryError := autorest.DetailedError{
+ Original: &azure.RequestError{
+ DetailedError: autorest.DetailedError{
+ Response: &http.Response{
+ StatusCode: 429,
+ Header: map[string][]string{"Retry-After": []string{"123"}},
+ },
+ },
+ ServiceError: &azure.ServiceError{},
+ },
+ }
+ wrapped := WrapAzureError(retryError)
+ _, ok := wrapped.(RateLimitError)
+ c.Check(ok, check.Equals, true)
+
+ quotaError := autorest.DetailedError{
+ Original: &azure.RequestError{
+ DetailedError: autorest.DetailedError{
+ Response: &http.Response{
+ StatusCode: 503,
+ },
+ },
+ ServiceError: &azure.ServiceError{
+ Message: "No more quota",
+ },
+ },
+ }
+ wrapped = WrapAzureError(quotaError)
+ _, ok = wrapped.(QuotaError)
+ c.Check(ok, check.Equals, true)
+}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index c5411128a..d8b1ad6c0 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -6,10 +6,32 @@ package dispatchcloud
import (
"context"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
)
+// A RateLimitError should be returned by a Provider when the cloud
+// service indicates it is rejecting all API calls for some time
+// interval.
+type RateLimitError interface {
+ // Time before which the caller should expect requests to
+ // fail.
+ EarliestRetry() time.Time
+ error
+}
+
+// A QuotaError should be returned by a Provider when the cloud
+// service indicates the account cannot create more VMs than already
+// exist.
+type QuotaError interface {
+ // If true, don't create more instances until some existing
+ // instances are destroyed. If false, don't handle the error
+ // as a quota error.
+ IsQuotaError() bool
+ error
+}
+
type InstanceTag string
type InstanceID string
type ImageID string
commit c548afa3245de240910c0b8ca6cf2a9e8800555a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Aug 22 22:42:26 2018 -0400
13964: Return Arvados instance type
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 29631e6dc..195bd8182 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -230,9 +230,10 @@ func (az *AzureProvider) Create(ctx context.Context,
vmParameters := compute.VirtualMachine{
Location: &az.azconfig.Location,
Tags: map[string]*string{
- "arvados-class": to.StringPtr("crunch-dynamic-compute"),
- "arvados-cluster": &az.arvconfig.ClusterID,
- "created-at": ×tamp,
+ "arvados-class": to.StringPtr("crunch-dynamic-compute"),
+ "arvados-instance-type": &instanceType.Name,
+ "arvados-cluster": &az.arvconfig.ClusterID,
+ "created-at": ×tamp,
},
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
@@ -311,11 +312,13 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
return nil, err
}
if result.Value().Tags["arvados-class"] != nil &&
+ result.Value().Tags["arvados-instance-type"] != nil &&
(*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
instances = append(instances, &AzureInstance{
- provider: az,
- vm: result.Value(),
- nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
+ provider: az,
+ vm: result.Value(),
+ nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID],
+ instanceType: az.arvconfig.InstanceTypes[(*result.Value().Tags["arvados-instance-type"])]})
}
}
return instances, nil
@@ -464,10 +467,6 @@ func (ai *AzureInstance) String() string {
return *ai.vm.Name
}
-func (ai *AzureInstance) ProviderType() string {
- return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
-}
-
func (ai *AzureInstance) InstanceType() arvados.InstanceType {
return ai.instanceType
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
index 2791ea6bf..bcba51bfd 100644
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/dispatchcloud/azure_test.go
@@ -61,45 +61,49 @@ func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName
var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
-func GetProvider() (Provider, ImageID, error) {
+func GetProvider() (Provider, ImageID, arvados.Cluster, error) {
+ cluster := arvados.Cluster{
+ InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
+ "tiny": arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "Standard_D1_v2",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ })}
if *live != "" {
cfg := AzureProviderConfig{}
err := config.LoadFile(&cfg, *live)
if err != nil {
- return nil, ImageID(""), err
+ return nil, ImageID(""), cluster, err
}
- ap, err := NewAzureProvider(cfg, arvados.Cluster{})
- return ap, ImageID(cfg.Image), err
+ ap, err := NewAzureProvider(cfg, cluster)
+ return ap, ImageID(cfg.Image), cluster, err
} else {
ap := AzureProvider{
azconfig: AzureProviderConfig{
BlobContainer: "vhds",
},
+ arvconfig: cluster,
}
ap.vmClient = &VirtualMachinesClientStub{}
ap.netClient = &InterfacesClientStub{}
- return &ap, ImageID("blob"), nil
+ return &ap, ImageID("blob"), cluster, nil
}
}
func (*AzureProviderSuite) TestCreate(c *check.C) {
- ap, img, err := GetProvider()
+ ap, img, cluster, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
inst, err := ap.Create(context.Background(),
- arvados.InstanceType{
- Name: "tiny",
- ProviderType: "Standard_D1_v2",
- VCPUs: 1,
- RAM: 4000000000,
- Scratch: 10000000000,
- Price: .02,
- Preemptible: false,
- },
- img,
- []InstanceTag{"tag1"})
+ cluster.InstanceTypes["tiny"],
+ img, []InstanceTag{"tag1"})
c.Assert(err, check.IsNil)
@@ -107,7 +111,7 @@ func (*AzureProviderSuite) TestCreate(c *check.C) {
}
func (*AzureProviderSuite) TestListInstances(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -117,12 +121,12 @@ func (*AzureProviderSuite) TestListInstances(c *check.C) {
c.Assert(err, check.IsNil)
for _, i := range l {
- log.Printf("%v %v", i.String(), i.Address())
+ log.Printf("%v %v %v", i.String(), i.Address(), i.InstanceType())
}
}
func (*AzureProviderSuite) TestManageNics(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -131,7 +135,7 @@ func (*AzureProviderSuite) TestManageNics(c *check.C) {
}
func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
@@ -140,7 +144,7 @@ func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
}
func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
- ap, _, err := GetProvider()
+ ap, _, _, err := GetProvider()
if err != nil {
c.Fatal("Error making provider", err)
}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index e896ac650..c5411128a 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -18,9 +18,8 @@ type ImageID string
type Instance interface {
// String typically returns the cloud-provided instance ID.
String() string
- // Cloud provider's "instance type" ID. Matches a key in
- // configured arvados.InstanceTypeMap.
- ProviderType() string
+ // Configured Arvados instance type
+ InstanceType() arvados.InstanceType
// Get tags
GetTags() ([]InstanceTag, error)
// Replace tags with the given tags
commit 3f649213bb9165b33a0ece597a4f852093cb409e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Aug 22 22:13:23 2018 -0400
13964: Full lifecycle for NICs/VHDs/VMs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index 6f769bfaa..29631e6dc 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -125,6 +125,7 @@ type AzureProvider struct {
netClient InterfacesClientWrapper
storageAcctClient storageacct.AccountsClient
azureEnv azure.Environment
+ interfaces map[string]network.Interface
}
func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (prv Provider, err error) {
@@ -279,7 +280,7 @@ func (az *AzureProvider) Create(ctx context.Context,
},
}
- vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name+"-compute", vmParameters)
+ vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name, vmParameters)
if err != nil {
return nil, err
}
@@ -293,6 +294,11 @@ func (az *AzureProvider) Create(ctx context.Context,
}
func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
+ interfaces, err := az.ManageNics(ctx)
+ if err != nil {
+ return nil, err
+ }
+
result, err := az.vmClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
return nil, err
@@ -304,59 +310,80 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
if err != nil {
return nil, err
}
- log.Printf("%v", *result.Value().Name)
if result.Value().Tags["arvados-class"] != nil &&
(*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
instances = append(instances, &AzureInstance{
provider: az,
- vm: result.Value()})
+ vm: result.Value(),
+ nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
}
}
return instances, nil
}
-func (az *AzureProvider) DeleteDanglingNics(ctx context.Context) {
+func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
- return
+ return nil, err
}
+ interfaces := make(map[string]network.Interface)
+
timestamp := time.Now()
wg := sync.WaitGroup{}
- defer wg.Wait()
+ deletechannel := make(chan string, 20)
+ defer func() {
+ wg.Wait()
+ close(deletechannel)
+ }()
+ for i := 0; i < 4; i += 1 {
+ go func() {
+ for {
+ nicname, ok := <-deletechannel
+ if !ok {
+ return
+ }
+ _, delerr := az.netClient.Delete(context.Background(), az.azconfig.ResourceGroup, nicname)
+ if delerr != nil {
+ log.Printf("Error deleting %v: %v", nicname, delerr)
+ } else {
+ log.Printf("Deleted %v", nicname)
+ }
+ wg.Done()
+ }
+ }()
+ }
+
for ; result.NotDone(); err = result.Next() {
if err != nil {
log.Printf("Error listing nics: %v", err)
- return
- }
- if !result.NotDone() {
- return
+ return interfaces, nil
}
if result.Value().Tags["arvados-class"] != nil &&
- (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" &&
- result.Value().VirtualMachine == nil {
-
- if result.Value().Tags["created-at"] != nil {
- created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
- if err == nil {
- log.Printf("found dangling NIC %v created %v seconds ago", *result.Value().Name, timestamp.Sub(created_at).Seconds())
- if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
- log.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
- wg.Add(1)
- go func(nicname string) {
- r, delerr := az.netClient.Delete(context.Background(), az.azconfig.ResourceGroup, nicname)
- log.Printf("%v %v", r, delerr)
- wg.Done()
- }(*result.Value().Name)
+ (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
+
+ if result.Value().VirtualMachine != nil {
+ interfaces[*result.Value().ID] = result.Value()
+ } else {
+
+ if result.Value().Tags["created-at"] != nil {
+ created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
+ if err == nil {
+ //log.Printf("found dangling NIC %v created %v seconds ago", *result.Value().Name, timestamp.Sub(created_at).Seconds())
+ if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ log.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
+ wg.Add(1)
+ deletechannel <- *result.Value().Name
+ }
}
}
}
}
}
-
+ return interfaces, nil
}
-func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
+func (az *AzureProvider) ManageBlobs(ctx context.Context) {
result, err := az.storageAcctClient.ListKeys(ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
if err != nil {
log.Printf("Couldn't get account keys %v", err)
@@ -374,6 +401,30 @@ func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
blobcont := blobsvc.GetContainerReference(az.azconfig.BlobContainer)
timestamp := time.Now()
+ wg := sync.WaitGroup{}
+ deletechannel := make(chan storage.Blob, 20)
+ defer func() {
+ wg.Wait()
+ close(deletechannel)
+ }()
+ for i := 0; i < 4; i += 1 {
+ go func() {
+ for {
+ blob, ok := <-deletechannel
+ if !ok {
+ return
+ }
+ err := blob.Delete(nil)
+ if err != nil {
+ log.Printf("error deleting %v: %v", blob.Name, err)
+ } else {
+ log.Printf("Deleted blob %v", blob.Name)
+ }
+ wg.Done()
+ }
+ }()
+ }
+
page := storage.ListBlobsParameters{Prefix: "compute-"}
for {
@@ -388,7 +439,10 @@ func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
b.Properties.LeaseState == "available" &&
b.Properties.LeaseStatus == "unlocked" &&
age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+
log.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
+ wg.Add(1)
+ deletechannel <- b
}
}
if response.NextMarker != "" {
@@ -407,7 +461,7 @@ type AzureInstance struct {
}
func (ai *AzureInstance) String() string {
- return *ai.vm.ID
+ return *ai.vm.Name
}
func (ai *AzureInstance) ProviderType() string {
@@ -428,7 +482,7 @@ func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
func (ai *AzureInstance) Destroy(ctx context.Context) error {
_, err := ai.provider.vmClient.Delete(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
- // check response code
+ // check response code?
return err
}
diff --git a/lib/dispatchcloud/azure_test.go b/lib/dispatchcloud/azure_test.go
new file mode 100644
index 000000000..2791ea6bf
--- /dev/null
+++ b/lib/dispatchcloud/azure_test.go
@@ -0,0 +1,154 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package dispatchcloud
+
+import (
+ "context"
+ "flag"
+ "log"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
+ "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
+ "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
+ "github.com/Azure/go-autorest/autorest/to"
+ check "gopkg.in/check.v1"
+)
+
+type AzureProviderSuite struct{}
+
+var _ = check.Suite(&AzureProviderSuite{})
+
+type VirtualMachinesClientStub struct{}
+
+func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ VMName string,
+ parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
+ parameters.ID = &VMName
+ return parameters, nil
+}
+
+func (*VirtualMachinesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ return nil, nil
+}
+
+func (*VirtualMachinesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
+ return compute.VirtualMachineListResultIterator{}, nil
+}
+
+type InterfacesClientStub struct{}
+
+func (*InterfacesClientStub) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ nicName string,
+ parameters network.Interface) (result network.Interface, err error) {
+ parameters.ID = to.StringPtr(nicName)
+ (*parameters.IPConfigurations)[0].PrivateIPAddress = to.StringPtr("192.168.5.5")
+ return parameters, nil
+}
+
+func (*InterfacesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ return nil, nil
+}
+
+func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
+ return network.InterfaceListResultIterator{}, nil
+}
+
+var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
+
+func GetProvider() (Provider, ImageID, error) {
+ if *live != "" {
+ cfg := AzureProviderConfig{}
+ err := config.LoadFile(&cfg, *live)
+ if err != nil {
+ return nil, ImageID(""), err
+ }
+ ap, err := NewAzureProvider(cfg, arvados.Cluster{})
+ return ap, ImageID(cfg.Image), err
+ } else {
+ ap := AzureProvider{
+ azconfig: AzureProviderConfig{
+ BlobContainer: "vhds",
+ },
+ }
+ ap.vmClient = &VirtualMachinesClientStub{}
+ ap.netClient = &InterfacesClientStub{}
+ return &ap, ImageID("blob"), nil
+ }
+}
+
+func (*AzureProviderSuite) TestCreate(c *check.C) {
+ ap, img, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ inst, err := ap.Create(context.Background(),
+ arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "Standard_D1_v2",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ img,
+ []InstanceTag{"tag1"})
+
+ c.Assert(err, check.IsNil)
+
+ log.Printf("Result %v %v", inst.String(), inst.Address())
+}
+
+func (*AzureProviderSuite) TestListInstances(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(context.Background())
+
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ log.Printf("%v %v", i.String(), i.Address())
+ }
+}
+
+func (*AzureProviderSuite) TestManageNics(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ ap.(*AzureProvider).ManageNics(context.Background())
+}
+
+func (*AzureProviderSuite) TestManageBlobs(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ ap.(*AzureProvider).ManageBlobs(context.Background())
+}
+
+func (*AzureProviderSuite) TestDestroyInstances(c *check.C) {
+ ap, _, err := GetProvider()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(context.Background())
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ c.Check(i.Destroy(context.Background()), check.IsNil)
+ }
+}
commit ee8dedc9286a12d95c5a4d9eefe4daf4a40d2528
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Aug 21 11:28:37 2018 -0400
13964: Can successfully create a VM, list existing VMs
Cleanup unused NICs. Blob cleanup in progress.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
index e397731e3..6f769bfaa 100644
--- a/lib/dispatchcloud/azure.go
+++ b/lib/dispatchcloud/azure.go
@@ -1,29 +1,43 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package dispatchcloud
import (
"context"
"fmt"
+ "log"
"net/http"
+ "sync"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
+ storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
+ "github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/Azure/go-autorest/autorest/to"
+ "github.com/jmcvetta/randutil"
)
type AzureProviderConfig struct {
- SubscriptionID string
- ClientID string
- ClientSecret string
- TenantID string
- CloudEnv string
- ResourceGroup string
- Location string
- Subnet string
- StorageAccount string
- BlobContainer string
+ SubscriptionID string `json:"subscription_id"`
+ ClientID string `json:"key"`
+ ClientSecret string `json:"secret"`
+ TenantID string `json:"tenant_id"`
+ CloudEnv string `json:"cloud_environment"`
+ ResourceGroup string `json:"resource_group"`
+ Location string `json:"region"`
+ Network string `json:"network"`
+ Subnet string `json:"subnet"`
+ StorageAccount string `json:"storage_account"`
+ BlobContainer string `json:"blob_container"`
+ Image string `json:"image"`
+ AuthorizedKey string `json:"authorized_key"`
+ DeleteDanglingResourcesAfter float64 `json:"delete_dangling_resources_after"`
}
type VirtualMachinesClientWrapper interface {
@@ -32,7 +46,7 @@ type VirtualMachinesClientWrapper interface {
VMName string,
parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
- List(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultPage, err error)
+ ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error)
}
type VirtualMachinesClientImpl struct {
@@ -42,11 +56,11 @@ type VirtualMachinesClientImpl struct {
func (cl *VirtualMachinesClientImpl) CreateOrUpdate(ctx context.Context,
resourceGroupName string,
VMName string,
- parameters VirtualMachine) (result compute.VirtualMachine, err error) {
+ parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
if err != nil {
- return nil, err
+ return compute.VirtualMachine{}, err
}
future.WaitForCompletionRef(ctx, cl.inner.Client)
return future.Result(cl.inner)
@@ -57,12 +71,12 @@ func (cl *VirtualMachinesClientImpl) Delete(ctx context.Context, resourceGroupNa
if err != nil {
return nil, err
}
- future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.GetResult()
+ err = future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.Response(), err
}
-func (cl *VirtualMachinesClientImpl) List(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultPage, err error) {
- return cl.inner.List(ctx, resourceGroupName)
+func (cl *VirtualMachinesClientImpl) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
+ return cl.inner.ListComplete(ctx, resourceGroupName)
}
type InterfacesClientWrapper interface {
@@ -71,7 +85,7 @@ type InterfacesClientWrapper interface {
networkInterfaceName string,
parameters network.Interface) (result network.Interface, err error)
Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
- List(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultPage, err error)
+ ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error)
}
type InterfacesClientImpl struct {
@@ -83,50 +97,63 @@ func (cl *InterfacesClientImpl) Delete(ctx context.Context, resourceGroupName st
if err != nil {
return nil, err
}
- future.WaitForCompletionRef(ctx, cl.inner.Client)
- return future.GetResult()
+ err = future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.Response(), err
}
func (cl *InterfacesClientImpl) CreateOrUpdate(ctx context.Context,
resourceGroupName string,
networkInterfaceName string,
- parameters network.Interface) (result compute.VirtualMachine, err error) {
+ parameters network.Interface) (result network.Interface, err error) {
future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
if err != nil {
- return nil, err
+ return network.Interface{}, err
}
future.WaitForCompletionRef(ctx, cl.inner.Client)
return future.Result(cl.inner)
}
-func (cl *InterfacesClientImpl) List(ctx context.Context, resourceGroupName string) (result compute.InterfaceListResultPage, err error) {
- return cl.inner.List(ctx, resourceGroupName)
+func (cl *InterfacesClientImpl) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
+ return cl.inner.ListComplete(ctx, resourceGroupName)
}
type AzureProvider struct {
- config AzureProviderConfig
- vmClient VirtualMachinesClientWrapper
- netClient InterfacesClientWrapper
- azureEnv auth.Environment
+ azconfig AzureProviderConfig
+ arvconfig arvados.Cluster
+ vmClient VirtualMachinesClientWrapper
+ netClient InterfacesClientWrapper
+ storageAcctClient storageacct.AccountsClient
+ azureEnv azure.Environment
+}
+
+func NewAzureProvider(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (prv Provider, err error) {
+ ap := AzureProvider{}
+ err = ap.setup(azcfg, arvcfg)
+ if err != nil {
+ return nil, err
+ }
+ return &ap, nil
}
-func (az *AzureProvider) Init(cfg AzureProviderConfig) error {
- az.config = cfg
- vmClient := compute.NewVirtualMachinesClient(az.config.SubscriptionId)
- netClient := network.NewInterfacesClient(az.config.SubscriptionId)
+func (az *AzureProvider) setup(azcfg AzureProviderConfig, arvcfg arvados.Cluster) (err error) {
+ az.azconfig = azcfg
+ az.arvconfig = arvcfg
+ vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
+ netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
+ storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
- az.azureEnv, err = azure.EnvironmentFromName(az.config.CloudEnv)
+ az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnv)
if err != nil {
return err
}
authorizer, err := auth.ClientCredentialsConfig{
- ClientID: az.config.ClientID,
- ClientSecret: az.config.ClientSecret,
- TenantID: az.config.TenantID,
- Resource: env.ResourceManagerEndpoint,
- AADEndpoint: env.ActiveDirectoryEndpoint,
+ ClientID: az.azconfig.ClientID,
+ ClientSecret: az.azconfig.ClientSecret,
+ TenantID: az.azconfig.TenantID,
+ Resource: az.azureEnv.ResourceManagerEndpoint,
+ AADEndpoint: az.azureEnv.ActiveDirectoryEndpoint,
}.Authorizer()
if err != nil {
return err
@@ -134,9 +161,11 @@ func (az *AzureProvider) Init(cfg AzureProviderConfig) error {
vmClient.Authorizer = authorizer
netClient.Authorizer = authorizer
+ storageAcctClient.Authorizer = authorizer
- az.vmClient = VirtualMachinesClientImpl{vmClient}
- az.netClient = InterfacesClientImpl{netClient}
+ az.vmClient = &VirtualMachinesClientImpl{vmClient}
+ az.netClient = &InterfacesClientImpl{netClient}
+ az.storageAcctClient = storageAcctClient
return nil
}
@@ -146,65 +175,85 @@ func (az *AzureProvider) Create(ctx context.Context,
imageId ImageID,
instanceTag []InstanceTag) (Instance, error) {
- name := "randomname"
+ name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
+ if err != nil {
+ return nil, err
+ }
+
+ name = "compute-" + name
+ log.Printf("name is %v", name)
+
+ timestamp := time.Now().Format(time.RFC3339Nano)
nicParameters := network.Interface{
- Location: az.config.Location,
- Tags: []map[string]string{
- "arvados-class": "dynamic-compute",
- "arvados-cluster": "",
+ Location: &az.azconfig.Location,
+ Tags: map[string]*string{
+ "arvados-class": to.StringPtr("crunch-dynamic-compute"),
+ "arvados-cluster": &az.arvconfig.ClusterID,
+ "created-at": ×tamp,
},
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
IPConfigurations: &[]network.InterfaceIPConfiguration{
- network.InterfaceIPConfiguration{},
- Name: "ip1",
- InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
- Subnet: &network.Subnet{
- ID: az.config.Subnet,
+ network.InterfaceIPConfiguration{
+ Name: to.StringPtr("ip1"),
+ InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
+ Subnet: &network.Subnet{
+ ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+
+ "/Microsoft.Network/virtualnetworks/%s/subnets/%s",
+ az.azconfig.SubscriptionID,
+ az.azconfig.ResourceGroup,
+ az.azconfig.Network,
+ az.azconfig.Subnet)),
+ },
+ PrivateIPAllocationMethod: network.Dynamic,
},
- PrivateIPAllocationMethod: network.Dynamic,
},
},
},
}
- nic, err := az.netClient.CreateOrUpdate(ctx, az.config.ResourceGroup, name+"-nic", nicParameters)
+ nic, err := az.netClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
if err != nil {
return nil, err
}
- instance_vhd = fmt.Sprintf("https://%s.blob.%s/%s/%s-os.vhd",
- az.config.StorageAccount,
+ log.Printf("Created NIC %v", *nic.ID)
+
+ instance_vhd := fmt.Sprintf("https://%s.blob.%s/%s/%s-os.vhd",
+ az.azconfig.StorageAccount,
az.azureEnv.StorageEndpointSuffix,
- az.config.BlobContainer,
+ az.azconfig.BlobContainer,
name)
+ log.Printf("URI instance vhd %v", instance_vhd)
+
vmParameters := compute.VirtualMachine{
- Location: az.config.Location,
- Tags: []map[string]string{
- "arvados-class": "dynamic-compute",
- "arvados-cluster": "",
+ Location: &az.azconfig.Location,
+ Tags: map[string]*string{
+ "arvados-class": to.StringPtr("crunch-dynamic-compute"),
+ "arvados-cluster": &az.arvconfig.ClusterID,
+ "created-at": ×tamp,
},
VirtualMachineProperties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
- VMSize: instanceType.ProviderType,
+ VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
},
StorageProfile: &compute.StorageProfile{
OsDisk: &compute.OSDisk{
OsType: compute.Linux,
- Name: "",
+ Name: to.StringPtr(fmt.Sprintf("%v-os", name)),
CreateOption: compute.FromImage,
Image: &compute.VirtualHardDisk{
- URI: imageId,
+ URI: to.StringPtr(string(imageId)),
},
Vhd: &compute.VirtualHardDisk{
- URI: vhd,
+ URI: &instance_vhd,
},
},
},
NetworkProfile: &compute.NetworkProfile{
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
compute.NetworkInterfaceReference{
- ID: "",
+ ID: nic.ID,
NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
Primary: to.BoolPtr(true),
},
@@ -212,22 +261,25 @@ func (az *AzureProvider) Create(ctx context.Context,
},
},
OsProfile: &compute.OSProfile{
+ ComputerName: &name,
+ AdminUsername: to.StringPtr("arvados"),
LinuxConfiguration: &compute.LinuxConfiguration{
- DisablePasswordAuthentication: true,
+ DisablePasswordAuthentication: to.BoolPtr(true),
SSH: &compute.SSHConfiguration{
PublicKeys: &[]compute.SSHPublicKey{
compute.SSHPublicKey{
- Path: "",
- KeyData: "",
+ Path: to.StringPtr("/home/arvados/.ssh/authorized_keys"),
+ KeyData: to.StringPtr(az.azconfig.AuthorizedKey),
},
},
},
},
+ //CustomData: to.StringPtr(""),
},
},
}
- vm, err := az.vmClient.CreateOrUpdate(ctx, az.config.ResourceGroup, name+"-compute", vmParameters)
+ vm, err := az.vmClient.CreateOrUpdate(ctx, az.azconfig.ResourceGroup, name+"-compute", vmParameters)
if err != nil {
return nil, err
}
@@ -237,22 +289,113 @@ func (az *AzureProvider) Create(ctx context.Context,
provider: az,
nic: nic,
vm: vm,
- }
+ }, nil
}
func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
- result, err := az.vmClient.List(ctx, az.config.ResourceGroup)
+ result, err := az.vmClient.ListComplete(ctx, az.azconfig.ResourceGroup)
if err != nil {
return nil, err
}
- instances := make([]Instance)
+
+ instances := make([]Instance, 0)
+
+ for ; result.NotDone(); err = result.Next() {
+ if err != nil {
+ return nil, err
+ }
+ log.Printf("%v", *result.Value().Name)
+ if result.Value().Tags["arvados-class"] != nil &&
+ (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" {
+ instances = append(instances, &AzureInstance{
+ provider: az,
+ vm: result.Value()})
+ }
+ }
+ return instances, nil
+}
+
+func (az *AzureProvider) DeleteDanglingNics(ctx context.Context) {
+ result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
+ if err != nil {
+ return
+ }
+
+ timestamp := time.Now()
+ wg := sync.WaitGroup{}
+ defer wg.Wait()
+ for ; result.NotDone(); err = result.Next() {
+ if err != nil {
+ log.Printf("Error listing nics: %v", err)
+ return
+ }
+ if !result.NotDone() {
+ return
+ }
+ if result.Value().Tags["arvados-class"] != nil &&
+ (*result.Value().Tags["arvados-class"]) == "crunch-dynamic-compute" &&
+ result.Value().VirtualMachine == nil {
+
+ if result.Value().Tags["created-at"] != nil {
+ created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
+ if err == nil {
+ log.Printf("found dangling NIC %v created %v seconds ago", *result.Value().Name, timestamp.Sub(created_at).Seconds())
+ if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ log.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
+ wg.Add(1)
+ go func(nicname string) {
+ r, delerr := az.netClient.Delete(context.Background(), az.azconfig.ResourceGroup, nicname)
+ log.Printf("%v %v", r, delerr)
+ wg.Done()
+ }(*result.Value().Name)
+ }
+ }
+ }
+ }
+ }
+
+}
+
+func (az *AzureProvider) DeleteDanglingBlobs(ctx context.Context) {
+ result, err := az.storageAcctClient.ListKeys(ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
+ if err != nil {
+ log.Printf("Couldn't get account keys %v", err)
+ return
+ }
+
+ key1 := *(*result.Keys)[0].Value
+ client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv)
+ if err != nil {
+ log.Printf("Couldn't make client %v", err)
+ return
+ }
+
+ blobsvc := client.GetBlobService()
+ blobcont := blobsvc.GetContainerReference(az.azconfig.BlobContainer)
+
+ timestamp := time.Now()
+ page := storage.ListBlobsParameters{Prefix: "compute-"}
+
for {
- if result.NotDone() {
- result.Next()
+ response, err := blobcont.ListBlobs(page)
+ if err != nil {
+ log.Printf("Error listing blobs %v", err)
+ return
+ }
+ for _, b := range response.Blobs {
+ age := timestamp.Sub(time.Time(b.Properties.LastModified))
+ if b.Properties.BlobType == storage.BlobTypePage &&
+ b.Properties.LeaseState == "available" &&
+ b.Properties.LeaseStatus == "unlocked" &&
+ age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ log.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
+ }
+ }
+ if response.NextMarker != "" {
+ page.Marker = response.NextMarker
} else {
- return instances, nil
+ break
}
- result.Values
}
}
@@ -264,11 +407,11 @@ type AzureInstance struct {
}
func (ai *AzureInstance) String() string {
- return ai.vm.ID
+ return *ai.vm.ID
}
func (ai *AzureInstance) ProviderType() string {
- return ai.vm.VirtualMachineProperties.HardwareProfile.VMSize
+ return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
}
func (ai *AzureInstance) InstanceType() arvados.InstanceType {
@@ -279,12 +422,16 @@ func (ai *AzureInstance) SetTags([]InstanceTag) error {
return nil
}
+func (ai *AzureInstance) GetTags() ([]InstanceTag, error) {
+ return nil, nil
+}
+
func (ai *AzureInstance) Destroy(ctx context.Context) error {
- response, err := ai.provider.vm.Delete(ctx, ai.provider.config.ResourceGroup, ai.vm.Name)
+ _, err := ai.provider.vmClient.Delete(ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
// check response code
return err
}
func (ai *AzureInstance) Address() string {
- return ai.nic.IPConfigurations[0].PrivateIPAddress
+ return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
index aebb93011..e896ac650 100644
--- a/lib/dispatchcloud/provider.go
+++ b/lib/dispatchcloud/provider.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package dispatchcloud
import (
@@ -17,6 +21,8 @@ type Instance interface {
// Cloud provider's "instance type" ID. Matches a key in
// configured arvados.InstanceTypeMap.
ProviderType() string
+ // Get tags
+ GetTags() ([]InstanceTag, error)
// Replace tags with the given tags
SetTags([]InstanceTag) error
// Shut down the node
@@ -26,6 +32,6 @@ type Instance interface {
}
type Provider interface {
- Create(arvados.InstanceType, ImageID, []InstanceTag) (Instance, error)
- Instances() ([]Instance, error)
+ Create(context.Context, arvados.InstanceType, ImageID, []InstanceTag) (Instance, error)
+ Instances(context.Context) ([]Instance, error)
}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index ec296d21d..6bf8c016d 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -24,6 +24,24 @@
"revisionTime": "2017-07-27T13:52:37Z"
},
{
+ "checksumSHA1": "KF4DsRUpZ+h+qRQ/umRAQZfVvw0=",
+ "path": "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute",
+ "revision": "4e8cbbfb1aeab140cd0fa97fd16b64ee18c3ca6a",
+ "revisionTime": "2018-07-27T22:05:59Z"
+ },
+ {
+ "checksumSHA1": "IZNzp1cYx+xYHd4gzosKpG6Jr/k=",
+ "path": "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network",
+ "revision": "4e8cbbfb1aeab140cd0fa97fd16b64ee18c3ca6a",
+ "revisionTime": "2018-07-27T22:05:59Z"
+ },
+ {
+ "checksumSHA1": "W4c2uTDJlwhfryWg9esshmJANo0=",
+ "path": "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage",
+ "revision": "4e8cbbfb1aeab140cd0fa97fd16b64ee18c3ca6a",
+ "revisionTime": "2018-07-27T22:05:59Z"
+ },
+ {
"checksumSHA1": "xHZe/h/tyrqmS9qiR03bLfRv5FI=",
"path": "github.com/Azure/azure-sdk-for-go/storage",
"revision": "f8eeb65a1a1f969696b49aada9d24073f2c2acd1",
@@ -36,28 +54,58 @@
"revisionTime": "2018-02-14T01:17:07Z"
},
{
- "checksumSHA1": "LQWU/2M2E4L/hVzT9BVW1SkLrpA=",
+ "checksumSHA1": "1Y2+bSzYrdPHQqRjR1OrBMHAvxY=",
"path": "github.com/Azure/go-autorest/autorest",
- "revision": "a91c94d19d5efcb398b3aab64b8766e724aa7442",
- "revisionTime": "2017-11-30T17:00:06Z"
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
},
{
- "checksumSHA1": "nBQ7cdhoeYUur6G6HG97uueoDmE=",
+ "checksumSHA1": "GxL0HHpZDj2milPhR3SPV6MWLPc=",
"path": "github.com/Azure/go-autorest/autorest/adal",
- "revision": "a91c94d19d5efcb398b3aab64b8766e724aa7442",
- "revisionTime": "2017-11-30T17:00:06Z"
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
},
{
- "checksumSHA1": "zXyLmDVpkYkIsL0yinNLoW82IZc=",
+ "checksumSHA1": "ZNgwJOdHZmm4k/HJIbT1L5giO6M=",
"path": "github.com/Azure/go-autorest/autorest/azure",
- "revision": "a91c94d19d5efcb398b3aab64b8766e724aa7442",
- "revisionTime": "2017-11-30T17:00:06Z"
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
+ },
+ {
+ "checksumSHA1": "6i7kwcXGTn55WqfubQs21swgr34=",
+ "path": "github.com/Azure/go-autorest/autorest/azure/auth",
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
},
{
"checksumSHA1": "9nXCi9qQsYjxCeajJKWttxgEt0I=",
"path": "github.com/Azure/go-autorest/autorest/date",
- "revision": "a91c94d19d5efcb398b3aab64b8766e724aa7442",
- "revisionTime": "2017-11-30T17:00:06Z"
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
+ },
+ {
+ "checksumSHA1": "SbBb2GcJNm5GjuPKGL2777QywR4=",
+ "path": "github.com/Azure/go-autorest/autorest/to",
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
+ },
+ {
+ "checksumSHA1": "HjdLfAF3oA2In8F3FKh/Y+BPyXk=",
+ "path": "github.com/Azure/go-autorest/autorest/validation",
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
+ },
+ {
+ "checksumSHA1": "b2lrPJRxf+MEfmMafN40wepi5WM=",
+ "path": "github.com/Azure/go-autorest/logger",
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
+ },
+ {
+ "checksumSHA1": "UtAIMAsMWLBJ6yO1qZ0soFnb0sI=",
+ "path": "github.com/Azure/go-autorest/version",
+ "revision": "39013ecb48eaf6ced3f4e3e1d95515140ce6b3cf",
+ "revisionTime": "2018-08-09T20:19:59Z"
},
{
"checksumSHA1": "o/3cn04KAiwC7NqNVvmfVTD+hgA=",
@@ -90,6 +138,12 @@
"revisionTime": "2017-10-19T21:57:19Z"
},
{
+ "checksumSHA1": "7EjxkAUND/QY/sN+2fNKJ52v1Rc=",
+ "path": "github.com/dimchansky/utfbom",
+ "revision": "5448fe645cb1964ba70ac8f9f2ffe975e61a536c",
+ "revisionTime": "2018-07-13T13:37:17Z"
+ },
+ {
"checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
"path": "github.com/docker/distribution/digestset",
"revision": "277ed486c948042cab91ad367c379524f3b25e18",
@@ -535,6 +589,18 @@
"revisionTime": "2017-11-25T19:00:56Z"
},
{
+ "checksumSHA1": "PJY7uCr3UnX4/Mf/RoWnbieSZ8o=",
+ "path": "golang.org/x/crypto/pkcs12",
+ "revision": "614d502a4dac94afa3a6ce146bd1736da82514c6",
+ "revisionTime": "2018-07-28T08:01:47Z"
+ },
+ {
+ "checksumSHA1": "p0GC51McIdA7JygoP223twJ1s0E=",
+ "path": "golang.org/x/crypto/pkcs12/internal/rc2",
+ "revision": "614d502a4dac94afa3a6ce146bd1736da82514c6",
+ "revisionTime": "2018-07-28T08:01:47Z"
+ },
+ {
"checksumSHA1": "NHjGg73p5iGZ+7tflJ4cVABNmKE=",
"path": "golang.org/x/crypto/ssh",
"revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
commit acc46e4fdbc9098ac2b4c522b17473b58b087013
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Aug 20 09:41:16 2018 -0400
13964: Azure provider WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/dispatchcloud/azure.go b/lib/dispatchcloud/azure.go
new file mode 100644
index 000000000..e397731e3
--- /dev/null
+++ b/lib/dispatchcloud/azure.go
@@ -0,0 +1,290 @@
+package dispatchcloud
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
+ "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
+ "github.com/Azure/go-autorest/autorest/azure"
+ "github.com/Azure/go-autorest/autorest/azure/auth"
+ "github.com/Azure/go-autorest/autorest/to"
+)
+
+type AzureProviderConfig struct {
+ SubscriptionID string
+ ClientID string
+ ClientSecret string
+ TenantID string
+ CloudEnv string
+ ResourceGroup string
+ Location string
+ Subnet string
+ StorageAccount string
+ BlobContainer string
+}
+
+type VirtualMachinesClientWrapper interface {
+ CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ VMName string,
+ parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
+ Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
+ List(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultPage, err error)
+}
+
+type VirtualMachinesClientImpl struct {
+ inner compute.VirtualMachinesClient
+}
+
+func (cl *VirtualMachinesClientImpl) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ VMName string,
+ parameters VirtualMachine) (result compute.VirtualMachine, err error) {
+
+ future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
+ if err != nil {
+ return nil, err
+ }
+ future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.Result(cl.inner)
+}
+
+func (cl *VirtualMachinesClientImpl) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
+ if err != nil {
+ return nil, err
+ }
+ future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.GetResult()
+}
+
+func (cl *VirtualMachinesClientImpl) List(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultPage, err error) {
+ return cl.inner.List(ctx, resourceGroupName)
+}
+
+type InterfacesClientWrapper interface {
+ CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ networkInterfaceName string,
+ parameters network.Interface) (result network.Interface, err error)
+ Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
+ List(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultPage, err error)
+}
+
+type InterfacesClientImpl struct {
+ inner network.InterfacesClient
+}
+
+func (cl *InterfacesClientImpl) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
+ future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
+ if err != nil {
+ return nil, err
+ }
+ future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.GetResult()
+}
+
+func (cl *InterfacesClientImpl) CreateOrUpdate(ctx context.Context,
+ resourceGroupName string,
+ networkInterfaceName string,
+ parameters network.Interface) (result compute.VirtualMachine, err error) {
+
+ future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
+ if err != nil {
+ return nil, err
+ }
+ future.WaitForCompletionRef(ctx, cl.inner.Client)
+ return future.Result(cl.inner)
+}
+
+func (cl *InterfacesClientImpl) List(ctx context.Context, resourceGroupName string) (result compute.InterfaceListResultPage, err error) {
+ return cl.inner.List(ctx, resourceGroupName)
+}
+
+type AzureProvider struct {
+ config AzureProviderConfig
+ vmClient VirtualMachinesClientWrapper
+ netClient InterfacesClientWrapper
+ azureEnv auth.Environment
+}
+
+func (az *AzureProvider) Init(cfg AzureProviderConfig) error {
+ az.config = cfg
+ vmClient := compute.NewVirtualMachinesClient(az.config.SubscriptionId)
+ netClient := network.NewInterfacesClient(az.config.SubscriptionId)
+
+ az.azureEnv, err = azure.EnvironmentFromName(az.config.CloudEnv)
+ if err != nil {
+ return err
+ }
+
+ authorizer, err := auth.ClientCredentialsConfig{
+ ClientID: az.config.ClientID,
+ ClientSecret: az.config.ClientSecret,
+ TenantID: az.config.TenantID,
+ Resource: env.ResourceManagerEndpoint,
+ AADEndpoint: env.ActiveDirectoryEndpoint,
+ }.Authorizer()
+ if err != nil {
+ return err
+ }
+
+ vmClient.Authorizer = authorizer
+ netClient.Authorizer = authorizer
+
+ az.vmClient = VirtualMachinesClientImpl{vmClient}
+ az.netClient = InterfacesClientImpl{netClient}
+
+ return nil
+}
+
+func (az *AzureProvider) Create(ctx context.Context,
+ instanceType arvados.InstanceType,
+ imageId ImageID,
+ instanceTag []InstanceTag) (Instance, error) {
+
+ name := "randomname"
+
+ nicParameters := network.Interface{
+ Location: az.config.Location,
+ Tags: []map[string]string{
+ "arvados-class": "dynamic-compute",
+ "arvados-cluster": "",
+ },
+ InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
+ IPConfigurations: &[]network.InterfaceIPConfiguration{
+ network.InterfaceIPConfiguration{},
+ Name: "ip1",
+ InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
+ Subnet: &network.Subnet{
+ ID: az.config.Subnet,
+ },
+ PrivateIPAllocationMethod: network.Dynamic,
+ },
+ },
+ },
+ }
+ nic, err := az.netClient.CreateOrUpdate(ctx, az.config.ResourceGroup, name+"-nic", nicParameters)
+ if err != nil {
+ return nil, err
+ }
+
+ instance_vhd = fmt.Sprintf("https://%s.blob.%s/%s/%s-os.vhd",
+ az.config.StorageAccount,
+ az.azureEnv.StorageEndpointSuffix,
+ az.config.BlobContainer,
+ name)
+
+ vmParameters := compute.VirtualMachine{
+ Location: az.config.Location,
+ Tags: []map[string]string{
+ "arvados-class": "dynamic-compute",
+ "arvados-cluster": "",
+ },
+ VirtualMachineProperties: &compute.VirtualMachineProperties{
+ HardwareProfile: &compute.HardwareProfile{
+ VMSize: instanceType.ProviderType,
+ },
+ StorageProfile: &compute.StorageProfile{
+ OsDisk: &compute.OSDisk{
+ OsType: compute.Linux,
+ Name: "",
+ CreateOption: compute.FromImage,
+ Image: &compute.VirtualHardDisk{
+ URI: imageId,
+ },
+ Vhd: &compute.VirtualHardDisk{
+ URI: vhd,
+ },
+ },
+ },
+ NetworkProfile: &compute.NetworkProfile{
+ NetworkInterfaces: &[]compute.NetworkInterfaceReference{
+ compute.NetworkInterfaceReference{
+ ID: "",
+ NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
+ Primary: to.BoolPtr(true),
+ },
+ },
+ },
+ },
+ OsProfile: &compute.OSProfile{
+ LinuxConfiguration: &compute.LinuxConfiguration{
+ DisablePasswordAuthentication: true,
+ SSH: &compute.SSHConfiguration{
+ PublicKeys: &[]compute.SSHPublicKey{
+ compute.SSHPublicKey{
+ Path: "",
+ KeyData: "",
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ vm, err := az.vmClient.CreateOrUpdate(ctx, az.config.ResourceGroup, name+"-compute", vmParameters)
+ if err != nil {
+ return nil, err
+ }
+
+ return &AzureInstance{
+ instanceType: instanceType,
+ provider: az,
+ nic: nic,
+ vm: vm,
+ }
+}
+
+func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
+ result, err := az.vmClient.List(ctx, az.config.ResourceGroup)
+ if err != nil {
+ return nil, err
+ }
+ instances := make([]Instance)
+ for {
+ if result.NotDone() {
+ result.Next()
+ } else {
+ return instances, nil
+ }
+ result.Values
+ }
+}
+
+type AzureInstance struct {
+ instanceType arvados.InstanceType
+ provider *AzureProvider
+ nic network.Interface
+ vm compute.VirtualMachine
+}
+
+func (ai *AzureInstance) String() string {
+ return ai.vm.ID
+}
+
+func (ai *AzureInstance) ProviderType() string {
+ return ai.vm.VirtualMachineProperties.HardwareProfile.VMSize
+}
+
+func (ai *AzureInstance) InstanceType() arvados.InstanceType {
+ return ai.instanceType
+}
+
+func (ai *AzureInstance) SetTags([]InstanceTag) error {
+ return nil
+}
+
+func (ai *AzureInstance) Destroy(ctx context.Context) error {
+ response, err := ai.provider.vm.Delete(ctx, ai.provider.config.ResourceGroup, ai.vm.Name)
+ // check response code
+ return err
+}
+
+func (ai *AzureInstance) Address() string {
+ return ai.nic.IPConfigurations[0].PrivateIPAddress
+}
diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
new file mode 100644
index 000000000..aebb93011
--- /dev/null
+++ b/lib/dispatchcloud/provider.go
@@ -0,0 +1,31 @@
+package dispatchcloud
+
+import (
+ "context"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+type InstanceTag string
+type InstanceID string
+type ImageID string
+
+// instance is implemented by the provider-specific instance types.
+type Instance interface {
+ // String typically returns the cloud-provided instance ID.
+ String() string
+ // Cloud provider's "instance type" ID. Matches a key in
+ // configured arvados.InstanceTypeMap.
+ ProviderType() string
+ // Replace tags with the given tags
+ SetTags([]InstanceTag) error
+ // Shut down the node
+ Destroy(ctx context.Context) error
+ // SSH server hostname or IP address, or empty string if unknown pending creation.
+ Address() string
+}
+
+type Provider interface {
+ Create(arvados.InstanceType, ImageID, []InstanceTag) (Instance, error)
+ Instances() ([]Instance, error)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list