[ARVADOS] created: 1.3.0-389-g676423e29
Git user
git at public.curoverse.com
Tue Feb 26 17:54:23 EST 2019
at 676423e298857c91d794dd9c6016b861304d45b1 (commit)
commit 676423e298857c91d794dd9c6016b861304d45b1
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Feb 26 17:54:08 2019 -0500
14291: EC2 driver WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
new file mode 100644
index 000000000..e4e8588d5
--- /dev/null
+++ b/lib/cloud/ec2/ec2.go
@@ -0,0 +1,217 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package ec2
+
+import (
+ "encoding/json"
+ "strings"
+
+ "git.curoverse.com/arvados.git/lib/cloud"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/sirupsen/logrus"
+ "golang.org/x/crypto/ssh"
+)
+
+const ARVADOS_DISPATCH_ID = "arvados-dispatch-id"
+const TAG_PREFIX = "disispatch-"
+
+// Driver is the ec2 implementation of the cloud.Driver interface.
+var Driver = cloud.DriverFunc(newEC2InstanceSet)
+
+type ec2InstanceSetConfig struct {
+ AccessKeyID string
+ SecretAccessKey string
+ Region string
+ SecurityGroupId string
+ SubnetId string
+ AdminUsername string
+ KeyPairName string
+}
+
+type ec2InstanceSet struct {
+ ec2config ec2InstanceSetConfig
+ dispatcherID cloud.InstanceSetID
+ logger logrus.FieldLogger
+ client *ec2.EC2
+ importedKey bool
+}
+
+func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
+ instanceSet := &ec2InstanceSet{
+ dispatcherID: dispatcherID,
+ logger: logger,
+ }
+ err = json.Unmarshal(config, &instanceSet.ec2config)
+ if err != nil {
+ return nil, err
+ }
+ awsConfig := aws.NewConfig().
+ WithCredentials(credentials.NewStaticCredentials(
+ instanceSet.ec2config.AccessKeyID,
+ instanceSet.ec2config.SecretAccessKey,
+ "")).
+ WithRegion(instanceSet.ec2config.Region)
+ instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
+ return instanceSet, nil
+}
+
+func (instanceSet *ec2InstanceSet) Create(
+ instanceType arvados.InstanceType,
+ imageID cloud.ImageID,
+ newTags cloud.InstanceTags,
+ initCommand cloud.InitCommand,
+ publicKey ssh.PublicKey) (cloud.Instance, error) {
+
+ if !instanceSet.importedKey {
+ instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+ KeyName: &instanceSet.ec2config.KeyPairName,
+ PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+ })
+ instanceSet.importedKey = true
+ }
+
+ ec2tags := []*ec2.Tag{
+ &ec2.Tag{
+ Key: aws.String(ARVADOS_DISPATCH_ID),
+ Value: aws.String(string(instanceSet.dispatcherID)),
+ },
+ }
+ for k, v := range newTags {
+ ec2tags = append(ec2tags, &ec2.Tag{
+ Key: aws.String(TAG_PREFIX + k),
+ Value: aws.String(v),
+ })
+ }
+
+ rsv, err := instanceSet.client.RunInstances(&ec2.RunInstancesInput{
+ ImageId: aws.String(string(imageID)),
+ InstanceType: &instanceType.ProviderType,
+ MaxCount: aws.Int64(1),
+ MinCount: aws.Int64(1),
+ KeyName: &instanceSet.ec2config.KeyPairName,
+ SecurityGroupIds: []*string{&instanceSet.ec2config.SecurityGroupId},
+ SubnetId: &instanceSet.ec2config.SubnetId,
+ TagSpecifications: []*ec2.TagSpecification{
+ &ec2.TagSpecification{
+ ResourceType: aws.String("instance"),
+ Tags: ec2tags,
+ }},
+ })
+
+ if err != nil {
+ return nil, err
+ }
+
+ return &ec2Instance{
+ provider: instanceSet,
+ instance: rsv.Instances[0],
+ }, nil
+}
+
+func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cloud.Instance, err error) {
+ dii := &ec2.DescribeInstancesInput{
+ Filters: []*ec2.Filter{&ec2.Filter{
+ Name: aws.String("tag:" + ARVADOS_DISPATCH_ID),
+ Values: []*string{aws.String(string(instanceSet.dispatcherID))},
+ }}}
+
+ for {
+ dio, err := instanceSet.client.DescribeInstances(dii)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, rsv := range dio.Reservations {
+ for _, inst := range rsv.Instances {
+ instances = append(instances, &ec2Instance{instanceSet, inst})
+ }
+ }
+ if dio.NextToken == nil {
+ return instances, err
+ }
+ dii.NextToken = dio.NextToken
+ }
+}
+
+func (az *ec2InstanceSet) Stop() {
+}
+
+type ec2Instance struct {
+ provider *ec2InstanceSet
+ instance *ec2.Instance
+}
+
+func (inst *ec2Instance) ID() cloud.InstanceID {
+ return cloud.InstanceID(*inst.instance.InstanceId)
+}
+
+func (inst *ec2Instance) String() string {
+ return *inst.instance.InstanceId
+}
+
+func (inst *ec2Instance) ProviderType() string {
+ return *inst.instance.InstanceType
+}
+
+func (inst *ec2Instance) SetTags(newTags cloud.InstanceTags) error {
+ ec2tags := []*ec2.Tag{
+ &ec2.Tag{
+ Key: aws.String(ARVADOS_DISPATCH_ID),
+ Value: aws.String(string(inst.provider.dispatcherID)),
+ },
+ }
+ for k, v := range newTags {
+ ec2tags = append(ec2tags, &ec2.Tag{
+ Key: aws.String(TAG_PREFIX + k),
+ Value: aws.String(v),
+ })
+ }
+
+ _, err := inst.provider.client.CreateTags(&ec2.CreateTagsInput{
+ Resources: []*string{inst.instance.InstanceId},
+ Tags: ec2tags,
+ })
+
+ return err
+}
+
+func (inst *ec2Instance) Tags() cloud.InstanceTags {
+ tags := make(map[string]string)
+
+ for _, t := range inst.instance.Tags {
+ if strings.HasPrefix(*t.Key, TAG_PREFIX) {
+ tags[(*t.Key)[len(TAG_PREFIX):]] = *t.Value
+ }
+ }
+
+ return tags
+}
+
+func (inst *ec2Instance) Destroy() error {
+ _, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
+ InstanceIds: []*string{inst.instance.InstanceId},
+ })
+ return err
+}
+
+func (inst *ec2Instance) Address() string {
+ if inst.instance.PrivateIpAddress != nil {
+ return *inst.instance.PrivateIpAddress
+ } else {
+ return ""
+ }
+}
+
+func (inst *ec2Instance) RemoteUser() string {
+ return inst.provider.ec2config.AdminUsername
+}
+
+func (inst *ec2Instance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
+ return cloud.ErrNotImplemented
+}
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
new file mode 100644
index 000000000..c0f72215e
--- /dev/null
+++ b/lib/cloud/ec2/ec2_test.go
@@ -0,0 +1,138 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+//
+//
+// How to manually run individual tests against the real cloud:
+//
+// $ go test -v git.curoverse.com/arvados.git/lib/cloud/ec2 -live-ec2-cfg ec2config.yml -check.f=TestCreate
+//
+// Tests should be run individually and in the order they are listed in the file:
+//
+// Example azconfig.yml:
+//
+// ImageIDForTestSuite: ami-xxxxxxxxxxxxxxxxx
+// DriverParameters:
+// AccessKeyID: XXXXXXXXXXXXXX
+// SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
+// Region: us-east-1
+// SecurityGroupId: sg-xxxxxxxx
+// SubnetId: subnet-xxxxxxxx
+// AdminUsername: crunch
+
+package ec2
+
+import (
+ "encoding/json"
+ "flag"
+ "log"
+ "testing"
+
+ "git.curoverse.com/arvados.git/lib/cloud"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
+ "github.com/sirupsen/logrus"
+ "golang.org/x/crypto/ssh"
+ check "gopkg.in/check.v1"
+)
+
+var live = flag.String("live-ec2-cfg", "", "Test with real EC2 API, provide config file")
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+type EC2InstanceSetSuite struct{}
+
+var _ = check.Suite(&EC2InstanceSetSuite{})
+
+type testConfig struct {
+ ImageIDForTestSuite string
+ DriverParameters json.RawMessage
+}
+
+func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
+ cluster := arvados.Cluster{
+ InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
+ "tiny": arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "m1.small",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ })}
+ if *live != "" {
+ var exampleCfg testConfig
+ err := config.LoadFile(&exampleCfg, *live)
+ if err != nil {
+ return nil, cloud.ImageID(""), cluster, err
+ }
+
+ ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", logrus.StandardLogger())
+ return ap, cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, err
+ }
+ ap := ec2InstanceSet{
+ ec2config: ec2InstanceSetConfig{},
+ dispatcherID: "test123",
+ logger: logrus.StandardLogger(),
+ }
+ return &ap, cloud.ImageID("blob"), cluster, nil
+}
+
+var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
+
+func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
+ ap, img, cluster, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ c.Assert(err, check.IsNil)
+
+ inst, err := ap.Create(cluster.InstanceTypes["tiny"],
+ img, map[string]string{
+ "TestTagName": "test tag value",
+ }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
+
+ c.Assert(err, check.IsNil)
+
+ tags := inst.Tags()
+ c.Check(tags["TestTagName"], check.Equals, "test tag value")
+ c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
+
+}
+
+func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(nil)
+
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ tg := i.Tags()
+ log.Printf("%v %v %v", i.String(), i.Address(), tg)
+ }
+}
+
+func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(nil)
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ c.Check(i.Destroy(), check.IsNil)
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list