[ARVADOS] updated: 1.3.0-410-gdc6c729a9

Git user git at public.curoverse.com
Thu Mar 7 16:27:47 EST 2019


Summary of changes:
 build/run-tests.sh            |  1 +
 lib/cloud/azure/azure_test.go |  5 +--
 lib/cloud/ec2/ec2.go          | 87 +++++++++++++++++++++++++++++++++++++------
 lib/cloud/ec2/ec2_test.go     | 17 +++++----
 sdk/go/arvados/config.go      |  6 ++-
 5 files changed, 91 insertions(+), 25 deletions(-)

       via  dc6c729a9e738d876642f2153e1fedd05878245a (commit)
       via  218866a5bb6d048576b8351515ea41c1e87eb43b (commit)
       via  62edb031350261966eec5dc9c5daa96ecc92765c (commit)
       via  ba5d3129f5f20124f9eb826a9f6b02c942c30945 (commit)
      from  1d347ca5163a984345b2541a434d04baa3ca8777 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit dc6c729a9e738d876642f2153e1fedd05878245a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Mar 7 16:27:18 2019 -0500

    14291: Generate key fingerprints that work with AWS.  Filter out terminated instances
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 464d42a36..d97099367 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -5,9 +5,14 @@
 package ec2
 
 import (
+	"crypto/md5"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"math/big"
 	"strings"
 	"sync"
 
@@ -38,6 +43,7 @@ type ec2InstanceSetConfig struct {
 }
 
 type ec2Interface interface {
+	DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error)
 	ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error)
 	RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error)
 	DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error)
@@ -77,6 +83,39 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
 	return instanceSet, nil
 }
 
+func awsKeyFingerprint(pk ssh.PublicKey) (md5fp string, sha1fp string, err error) {
+	// AWS key fingerprints don't use the usual key fingerprint
+	// you get from ssh-keygen or ssh.FingerprintLegacyMD5()
+	// (you can get that from md5.Sum(pk.Marshal())
+	//
+	// AWS uses the md5 or sha1 of the PKIX DER encoding of the
+	// public key, so calculate those fingerprints here.
+	var rsaPub struct {
+		Name string
+		E    *big.Int
+		N    *big.Int
+	}
+	if err := ssh.Unmarshal(pk.Marshal(), &rsaPub); err != nil {
+		return "", "", fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err)
+	}
+	rsaPk := rsa.PublicKey{
+		E: int(rsaPub.E.Int64()),
+		N: rsaPub.N,
+	}
+	pkix, _ := x509.MarshalPKIXPublicKey(&rsaPk)
+	md5pkix := md5.Sum([]byte(pkix))
+	sha1pkix := sha1.Sum([]byte(pkix))
+	md5fp = ""
+	sha1fp = ""
+	for i := 0; i < len(md5pkix); i += 1 {
+		md5fp += fmt.Sprintf(":%02x", md5pkix[i])
+	}
+	for i := 0; i < len(sha1pkix); i += 1 {
+		sha1fp += fmt.Sprintf(":%02x", sha1pkix[i])
+	}
+	return md5fp[1:], sha1fp[1:], nil
+}
+
 func (instanceSet *ec2InstanceSet) Create(
 	instanceType arvados.InstanceType,
 	imageID cloud.ImageID,
@@ -84,20 +123,37 @@ func (instanceSet *ec2InstanceSet) Create(
 	initCommand cloud.InitCommand,
 	publicKey ssh.PublicKey) (cloud.Instance, error) {
 
-	keyFingerprint := ssh.FingerprintSHA256(publicKey)
+	md5keyFingerprint, sha1keyFingerprint, err := awsKeyFingerprint(publicKey)
+	if err != nil {
+		return nil, fmt.Errorf("Could not make key fingerprint: %v", err)
+	}
 	instanceSet.keysMtx.Lock()
 	var keyname string
 	var ok bool
-	if keyname, ok = instanceSet.keys[keyFingerprint]; !ok {
-		keyname = "arvados-dispatch-keypair-" + keyFingerprint
-		_, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
-			KeyName:           &keyname,
-			PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+	if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok {
+		keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{
+			Filters: []*ec2.Filter{&ec2.Filter{
+				Name:   aws.String("fingerprint"),
+				Values: []*string{&md5keyFingerprint, &sha1keyFingerprint},
+			}},
 		})
 		if err != nil {
-			return nil, fmt.Errorf("Could not import keypair: %v", err)
+			return nil, fmt.Errorf("Could not search for keypair: %v", err)
+		}
+
+		if len(keyout.KeyPairs) > 0 {
+			keyname = *(keyout.KeyPairs[0].KeyName)
+		} else {
+			keyname = "arvados-dispatch-keypair-" + md5keyFingerprint
+			_, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+				KeyName:           &keyname,
+				PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+			})
+			if err != nil {
+				return nil, fmt.Errorf("Could not import keypair: %v", err)
+			}
 		}
-		instanceSet.keys[keyFingerprint] = keyname
+		instanceSet.keys[md5keyFingerprint] = keyname
 	}
 	instanceSet.keysMtx.Unlock()
 
@@ -189,7 +245,9 @@ func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cl
 
 		for _, rsv := range dio.Reservations {
 			for _, inst := range rsv.Instances {
-				instances = append(instances, &ec2Instance{instanceSet, inst})
+				if *inst.State.Name != "terminated" {
+					instances = append(instances, &ec2Instance{instanceSet, inst})
+				}
 			}
 		}
 		if dio.NextToken == nil {
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 3138cba3d..50ba01174 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -60,6 +60,10 @@ func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPa
 	return nil, nil
 }
 
+func (e *ec2stub) DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
+	return &ec2.DescribeKeyPairsOutput{}, nil
+}
+
 func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
 	return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{
 		InstanceId: aws.String("i-123"),

commit 218866a5bb6d048576b8351515ea41c1e87eb43b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Mar 7 14:16:40 2019 -0500

    14291: Add EbsVolumeType, also test fixes
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index caaca1f31..095d32eaa 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -933,6 +933,7 @@ gostuff=(
     lib/crunchstat
     lib/cloud
     lib/cloud/azure
+    lib/cloud/ec2
     lib/dispatchcloud
     lib/dispatchcloud/container
     lib/dispatchcloud/scheduler
diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index 61649c398..bd82a424f 100644
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -43,6 +43,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/lib/cloud"
+	"git.curoverse.com/arvados.git/lib/dispatchcloud/test"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/config"
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
@@ -67,8 +68,6 @@ var _ = check.Suite(&AzureInstanceSetSuite{})
 
 type VirtualMachinesClientStub struct{}
 
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
-
 func (*VirtualMachinesClientStub) createOrUpdate(ctx context.Context,
 	resourceGroupName string,
 	VMName string,
@@ -157,7 +156,7 @@ func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
 		c.Fatal("Error making provider", err)
 	}
 
-	pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+	pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 	c.Assert(err, check.IsNil)
 
 	inst, err := ap.Create(cluster.InstanceTypes["tiny"],
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 35153b3ff..464d42a36 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -8,7 +8,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 	"sync"
 
@@ -35,6 +34,7 @@ type ec2InstanceSetConfig struct {
 	SecurityGroupIDs []string
 	SubnetID         string
 	AdminUsername    string
+	EbsVolumeType    string
 }
 
 type ec2Interface interface {
@@ -71,6 +71,9 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
 		WithRegion(instanceSet.ec2config.Region)
 	instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
 	instanceSet.keys = make(map[string]string)
+	if instanceSet.ec2config.EbsVolumeType == "" {
+		instanceSet.ec2config.EbsVolumeType = "gp2"
+	}
 	return instanceSet, nil
 }
 
@@ -145,8 +148,8 @@ func (instanceSet *ec2InstanceSet) Create(
 			DeviceName: aws.String("/dev/xvdt"),
 			Ebs: &ec2.EbsBlockDevice{
 				DeleteOnTermination: aws.Bool(true),
-				VolumeSize:          aws.Int64((int64(instanceType.AddedScratch) / 1000000000) + 1),
-				VolumeType:          aws.String("gp2"),
+				VolumeSize:          aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
+				VolumeType:          &instanceSet.ec2config.EbsVolumeType,
 			}}}
 	}
 
@@ -251,7 +254,6 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags {
 }
 
 func (inst *ec2Instance) Destroy() error {
-	log.Printf("terminating %v", *inst.instance.InstanceId)
 	_, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
 		InstanceIds: []*string{inst.instance.InstanceId},
 	})
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index ba65758cc..3138cba3d 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -25,16 +25,15 @@ package ec2
 import (
 	"encoding/json"
 	"flag"
-	"log"
 	"testing"
 
 	"git.curoverse.com/arvados.git/lib/cloud"
+	"git.curoverse.com/arvados.git/lib/dispatchcloud/test"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/config"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/ssh"
 	check "gopkg.in/check.v1"
 )
 
@@ -131,15 +130,13 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
 	return &ap, cloud.ImageID("blob"), cluster, nil
 }
 
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
-
 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 	ap, img, cluster, err := GetInstanceSet()
 	if err != nil {
 		c.Fatal("Error making provider", err)
 	}
 
-	pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+	pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 	c.Assert(err, check.IsNil)
 
 	inst, err := ap.Create(cluster.InstanceTypes["tiny"],
@@ -161,7 +158,7 @@ func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 		c.Fatal("Error making provider", err)
 	}
 
-	pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+	pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 	c.Assert(err, check.IsNil)
 
 	inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
@@ -183,7 +180,7 @@ func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
 		c.Fatal("Error making provider", err)
 	}
 
-	pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+	pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 	c.Assert(err, check.IsNil)
 
 	inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
@@ -227,7 +224,7 @@ func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 
 	for _, i := range l {
 		tg := i.Tags()
-		log.Printf("%v %v %v", i.String(), i.Address(), tg)
+		c.Logf("%v %v %v", i.String(), i.Address(), tg)
 	}
 }
 

commit 62edb031350261966eec5dc9c5daa96ecc92765c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Mar 7 14:16:11 2019 -0500

    14291: Tweak Scratch/AddedScratch/IncludedScratch behavior
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b0ada5c92..f16f98a94 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -181,10 +181,12 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 			}
 			if t.Scratch == 0 {
 				t.Scratch = t.IncludedScratch + t.AddedScratch
-			}
-			if (t.Scratch - t.IncludedScratch) > t.AddedScratch {
+			} else if t.AddedScratch == 0 {
 				t.AddedScratch = t.Scratch - t.IncludedScratch
+			} else if t.IncludedScratch == 0 {
+				t.IncludedScratch = t.Scratch - t.AddedScratch
 			}
+
 			if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 				return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 			}

commit ba5d3129f5f20124f9eb826a9f6b02c942c30945
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Mar 7 11:43:05 2019 -0500

    14291: Report errors from ImportKeyPair
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 1dd398db9..35153b3ff 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -87,10 +87,13 @@ func (instanceSet *ec2InstanceSet) Create(
 	var ok bool
 	if keyname, ok = instanceSet.keys[keyFingerprint]; !ok {
 		keyname = "arvados-dispatch-keypair-" + keyFingerprint
-		instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+		_, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
 			KeyName:           &keyname,
 			PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
 		})
+		if err != nil {
+			return nil, fmt.Errorf("Could not import keypair: %v", err)
+		}
 		instanceSet.keys[keyFingerprint] = keyname
 	}
 	instanceSet.keysMtx.Unlock()

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list