[ARVADOS] updated: 1.3.0-410-gdc6c729a9
Git user
git at public.curoverse.com
Thu Mar 7 16:27:47 EST 2019
Summary of changes:
build/run-tests.sh | 1 +
lib/cloud/azure/azure_test.go | 5 +--
lib/cloud/ec2/ec2.go | 87 +++++++++++++++++++++++++++++++++++++------
lib/cloud/ec2/ec2_test.go | 17 +++++----
sdk/go/arvados/config.go | 6 ++-
5 files changed, 91 insertions(+), 25 deletions(-)
via dc6c729a9e738d876642f2153e1fedd05878245a (commit)
via 218866a5bb6d048576b8351515ea41c1e87eb43b (commit)
via 62edb031350261966eec5dc9c5daa96ecc92765c (commit)
via ba5d3129f5f20124f9eb826a9f6b02c942c30945 (commit)
from 1d347ca5163a984345b2541a434d04baa3ca8777 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit dc6c729a9e738d876642f2153e1fedd05878245a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Mar 7 16:27:18 2019 -0500
14291: Generate key fingerprints that work with AWS. Filter out terminated instances
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 464d42a36..d97099367 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -5,9 +5,14 @@
package ec2
import (
+ "crypto/md5"
+ "crypto/rsa"
+ "crypto/sha1"
+ "crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
+ "math/big"
"strings"
"sync"
@@ -38,6 +43,7 @@ type ec2InstanceSetConfig struct {
}
type ec2Interface interface {
+ DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error)
ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error)
RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error)
DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error)
@@ -77,6 +83,39 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
return instanceSet, nil
}
+func awsKeyFingerprint(pk ssh.PublicKey) (md5fp string, sha1fp string, err error) {
+ // AWS key fingerprints don't use the usual key fingerprint
+ // you get from ssh-keygen or ssh.FingerprintLegacyMD5()
+ // (you can get that from md5.Sum(pk.Marshal())
+ //
+ // AWS uses the md5 or sha1 of the PKIX DER encoding of the
+ // public key, so calculate those fingerprints here.
+ var rsaPub struct {
+ Name string
+ E *big.Int
+ N *big.Int
+ }
+ if err := ssh.Unmarshal(pk.Marshal(), &rsaPub); err != nil {
+ return "", "", fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err)
+ }
+ rsaPk := rsa.PublicKey{
+ E: int(rsaPub.E.Int64()),
+ N: rsaPub.N,
+ }
+ pkix, _ := x509.MarshalPKIXPublicKey(&rsaPk)
+ md5pkix := md5.Sum([]byte(pkix))
+ sha1pkix := sha1.Sum([]byte(pkix))
+ md5fp = ""
+ sha1fp = ""
+ for i := 0; i < len(md5pkix); i += 1 {
+ md5fp += fmt.Sprintf(":%02x", md5pkix[i])
+ }
+ for i := 0; i < len(sha1pkix); i += 1 {
+ sha1fp += fmt.Sprintf(":%02x", sha1pkix[i])
+ }
+ return md5fp[1:], sha1fp[1:], nil
+}
+
func (instanceSet *ec2InstanceSet) Create(
instanceType arvados.InstanceType,
imageID cloud.ImageID,
@@ -84,20 +123,37 @@ func (instanceSet *ec2InstanceSet) Create(
initCommand cloud.InitCommand,
publicKey ssh.PublicKey) (cloud.Instance, error) {
- keyFingerprint := ssh.FingerprintSHA256(publicKey)
+ md5keyFingerprint, sha1keyFingerprint, err := awsKeyFingerprint(publicKey)
+ if err != nil {
+ return nil, fmt.Errorf("Could not make key fingerprint: %v", err)
+ }
instanceSet.keysMtx.Lock()
var keyname string
var ok bool
- if keyname, ok = instanceSet.keys[keyFingerprint]; !ok {
- keyname = "arvados-dispatch-keypair-" + keyFingerprint
- _, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
- KeyName: &keyname,
- PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+ if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok {
+ keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{
+ Filters: []*ec2.Filter{&ec2.Filter{
+ Name: aws.String("fingerprint"),
+ Values: []*string{&md5keyFingerprint, &sha1keyFingerprint},
+ }},
})
if err != nil {
- return nil, fmt.Errorf("Could not import keypair: %v", err)
+ return nil, fmt.Errorf("Could not search for keypair: %v", err)
+ }
+
+ if len(keyout.KeyPairs) > 0 {
+ keyname = *(keyout.KeyPairs[0].KeyName)
+ } else {
+ keyname = "arvados-dispatch-keypair-" + md5keyFingerprint
+ _, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+ KeyName: &keyname,
+ PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("Could not import keypair: %v", err)
+ }
}
- instanceSet.keys[keyFingerprint] = keyname
+ instanceSet.keys[md5keyFingerprint] = keyname
}
instanceSet.keysMtx.Unlock()
@@ -189,7 +245,9 @@ func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cl
for _, rsv := range dio.Reservations {
for _, inst := range rsv.Instances {
- instances = append(instances, &ec2Instance{instanceSet, inst})
+ if *inst.State.Name != "terminated" {
+ instances = append(instances, &ec2Instance{instanceSet, inst})
+ }
}
}
if dio.NextToken == nil {
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 3138cba3d..50ba01174 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -60,6 +60,10 @@ func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPa
return nil, nil
}
+func (e *ec2stub) DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
+ return &ec2.DescribeKeyPairsOutput{}, nil
+}
+
func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{
InstanceId: aws.String("i-123"),
commit 218866a5bb6d048576b8351515ea41c1e87eb43b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Mar 7 14:16:40 2019 -0500
14291: Add EbsVolumeType, also test fixes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index caaca1f31..095d32eaa 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -933,6 +933,7 @@ gostuff=(
lib/crunchstat
lib/cloud
lib/cloud/azure
+ lib/cloud/ec2
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index 61649c398..bd82a424f 100644
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -43,6 +43,7 @@ import (
"time"
"git.curoverse.com/arvados.git/lib/cloud"
+ "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/config"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
@@ -67,8 +68,6 @@ var _ = check.Suite(&AzureInstanceSetSuite{})
type VirtualMachinesClientStub struct{}
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
-
func (*VirtualMachinesClientStub) createOrUpdate(ctx context.Context,
resourceGroupName string,
VMName string,
@@ -157,7 +156,7 @@ func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
c.Fatal("Error making provider", err)
}
- pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
c.Assert(err, check.IsNil)
inst, err := ap.Create(cluster.InstanceTypes["tiny"],
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 35153b3ff..464d42a36 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -8,7 +8,6 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
- "log"
"strings"
"sync"
@@ -35,6 +34,7 @@ type ec2InstanceSetConfig struct {
SecurityGroupIDs []string
SubnetID string
AdminUsername string
+ EbsVolumeType string
}
type ec2Interface interface {
@@ -71,6 +71,9 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
WithRegion(instanceSet.ec2config.Region)
instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
instanceSet.keys = make(map[string]string)
+ if instanceSet.ec2config.EbsVolumeType == "" {
+ instanceSet.ec2config.EbsVolumeType = "gp2"
+ }
return instanceSet, nil
}
@@ -145,8 +148,8 @@ func (instanceSet *ec2InstanceSet) Create(
DeviceName: aws.String("/dev/xvdt"),
Ebs: &ec2.EbsBlockDevice{
DeleteOnTermination: aws.Bool(true),
- VolumeSize: aws.Int64((int64(instanceType.AddedScratch) / 1000000000) + 1),
- VolumeType: aws.String("gp2"),
+ VolumeSize: aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
+ VolumeType: &instanceSet.ec2config.EbsVolumeType,
}}}
}
@@ -251,7 +254,6 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags {
}
func (inst *ec2Instance) Destroy() error {
- log.Printf("terminating %v", *inst.instance.InstanceId)
_, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{inst.instance.InstanceId},
})
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index ba65758cc..3138cba3d 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -25,16 +25,15 @@ package ec2
import (
"encoding/json"
"flag"
- "log"
"testing"
"git.curoverse.com/arvados.git/lib/cloud"
+ "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/config"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/sirupsen/logrus"
- "golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
)
@@ -131,15 +130,13 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
return &ap, cloud.ImageID("blob"), cluster, nil
}
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
-
func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
ap, img, cluster, err := GetInstanceSet()
if err != nil {
c.Fatal("Error making provider", err)
}
- pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
c.Assert(err, check.IsNil)
inst, err := ap.Create(cluster.InstanceTypes["tiny"],
@@ -161,7 +158,7 @@ func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
c.Fatal("Error making provider", err)
}
- pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
c.Assert(err, check.IsNil)
inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
@@ -183,7 +180,7 @@ func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
c.Fatal("Error making provider", err)
}
- pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
c.Assert(err, check.IsNil)
inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
@@ -227,7 +224,7 @@ func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
for _, i := range l {
tg := i.Tags()
- log.Printf("%v %v %v", i.String(), i.Address(), tg)
+ c.Logf("%v %v %v", i.String(), i.Address(), tg)
}
}
commit 62edb031350261966eec5dc9c5daa96ecc92765c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Mar 7 14:16:11 2019 -0500
14291: Tweak Scratch/AddedScratch/IncludedScratch behavior
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b0ada5c92..f16f98a94 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -181,10 +181,12 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
}
if t.Scratch == 0 {
t.Scratch = t.IncludedScratch + t.AddedScratch
- }
- if (t.Scratch - t.IncludedScratch) > t.AddedScratch {
+ } else if t.AddedScratch == 0 {
t.AddedScratch = t.Scratch - t.IncludedScratch
+ } else if t.IncludedScratch == 0 {
+ t.IncludedScratch = t.Scratch - t.AddedScratch
}
+
if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
}
commit ba5d3129f5f20124f9eb826a9f6b02c942c30945
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Mar 7 11:43:05 2019 -0500
14291: Report errors from ImportKeyPair
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 1dd398db9..35153b3ff 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -87,10 +87,13 @@ func (instanceSet *ec2InstanceSet) Create(
var ok bool
if keyname, ok = instanceSet.keys[keyFingerprint]; !ok {
keyname = "arvados-dispatch-keypair-" + keyFingerprint
- instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+ _, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
KeyName: &keyname,
PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
})
+ if err != nil {
+ return nil, fmt.Errorf("Could not import keypair: %v", err)
+ }
instanceSet.keys[keyFingerprint] = keyname
}
instanceSet.keysMtx.Unlock()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list