[ARVADOS] updated: 1.3.0-401-g3246e20c2
Git user
git at public.curoverse.com
Wed Mar 6 15:40:49 EST 2019
Summary of changes:
lib/cloud/ec2/ec2.go | 24 ++++++++++++++++--------
lib/cloud/ec2/ec2_test.go | 21 ++++++++++-----------
sdk/go/arvados/config.go | 26 ++++++++++++++++++--------
3 files changed, 44 insertions(+), 27 deletions(-)
via 3246e20c201aedc9fd448ab99d7d0950654e218b (commit)
from 85cd30dffafaba80483e28ad6d4cac12b49a6d09 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3246e20c201aedc9fd448ab99d7d0950654e218b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Mar 6 15:39:44 2019 -0500
14291: Introduce "AddedScratch" and "IncludedScratch" to InstanceType
EC2 driver imports public keys on the fly.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index b8a360e19..1dd398db9 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -10,6 +10,7 @@ import (
"fmt"
"log"
"strings"
+ "sync"
"git.curoverse.com/arvados.git/lib/cloud"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -34,7 +35,6 @@ type ec2InstanceSetConfig struct {
SecurityGroupIDs []string
SubnetID string
AdminUsername string
- KeyPairName string
}
type ec2Interface interface {
@@ -50,7 +50,8 @@ type ec2InstanceSet struct {
dispatcherID cloud.InstanceSetID
logger logrus.FieldLogger
client ec2Interface
- importedKey bool
+ keysMtx sync.Mutex
+ keys map[string]string
}
func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
@@ -69,6 +70,7 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
"")).
WithRegion(instanceSet.ec2config.Region)
instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
+ instanceSet.keys = make(map[string]string)
return instanceSet, nil
}
@@ -79,13 +81,19 @@ func (instanceSet *ec2InstanceSet) Create(
initCommand cloud.InitCommand,
publicKey ssh.PublicKey) (cloud.Instance, error) {
- if !instanceSet.importedKey {
+ keyFingerprint := ssh.FingerprintSHA256(publicKey)
+ instanceSet.keysMtx.Lock()
+ var keyname string
+ var ok bool
+ if keyname, ok = instanceSet.keys[keyFingerprint]; !ok {
+ keyname = "arvados-dispatch-keypair-" + keyFingerprint
instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
- KeyName: &instanceSet.ec2config.KeyPairName,
+ KeyName: &keyname,
PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
})
- instanceSet.importedKey = true
+ instanceSet.keys[keyFingerprint] = keyname
}
+ instanceSet.keysMtx.Unlock()
ec2tags := []*ec2.Tag{
&ec2.Tag{
@@ -109,7 +117,7 @@ func (instanceSet *ec2InstanceSet) Create(
InstanceType: &instanceType.ProviderType,
MaxCount: aws.Int64(1),
MinCount: aws.Int64(1),
- KeyName: &instanceSet.ec2config.KeyPairName,
+ KeyName: &keyname,
NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
&ec2.InstanceNetworkInterfaceSpecification{
@@ -129,12 +137,12 @@ func (instanceSet *ec2InstanceSet) Create(
}},
}
- if instanceType.AttachScratch {
+ if instanceType.AddedScratch > 0 {
rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{&ec2.BlockDeviceMapping{
DeviceName: aws.String("/dev/xvdt"),
Ebs: &ec2.EbsBlockDevice{
DeleteOnTermination: aws.Bool(true),
- VolumeSize: aws.Int64((int64(instanceType.Scratch) / 1000000000) + 1),
+ VolumeSize: aws.Int64((int64(instanceType.AddedScratch) / 1000000000) + 1),
VolumeType: aws.String("gp2"),
}}}
}
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index f5352d121..ba65758cc 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -16,10 +16,9 @@
// AccessKeyID: XXXXXXXXXXXXXX
// SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
// Region: us-east-1
-// SecurityGroupId: sg-xxxxxxxx
-// SubnetId: subnet-xxxxxxxx
+// SecurityGroupIDs: [sg-xxxxxxxx]
+// SubnetID: subnet-xxxxxxxx
// AdminUsername: crunch
-// KeyPairName: arvados-dispatcher-keypair
package ec2
@@ -94,14 +93,13 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
Preemptible: false,
},
"tiny-with-extra-scratch": arvados.InstanceType{
- Name: "tiny",
- ProviderType: "t2.micro",
- VCPUs: 1,
- RAM: 4000000000,
- Scratch: 20000000000,
- Price: .02,
- Preemptible: false,
- AttachScratch: true,
+ Name: "tiny",
+ ProviderType: "t2.micro",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Price: .02,
+ Preemptible: false,
+ AddedScratch: 20000000000,
},
"tiny-preemptible": arvados.InstanceType{
Name: "tiny",
@@ -128,6 +126,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
dispatcherID: "test123",
logger: logrus.StandardLogger(),
client: &ec2stub{},
+ keys: make(map[string]string),
}
return &ap, cloud.ImageID("blob"), cluster, nil
}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index fddf5100a..b0ada5c92 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -94,14 +94,15 @@ type RemoteCluster struct {
}
type InstanceType struct {
- Name string
- ProviderType string
- VCPUs int
- RAM ByteSize
- Scratch ByteSize
- Price float64
- Preemptible bool
- AttachScratch bool
+ Name string
+ ProviderType string
+ VCPUs int
+ RAM ByteSize
+ Scratch ByteSize
+ IncludedScratch ByteSize
+ AddedScratch ByteSize
+ Price float64
+ Preemptible bool
}
type Dispatch struct {
@@ -178,6 +179,15 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
if t.ProviderType == "" {
t.ProviderType = t.Name
}
+ if t.Scratch == 0 {
+ t.Scratch = t.IncludedScratch + t.AddedScratch
+ }
+ if (t.Scratch - t.IncludedScratch) > t.AddedScratch {
+ t.AddedScratch = t.Scratch - t.IncludedScratch
+ }
+ if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
+ return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
+ }
(*it)[t.Name] = t
}
return nil
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list