[ARVADOS] updated: 1.3.0-396-g544ad965e
Git user
git at public.curoverse.com
Thu Feb 28 10:16:43 EST 2019
Summary of changes:
lib/cloud/ec2/ec2.go | 30 +++++++++++++++--
lib/cloud/ec2/ec2_test.go | 83 +++++++++++++++++++++++++++++++++++++++++++++--
sdk/go/arvados/config.go | 1 +
3 files changed, 110 insertions(+), 4 deletions(-)
via 544ad965e6d0a8f75268d2daea3a4c8c2748490c (commit)
via aa541480ba27f2dfec85cbe4f07c07743bb901b2 (commit)
from dc7fe787f4b6bae1ee6e740d8d5982e192c3fd75 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 544ad965e6d0a8f75268d2daea3a4c8c2748490c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Feb 28 10:16:22 2019 -0500
14291: Change instance type to a supported one, add SetTags test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index ab6c6b597..e76ec724c 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -58,7 +58,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
"tiny": arvados.InstanceType{
Name: "tiny",
- ProviderType: "m1.small",
+ ProviderType: "t2.micro",
VCPUs: 1,
RAM: 4000000000,
Scratch: 10000000000,
@@ -67,7 +67,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
},
"tiny-with-extra-scratch": arvados.InstanceType{
Name: "tiny",
- ProviderType: "m1.small",
+ ProviderType: "t2.micro",
VCPUs: 1,
RAM: 4000000000,
Scratch: 10000000000,
@@ -77,7 +77,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
},
"tiny-preemptible": arvados.InstanceType{
Name: "tiny",
- ProviderType: "m1.small",
+ ProviderType: "t2.micro",
VCPUs: 1,
RAM: 4000000000,
Scratch: 10000000000,
@@ -171,6 +171,22 @@ func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
}
+func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
+ ap, _, _, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ l, err := ap.Instances(nil)
+ c.Assert(err, check.IsNil)
+
+ for _, i := range l {
+ tg := i.Tags()
+ tg["TestTag2"] = "123 test tag 2"
+ c.Check(i.SetTags(tg), check.IsNil)
+ }
+}
+
func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
ap, _, _, err := GetInstanceSet()
if err != nil {
commit aa541480ba27f2dfec85cbe4f07c07743bb901b2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Feb 27 16:38:33 2019 -0500
14291: Support EBS attached storage and preemptible instances
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 115c1a0c0..664c51cd4 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -7,6 +7,7 @@ package ec2
import (
"encoding/base64"
"encoding/json"
+ "fmt"
"log"
"strings"
@@ -83,6 +84,10 @@ func (instanceSet *ec2InstanceSet) Create(
Key: aws.String(ARVADOS_DISPATCH_ID),
Value: aws.String(string(instanceSet.dispatcherID)),
},
+ &ec2.Tag{
+ Key: aws.String("arvados-class"),
+ Value: aws.String("dynamic-compute"),
+ },
}
for k, v := range newTags {
ec2tags = append(ec2tags, &ec2.Tag{
@@ -91,7 +96,7 @@ func (instanceSet *ec2InstanceSet) Create(
})
}
- rsv, err := instanceSet.client.RunInstances(&ec2.RunInstancesInput{
+ rii := ec2.RunInstancesInput{
ImageId: aws.String(string(imageID)),
InstanceType: &instanceType.ProviderType,
MaxCount: aws.Int64(1),
@@ -114,7 +119,28 @@ func (instanceSet *ec2InstanceSet) Create(
ResourceType: aws.String("instance"),
Tags: ec2tags,
}},
- })
+ }
+
+ if instanceType.ExtraScratch > 0 {
+ rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{&ec2.BlockDeviceMapping{
+ DeviceName: aws.String("/dev/xvdt"),
+ Ebs: &ec2.EbsBlockDevice{
+ DeleteOnTermination: aws.Bool(true),
+ VolumeSize: aws.Int64((int64(instanceType.ExtraScratch) / 1000000000) + 1),
+ VolumeType: aws.String("gp2"),
+ }}}
+ }
+
+ if instanceType.Preemptible {
+ rii.InstanceMarketOptions = &ec2.InstanceMarketOptionsRequest{
+ MarketType: aws.String("spot"),
+ SpotOptions: &ec2.SpotMarketOptions{
+ InstanceInterruptionBehavior: aws.String("terminate"),
+ MaxPrice: aws.String(fmt.Sprintf("%v", instanceType.Price)),
+ }}
+ }
+
+ rsv, err := instanceSet.client.RunInstances(&rii)
if err != nil {
return nil, err
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index cd469af1d..ab6c6b597 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -65,6 +65,25 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
Price: .02,
Preemptible: false,
},
+ "tiny-with-extra-scratch": arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "m1.small",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ ExtraScratch: 20000000000,
+ Price: .02,
+ Preemptible: false,
+ },
+ "tiny-preemptible": arvados.InstanceType{
+ Name: "tiny",
+ ProviderType: "m1.small",
+ VCPUs: 1,
+ RAM: 4000000000,
+ Scratch: 10000000000,
+ Price: .02,
+ Preemptible: true,
+ },
})}
if *live != "" {
var exampleCfg testConfig
@@ -108,10 +127,54 @@ func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
}
+func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
+ ap, img, cluster, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ c.Assert(err, check.IsNil)
+
+ inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
+ img, map[string]string{
+ "TestTagName": "test tag value",
+ }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
+
+ c.Assert(err, check.IsNil)
+
+ tags := inst.Tags()
+ c.Check(tags["TestTagName"], check.Equals, "test tag value")
+ c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
+
+}
+
+func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
+ ap, img, cluster, err := GetInstanceSet()
+ if err != nil {
+ c.Fatal("Error making provider", err)
+ }
+
+ pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+ c.Assert(err, check.IsNil)
+
+ inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
+ img, map[string]string{
+ "TestTagName": "test tag value",
+ }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
+
+ c.Assert(err, check.IsNil)
+
+ tags := inst.Tags()
+ c.Check(tags["TestTagName"], check.Equals, "test tag value")
+ c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
+
+}
+
func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
ap, _, _, err := GetInstanceSet()
if err != nil {
- c.Fatal("Error making provider", err)
+ c.Fatal("Error making provider: ", err)
}
l, err := ap.Instances(nil)
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index c2154d0f2..5f6235cf0 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -99,6 +99,7 @@ type InstanceType struct {
VCPUs int
RAM ByteSize
Scratch ByteSize
+ ExtraScratch ByteSize
Price float64
Preemptible bool
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list