[ARVADOS] updated: 1.3.0-314-g11ba2964a
Git user
git at public.curoverse.com
Mon Feb 11 14:59:22 EST 2019
Summary of changes:
lib/cloud/azure/azure.go | 55 ++++++++++++++++++--------------
lib/cloud/azure/azure_test.go | 74 +++++++++++++++++++++++--------------------
sdk/go/arvados/duration.go | 16 +++++++++-
3 files changed, 86 insertions(+), 59 deletions(-)
via 11ba2964a6cc44bd5c02fbb511bf413e52d03774 (commit)
via 15a809ab5d5cd892b7f3097f82ba271890721cd1 (commit)
via 3fc84235dd2ce76eaf7ec140ac1754b5370305bb (commit)
from d38793d3af2b0563196607a4e52bf07737ec55df (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 11ba2964a6cc44bd5c02fbb511bf413e52d03774
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Mon Feb 11 14:59:15 2019 -0500
14745: Makes DeleteDanglingResourcesAfter an arvados.Duration
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <eric.biagiotti at gmail.com>
diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go
index 2a63d28f6..be7aa3cf4 100644
--- a/lib/cloud/azure/azure.go
+++ b/lib/cloud/azure/azure.go
@@ -32,18 +32,18 @@ import (
)
type AzureInstanceSetConfig struct {
- SubscriptionID string "SubscriptionID"
- ClientID string "ClientID"
- ClientSecret string "ClientSecret"
- TenantID string "TenantID"
- CloudEnvironment string "CloudEnvironment"
- ResourceGroup string "ResourceGroup"
- Location string "Location"
- Network string "Network"
- Subnet string "Subnet"
- StorageAccount string "StorageAccount"
- BlobContainer string "BlobContainer"
- DeleteDanglingResourcesAfter float64 "DeleteDanglingResourcesAfter"
+ SubscriptionID string
+ ClientID string
+ ClientSecret string
+ TenantID string
+ CloudEnvironment string
+ ResourceGroup string
+ Location string
+ Network string
+ Subnet string
+ StorageAccount string
+ BlobContainer string
+ DeleteDanglingResourcesAfter arvados.Duration
}
type VirtualMachinesClientWrapper interface {
@@ -204,9 +204,19 @@ type AzureInstanceSet struct {
func NewAzureInstanceSet(config map[string]interface{}, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
azcfg := AzureInstanceSetConfig{}
- if err = mapstructure.Decode(config, &azcfg); err != nil {
+
+ decoderConfig := mapstructure.DecoderConfig{
+ DecodeHook: arvados.DurationMapStructureDecodeHook(),
+ Result: &azcfg}
+
+ decoder, err := mapstructure.NewDecoder(&decoderConfig)
+ if err != nil {
return nil, err
}
+ if err = decoder.Decode(config); err != nil {
+ return nil, err
+ }
+
ap := AzureInstanceSet{logger: logger}
err = ap.setup(azcfg, string(dispatcherID))
if err != nil {
@@ -489,7 +499,7 @@ func (az *AzureInstanceSet) ManageNics() (map[string]network.Interface, error) {
if result.Value().Tags["created-at"] != nil {
createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
if err == nil {
- if timestamp.Sub(createdAt).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ if timestamp.Sub(createdAt).Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
az.logger.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
az.deleteNIC <- *result.Value().Name
}
@@ -537,7 +547,7 @@ func (az *AzureInstanceSet) ManageBlobs() {
if b.Properties.BlobType == storage.BlobTypePage &&
b.Properties.LeaseState == "available" &&
b.Properties.LeaseStatus == "unlocked" &&
- age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
az.logger.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
az.deleteBlob <- b
diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index dfd9b95e0..e5e8056a2 100644
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -21,7 +21,7 @@
// Subnet: zzzzz-subnet-private
// StorageAccount: example
// BlobContainer: vhds
-// DeleteDanglingResourcesAfter: 20
+// DeleteDanglingResourcesAfter: 20s
package azure
diff --git a/sdk/go/arvados/duration.go b/sdk/go/arvados/duration.go
index d2a19a024..d7dff6ddc 100644
--- a/sdk/go/arvados/duration.go
+++ b/sdk/go/arvados/duration.go
@@ -7,6 +7,7 @@ package arvados
import (
"encoding/json"
"fmt"
+ "reflect"
"time"
)
@@ -37,9 +38,22 @@ func (d Duration) Duration() time.Duration {
return time.Duration(d)
}
-// Value implements flag.Value
+// Set sets the current duration by parsing the string using time.ParseDuration
func (d *Duration) Set(s string) error {
dur, err := time.ParseDuration(s)
*d = Duration(dur)
return err
}
+
+// DurationMapStructureDecodeHook can be used to create a decoder for arvados.duration when using mapstructure
+func DurationMapStructureDecodeHook() interface{} {
+ return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+ var duration Duration
+ if f.Kind() != reflect.String || t != reflect.TypeOf(duration) {
+ return data, nil
+ }
+
+ duration.Set(data.(string))
+ return duration, nil
+ }
+}
commit 15a809ab5d5cd892b7f3097f82ba271890721cd1
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Mon Feb 11 11:07:28 2019 -0500
14745: Updates config keys, documentation, and addresses golint suggestions
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <eric.biagiotti at gmail.com>
diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go
index 5bc7ccfd4..2a63d28f6 100644
--- a/lib/cloud/azure/azure.go
+++ b/lib/cloud/azure/azure.go
@@ -32,19 +32,18 @@ import (
)
type AzureInstanceSetConfig struct {
- SubscriptionID string `mapstructure:"subscription_id"`
- ClientID string `mapstructure:"key"`
- ClientSecret string `mapstructure:"secret"`
- TenantID string `mapstructure:"tenant_id"`
- CloudEnv string `mapstructure:"cloud_environment"`
- ResourceGroup string `mapstructure:"resource_group"`
- Location string `mapstructure:"region"`
- Network string `mapstructure:"network"`
- Subnet string `mapstructure:"subnet"`
- StorageAccount string `mapstructure:"storage_account"`
- BlobContainer string `mapstructure:"blob_container"`
- Image string `mapstructure:"image"`
- DeleteDanglingResourcesAfter float64 `mapstructure:"delete_dangling_resources_after"`
+ SubscriptionID string "SubscriptionID"
+ ClientID string "ClientID"
+ ClientSecret string "ClientSecret"
+ TenantID string "TenantID"
+ CloudEnvironment string "CloudEnvironment"
+ ResourceGroup string "ResourceGroup"
+ Location string "Location"
+ Network string "Network"
+ Subnet string "Subnet"
+ StorageAccount string "StorageAccount"
+ BlobContainer string "BlobContainer"
+ DeleteDanglingResourcesAfter float64 "DeleteDanglingResourcesAfter"
}
type VirtualMachinesClientWrapper interface {
@@ -222,7 +221,7 @@ func (az *AzureInstanceSet) setup(azcfg AzureInstanceSetConfig, dispatcherID str
netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
- az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnv)
+ az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment)
if err != nil {
return err
}
@@ -269,7 +268,7 @@ func (az *AzureInstanceSet) setup(azcfg AzureInstanceSetConfig, dispatcherID str
az.deleteNIC = make(chan string)
az.deleteBlob = make(chan storage.Blob)
- for i := 0; i < 4; i += 1 {
+ for i := 0; i < 4; i++ {
go func() {
for {
nicname, ok := <-az.deleteNIC
@@ -305,7 +304,7 @@ func (az *AzureInstanceSet) setup(azcfg AzureInstanceSetConfig, dispatcherID str
func (az *AzureInstanceSet) Create(
instanceType arvados.InstanceType,
- imageId cloud.ImageID,
+ imageID cloud.ImageID,
newTags cloud.InstanceTags,
publicKey ssh.PublicKey) (cloud.Instance, error) {
@@ -361,7 +360,7 @@ func (az *AzureInstanceSet) Create(
return nil, WrapAzureError(err)
}
- instance_vhd := fmt.Sprintf("https://%s.blob.%s/%s/%s-os.vhd",
+ instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s-os.vhd",
az.azconfig.StorageAccount,
az.azureEnv.StorageEndpointSuffix,
az.azconfig.BlobContainer,
@@ -383,10 +382,10 @@ echo '%s-%s' > /home/crunch/node-token`, name, newTags["node-token"])))
Name: to.StringPtr(name + "-os"),
CreateOption: compute.FromImage,
Image: &compute.VirtualHardDisk{
- URI: to.StringPtr(string(imageId)),
+ URI: to.StringPtr(string(imageID)),
},
Vhd: &compute.VirtualHardDisk{
- URI: &instance_vhd,
+ URI: &instanceVhd,
},
},
},
@@ -488,9 +487,9 @@ func (az *AzureInstanceSet) ManageNics() (map[string]network.Interface, error) {
interfaces[*result.Value().ID] = result.Value()
} else {
if result.Value().Tags["created-at"] != nil {
- created_at, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
+ createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
if err == nil {
- if timestamp.Sub(created_at).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
+ if timestamp.Sub(createdAt).Seconds() > az.azconfig.DeleteDanglingResourcesAfter {
az.logger.Printf("Will delete %v because it is older than %v s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
az.deleteNIC <- *result.Value().Name
}
diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index e924aa610..dfd9b95e0 100644
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -5,24 +5,23 @@
//
// How to manually run individual tests against the real cloud
//
-// $ go test -v git.curoverse.com/arvados.git/lib/cloud -live-azure-cfg azconfig.yml -check.f=TestListInstances
+// $ go test -v git.curoverse.com/arvados.git/lib/cloud/azure -live-azure-cfg azconfig.yml -check.f=TestListInstances
//
// Example azconfig.yml:
//
-// subscription_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-// key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-// region: centralus
-// cloud_environment: AzurePublicCloud
-// secret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-// tenant_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-// resource_group: zzzzz
-// network: zzzzz
-// subnet: zzzzz-subnet-private
-// storage_account: example
-// blob_container: vhds
-// image: "https://example.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX.vhd"
-// delete_dangling_resources_after: 20
-// authorized_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example"
+// ImageIDForTestSuite: "https://example.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX.vhd"
+// SubscriptionID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+// ClientID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+// Location: centralus
+// CloudEnvironment: AzurePublicCloud
+// ClientSecret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+// TenantId: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+// ResourceGroup: zzzzz
+// Network: zzzzz0:10 / 3:26:1
+// Subnet: zzzzz-subnet-private
+// StorageAccount: example
+// BlobContainer: vhds
+// DeleteDanglingResourcesAfter: 20
package azure
@@ -64,7 +63,7 @@ var _ = check.Suite(&AzureInstanceSetSuite{})
type VirtualMachinesClientStub struct{}
-var testKey []byte = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
+var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example at example`)
func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
resourceGroupName string,
@@ -118,29 +117,28 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
},
})}
if *live != "" {
- cfg := make(map[string]interface{})
- err := config.LoadFile(&cfg, *live)
+ exampleCfg := make(map[string]interface{})
+ err := config.LoadFile(&exampleCfg, *live)
if err != nil {
return nil, cloud.ImageID(""), cluster, err
}
- ap, err := NewAzureInstanceSet(cfg, "test123", logrus.StandardLogger())
- return ap, cloud.ImageID(cfg["image"].(string)), cluster, err
- } else {
- ap := AzureInstanceSet{
- azconfig: AzureInstanceSetConfig{
- BlobContainer: "vhds",
- },
- dispatcherID: "test123",
- namePrefix: "compute-test123-",
- logger: logrus.StandardLogger(),
- deleteNIC: make(chan string),
- deleteBlob: make(chan storage.Blob),
- }
- ap.ctx, ap.stopFunc = context.WithCancel(context.Background())
- ap.vmClient = &VirtualMachinesClientStub{}
- ap.netClient = &InterfacesClientStub{}
- return &ap, cloud.ImageID("blob"), cluster, nil
+ ap, err := NewAzureInstanceSet(exampleCfg, "test123", logrus.StandardLogger())
+ return ap, cloud.ImageID(exampleCfg["ImageIDForTestSuite"].(string)), cluster, err
+ }
+ ap := AzureInstanceSet{
+ azconfig: AzureInstanceSetConfig{
+ BlobContainer: "vhds",
+ },
+ dispatcherID: "test123",
+ namePrefix: "compute-test123-",
+ logger: logrus.StandardLogger(),
+ deleteNIC: make(chan string),
+ deleteBlob: make(chan storage.Blob),
}
+ ap.ctx, ap.stopFunc = context.WithCancel(context.Background())
+ ap.vmClient = &VirtualMachinesClientStub{}
+ ap.netClient = &InterfacesClientStub{}
+ return &ap, cloud.ImageID("blob"), cluster, nil
}
func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
commit 3fc84235dd2ce76eaf7ec140ac1754b5370305bb
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Mon Feb 11 09:58:11 2019 -0500
14745: Adds gocheck hook to go test for azure dirver
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <eric.biagiotti at gmail.com>
diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index b496d3b3f..e924aa610 100644
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -35,6 +35,7 @@ import (
"net"
"net/http"
"os"
+ "testing"
"time"
"git.curoverse.com/arvados.git/lib/cloud"
@@ -52,6 +53,11 @@ import (
check "gopkg.in/check.v1"
)
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
type AzureInstanceSetSuite struct{}
var _ = check.Suite(&AzureInstanceSetSuite{})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list