[ARVADOS-DEV] updated: 29caf4ee210e2b6780cb19a489efd439dd88bcfc
Git user
git at public.arvados.org
Thu May 7 01:30:57 UTC 2020
Summary of changes:
compute-image-cleaner/.gitignore | 1 +
compute-image-cleaner/Makefile | 11 ++
compute-image-cleaner/azure.go | 114 ++++++++++++++
compute-image-cleaner/compute-image-cleaner.go | 201 +++++++++++++++++++++++++
compute-image-cleaner/config/azure-config.go | 84 +++++++++++
compute-image-cleaner/go.mod | 24 +++
compute-image-cleaner/go.sum | 106 +++++++++++++
compute-image-cleaner/usage.go | 30 ++++
8 files changed, 571 insertions(+)
create mode 100644 compute-image-cleaner/.gitignore
create mode 100644 compute-image-cleaner/Makefile
create mode 100644 compute-image-cleaner/azure.go
create mode 100644 compute-image-cleaner/compute-image-cleaner.go
create mode 100644 compute-image-cleaner/config/azure-config.go
create mode 100644 compute-image-cleaner/go.mod
create mode 100644 compute-image-cleaner/go.sum
create mode 100644 compute-image-cleaner/usage.go
via 29caf4ee210e2b6780cb19a489efd439dd88bcfc (commit)
from 7e09d23c02f033536434067422dce8fa94bf05ea (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 29caf4ee210e2b6780cb19a489efd439dd88bcfc
Author: Ward Vandewege <ward at jhvc.com>
Date: Wed May 6 21:30:04 2020 -0400
Add a script to clean up old compute node images on Azure.
closes #16418
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at jhvc.com>
diff --git a/compute-image-cleaner/.gitignore b/compute-image-cleaner/.gitignore
new file mode 100644
index 0000000..b017886
--- /dev/null
+++ b/compute-image-cleaner/.gitignore
@@ -0,0 +1 @@
+compute-image-cleaner
diff --git a/compute-image-cleaner/Makefile b/compute-image-cleaner/Makefile
new file mode 100644
index 0000000..3b42bc0
--- /dev/null
+++ b/compute-image-cleaner/Makefile
@@ -0,0 +1,11 @@
+
+build:
+ @go build -ldflags "-s -w"
+
+lint:
+ @gofmt -s -w *go
+ @golint
+ @cd config/; golint
+ @golangci-lint run
+ @cd config/; golangci-lint run
+
diff --git a/compute-image-cleaner/azure.go b/compute-image-cleaner/azure.go
new file mode 100644
index 0000000..c42dfcc
--- /dev/null
+++ b/compute-image-cleaner/azure.go
@@ -0,0 +1,114 @@
+// Copyright (C) The Azure-Samples Authors. All rights reserved.
+//
+// SPDX-License-Identifier: MIT
+
+// Largely borrowed from
+// https://github.com/Azure-Samples/azure-sdk-for-go-samples/blob/master/internal/iam/authorizers.go
+
+package main
+
+import (
+ "fmt"
+ "log"
+
+ "github.com/arvados/arvados-dev/compute-image-cleaner/config"
+
+ "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-06-01/storage"
+
+ "github.com/Azure/go-autorest/autorest"
+ "github.com/Azure/go-autorest/autorest/adal"
+ "github.com/Azure/go-autorest/autorest/azure/auth"
+)
+
+// OAuthGrantType specifies which grant type to use.
+type OAuthGrantType int
+
+const (
+ // OAuthGrantTypeServicePrincipal for client credentials flow
+ OAuthGrantTypeServicePrincipal OAuthGrantType = iota
+ // OAuthGrantTypeDeviceFlow for device flow
+ OAuthGrantTypeDeviceFlow
+)
+
+var (
+ armAuthorizer autorest.Authorizer
+)
+
+// GrantType returns what grant type has been configured.
+func grantType() OAuthGrantType {
+ if config.UseDeviceFlow() {
+ return OAuthGrantTypeDeviceFlow
+ }
+ return OAuthGrantTypeServicePrincipal
+}
+
+func getAuthorizerForResource(grantType OAuthGrantType, resource string) (autorest.Authorizer, error) {
+ var a autorest.Authorizer
+ var err error
+
+ switch grantType {
+
+ case OAuthGrantTypeServicePrincipal:
+ oauthConfig, err := adal.NewOAuthConfig(
+ config.Environment().ActiveDirectoryEndpoint, config.TenantID())
+ if err != nil {
+ return nil, err
+ }
+
+ token, err := adal.NewServicePrincipalToken(
+ *oauthConfig, config.ClientID(), config.ClientSecret(), resource)
+ if err != nil {
+ return nil, err
+ }
+ a = autorest.NewBearerAuthorizer(token)
+
+ case OAuthGrantTypeDeviceFlow:
+ deviceconfig := auth.NewDeviceFlowConfig(config.ClientID(), config.TenantID())
+ deviceconfig.Resource = resource
+ a, err = deviceconfig.Authorizer()
+ if err != nil {
+ return nil, err
+ }
+
+ default:
+ return a, fmt.Errorf("invalid grant type specified")
+ }
+
+ return a, err
+}
+
+// GetResourceManagementAuthorizer gets an OAuthTokenAuthorizer for Azure Resource Manager
+func GetResourceManagementAuthorizer() (autorest.Authorizer, error) {
+ if armAuthorizer != nil {
+ return armAuthorizer, nil
+ }
+
+ var a autorest.Authorizer
+ var err error
+
+ a, err = getAuthorizerForResource(
+ grantType(), config.Environment().ResourceManagerEndpoint)
+
+ if err == nil {
+ // cache
+ armAuthorizer = a
+ } else {
+ // clear cache
+ armAuthorizer = nil
+ }
+ return armAuthorizer, err
+}
+
+func getStorageAccountsClient() storage.AccountsClient {
+ storageAccountsClient := storage.NewAccountsClient(config.SubscriptionID())
+ auth, err := GetResourceManagementAuthorizer()
+ if err != nil {
+ log.Fatal(err)
+ }
+ storageAccountsClient.Authorizer = auth
+ err = storageAccountsClient.AddToUserAgent("compute-image-cleaner")
+ if err != nil {
+ log.Fatal(err)
+ }
+ return storageAccountsClient
+}
diff --git a/compute-image-cleaner/compute-image-cleaner.go b/compute-image-cleaner/compute-image-cleaner.go
new file mode 100644
index 0000000..b812a38
--- /dev/null
+++ b/compute-image-cleaner/compute-image-cleaner.go
@@ -0,0 +1,201 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "net/url"
+ "os"
+ "regexp"
+ "sort"
+ "time"
+
+ "github.com/arvados/arvados-dev/compute-image-cleaner/config"
+
+ "github.com/Azure/azure-pipeline-go/pipeline"
+ "github.com/Azure/azure-storage-blob-go/azblob"
+
+ "code.cloudfoundry.org/bytefmt"
+)
+
+type blob struct {
+ name string
+ created time.Time
+ contentLength int64
+ deletionCandidate bool
+}
+
+func prepAzBlob(storageKey string, account string, container string) (p pipeline.Pipeline, containerURL azblob.ContainerURL) {
+ // Create a default request pipeline using your storage account name and account key.
+ credential, err := azblob.NewSharedKeyCredential(account, storageKey)
+ if err != nil {
+ log.Fatal("Invalid credentials with error: " + err.Error())
+ }
+ p = azblob.NewPipeline(credential, azblob.PipelineOptions{})
+ // From the Azure portal, get your storage account blob service URL endpoint.
+ URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", account, container))
+
+ // Create a ContainerURL object that wraps the container URL and a request
+ // pipeline to make requests.
+ containerURL = azblob.NewContainerURL(*URL, p)
+
+ return
+}
+
+func loadBlobs(p pipeline.Pipeline, containerURL azblob.ContainerURL) (blobs []blob, blobNames map[string]*blob) {
+ blobNames = make(map[string]*blob)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ for marker := (azblob.Marker{}); marker.NotDone(); {
+ // Get a result segment starting with the blob indicated by the current Marker.
+ listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+ if err != nil {
+ log.Fatal("Error getting blob list: " + err.Error())
+ }
+
+ // ListBlobs returns the start of the next segment; you MUST use this to get
+ // the next segment (after processing the current result segment).
+ marker = listBlob.NextMarker
+
+ // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute)
+ for _, blobInfo := range listBlob.Segment.BlobItems {
+ blobs = append(blobs, blob{name: blobInfo.Name, created: *blobInfo.Properties.CreationTime, contentLength: *blobInfo.Properties.ContentLength})
+ blobNames[blobInfo.Name] = &blobs[len(blobs)-1]
+ }
+ }
+ sort.Slice(blobs, func(i, j int) bool { return blobs[i].created.After(blobs[j].created) })
+
+ return
+}
+
+func weedBlobs(blobs []blob, blobNames map[string]*blob, containerURL azblob.ContainerURL, account string, container string, doIt bool) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ var pairedFileName string
+ skipCount := 10
+ t := time.Now()
+ thirtyDaysAgo := t.AddDate(0, 0, -30)
+
+ // e.g. su92l-compute-osDisk.866eb426-8d1e-45ad-91be-2bb55b5a8147.vhd
+ vhd := regexp.MustCompile(`^(.*)-compute-osDisk\.(.*)\.vhd$`)
+ // e.g. su92l-compute-vmTemplate.866eb426-8d1e-45ad-91be-2bb55b5a8147.json
+ json := regexp.MustCompile(`^(.*)-compute-vmTemplate\.(.*)\.json$`)
+
+ for i, blob := range blobs {
+ matches := vhd.FindStringSubmatch(blob.name)
+ if len(matches) > 1 {
+ // osDisk image file
+ pairedFileName = matches[1] + "-compute-vmTemplate." + matches[2] + ".json"
+ } else {
+ matches := json.FindStringSubmatch(blob.name)
+ if len(matches) > 1 {
+ // vmTemplate file
+ pairedFileName = matches[1] + "-compute-osDisk." + matches[2] + ".vhd"
+ } else {
+ log.Println("Skipping blob because name does not match a known file name pattern:", blob.name, " ", blob.created)
+ continue
+ }
+ }
+ if blob.created.After(thirtyDaysAgo) {
+ log.Println("Skipping blob because it was created less than 30 days ago:", blob.name, " ", blob.created)
+ skipCount = skipCount - 1
+ continue
+ }
+ if skipCount > 0 {
+ log.Println("Skipping blob because it's in the top 10 most recent list:", blob.name, " ", blob.created)
+ skipCount = skipCount - 1
+ continue
+ }
+ if _, ok := blobNames[pairedFileName]; !ok {
+ log.Println("Warning: paired file", pairedFileName, "not found for blob", blob.name, " ", blob.created)
+ }
+ blobs[i].deletionCandidate = true
+ }
+
+ var reclaimedSpace, otherSpace int64
+
+ for _, blob := range blobs {
+ if blob.deletionCandidate {
+ log.Println("Candidate for deletion:", blob.name, " ", blob.created)
+ reclaimedSpace = reclaimedSpace + blob.contentLength
+
+ if doIt {
+ log.Println("Deleting:", blob.name, " ", blob.created)
+ blockBlobURL := containerURL.NewBlockBlobURL(blob.name)
+ result, err := blockBlobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
+ if err != nil {
+ log.Println(result)
+ log.Fatal("Error deleting blob: ", err.Error(), "\n", result)
+ }
+ }
+ } else {
+ otherSpace = otherSpace + blob.contentLength
+ }
+ }
+
+ if doIt {
+ log.Println("Reclaimed", bytefmt.ByteSize(uint64(reclaimedSpace)), "or", reclaimedSpace, "bytes.")
+ } else {
+ log.Println("Deletion not requested. Able to reclaim", bytefmt.ByteSize(uint64(reclaimedSpace)), "or", reclaimedSpace, "bytes.")
+ }
+ log.Println("Kept", bytefmt.ByteSize(uint64(otherSpace)), "or", otherSpace, "bytes.")
+
+}
+
+func loadStorageAccountKey(resourceGroup string, account string) (key string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ storageClient := getStorageAccountsClient()
+ keys, err := storageClient.ListKeys(ctx, resourceGroup, account)
+ if err != nil {
+ log.Fatal("Error getting storage account key:", err.Error())
+ }
+
+ key = *(*keys.Keys)[0].Value
+
+ return
+}
+
+func validateInputs() (resourceGroup string, account string, container string, doIt bool) {
+ err := config.ParseEnvironment()
+ if err != nil {
+ log.Fatal("Unable to parse environment")
+ }
+
+ if config.ClientID() == "" || config.ClientSecret() == "" || config.TenantID() == "" || config.SubscriptionID() == "" {
+ log.Fatal("Please make sure the environment variables AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID and AZURE_SUBSCRIPTION_ID are set")
+ }
+
+ flags := flag.NewFlagSet("compute-image-cleaner", flag.ExitOnError)
+ flags.StringVar(&resourceGroup, "resourceGroup", "", "Name of the Azure resource group")
+ flags.StringVar(&account, "account", "", "Name of the Azure storage account")
+ flags.StringVar(&container, "container", "", "Name of the container in the Azure storage account")
+ flags.BoolVar(&doIt, "delete", false, "Delete blobs that meet criteria (default: false)")
+ flags.Usage = func() { usage(flags) }
+ err = flags.Parse(os.Args[1:])
+
+ if err != nil || resourceGroup == "" || account == "" || container == "" {
+ usage(flags)
+ os.Exit(1)
+ }
+
+ return
+}
+
+func main() {
+ resourceGroup, account, container, doIt := validateInputs()
+ storageKey := loadStorageAccountKey(resourceGroup, account)
+ p, containerURL := prepAzBlob(storageKey, account, container)
+
+ blobs, blobNames := loadBlobs(p, containerURL)
+ weedBlobs(blobs, blobNames, containerURL, account, container, doIt)
+}
diff --git a/compute-image-cleaner/config/azure-config.go b/compute-image-cleaner/config/azure-config.go
new file mode 100644
index 0000000..d411513
--- /dev/null
+++ b/compute-image-cleaner/config/azure-config.go
@@ -0,0 +1,84 @@
+// Copyright (C) The Azure-Samples Authors. All rights reserved.
+//
+// SPDX-License-Identifier: MIT
+
+// Largely borrowed from
+// https://github.com/Azure-Samples/azure-sdk-for-go-samples/tree/master/internal/config
+
+package config
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/Azure/go-autorest/autorest/azure"
+)
+
+var (
+ clientID string
+ clientSecret string
+ tenantID string
+ subscriptionID string
+ cloudName string = "AzurePublicCloud"
+ useDeviceFlow bool
+ environment *azure.Environment
+)
+
+// ClientID is the OAuth client ID.
+func ClientID() string {
+ return clientID
+}
+
+// ClientSecret is the OAuth client secret.
+func ClientSecret() string {
+ return clientSecret
+}
+
+// TenantID is the AAD tenant to which this client belongs.
+func TenantID() string {
+ return tenantID
+}
+
+// SubscriptionID is a target subscription for Azure resources.
+func SubscriptionID() string {
+ return subscriptionID
+}
+
+// UseDeviceFlow specifies if interactive auth should be used. Interactive
+// auth uses the OAuth Device Flow grant type.
+func UseDeviceFlow() bool {
+ return useDeviceFlow
+}
+
+// Environment returns an `azure.Environment{...}` for the current cloud.
+func Environment() *azure.Environment {
+ if environment != nil {
+ return environment
+ }
+ env, err := azure.EnvironmentFromName(cloudName)
+ if err != nil {
+ // TODO: move to initialization of var
+ panic(fmt.Sprintf(
+ "invalid cloud name '%s' specified, cannot continue\n", cloudName))
+ }
+ environment = &env
+ return environment
+}
+
+// ParseEnvironment loads the Azure environment variables for authentication
+func ParseEnvironment() error {
+ // these must be provided by environment
+ // clientID
+ clientID = os.Getenv("AZURE_CLIENT_ID")
+
+ // clientSecret
+ clientSecret = os.Getenv("AZURE_CLIENT_SECRET")
+
+ // tenantID (AAD)
+ tenantID = os.Getenv("AZURE_TENANT_ID")
+
+ // subscriptionID (ARM)
+ subscriptionID = os.Getenv("AZURE_SUBSCRIPTION_ID")
+
+ return nil
+}
diff --git a/compute-image-cleaner/go.mod b/compute-image-cleaner/go.mod
new file mode 100644
index 0000000..9330775
--- /dev/null
+++ b/compute-image-cleaner/go.mod
@@ -0,0 +1,24 @@
+module github.com/arvados/arvados-dev/compute-image-cleaner
+
+go 1.14
+
+require (
+ code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48
+ github.com/Azure/azure-pipeline-go v0.1.9
+ github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
+ github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc
+ github.com/Azure/go-autorest/autorest v0.10.1
+ github.com/Azure/go-autorest/autorest/adal v0.8.2
+ github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
+ github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/golang/protobuf v1.3.1 // indirect
+ github.com/kr/pretty v0.1.0 // indirect
+ github.com/onsi/ginkgo v1.12.0 // indirect
+ github.com/onsi/gomega v1.9.0 // indirect
+ github.com/pkg/errors v0.8.1 // indirect
+ github.com/stretchr/testify v1.5.1 // indirect
+ golang.org/x/net v0.0.0-20190520210107-018c4d40a106 // indirect
+ golang.org/x/text v0.3.2 // indirect
+ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+)
diff --git a/compute-image-cleaner/go.sum b/compute-image-cleaner/go.sum
new file mode 100644
index 0000000..0e8c43b
--- /dev/null
+++ b/compute-image-cleaner/go.sum
@@ -0,0 +1,106 @@
+code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE=
+code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
+github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg=
+github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc h1:BElWmFfsryQD72OcovStKpkIcd4e9ozSkdsTNQDSHGk=
+github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
+github.com/Azure/go-autorest/autorest v0.10.1 h1:uaB8A32IZU9YKs9v50+/LWIWTDHJk2vlGzbfd7FfESI=
+github.com/Azure/go-autorest/autorest v0.10.1/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
+github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk=
+github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM=
+github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U=
+github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4=
+github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI=
+github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190520210107-018c4d40a106 h1:EZofHp/BzEf3j39/+7CX1JvH0WaPG+ikBrqAdAPf+GM=
+golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/compute-image-cleaner/usage.go b/compute-image-cleaner/usage.go
new file mode 100644
index 0000000..13a6e4d
--- /dev/null
+++ b/compute-image-cleaner/usage.go
@@ -0,0 +1,30 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+)
+
+func usage(fs *flag.FlagSet) {
+ fmt.Fprintf(os.Stderr, `
+compute-image-cleaner removes old compute images from the specified storage account/container.
+
+The following environment variables must be set:
+
+ AZURE_TENANT_ID
+ AZURE_SUBSCRIPTION_ID
+ AZURE_CLIENT_ID
+ AZURE_CLIENT_SECRET
+
+For more information about those values and for instructions to create a service principal, see
+https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal
+
+Usage:
+`)
+ fs.PrintDefaults()
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list