[ARVADOS] created: 2.1.0-183-gbbb12c3c1

Git user git at public.arvados.org
Sat Dec 5 21:38:09 UTC 2020


        at  bbb12c3c1600b88c3d5bc9852883c39935f6a70a (commit)


commit bbb12c3c1600b88c3d5bc9852883c39935f6a70a
Author: Ward Vandewege <ward at curii.com>
Date:   Sat Dec 5 16:37:00 2020 -0500

    17187: costanalyzer: add support for collection uuids.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 4284542b8..75cbdc037 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -62,12 +62,18 @@ Usage:
 	each container.
 
 	When supplied with the uuid of a container request, it will calculate the
-	cost of that container request and all its children. When suplied with a
-	project uuid or when supplied with multiple container request uuids, it will
-	create a CSV report for each supplied uuid, as well as a CSV file with
-	aggregate cost accounting for all supplied uuids. The aggregate cost report
-	takes container reuse into account: if a container was reused between several
-	container requests, its cost will only be counted once.
+	cost of that container request and all its children.
+
+	When supplied with the uuid of a collection, it will see if there is a
+	container_request uuid in the properties of the collection, and if so, it
+	will calculate the cost of that container request and all its children.
+
+	When supplied with a project uuid or when supplied with multiple container
+	request or collection uuids, it will create a CSV report for each supplied
+	uuid, as well as a CSV file with aggregate cost accounting for all supplied
+	uuids. The aggregate cost report takes container reuse into account: if a
+	container was reused between several container requests, its cost will only
+	be counted once.
 
 	To get the node costs, the progam queries the Arvados API for current cost
 	data for each node type used. This means that the reported cost always
@@ -180,8 +186,8 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container
 
 func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
 	reload = true
-	if strings.Contains(uuid, "-j7d0g-") {
-		// We do not cache projects, they have no final state
+	if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+		// We do not cache projects or collections, they have no final state
 		return
 	}
 	// See if we have a cached copy of this object
@@ -251,6 +257,8 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
 		err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
 	} else if strings.Contains(uuid, "-dz642-") {
 		err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+	} else if strings.Contains(uuid, "-4zz18-") {
+		err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
 	} else {
 		err = fmt.Errorf("unsupported object type with UUID %q:\n  %s", uuid, err)
 		return
@@ -309,7 +317,6 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien
 }
 
 func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
-
 	cost = make(map[string]float64)
 
 	var project arvados.Group
@@ -366,9 +373,24 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 	var tmpTotalCost float64
 	var totalCost float64
 
+	var crUUID = uuid
+	if strings.Contains(uuid, "-4zz18-") {
+		// This is a collection, find the associated container request (if any)
+		var c arvados.Collection
+		err = loadObject(logger, ac, uuid, uuid, cache, &c)
+		if err != nil {
+			return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+		}
+		value, ok := c.Properties["container_request"]
+		if !ok {
+			return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+		}
+		crUUID = value.(string)
+	}
+
 	// This is a container request, find the container
 	var cr arvados.ContainerRequest
-	err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+	err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
 	if err != nil {
 		return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
 	}
@@ -474,12 +496,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
 			for k, v := range cost {
 				cost[k] = v
 			}
-		} else if strings.Contains(uuid, "-xvhdp-") {
+		} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
 			// This is a container request
 			var crCsv map[string]float64
 			crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
 			if err != nil {
-				err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+				err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
 				exitcode = 2
 				return
 			}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 4fab93bf4..03bf39d1c 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -177,6 +177,45 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
 	c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
 }
 
+func (*Suite) TestCollectionUUID(c *check.C) {
+	var stdout, stderr bytes.Buffer
+
+	// Run costanalyzer with 1 collection uuid, without 'container_request' property
+	exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 2)
+	c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+	// Update the collection, attach a 'container_request' property
+	ac := arvados.NewClientFromEnv()
+	var coll arvados.Collection
+
+	// Update collection record
+	err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{
+		"collection": map[string]interface{}{
+			"properties": map[string]interface{}{
+				"container_request": arvadostest.CompletedContainerRequestUUID,
+			},
+		},
+	})
+	c.Assert(err, check.IsNil)
+
+	// Run costanalyzer with 1 collection uuid
+	exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 0)
+	c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+	uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+	c.Assert(err, check.IsNil)
+	c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+	re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+	matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+	aggregateCostReport, err := ioutil.ReadFile(matches[1])
+	c.Assert(err, check.IsNil)
+
+	c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
 func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
 	var stdout, stderr bytes.Buffer
 	// Run costanalyzer with 2 container request uuids

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list