[ARVADOS] updated: 1.3.0-3259-g9adb8ea50
Git user
git at public.arvados.org
Tue Nov 3 22:00:46 UTC 2020
Summary of changes:
lib/costanalyzer/costanalyzer_test.go | 2 +-
services/api/test/fixtures/collections.yml | 5 +++++
2 files changed, 6 insertions(+), 1 deletion(-)
discards 64cbc85cf36c5dc6731cc8854ae09b2c887695f0 (commit)
via 9adb8ea50fcf555c7ec73cb0924c869af2345f86 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (64cbc85cf36c5dc6731cc8854ae09b2c887695f0)
\
N -- N -- N (9adb8ea50fcf555c7ec73cb0924c869af2345f86)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9adb8ea50fcf555c7ec73cb0924c869af2345f86
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 3 16:34:16 2020 -0500
16950: add tests for the costanalyzer.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/command.go b/lib/costanalyzer/command.go
index 3cca16ea0..0760b4fd2 100644
--- a/lib/costanalyzer/command.go
+++ b/lib/costanalyzer/command.go
@@ -22,7 +22,7 @@ func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
return []byte(entry.Message), nil
}
-// RunCommand implements the subcommand "deduplication-report <collection> <collection> ..."
+// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index d754e8875..c86e26769 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -119,7 +119,9 @@ Usage:
provider.
- when generating reports for older container requests, the cost data in the
Arvados API configuration file may have changed since the container request
- was fulfilled.
+ was fulfilled. This program uses the cost data stored at the time of the
+ execution of the container, stored in the 'node.json' file in its log
+ collection.
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
@@ -133,7 +135,7 @@ Options:
flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
err := flags.Parse(args)
if err == flag.ErrHelp {
- exitCode = 0
+ exitCode = 1
return
} else if err != nil {
exitCode = 2
@@ -156,20 +158,21 @@ Options:
return
}
-func ensureDirectory(logger *logrus.Logger, dir string) {
+func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
statData, err := os.Stat(dir)
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0700)
if err != nil {
logger.Errorf("Error creating directory %s: %s\n", dir, err.Error())
- os.Exit(1)
+ return
}
} else {
if !statData.IsDir() {
logger.Errorf("The path %s is not a directory\n", dir)
- os.Exit(1)
+ return
}
}
+ return
}
func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) {
@@ -245,9 +248,11 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload b
}
// Load an Arvados object.
-func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict) {
-
- ensureDirectory(logger, path)
+func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict, err error) {
+ err = ensureDirectory(logger, path)
+ if err != nil {
+ return
+ }
file := path + "/" + uuid + ".json"
@@ -256,9 +261,7 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
if reload {
var err error
- if strings.Contains(uuid, "-d1hrv-") {
- err = arv.Get("pipeline_instances", uuid, nil, &object)
- } else if strings.Contains(uuid, "-j7d0g-") {
+ if strings.Contains(uuid, "-j7d0g-") {
err = arv.Get("groups", uuid, nil, &object)
} else if strings.Contains(uuid, "-xvhdp-") {
err = arv.Get("container_requests", uuid, nil, &object)
@@ -268,24 +271,21 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
err = arv.Get("jobs", uuid, nil, &object)
}
if err != nil {
- logger.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err)
- os.Exit(1)
+ logger.Fatalf("Error loading object with UUID %q:\n %s\n", uuid, err)
}
encoded, err := json.MarshalIndent(object, "", " ")
if err != nil {
- logger.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
- os.Exit(1)
+ logger.Fatalf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
}
err = ioutil.WriteFile(file, encoded, 0644)
if err != nil {
- logger.Errorf("Error writing file %s:\n %s\n", file, err)
- os.Exit(1)
+ logger.Fatalf("Error writing file %s:\n %s\n", file, err)
}
}
return
}
-func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
+func getNode(arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
if _, ok := itemMap["log_uuid"]; ok {
if itemMap["log_uuid"] == nil {
err = errors.New("No log collection")
@@ -295,29 +295,28 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
var collection arvados.Collection
err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection)
if err != nil {
- logger.Errorf("error getting collection: %s\n", err)
+ err = fmt.Errorf("Error getting collection: %s", err)
return
}
var fs arvados.CollectionFileSystem
fs, err = collection.FileSystem(arv2, kc)
if err != nil {
- logger.Errorf("error opening collection as filesystem: %s\n", err)
+ err = fmt.Errorf("Error opening collection as filesystem: %s", err)
return
}
var f http.File
f, err = fs.Open("node.json")
if err != nil {
- logger.Errorf("error opening file in collection: %s\n", err)
+ err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
return
}
var nodeDict Dict
- // TODO: checkout io (ioutil?) readall function
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(f)
if err != nil {
- logger.Errorf("error reading %q: %s\n", f, err)
+ err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
return
}
contents := buf.String()
@@ -325,21 +324,21 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
err = json.Unmarshal([]byte(contents), &nodeDict)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
if val, ok := nodeDict["properties"]; ok {
var encoded []byte
encoded, err = json.MarshalIndent(val, "", " ")
if err != nil {
- logger.Errorf("error marshalling: %s\n", err)
+ err = fmt.Errorf("Error marshalling: %s", err)
return
}
// node is type LegacyNodeInfo
var newNode LegacyNodeInfo
err = json.Unmarshal(encoded, &newNode)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
node = newNode
@@ -348,7 +347,7 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
var newNode Node
err = json.Unmarshal([]byte(contents), &newNode)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
node = newNode
@@ -361,7 +360,10 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
cost = make(map[string]float64)
- project := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ if err != nil {
+ logger.Fatalf("Error loading object %s: %s\n", uuid, err.Error())
+ }
// arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
@@ -379,7 +381,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operand: nil,
},
}
- err := arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
if err != nil {
logger.Fatalf("Error querying container_requests: %s\n", err.Error())
}
@@ -408,12 +410,18 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var totalCost float64
// This is a container request, find the container
- cr := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
- container := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+ cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", uuid, err)
+ }
+ container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ }
- topNode, err := getNode(logger, arv, arv2, kc, cr)
+ topNode, err := getNode(arv, arv2, kc, cr)
if err != nil {
- log.Fatalf("error getting node: %s", err)
+ log.Fatalf("Error getting node %s: %s\n", cr["uuid"], err)
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
@@ -439,9 +447,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
for _, item := range items {
logger.Info(".")
itemMap := item.(map[string]interface{})
- node, _ := getNode(logger, arv, arv2, kc, itemMap)
+ node, err := getNode(arv, arv2, kc, itemMap)
+ if err != nil {
+ log.Fatalf("Error getting node %s: %s\n", itemMap["uuid"], err)
+ }
logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
- c2 := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ }
tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
cost[itemMap["container_uuid"].(string)] = tmpTotalCost
csv += tmpCsv
@@ -468,8 +482,11 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
if exitcode != 0 {
return
}
-
- ensureDirectory(logger, resultsDir)
+ err := ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
@@ -486,23 +503,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
arv2 := arvados.NewClientFromEnv()
cost := make(map[string]float64)
-
for _, uuid := range uuids {
- //csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
-
- if strings.Contains(uuid, "-d1hrv-") {
- // This is a pipeline instance, not a job! Find the cwl-runner job.
- pi := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
- for _, v := range pi["components"].(map[string]interface{}) {
- x := v.(map[string]interface{})
- y := x["job"].(map[string]interface{})
- uuid = y["uuid"].(string)
- }
- }
-
- // for projects:
- // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
-
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) {
@@ -528,7 +529,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
if len(cost) == 0 {
logger.Info("Nothing to do!\n")
- os.Exit(0)
+ return
}
var csv string
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
new file mode 100644
index 000000000..2ef8733b0
--- /dev/null
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -0,0 +1,242 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "regexp"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the database so they don't affect subsequent tests.
+ arvadostest.ResetEnv()
+}
+
+func (s *Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+
+ // Get the various arvados, arvadosclient, and keep client objects
+ ac := arvados.NewClientFromEnv()
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ standardE4sV3JSON := `{
+ "Name": "Standard_E4s_v3",
+ "ProviderType": "Standard_E4s_v3",
+ "VCPUs": 4,
+ "RAM": 34359738368,
+ "Scratch": 64000000000,
+ "IncludedScratch": 64000000000,
+ "AddedScratch": 0,
+ "Price": 0.292,
+ "Preemptible": false
+}`
+ standardD32sV3JSON := `{
+ "Name": "Standard_D32s_v3",
+ "ProviderType": "Standard_D32s_v3",
+ "VCPUs": 32,
+ "RAM": 137438953472,
+ "Scratch": 256000000000,
+ "IncludedScratch": 256000000000,
+ "AddedScratch": 0,
+ "Price": 1.76,
+ "Preemptible": false
+}`
+
+ standardA1V2JSON := `{
+ "Name": "a1v2",
+ "ProviderType": "Standard_A1_v2",
+ "VCPUs": 1,
+ "RAM": 2147483648,
+ "Scratch": 10000000000,
+ "IncludedScratch": 10000000000,
+ "AddedScratch": 0,
+ "Price": 0.043,
+ "Preemptible": false
+}`
+
+ standardA2V2JSON := `{
+ "Name": "a2v2",
+ "ProviderType": "Standard_A2_v2",
+ "VCPUs": 2,
+ "RAM": 4294967296,
+ "Scratch": 20000000000,
+ "IncludedScratch": 20000000000,
+ "AddedScratch": 0,
+ "Price": 0.091,
+ "Preemptible": false
+}`
+
+ // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
+
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON)
+}
+
+func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
+ // Get the CR
+ var cr arvados.ContainerRequest
+ err := arv.Get("container_requests", crUUID, arvadosclient.Dict{}, &cr)
+ c.Assert(err, check.Equals, nil)
+ c.Assert(cr.LogUUID, check.Equals, logUUID)
+
+ // Get the log collection
+ var coll arvados.Collection
+ err = arv.Get("collections", cr.LogUUID, arvadosclient.Dict{}, &coll)
+ c.Assert(err, check.IsNil)
+
+ // Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, nodeJSON)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+
+ // Flush the data to Keep
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ c.Assert(mtxt, check.NotNil)
+
+ // Update collection record
+ err = arv.Update("collections", cr.LogUUID, arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": mtxt}}, &coll)
+ c.Assert(err, check.IsNil)
+}
+
+func (*Suite) TestUsage(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 1 container request uuid
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+
+ // Now move both container requests into an existing project, and then re-run
+ // the analysis with the project uuid. The results should be identical.
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+
+ var cr arvados.ContainerRequest
+ err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ c.Assert(err, check.IsNil)
+ err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID2, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ c.Assert(err, check.IsNil)
+
+ // Run costanalyzer with the project uuid
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err = ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+}
+
+func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377")
+}
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 9049c73c4..aeb5a47e6 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -44,8 +44,27 @@ const (
RunningContainerUUID = "zzzzz-dz642-runningcontainr"
- CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
- CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2"
+
+ CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001"
+ CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002"
+ CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001"
+ CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002"
+ DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1"
+ DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2"
+
+ CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001"
+ CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002"
+ CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003"
+ CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1"
+ CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2"
+ CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3"
+
+ Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001"
+ Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
+ Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
@@ -75,7 +94,8 @@ const (
CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud"
- LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID2 = "zzzzz-4zz18-logcollection02"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 2243d6a44..767f035b8 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -1043,6 +1043,78 @@ log_collection:
manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
name: a real log collection for a completed container
+log_collection2:
+ uuid: zzzzz-4zz18-logcollection02
+ current_version_uuid: zzzzz-4zz18-logcollection02
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: another real log collection for a completed container
+
+diagnostics_request_container_log_collection:
+ uuid: zzzzz-4zz18-diagcompreqlog1
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog1
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:44.007557000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:44.005381000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0001
+
+hasher1_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash001
+ current_version_uuid: zzzzz-4zz18-dlogcollhash001
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:16:55.272606000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:16:55.267006000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher1 log collection
+
+hasher2_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash002
+ current_version_uuid: zzzzz-4zz18-dlogcollhash002
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:23.547251000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:23.545275000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher2 log collection
+
+hasher3_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash003
+ current_version_uuid: zzzzz-4zz18-dlogcollhash003
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:38.789204000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:38.787329000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher3 log collection
+
+diagnostics_request_container_log_collection2:
+ uuid: zzzzz-4zz18-diagcompreqlog2
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog2
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-03T16:17:53.351593000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-03T16:17:53.346969000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0002
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index b3fd6b9dd..ab0400a67 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -115,10 +115,238 @@ completed-older:
output_path: test
command: ["arvados-cwl-runner", "echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainr2
+ log_uuid: zzzzz-4zz18-logcollection02
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
ram: 123
+completed_diagnostics:
+ name: CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-diagcompreqlog1
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag1hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag1hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag1hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics2:
+ name: Copy of CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-03T15:54:30.098485000Z
+ modified_at: 2020-11-03T16:17:53.406809000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-diagcompreqlog2
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1_reuse:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag2hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2_reuse:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag2hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3_reuse:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag2hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
requester:
uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index f18adb5db..b7d082771 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -126,6 +126,153 @@ completed_older:
secret_mounts: {}
secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b
+diagnostics_completed_requester:
+ uuid: zzzzz-dz642-diagcompreq0001
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:03:50.192697000Z
+ modified_at: 2020-11-02T00:20:43.987275000Z
+ started_at: 2020-11-02T00:08:07.186711000Z
+ finished_at: 2020-11-02T00:20:43.975416000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 6129e376cb05c942f75a0c36083383e8+244
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
+diagnostics_completed_hasher1:
+ uuid: zzzzz-dz642-diagcomphasher1
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:08:18.829222000Z
+ modified_at: 2020-11-02T00:16:55.142023000Z
+ started_at: 2020-11-02T00:16:52.375871000Z
+ finished_at: 2020-11-02T00:16:55.105985000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: fed8fb19fe8e3a320c29fed0edab12dd+220
+ output: d3a687732e84061f3bae15dc7e313483+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_hasher2:
+ uuid: zzzzz-dz642-diagcomphasher2
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:17:07.026493000Z
+ modified_at: 2020-11-02T00:20:23.505908000Z
+ started_at: 2020-11-02T00:20:21.513185000Z
+ finished_at: 2020-11-02T00:20:23.478317000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 4fc03b95fc2646b0dec7383dbb7d56d8+221
+ output: 6bd770f6cf8f83e7647c602eecfaeeb8+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 2
+
+diagnostics_completed_hasher3:
+ uuid: zzzzz-dz642-diagcomphasher3
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:20:30.943856000Z
+ modified_at: 2020-11-02T00:20:38.746541000Z
+ started_at: 2020-11-02T00:20:36.748957000Z
+ finished_at: 2020-11-02T00:20:38.732199000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 1eeaf70de0f65b1346e54c59f09e848d+210
+ output: 11b5fdaa380102e760c3eb6de80a9876+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_requester2:
+ uuid: zzzzz-dz642-diagcompreq0002
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 1124295487972526
+ created_at: 2020-11-03T15:54:36.504661000Z
+ modified_at: 2020-11-03T16:17:53.242868000Z
+ started_at: 2020-11-03T16:09:51.123659000Z
+ finished_at: 2020-11-03T16:17:53.220358000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: f1933bf5191f576613ea7f65bd0ead53+244
+ output: 941b71a57208741ce8742eca62352fb1+123
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
requester:
uuid: zzzzz-dz642-requestingcntnr
owner_uuid: zzzzz-tpzed-000000000000000
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list