[arvados] created: 2.1.0-3174-g7fcd7676c
git repository hosting
git at public.arvados.org
Fri Dec 9 16:35:45 UTC 2022
at 7fcd7676cfa748d3a25fa0f247dba5f50edd99b4 (commit)
commit 7fcd7676cfa748d3a25fa0f247dba5f50edd99b4
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 8 14:41:50 2022 -0500
19844: Get multiple pages of child CRs if necessary.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index a3673c979..e68e2cb8c 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -26,6 +26,8 @@ import (
const timestampFormat = "2006-01-02T15:04:05"
+var pagesize = 1000
+
type nodeInfo struct {
// Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
Properties struct {
@@ -355,6 +357,35 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien
return
}
+func getContainerRequests(ac *arvados.Client, filters []arvados.Filter) ([]arvados.ContainerRequest, error) {
+ var allItems []arvados.ContainerRequest
+ for {
+ pagefilters := append([]arvados.Filter(nil), filters...)
+ if len(allItems) > 0 {
+ pagefilters = append(pagefilters, arvados.Filter{
+ Attr: "uuid",
+ Operator: ">",
+ Operand: allItems[len(allItems)-1].UUID,
+ })
+ }
+ var resp arvados.ContainerRequestList
+ err := ac.RequestAndDecode(&resp, "GET", "arvados/v1/container_requests", nil, arvados.ResourceListParams{
+ Filters: pagefilters,
+ Limit: &pagesize,
+ Order: "uuid",
+ Count: "none",
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error querying container_requests: %w", err)
+ }
+ if len(resp.Items) == 0 {
+ // no more pages
+ return allItems, nil
+ }
+ allItems = append(allItems, resp.Items...)
+ }
+}
+
func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) {
cost = make(map[string]consumption)
@@ -363,9 +394,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
}
-
- var childCrs map[string]interface{}
- filterset := []arvados.Filter{
+ allItems, err := getContainerRequests(ac, []arvados.Filter{
{
Attr: "owner_uuid",
Operator: "=",
@@ -376,29 +405,23 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operator: "=",
Operand: nil,
},
- }
- err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
- "filters": filterset,
- "limit": 10000,
})
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- if value, ok := childCrs["items"]; ok {
- logger.Infof("Collecting top level container requests in project %s", uuid)
- items := value.([]interface{})
- for _, item := range items {
- itemMap := item.(map[string]interface{})
- crInfo, err := generateCrInfo(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
- if err != nil {
- return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
- }
- for k, v := range crInfo {
- cost[k] = v
- }
- }
- } else {
+ if len(allItems) == 0 {
logger.Infof("No top level container requests found in project %s", uuid)
+ return
+ }
+ logger.Infof("Collecting top level container requests in project %s", uuid)
+ for _, cr := range allItems {
+ crInfo, err := generateCrInfo(logger, cr.UUID, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ return nil, fmt.Errorf("error generating container_request CSV for %s: %s", cr.UUID, err)
+ }
+ for k, v := range crInfo {
+ cost[k] = v
+ }
}
return
}
@@ -456,28 +479,20 @@ func generateCrInfo(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvad
csv += tmpCsv
cost[container.UUID] = total
- // Find all container requests that have the container we found above as requesting_container_uuid
- var childCrs arvados.ContainerRequestList
- filterset := []arvados.Filter{
- {
- Attr: "requesting_container_uuid",
- Operator: "=",
- Operand: container.UUID,
- }}
- err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
- "filters": filterset,
- "limit": 10000,
- })
- if err != nil {
- return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
- }
- logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
+ // Find all container requests that have the container we
+ // found above as requesting_container_uuid.
+ allItems, err := getContainerRequests(ac, []arvados.Filter{{
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: container.UUID,
+ }})
+ logger.Infof("Looking up %d child containers for container %s (%s)", len(allItems), container.UUID, container.FinishedAt)
progressTicker := time.NewTicker(5 * time.Second)
defer progressTicker.Stop()
- for i, cr2 := range childCrs.Items {
+ for i, cr2 := range allItems {
select {
case <-progressTicker.C:
- logger.Infof("... %d of %d", i+1, len(childCrs.Items))
+ logger.Infof("... %d of %d", i+1, len(allItems))
default:
}
node, err := getNode(arv, ac, kc, cr2)
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index b78b288ab..1054870ad 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -35,6 +35,10 @@ func (s *Suite) TearDownSuite(c *check.C) {
func (s *Suite) SetUpSuite(c *check.C) {
arvadostest.StartKeep(2, true)
+ // Use a small page size to exercise paging without adding
+ // lots of fixtures
+ pagesize = 2
+
// Get the various arvados, arvadosclient, and keep client objects
ac := arvados.NewClientFromEnv()
arv, err := arvadosclient.MakeArvadosClient()
commit f42a4541d44e71965b7a07380e3f81a395fcfb40
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 8 14:06:15 2022 -0500
19844: Use 127.0.0.1 instead of localhost for test servers.
Otherwise, depending on host config, nginx futilely tries ::1 first.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 2bb20ca5d..33b9cdfdb 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -635,8 +635,8 @@ def run_nginx():
return
stop_nginx()
nginxconf = {}
- nginxconf['UPSTREAMHOST'] = 'localhost'
- nginxconf['LISTENHOST'] = 'localhost'
+ nginxconf['UPSTREAMHOST'] = '127.0.0.1'
+ nginxconf['LISTENHOST'] = '127.0.0.1'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))
nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list