[ARVADOS] updated: 2.1.0-933-g056b3d236
Git user
git at public.arvados.org
Fri Jun 11 14:29:24 UTC 2021
Summary of changes:
apps/workbench/Gemfile.lock | 91 ++++++------
go.mod | 4 +-
go.sum | 16 ++
lib/costanalyzer/cmd.go | 19 ++-
lib/costanalyzer/costanalyzer.go | 173 ++++++++++++++++------
lib/costanalyzer/costanalyzer_test.go | 24 +++
sdk/python/arvados/commands/put.py | 14 +-
sdk/python/tests/test_arv_put.py | 12 +-
services/api/Gemfile.lock | 91 ++++++------
services/api/app/models/container_request.rb | 14 ++
services/api/test/fixtures/container_requests.yml | 30 +++-
services/api/test/fixtures/containers.yml | 24 ++-
services/api/test/unit/container_request_test.rb | 25 ++++
13 files changed, 361 insertions(+), 176 deletions(-)
via 056b3d2368b151a626fbf79025d9989a4d29a018 (commit)
via 23f0fc06dbb6d7e82d820a8c65997f32c760f34e (commit)
via 523d1c2a9963edc25becf7958e024992ed8a6e66 (commit)
via b7afa91b549fe3e49e11341196d8d7ab02ba78be (commit)
via bf963a4027160e35b7f526328f2c61f3aac78121 (commit)
via 4528ae5bcdfa43cd31a0d0419aa5c7caea1f017a (commit)
via f9ee759496885b89d034888227cb69955080a2a4 (commit)
via 59b4ca616c7d0c43da4481c77b4479ee59b94697 (commit)
via 75a18b03319f6b9b795d7fe46403c04fe795956f (commit)
via 2c843ad3115a94385815dc6925ecee219e3e1a93 (commit)
via 0b0b4c7b23e96a6efb3cfd88b0ba7224158e9544 (commit)
via cd6647b38be0ca224f98fe9e38ec315947e0f7f2 (commit)
via 41e0df276eaaa548e692e44cd8f3c27c4692375e (commit)
via 0e26c53b179763f279a9f6014fec0ac96c6dbeab (commit)
via 57a26e595dc47865da5d929271e269facd14d4af (commit)
via 05607eae460bab7efc61dd33d1e88b31139a97c7 (commit)
from 38218a359786e327bbd4f3379be19e25a5a005f9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 056b3d2368b151a626fbf79025d9989a4d29a018
Author: Tom Clegg <tom at curii.com>
Date: Fri Jun 11 10:18:19 2021 -0400
17609: Make costanalyzer logging a bit less verbose.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index f9875c6a8..dfe3d584c 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -410,7 +410,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
- fmt.Printf("Processing %s\n", uuid)
+ logger.Debugf("Processing %s", uuid)
var crUUID = uuid
if strings.Contains(uuid, "-4zz18-") {
@@ -473,14 +473,20 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
- for _, cr2 := range childCrs.Items {
- logger.Info(".")
+ progressTicker := time.NewTicker(5 * time.Second)
+ defer progressTicker.Stop()
+ for i, cr2 := range childCrs.Items {
+ select {
+ case <-progressTicker.C:
+ logger.Infof("... %d of %d", i+1, len(childCrs.Items))
+ default:
+ }
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
continue
}
- logger.Debug("\nChild container: " + cr2.ContainerUUID)
+ logger.Debug("Child container: " + cr2.ContainerUUID)
var c2 arvados.Container
err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
@@ -491,7 +497,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
csv += tmpCsv
totalCost += tmpTotalCost
}
- logger.Info(" done")
+ logger.Debug("Done collecting child containers")
csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
@@ -502,7 +508,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if err != nil {
return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
}
- logger.Infof("\nUUID report in %s\n", fName)
+ logger.Infof("\nUUID report in %s", fName)
}
return
@@ -562,7 +568,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
if err != nil {
- logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+ logger.Errorf("Error getting container request list from Arvados API: %s", err)
break
}
if len(list.Items) == 0 {
@@ -581,7 +587,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
cost := make(map[string]float64)
for uuid := range uuidChannel {
- fmt.Printf("Considering %s\n", uuid)
+ logger.Debugf("Considering %s", uuid)
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
commit 23f0fc06dbb6d7e82d820a8c65997f32c760f34e
Merge: 38218a359 523d1c2a9
Author: Tom Clegg <tom at curii.com>
Date: Fri Jun 11 10:02:58 2021 -0400
17609: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc go.mod
index 9062c667d,0ff679a57..b70f6f3b4
--- a/go.mod
+++ b/go.mod
@@@ -59,10 -59,10 +59,12 @@@ require
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
- golang.org/x/net v0.0.0-20201021035429-f5854403a974
+ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+ golang.org/x/sys v0.0.0-20210603125802-9665404d3644
+ golang.org/x/tools v0.1.0 // indirect
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+ golang.org/x/tools v0.1.2 // indirect
google.golang.org/api v0.13.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
diff --cc go.sum
index 23731f69a,c28ab4624..1fd37ed11
--- a/go.sum
+++ b/go.sum
@@@ -303,12 -309,15 +311,18 @@@ golang.org/x/sys v0.0.0-20190422165155-
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
diff --cc lib/costanalyzer/cmd.go
index b08e943c0,525ec619b..6065ad2c0
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@@ -6,31 -6,41 +6,34 @@@ package costanalyze
import (
"io"
+ "time"
+ "git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/sirupsen/logrus"
)
- var Command command
+ var Command = command{}
- type command struct{}
+ type command struct {
+ uuids arrayFlags
+ resultsDir string
+ cache bool
+ begin time.Time
+ end time.Time
+ }
-type NoPrefixFormatter struct{}
-
-func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
- return []byte(entry.Message), nil
-}
-
// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
- func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ func (c command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
+ logger.SetFormatter(cmd.NoPrefixFormatter{})
defer func() {
if err != nil {
- logger.Error("\n" + err.Error() + "\n")
+ logger.Error("\n" + err.Error())
}
}()
- loader := config.NewLoader(stdin, logger)
- loader.SkipLegacy = true
- logger.SetFormatter(new(NoPrefixFormatter))
--
- exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+ exitcode, err := c.costAnalyzer(prog, args, logger, stdout, stderr)
return exitcode
}
diff --cc lib/costanalyzer/costanalyzer.go
index 23df754bc,edaaa5bd1..f9875c6a8
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@@ -17,10 -20,6 +17,9 @@@ import
"strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
@@@ -139,8 -168,8 +168,8 @@@ Options
return
}
logger.SetLevel(lvl)
- if !cache {
+ if !c.cache {
- logger.Debug("Caching disabled\n")
+ logger.Debug("Caching disabled")
}
return
}
@@@ -446,9 -477,10 +477,10 @@@ func generateCrCsv(logger *logrus.Logge
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ continue
}
- logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+ logger.Debug("\nChild container: " + cr2.ContainerUUID)
var c2 arvados.Container
err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list