[ARVADOS] updated: 2.1.0-933-g056b3d236

Git user git at public.arvados.org
Fri Jun 11 14:29:24 UTC 2021


Summary of changes:
 apps/workbench/Gemfile.lock                       |  91 ++++++------
 go.mod                                            |   4 +-
 go.sum                                            |  16 ++
 lib/costanalyzer/cmd.go                           |  19 ++-
 lib/costanalyzer/costanalyzer.go                  | 173 ++++++++++++++++------
 lib/costanalyzer/costanalyzer_test.go             |  24 +++
 sdk/python/arvados/commands/put.py                |  14 +-
 sdk/python/tests/test_arv_put.py                  |  12 +-
 services/api/Gemfile.lock                         |  91 ++++++------
 services/api/app/models/container_request.rb      |  14 ++
 services/api/test/fixtures/container_requests.yml |  30 +++-
 services/api/test/fixtures/containers.yml         |  24 ++-
 services/api/test/unit/container_request_test.rb  |  25 ++++
 13 files changed, 361 insertions(+), 176 deletions(-)

       via  056b3d2368b151a626fbf79025d9989a4d29a018 (commit)
       via  23f0fc06dbb6d7e82d820a8c65997f32c760f34e (commit)
       via  523d1c2a9963edc25becf7958e024992ed8a6e66 (commit)
       via  b7afa91b549fe3e49e11341196d8d7ab02ba78be (commit)
       via  bf963a4027160e35b7f526328f2c61f3aac78121 (commit)
       via  4528ae5bcdfa43cd31a0d0419aa5c7caea1f017a (commit)
       via  f9ee759496885b89d034888227cb69955080a2a4 (commit)
       via  59b4ca616c7d0c43da4481c77b4479ee59b94697 (commit)
       via  75a18b03319f6b9b795d7fe46403c04fe795956f (commit)
       via  2c843ad3115a94385815dc6925ecee219e3e1a93 (commit)
       via  0b0b4c7b23e96a6efb3cfd88b0ba7224158e9544 (commit)
       via  cd6647b38be0ca224f98fe9e38ec315947e0f7f2 (commit)
       via  41e0df276eaaa548e692e44cd8f3c27c4692375e (commit)
       via  0e26c53b179763f279a9f6014fec0ac96c6dbeab (commit)
       via  57a26e595dc47865da5d929271e269facd14d4af (commit)
       via  05607eae460bab7efc61dd33d1e88b31139a97c7 (commit)
      from  38218a359786e327bbd4f3379be19e25a5a005f9 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 056b3d2368b151a626fbf79025d9989a4d29a018
Author: Tom Clegg <tom at curii.com>
Date:   Fri Jun 11 10:18:19 2021 -0400

    17609: Make costanalyzer logging a bit less verbose.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index f9875c6a8..dfe3d584c 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -410,7 +410,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 	var tmpCsv string
 	var tmpTotalCost float64
 	var totalCost float64
-	fmt.Printf("Processing %s\n", uuid)
+	logger.Debugf("Processing %s", uuid)
 
 	var crUUID = uuid
 	if strings.Contains(uuid, "-4zz18-") {
@@ -473,14 +473,20 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
 	}
 	logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
-	for _, cr2 := range childCrs.Items {
-		logger.Info(".")
+	progressTicker := time.NewTicker(5 * time.Second)
+	defer progressTicker.Stop()
+	for i, cr2 := range childCrs.Items {
+		select {
+		case <-progressTicker.C:
+			logger.Infof("... %d of %d", i+1, len(childCrs.Items))
+		default:
+		}
 		node, err := getNode(arv, ac, kc, cr2)
 		if err != nil {
 			logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
 			continue
 		}
-		logger.Debug("\nChild container: " + cr2.ContainerUUID)
+		logger.Debug("Child container: " + cr2.ContainerUUID)
 		var c2 arvados.Container
 		err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
 		if err != nil {
@@ -491,7 +497,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		csv += tmpCsv
 		totalCost += tmpTotalCost
 	}
-	logger.Info(" done")
+	logger.Debug("Done collecting child containers")
 
 	csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 
@@ -502,7 +508,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		if err != nil {
 			return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
 		}
-		logger.Infof("\nUUID report in %s\n", fName)
+		logger.Infof("\nUUID report in %s", fName)
 	}
 
 	return
@@ -562,7 +568,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 
 				err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
 				if err != nil {
-					logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+					logger.Errorf("Error getting container request list from Arvados API: %s", err)
 					break
 				}
 				if len(list.Items) == 0 {
@@ -581,7 +587,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 	cost := make(map[string]float64)
 
 	for uuid := range uuidChannel {
-		fmt.Printf("Considering %s\n", uuid)
+		logger.Debugf("Considering %s", uuid)
 		if strings.Contains(uuid, "-j7d0g-") {
 			// This is a project (group)
 			cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)

commit 23f0fc06dbb6d7e82d820a8c65997f32c760f34e
Merge: 38218a359 523d1c2a9
Author: Tom Clegg <tom at curii.com>
Date:   Fri Jun 11 10:02:58 2021 -0400

    17609: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --cc go.mod
index 9062c667d,0ff679a57..b70f6f3b4
--- a/go.mod
+++ b/go.mod
@@@ -59,10 -59,10 +59,12 @@@ require 
  	github.com/src-d/gcfg v1.3.0 // indirect
  	github.com/xanzy/ssh-agent v0.1.0 // indirect
  	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
- 	golang.org/x/net v0.0.0-20201021035429-f5854403a974
+ 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
  	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 +	golang.org/x/sys v0.0.0-20210603125802-9665404d3644
 +	golang.org/x/tools v0.1.0 // indirect
+ 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+ 	golang.org/x/tools v0.1.2 // indirect
  	google.golang.org/api v0.13.0
  	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
  	gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
diff --cc go.sum
index 23731f69a,c28ab4624..1fd37ed11
--- a/go.sum
+++ b/go.sum
@@@ -303,12 -309,15 +311,18 @@@ golang.org/x/sys v0.0.0-20190422165155-
  golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
  golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 +golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
 +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
diff --cc lib/costanalyzer/cmd.go
index b08e943c0,525ec619b..6065ad2c0
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@@ -6,31 -6,41 +6,34 @@@ package costanalyze
  
  import (
  	"io"
+ 	"time"
  
 +	"git.arvados.org/arvados.git/lib/cmd"
- 	"git.arvados.org/arvados.git/lib/config"
  	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 -	"github.com/sirupsen/logrus"
  )
  
- var Command command
+ var Command = command{}
  
- type command struct{}
+ type command struct {
+ 	uuids      arrayFlags
+ 	resultsDir string
+ 	cache      bool
+ 	begin      time.Time
+ 	end        time.Time
+ }
  
 -type NoPrefixFormatter struct{}
 -
 -func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
 -	return []byte(entry.Message), nil
 -}
 -
  // RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
- func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ func (c command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
  	var err error
  	logger := ctxlog.New(stderr, "text", "info")
 +	logger.SetFormatter(cmd.NoPrefixFormatter{})
  	defer func() {
  		if err != nil {
 -			logger.Error("\n" + err.Error() + "\n")
 +			logger.Error("\n" + err.Error())
  		}
  	}()
  
- 	loader := config.NewLoader(stdin, logger)
- 	loader.SkipLegacy = true
 -	logger.SetFormatter(new(NoPrefixFormatter))
--
- 	exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+ 	exitcode, err := c.costAnalyzer(prog, args, logger, stdout, stderr)
  
  	return exitcode
  }
diff --cc lib/costanalyzer/costanalyzer.go
index 23df754bc,edaaa5bd1..f9875c6a8
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@@ -17,10 -20,6 +17,9 @@@ import 
  	"strings"
  	"time"
  
- 	"git.arvados.org/arvados.git/lib/config"
 +	"git.arvados.org/arvados.git/sdk/go/arvados"
 +	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 +	"git.arvados.org/arvados.git/sdk/go/keepclient"
  	"github.com/sirupsen/logrus"
  )
  
@@@ -139,8 -168,8 +168,8 @@@ Options
  		return
  	}
  	logger.SetLevel(lvl)
- 	if !cache {
+ 	if !c.cache {
 -		logger.Debug("Caching disabled\n")
 +		logger.Debug("Caching disabled")
  	}
  	return
  }
@@@ -446,9 -477,10 +477,10 @@@ func generateCrCsv(logger *logrus.Logge
  		logger.Info(".")
  		node, err := getNode(arv, ac, kc, cr2)
  		if err != nil {
- 			return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ 			logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ 			continue
  		}
 -		logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
 +		logger.Debug("\nChild container: " + cr2.ContainerUUID)
  		var c2 arvados.Container
  		err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
  		if err != nil {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list