[ARVADOS] created: 2.1.0-2244-g7f439b0bf

Git user git at public.arvados.org
Fri Apr 8 15:08:27 UTC 2022


        at  7f439b0bfe215b3641c054a73677ba118023f596 (commit)


commit 7f439b0bfe215b3641c054a73677ba118023f596
Author: Tom Clegg <tom at curii.com>
Date:   Fri Apr 8 11:08:14 2022 -0400

    18794: Add SourceTimestamp field.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/config/load.go b/lib/config/load.go
index 5afb51c5a..d3592be41 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -17,6 +17,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"github.com/ghodss/yaml"
@@ -46,6 +47,10 @@ type Loader struct {
 	KeepBalancePath         string
 
 	configdata []byte
+	// UTC time for configdata: either the modtime of the file we
+	// read configdata from, or the time when we read configdata
+	// from a pipe.
+	sourceTimestamp time.Time
 }
 
 // NewLoader returns a new Loader with Stdin and Logger set to the
@@ -166,25 +171,32 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
 	return munged
 }
 
-func (ldr *Loader) loadBytes(path string) ([]byte, error) {
+func (ldr *Loader) loadBytes(path string) ([]byte, time.Time, error) {
 	if path == "-" {
-		return ioutil.ReadAll(ldr.Stdin)
+		buf, err := ioutil.ReadAll(ldr.Stdin)
+		return buf, time.Now().UTC(), err
 	}
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	defer f.Close()
-	return ioutil.ReadAll(f)
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, time.Time{}, err
+	}
+	buf, err := ioutil.ReadAll(f)
+	return buf, fi.ModTime().UTC(), err
 }
 
 func (ldr *Loader) Load() (*arvados.Config, error) {
 	if ldr.configdata == nil {
-		buf, err := ldr.loadBytes(ldr.Path)
+		buf, t, err := ldr.loadBytes(ldr.Path)
 		if err != nil {
 			return nil, err
 		}
 		ldr.configdata = buf
+		ldr.sourceTimestamp = t
 	}
 
 	// FIXME: We should reject YAML if the same key is used twice
@@ -330,6 +342,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 			}
 		}
 	}
+	cfg.SourceTimestamp = ldr.sourceTimestamp
 	return &cfg, nil
 }
 
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 5270dcccc..c28626f6e 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -12,13 +12,16 @@ import (
 	"os"
 	"os/exec"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"github.com/ghodss/yaml"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 	check "gopkg.in/check.v1"
 )
 
@@ -315,8 +318,14 @@ Clusters:
 	c.Assert(err, check.IsNil)
 	yaml, err := yaml.Marshal(cfg)
 	c.Assert(err, check.IsNil)
+	// Well, *nearly* no warnings. SourceTimestamp is included in
+	// a config-dump, but isn't expected in a real config file.
+	yaml = regexp.MustCompile(`SourceTimestamp: [^\n]+\n`).ReplaceAll(yaml, []byte{})
 	cfgDumped, err := testLoader(c, string(yaml), &logbuf).Load()
 	c.Assert(err, check.IsNil)
+	// SourceTimestamp isn't expected to be preserved when reading
+	// from stdin
+	cfgDumped.SourceTimestamp = cfg.SourceTimestamp
 	c.Check(cfg, check.DeepEquals, cfgDumped)
 	c.Check(logbuf.String(), check.Equals, "")
 }
@@ -503,6 +512,9 @@ func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
 	c.Assert(err, check.IsNil)
 	expected, err := expectedldr.Load()
 	c.Assert(err, check.IsNil)
+	// The inputs generally aren't even files, so SourceTimestamp
+	// can't be expected to match.
+	got.SourceTimestamp = expected.SourceTimestamp
 	checkEqualYAML(c, got, expected)
 }
 
@@ -762,3 +774,28 @@ Clusters:
 	c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`)
 	c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`)
 }
+
+func (s *LoadSuite) TestSourceTimestamp(c *check.C) {
+	conftime, err := time.Parse(time.RFC3339, "2022-03-04T05:06:07-08:00")
+	c.Assert(err, check.IsNil)
+	confdata := `Clusters: {zzzzz: {}}`
+	conffile := c.MkDir() + "/config.yml"
+	ioutil.WriteFile(conffile, []byte(confdata), 0777)
+	tv := unix.NsecToTimeval(conftime.UnixNano())
+	unix.Lutimes(conffile, []unix.Timeval{tv, tv})
+	for _, trial := range []struct {
+		configarg  string
+		expectTime time.Time
+	}{
+		{"-", time.Now()},
+		{conffile, conftime},
+	} {
+		c.Logf("trial: %+v", trial)
+		ldr := NewLoader(strings.NewReader(confdata), ctxlog.TestLogger(c))
+		ldr.Path = trial.configarg
+		cfg, err := ldr.Load()
+		c.Assert(err, check.IsNil)
+		c.Check(cfg.SourceTimestamp, check.Equals, cfg.SourceTimestamp.UTC())
+		c.Check(int(cfg.SourceTimestamp.Sub(trial.expectTime).Seconds()), check.Equals, 0)
+	}
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6c9324e47..eed56baa8 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/config"
 )
@@ -24,6 +25,7 @@ var DefaultConfigFile = func() string {
 type Config struct {
 	Clusters         map[string]Cluster
 	AutoReloadConfig bool
+	SourceTimestamp  time.Time
 }
 
 // GetConfig returns the current system config, loading it from
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index 05f0bdd31..62eca894b 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"regexp"
 	"strings"
 	"time"
 
@@ -142,6 +143,7 @@ func (s *AggregatorSuite) TestCheckCommand(c *check.C) {
 	tmpdir := c.MkDir()
 	confdata, err := yaml.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{s.handler.Cluster.ClusterID: *s.handler.Cluster}})
 	c.Assert(err, check.IsNil)
+	confdata = regexp.MustCompile(`SourceTimestamp: [^\n]+\n`).ReplaceAll(confdata, []byte{})
 	err = ioutil.WriteFile(tmpdir+"/config.yml", confdata, 0777)
 	c.Assert(err, check.IsNil)
 	var stdout, stderr bytes.Buffer

commit 541c300ca6abb15449e917f648ae5ffd68087ff9
Author: Tom Clegg <tom at curii.com>
Date:   Thu Apr 7 16:11:55 2022 -0400

    18794: Add "arvados-server check" subcommand.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index c8b945bea..4b48301eb 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -17,6 +17,7 @@ import (
 	"git.arvados.org/arvados.git/lib/install"
 	"git.arvados.org/arvados.git/lib/lsf"
 	"git.arvados.org/arvados.git/lib/recovercollection"
+	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/services/keepstore"
 	"git.arvados.org/arvados.git/services/ws"
 )
@@ -28,6 +29,7 @@ var (
 		"--version": cmd.Version,
 
 		"boot":               boot.Command,
+		"check":              health.CheckCommand,
 		"cloudtest":          cloudtest.Command,
 		"config-check":       config.CheckCommand,
 		"config-defaults":    config.DumpDefaultsCommand,
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index a666ef8ec..4175bc544 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -8,19 +8,26 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"sync"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/ghodss/yaml"
+	"github.com/sirupsen/logrus"
 )
 
 const defaultTimeout = arvados.Duration(2 * time.Second)
 
-// Aggregator implements http.Handler. It handles "GET /_health/all"
+// Aggregator implements service.Handler. It handles "GET /_health/all"
 // by checking the health of all configured services on the cluster
 // and responding 200 if everything is healthy.
 type Aggregator struct {
@@ -229,3 +236,61 @@ func (agg *Aggregator) checkAuth(req *http.Request) bool {
 	}
 	return false
 }
+
+var errSilent = errors.New("")
+
+var CheckCommand cmd.Handler = checkCommand{}
+
+type checkCommand struct{}
+
+func (ccmd checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "json", "info")
+	ctx := ctxlog.Context(context.Background(), logger)
+	err := ccmd.run(ctx, prog, args, stdin, stdout, stderr)
+	if err != nil {
+		if err != errSilent {
+			fmt.Fprintln(stdout, err.Error())
+		}
+		return 1
+	}
+	return 0
+}
+
+func (ccmd checkCommand) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	loader := config.NewLoader(stdin, ctxlog.New(stderr, "text", "info"))
+	loader.SetupFlags(flags)
+	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+	timeout := flags.Duration("timeout", defaultTimeout.Duration(), "Maximum time to wait for health responses")
+	if ok, _ := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
+		// cmd.ParseFlags already reported the error
+		return errSilent
+	} else if *versionFlag {
+		cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+		return nil
+	}
+	cfg, err := loader.Load()
+	if err != nil {
+		return err
+	}
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
+	}
+	logger := ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+		"ClusterID": cluster.ClusterID,
+	})
+	ctx = ctxlog.Context(ctx, logger)
+	agg := Aggregator{Cluster: cluster, timeout: arvados.Duration(*timeout)}
+	resp := agg.ClusterHealth()
+	buf, err := yaml.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	stdout.Write(buf)
+	if resp.Health != "OK" {
+		return fmt.Errorf("health check failed")
+	}
+	return nil
+}
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index 04106caa4..05f0bdd31 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -5,14 +5,19 @@
 package health
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/ghodss/yaml"
 	"gopkg.in/check.v1"
 )
 
@@ -30,9 +35,17 @@ func (s *AggregatorSuite) TestInterface(c *check.C) {
 }
 
 func (s *AggregatorSuite) SetUpTest(c *check.C) {
-	s.handler = &Aggregator{Cluster: &arvados.Cluster{
-		ManagementToken: arvadostest.ManagementToken,
-	}}
+	ldr := config.NewLoader(bytes.NewBufferString(`Clusters: {zzzzz: {}}`), ctxlog.TestLogger(c))
+	ldr.Path = "-"
+	cfg, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+	cluster, err := cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	cluster.ManagementToken = arvadostest.ManagementToken
+	cluster.SystemRootToken = arvadostest.SystemRootToken
+	cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+	cluster.Volumes["z"] = arvados.Volume{StorageClasses: map[string]bool{"default": true}}
+	s.handler = &Aggregator{Cluster: cluster}
 	s.req = httptest.NewRequest("GET", "/_health/all", nil)
 	s.req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 	s.resp = httptest.NewRecorder()
@@ -122,6 +135,22 @@ func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
 	c.Check(rt > 0.005, check.Equals, true)
 }
 
+func (s *AggregatorSuite) TestCheckCommand(c *check.C) {
+	srv, listen := s.stubServer(&healthyHandler{})
+	defer srv.Close()
+	s.setAllServiceURLs(listen)
+	tmpdir := c.MkDir()
+	confdata, err := yaml.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{s.handler.Cluster.ClusterID: *s.handler.Cluster}})
+	c.Assert(err, check.IsNil)
+	err = ioutil.WriteFile(tmpdir+"/config.yml", confdata, 0777)
+	c.Assert(err, check.IsNil)
+	var stdout, stderr bytes.Buffer
+	exitcode := CheckCommand.RunCommand("check", []string{"-config=" + tmpdir + "/config.yml"}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 0)
+	c.Check(stderr.String(), check.Equals, "")
+	c.Check(stdout.String(), check.Matches, `(?ms).*(\n|^)health: OK\n.*`)
+}
+
 func (s *AggregatorSuite) checkError(c *check.C) {
 	c.Check(s.resp.Code, check.Not(check.Equals), http.StatusOK)
 	var resp ClusterHealthResponse

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list