[ARVADOS] created: 1.3.0-2356-g5e31613d2
Git user
git at public.arvados.org
Sat Mar 14 01:17:26 UTC 2020
at 5e31613d2a0647647b710a0558e11408bc157406 (commit)
commit 5e31613d2a0647647b710a0558e11408bc157406
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:16:56 2020 -0400
16053: Add "install" command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index a9d927d87..328599826 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -14,6 +14,7 @@ import (
"git.arvados.org/arvados.git/lib/controller"
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
+ "git.arvados.org/arvados.git/lib/install"
)
var (
@@ -30,6 +31,7 @@ var (
"controller": controller.Command,
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
+ "install": install.Command,
})
)
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index d230dcc3e..e75de3244 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -126,7 +126,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
+ super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
super.cluster, err = cfg.GetCluster("")
if err != nil {
@@ -360,7 +360,11 @@ func (super *Supervisor) setupRubyEnv() error {
"GEM_HOME=",
"GEM_PATH=",
})
- cmd := exec.Command("gem", "env", "gempath")
+ gem := "gem"
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ gem = "/var/lib/arvados/bin/gem"
+ }
+ cmd := exec.Command(gem, "env", "gempath")
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
diff --git a/lib/install/deps.go b/lib/install/deps.go
new file mode 100644
index 000000000..50eab6aef
--- /dev/null
+++ b/lib/install/deps.go
@@ -0,0 +1,302 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+var Command cmd.Handler = installCommand{}
+
+type installCommand struct{}
+
+func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ }
+
+ var dev, test, prod bool
+ switch *clusterType {
+ case "development":
+ dev = true
+ case "test":
+ test = true
+ case "production":
+ prod = true
+ default:
+ err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
+ return 2
+ }
+
+ osv, err := identifyOS()
+ if err != nil {
+ return 1
+ }
+
+ listdir, err := os.Open("/var/lib/apt/lists")
+ if err != nil {
+ logger.Warnf("error while checking whether to run apt-get update: %s", err)
+ } else if names, _ := listdir.Readdirnames(1); len(names) == 0 {
+ // Special case for a base docker image where the
+ // package cache has been deleted and all "apt-get
+ // install" commands will fail unless we fetch repos.
+ cmd := exec.CommandContext(ctx, "apt-get", "update")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ if dev || test {
+ debs := []string{
+ "bison",
+ "bsdmainutils",
+ "build-essential",
+ "cadaver",
+ "curl",
+ "cython",
+ "fuse",
+ "gettext",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "iceweasel",
+ "libattr1-dev",
+ "libcrypt-ssleay-perl",
+ "libcrypt-ssleay-perl",
+ "libcurl3-gnutls",
+ "libcurl4-openssl-dev",
+ "libfuse-dev",
+ "libgnutls28-dev",
+ "libjson-perl",
+ "libjson-perl",
+ "libpam-dev",
+ "libpcre3-dev",
+ "libpq-dev",
+ "libpython2.7-dev",
+ "libreadline-dev",
+ "libssl-dev",
+ "libwww-perl",
+ "libxml2-dev",
+ "libxslt1.1",
+ "linkchecker",
+ "lsof",
+ "net-tools",
+ "nginx",
+ "pandoc",
+ "perl-modules",
+ "pkg-config",
+ "postgresql",
+ "postgresql-contrib",
+ "python",
+ "python3-dev",
+ "python-epydoc",
+ "r-base",
+ "r-cran-testthat",
+ "sudo",
+ "virtualenv",
+ "wget",
+ "xvfb",
+ "zlib1g-dev",
+ }
+ switch {
+ case osv.Debian && osv.Major >= 10:
+ debs = append(debs, "libcurl4")
+ default:
+ debs = append(debs, "libcurl3")
+ }
+ cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
+ cmd.Args = append(cmd.Args, debs...)
+ cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ os.Mkdir("/var/lib/arvados", 0755)
+ rubyversion := "2.5.7"
+ if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
+ logger.Print("ruby " + rubyversion + " already installed")
+ } else {
+ err = runBash(`
+mkdir -p /var/lib/arvados/src
+cd /var/lib/arvados/src
+wget -c https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz
+tar xzf ruby-`+rubyversion+`.tar.gz
+cd ruby-`+rubyversion+`
+./configure --disable-install-doc --prefix /var/lib/arvados
+make -j4
+make install
+/var/lib/arvados/bin/gem install bundler
+cd ..
+rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if !prod {
+ goversion := "1.14"
+ if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
+ logger.Print("go " + goversion + " already installed")
+ } else {
+ err = runBash(`
+cd /tmp
+wget -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
+ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ pjsversion := "1.9.8"
+ if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
+ logger.Print("phantomjs " + pjsversion + " already installed")
+ } else {
+ err = runBash(`
+PJS=phantomjs-`+pjsversion+`-linux-x86_64
+wget -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
+ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ geckoversion := "0.24.0"
+ if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
+ logger.Print("geckodriver " + geckoversion + " already installed")
+ } else {
+ err = runBash(`
+GD=v`+geckoversion+`
+wget -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
+ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ nodejsversion := "v8.15.1"
+ if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
+ logger.Print("nodejs " + nodejsversion + " already installed")
+ } else {
+ err = runBash(`
+NJS=`+nodejsversion+`
+wget -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+ }
+
+ return 0
+}
+
+type osversion struct {
+ Debian bool
+ Ubuntu bool
+ Major int
+}
+
+func identifyOS() (osversion, error) {
+ var osv osversion
+ f, err := os.Open("/etc/os-release")
+ if err != nil {
+ return osv, err
+ }
+ defer f.Close()
+
+ kv := map[string]string{}
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+ toks := strings.SplitN(line, "=", 2)
+ if len(toks) != 2 {
+ return osv, fmt.Errorf("invalid line in /etc/os-release: %q", line)
+ }
+ k := toks[0]
+ v := strings.Trim(toks[1], `"`)
+ if v == toks[1] {
+ v = strings.Trim(v, `'`)
+ }
+ kv[k] = v
+ }
+ if err = scanner.Err(); err != nil {
+ return osv, err
+ }
+ switch kv["ID"] {
+ case "ubuntu":
+ osv.Ubuntu = true
+ case "debian":
+ osv.Debian = true
+ default:
+ return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
+ }
+ vstr := kv["VERSION_ID"]
+ if i := strings.Index(vstr, "."); i > 0 {
+ vstr = vstr[:i]
+ }
+ osv.Major, err = strconv.Atoi(vstr)
+ if err != nil {
+ return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os/release: %q", kv["VERSION_ID"])
+ }
+ return osv, nil
+}
+
+func runBash(script string, stdout, stderr io.Writer) error {
+ cmd := exec.Command("bash", "-")
+ cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ return cmd.Run()
+}
commit af4d0a2c71b4b633996fe33a9f63295244ea978f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:16:36 2020 -0400
16053: Run postgresql as "postgres" user if supervisor is root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index df9890415..a08180e0c 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -11,7 +11,9 @@ import (
"fmt"
"os"
"os/exec"
+ "os/user"
"path/filepath"
+ "strconv"
"strings"
"time"
@@ -34,6 +36,13 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
return err
}
+ iamroot := false
+ if u, err := user.Current(); err != nil {
+ return fmt.Errorf("user.Current(): %s", err)
+ } else if u.Uid == "0" {
+ iamroot = true
+ }
+
buf := bytes.NewBuffer(nil)
err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
if err != nil {
@@ -42,11 +51,40 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
bindir := strings.TrimSpace(buf.String())
datadir := filepath.Join(super.tempdir, "pgdata")
- err = os.Mkdir(datadir, 0755)
+ err = os.Mkdir(datadir, 0700)
if err != nil {
return err
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "initdb"), "-D", datadir)
+ prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+ if iamroot {
+ postgresUser, err := user.Lookup("postgres")
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
+ }
+ postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
+ }
+ postgresGid, err := strconv.Atoi(postgresUser.Gid)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric gid?: %q", postgresUser.Gid)
+ }
+ err = os.Chown(super.tempdir, 0, postgresGid)
+ if err != nil {
+ return err
+ }
+ err = os.Chmod(super.tempdir, 0710)
+ if err != nil {
+ return err
+ }
+ err = os.Chown(datadir, postgresUid, 0)
+ if err != nil {
+ return err
+ }
+ args = append([]string{"-u", "postgres", prog}, args...)
+ prog = "sudo"
+ }
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
if err != nil {
return err
}
@@ -55,18 +93,29 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
+ if iamroot {
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+ if err != nil {
+ return err
+ }
+ }
port := super.cluster.PostgreSQL.Connection["port"]
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "postgres"),
+ prog, args := filepath.Join(bindir, "postgres"), []string{
"-l", // enable ssl
"-D", datadir, // data dir
"-k", datadir, // socket dir
"-p", super.cluster.PostgreSQL.Connection["port"],
- ))
+ }
+ if iamroot {
+ args = append([]string{"-u", "postgres", prog}, args...)
+ prog = "sudo"
+ }
+ fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
}()
for {
@@ -78,11 +127,15 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
}
time.Sleep(time.Second / 2)
}
- db, err := sql.Open("postgres", arvados.PostgreSQLConnection{
+ pgconn := arvados.PostgreSQLConnection{
"host": datadir,
"port": port,
"dbname": "postgres",
- }.String())
+ }
+ if iamroot {
+ pgconn["user"] = "postgres"
+ }
+ db, err := sql.Open("postgres", pgconn.String())
if err != nil {
return fmt.Errorf("db open failed: %s", err)
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 4a4ab4d98..d230dcc3e 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -406,7 +406,11 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
- logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
+ logprefix := prog
+ if logprefix == "sudo" && len(args) >= 3 && args[0] == "-u" {
+ logprefix = args[2]
+ }
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
logprefix = args[1]
} else if logprefix == "arvados-server" && len(args) > 1 {
commit 919428dfcbb1f7d13a6082e4ffcb094059a773de
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:13:37 2020 -0400
16053: 15954: Use auto-generated certificates in nginx config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 6b2d6777f..ecbb7a9d3 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -26,15 +26,18 @@ func (runNginx) String() string {
}
func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, createCertificates{})
+ if err != nil {
+ return err
+ }
vars := map[string]string{
"LISTENHOST": super.ListenHost,
- "SSLCERT": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
- "SSLKEY": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
+ "SSLCERT": filepath.Join(super.tempdir, "server.crt"),
+ "SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
"TMPDIR": super.tempdir,
}
- var err error
for _, cmpt := range []struct {
varname string
svc arvados.Service
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list