[ARVADOS] created: 2.1.0-336-g4865911a6
Git user
git at public.arvados.org
Wed Jan 27 08:15:13 UTC 2021
at 4865911a605128adb454b3280b1cf9dcd38f499e (commit)
commit 4865911a605128adb454b3280b1cf9dcd38f499e
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 27 03:14:44 2021 -0500
16306: Accept -package-version arg to testinstall.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index cf9cbfa6c..e826408c1 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -126,7 +126,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
return err
}
- cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
+ cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages --multiversion . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
cmd.Stdout = stdout
cmd.Stderr = stderr
cmd.Dir = opts.PackageDir
diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index b4ad68b1d..75e8f7604 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -56,6 +56,7 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
cmd = exec.Command("go", "run", ".",
"testinstall",
"-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 774771efb..984779950 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -6,7 +6,6 @@ package main
import (
"context"
- "errors"
"fmt"
"io"
"io/ioutil"
@@ -20,9 +19,6 @@ import (
)
func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
- if opts.PackageVersion != "" {
- return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
- }
depsImageName := "arvados-package-deps-" + opts.TargetOS
depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
@@ -77,6 +73,10 @@ eatmydata apt-get remove -y arvados-server-easy
}
}
+ versionsuffix := ""
+ if opts.PackageVersion != "" {
+ versionsuffix = "=" + opts.PackageVersion
+ }
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
"--tmpfs", "/tmp:exec,mode=01777",
"-v", opts.PackageDir+":/pkg:ro",
@@ -87,7 +87,7 @@ eatmydata apt-get remove -y arvados-server-easy
set -e
PATH="/var/lib/arvados/bin:$PATH"
apt-get update
-eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy`+versionsuffix+`
apt-get -y autoremove
/etc/init.d/postgresql start
arvados-server init -cluster-id x1234
commit 01e15db1f4a331508117bc841256acec8ca361de
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 25 10:45:30 2021 -0500
16306: Fix runPassenger HOME env in non-production case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 84f51d0a7..420393997 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -141,12 +141,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
}
opts := runOptions{
env: append([]string{
- "HOME=/var/www",
"TMPDIR=" + super.wwwtempdir,
}, railsEnv...),
}
if super.ClusterType == "production" {
opts.user = "www-data"
+ opts.env = append(opts.env, "HOME=/var/www")
} else {
// This would be desirable when changing uid
// too, but it fails because /dev/stderr is a
commit 18def2a271e02fd64749fe650034f50d1b659e45
Merge: 591922d85 fbc95892b
Author: Tom Clegg <tom at curii.com>
Date: Fri Jan 22 00:58:58 2021 -0500
16306: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit 591922d85a4cbb6fc890bde7b5f4848f4c92468a
Author: Tom Clegg <tom at curii.com>
Date: Fri Jan 22 00:53:42 2021 -0500
16306: Remove obsolete RailsSessionSecretToken config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index d4d9ef47e..7ae42c531 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -156,8 +156,6 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
Health:
InternalURLs:
"http://0.0.0.0:9007/": {}
- API:
- RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
Collections:
BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
Containers:
commit 8cd08f2ce640e0b1967db489d29e3761ac63f0d7
Merge: a4a17913e 4c6c49190
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 21 16:33:40 2021 -0500
16306: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit a4a17913e299851f0b5979189d36997a62ff43e1
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 19 15:42:50 2021 -0500
16306: Hide "easy install" option from the install docs.
To be un-hidden when it is fully implemented and usable.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index 5386e8797..75a55b469 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -208,8 +208,6 @@ navbar:
- install/arvados-on-kubernetes.html.textile.liquid
- install/arvados-on-kubernetes-minikube.html.textile.liquid
- install/arvados-on-kubernetes-GKE.html.textile.liquid
- - Automatic installation:
- - install/automatic.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
index 049bf6891..79e850538 100644
--- a/doc/install/automatic.html.textile.liquid
+++ b/doc/install/automatic.html.textile.liquid
@@ -9,6 +9,10 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin' %}
+This installation method is not fully implemented, which is why this page is not yet listed in the "table of installation options":{{site.baseurl}}/install/index.html or in the left nav.
+{% include 'notebox_end' %}
+
This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
@@ -24,7 +28,9 @@ You will need:
h2. Initialize the cluster
<pre>
-# apt-get install arvados-server
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# apt-get update
+# apt-get install arvados-server-easy
# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example at gmail.com.example
</pre>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index f16ae2dad..1b27ca6ed 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -25,7 +25,6 @@ table(table table-bordered table-condensed).
|"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
|"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
-|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
|"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
|"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
commit edc2687671a2fec74304c6e00092034d50c8f667
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 19 15:32:15 2021 -0500
16306: Fixup test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index 9f002b535..b4ad68b1d 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -32,6 +32,8 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
}
tmpdir := c.MkDir()
defer os.RemoveAll(tmpdir)
+ err := os.Chmod(tmpdir, 0755)
+ c.Assert(err, check.IsNil)
cmd := exec.Command("go", "run", ".",
"build",
@@ -41,13 +43,16 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
- err := cmd.Run()
+ err = cmd.Run()
c.Check(err, check.IsNil)
fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
c.Assert(err, check.IsNil)
c.Logf("%#v", fi)
+ buf, _ := exec.Command("ls", "-l", tmpdir).CombinedOutput()
+ c.Logf("%s", buf)
+
cmd = exec.Command("go", "run", ".",
"testinstall",
"-package-dir", tmpdir,
commit d20342c56db7e148f793a29fd79619ce2e3f0243
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 18 16:33:16 2021 -0500
16306: Remove old bash scripts.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/docker-boot.sh b/cmd/arvados-package/docker-boot.sh
deleted file mode 100755
index 39a8a56c8..000000000
--- a/cmd/arvados-package/docker-boot.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Bring up a docker container with some locally-built commands (e.g.,
-# cmd/arvados-server) replacing the ones that came with
-# arvados-server-easy when the arvados-installpackage-* image was
-# built.
-#
-# Assumes docker-build-install.sh has already succeeded.
-#
-# Example:
-#
-# docker-boot.sh cmd/arvados-server services/keep-balance
-
-set -e -o pipefail
-
-declare -A opts=()
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --os)
- shift
- opts[os]="$1"
- shift
- ;;
- --shell)
- shift
- opts[shell]=1
- ;;
- *)
- break
- ;;
- esac
-done
-
-cleanup() {
- if [[ -n "${tmpdir}" ]]; then
- rm -rf "${tmpdir}"
- fi
-}
-trap cleanup ERR EXIT
-
-tmpdir=$(mktemp -d)
-version=$(git describe --tag --dirty)
-
-declare -a volargs=()
-for inject in "$@"; do
- case "$inject" in
- nginx.conf)
- volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
- ;;
- *)
- echo >&2 "building $inject..."
- (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
- cmd="$(basename "$inject")"
- volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
- ;;
- esac
-done
-
-osbase=${opts[os]:-debian:10}
-installimage=arvados-installpackage-${osbase}
-command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
-if [[ "${opts[shell]}" ]]; then
- command="bash -login"
-fi
-docker run -it --rm \
- "${volargs[@]}" \
- "${installimage}" \
- bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-package/docker-build-install.sh b/cmd/arvados-package/docker-build-install.sh
deleted file mode 100755
index 4fdcccbce..000000000
--- a/cmd/arvados-package/docker-build-install.sh
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Build an arvados-server-easy package, then install and run it on a
-# base OS image.
-#
-# Examples:
-#
-# docker-build-install.sh --force-buildimage --force-installimage # always build fresh docker images
-#
-# docker-build-install.sh # reuse cached docker images if possible
-
-set -e -o pipefail
-
-declare -A opts=()
-while [[ $# -gt 0 ]]; do
- arg="$1"
- shift
- case "$arg" in
- --force-buildimage)
- opts[force-buildimage]=1
- ;;
- --force-installimage)
- opts[force-installimage]=1
- ;;
- --os)
- opts[os]="$1"
- shift
- ;;
- *)
- echo >&2 "invalid argument '$arg'"
- exit 2
- ;;
- esac
-done
-
-cleanup() {
- if [[ -n "${buildctr}" ]]; then
- docker rm "${buildctr}" || true
- fi
- if [[ -n "${installctr}" ]]; then
- docker rm "${installctr}" || true
- fi
-}
-trap cleanup ERR EXIT
-
-version=$(git describe --tag --dirty)
-osbase=${opts[os]:-debian:10}
-
-mkdir -p /tmp/pkg
-
-buildimage=arvados-package-build-${osbase}
-if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
- (
- echo >&2 building arvados-server...
- cd cmd/arvados-server
- go install
- )
- echo >&2 building ${buildimage}...
- buildctr=${buildimage/:/-}
- docker rm "${buildctr}" || true
- docker run \
- --name "${buildctr}" \
- --tmpfs /tmp:exec,mode=01777 \
- -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
- -v "$(pwd)":/arvados:ro \
- "${osbase}" \
- /arvados-server install \
- -eatmydata \
- -type package \
- -source /arvados \
- -package-version "${version}"
- docker commit "${buildctr}" "${buildimage}"
- docker rm "${buildctr}"
- buildctr=
-fi
-
-pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
-rm -v -f "${pkgfile}"
-
-(
- echo >&2 building arvados-package...
- cd cmd/arvados-package
- go install
-)
-echo >&2 building ${pkgfile}...
-docker run --rm \
- --tmpfs /tmp:exec,mode=01777 \
- -v /tmp/pkg:/pkg \
- -v "${GOPATH:-${HOME}/go}"/bin/arvados-package:/arvados-package:ro \
- -v "$(pwd)":/arvados:ro \
- "${buildimage}" \
- eatmydata \
- /arvados-package build \
- -source /arvados \
- -package-version "${version}" \
- -output-directory /pkg
-
-ls -l ${pkgfile}
-(
- echo >&2 dpkg-scanpackages...
- cd /tmp/pkg
- dpkg-scanpackages . | gzip > Packages.gz
-)
-sourcesfile=/tmp/sources.conf.d-arvados
-echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-
-installimage="arvados-installpackage-${osbase}"
-if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
- echo >&2 building ${installimage}...
- installctr=${installimage/:/-}
- docker rm "${installctr}" || true
- docker run -it \
- --name "${installctr}" \
- --tmpfs /tmp \
- -v /tmp/pkg:/pkg:ro \
- -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
- --env DEBIAN_FRONTEND=noninteractive \
- "${osbase}" \
- bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
- docker commit "${installctr}" "${installimage}"
- docker rm "${installctr}"
- installctr=
-fi
-
-echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
-docker run -it --rm \
- -v /tmp/pkg:/pkg:ro \
- -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
- "${installimage}" \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive eatmydata apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 719258a9b..774771efb 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -19,27 +19,6 @@ import (
"github.com/docker/docker/client"
)
-// sourcesfile=/tmp/sources.conf.d-arvados
-// echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-
-// installimage="arvados-installpackage-${osbase}"
-// if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
-// echo >&2 building ${installimage}...
-// installctr=${installimage/:/-}
-// docker rm "${installctr}" || true
-// docker run -it \
-// --name "${installctr}" \
-// --tmpfs /tmp \
-// -v /tmp/pkg:/pkg:ro \
-// -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
-// --env DEBIAN_FRONTEND=noninteractive \
-// "${osbase}" \
-// bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
-// docker commit "${installctr}" "${installimage}"
-// docker rm "${installctr}"
-// installctr=
-// fi
-
func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
if opts.PackageVersion != "" {
return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
commit 4085665f87c82939961e55bf004ca47c03ca30ac
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 18 16:31:31 2021 -0500
16306: Update tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index abb54532d..9f002b535 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -5,10 +5,8 @@
package main
import (
- "flag"
"os"
"os/exec"
- "path/filepath"
"testing"
"gopkg.in/check.v1"
@@ -18,13 +16,6 @@ var buildimage string
func init() {
os.Args = append(os.Args, "-test.timeout=30m") // kludge
-
- // This enables a hack to speed up repeated tests: hit "docker
- // commit --pause {containername} checkpointtag" after the
- // test container has downloaded/compiled some stuff, then run
- // tests with "-test.buildimage=checkpointtag" next time to
- // retry/resume/update from that point.
- flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
}
type BuildSuite struct{}
@@ -42,36 +33,30 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
tmpdir := c.MkDir()
defer os.RemoveAll(tmpdir)
- err := os.Mkdir(tmpdir+"/pkg", 0755)
- c.Assert(err, check.IsNil)
- err = os.Mkdir(tmpdir+"/bin", 0755)
- c.Assert(err, check.IsNil)
-
- cmd := exec.Command("go", "install")
- cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
- cmd.Stdout = os.Stdout
+ cmd := exec.Command("go", "run", ".",
+ "build",
+ "-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
+ "-source", "../..",
+ )
+ cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
- err = cmd.Run()
- c.Assert(err, check.IsNil)
+ err := cmd.Run()
+ c.Check(err, check.IsNil)
- srctree, err := filepath.Abs("../..")
+ fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
c.Assert(err, check.IsNil)
+ c.Logf("%#v", fi)
- cmd = exec.Command("docker", "run", "--rm",
- "-v", tmpdir+"/pkg:/pkg",
- "-v", tmpdir+"/bin/arvados-package:/arvados-package:ro",
- "-v", srctree+":/usr/local/src/arvados:ro",
- buildimage,
- "/arvados-package", "build",
- "-package-version", "0.9.99",
- "-source", "/usr/local/src/arvados",
- "-output-directory", "/pkg")
- cmd.Stdout = os.Stdout
+ cmd = exec.Command("go", "run", ".",
+ "testinstall",
+ "-package-dir", tmpdir,
+ )
+ cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
err = cmd.Run()
- c.Assert(err, check.IsNil)
+ c.Check(err, check.IsNil)
- fi, err := os.Stat(tmpdir + "/pkg/arvados-server-easy_0.9.99_amd64.deb")
- c.Assert(err, check.IsNil)
- c.Logf("%#v", fi)
+ err = os.RemoveAll(tmpdir)
+ c.Check(err, check.IsNil)
}
commit 1297d71125f5a694996147406edf38bdc913396c
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 18 16:31:40 2021 -0500
16306: Use cwd as default source tree.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 6cf3ba3c0..b0c0e9e3c 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -68,7 +68,8 @@ type opts struct {
func parseFlags(args []string) (opts, error) {
opts := opts{
- TargetOS: "debian:10",
+ SourceDir: ".",
+ TargetOS: "debian:10",
}
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
@@ -123,5 +124,9 @@ Options:
opts.SourceDir = d
}
opts.PackageDir = filepath.Clean(opts.PackageDir)
+ opts.SourceDir, err = filepath.Abs(opts.SourceDir)
+ if err != nil {
+ return opts, err
+ }
return opts, nil
}
commit 41305b5ac71cc9a306dc654c42c11ffcc4258a47
Merge: b979f8011 025639399
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 18 13:55:27 2021 -0500
16306: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc lib/boot/supervisor.go
index e892d3e62,f2e715a76..838808df5
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@@ -244,11 -222,11 +244,11 @@@ func (super *Supervisor) run(cfg *arvad
runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
- runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
+ runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
- installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
- runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
++ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
- runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
if super.ClusterType != "test" {
@@@ -682,26 -605,27 +682,26 @@@ func (super *Supervisor) autofillConfig
}
}
}
- if cluster.SystemRootToken == "" {
- cluster.SystemRootToken = randomHexString(64)
- }
- if cluster.ManagementToken == "" {
- cluster.ManagementToken = randomHexString(64)
- }
- if cluster.Collections.BlobSigningKey == "" {
- cluster.Collections.BlobSigningKey = randomHexString(64)
- }
- if cluster.Users.AnonymousUserToken == "" {
- cluster.Users.AnonymousUserToken = randomHexString(64)
- }
-
- if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
- buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
- if err != nil {
- return err
- }
- cluster.Containers.DispatchPrivateKey = string(buf)
- }
if super.ClusterType != "production" {
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
- if cluster.API.RailsSessionSecretToken == "" {
- cluster.API.RailsSessionSecretToken = randomHexString(64)
- }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
++ if cluster.Users.AnonymousUserToken == "" {
++ cluster.Users.AnonymousUserToken = randomHexString(64)
++ }
+ if cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
cluster.TLS.Insecure = true
}
if super.ClusterType == "test" {
commit b979f80113c4f8a4c13c96b4679c28788c099333
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 7 17:24:29 2021 -0500
16306: Command usage help.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index c85979268..cf9cbfa6c 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -108,7 +108,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
"-v", selfbin+":/arvados-package:ro",
"-v", opts.SourceDir+":/arvados:ro",
buildImageName,
- "eatmydata", "/arvados-package", "fpm",
+ "eatmydata", "/arvados-package", "_fpm",
"-source", "/arvados",
"-package-version", opts.PackageVersion,
"-package-dir", "/pkg",
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 9b9971e92..6cf3ba3c0 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -11,6 +11,7 @@ import (
"io"
"os"
"path/filepath"
+ "strings"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/install"
@@ -24,13 +25,17 @@ var (
"--version": cmd.Version,
"build": cmdFunc(build),
- "fpm": cmdFunc(fpm),
"testinstall": cmdFunc(testinstall),
+ "_fpm": cmdFunc(fpm), // internal use
"_install": install.Command, // internal use
})
)
func main() {
+ if len(os.Args) < 2 || strings.HasPrefix(os.Args[1], "-") {
+ parseFlags([]string{"-help"})
+ os.Exit(2)
+ }
os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
@@ -72,6 +77,37 @@ func parseFlags(args []string) (opts, error) {
flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+ flags.Usage = func() {
+ fmt.Fprint(flags.Output(), `Usage: arvados-package <subcommand> [options]
+
+Subcommands:
+ build
+ use a docker container to build a package from a checked
+ out version of the arvados source tree
+ testinstall
+ use a docker container to install a package and confirm
+ the resulting installation is functional
+ version
+ show program version
+
+Internally used subcommands:
+ _fpm
+ build a package
+ _install
+ equivalent to "arvados-server install"
+
+Automation/integration notes:
+ The first time a given machine runs "build" or "testinstall" (and
+ any time the -rebuild-image is used), new docker images are built,
+ which is quite slow. If you use on-demand VMs to run automated builds,
+ run "build" and "testinstall" once when setting up your initial VM
+ image, and be prepared to rebuild that VM image when package-building
+ slows down (this will happen when new dependencies are introduced).
+
+Options:
+`)
+ flags.PrintDefaults()
+ }
err := flags.Parse(args)
if err != nil {
return opts, err
commit 57c3fbfb0f75dc38be717d52f29abe5bcefea1bb
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 4 15:58:24 2021 -0500
16306: Remove daemontools dependency.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 4a605e35a..84f51d0a7 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -148,11 +148,11 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
if super.ClusterType == "production" {
opts.user = "www-data"
} else {
- // This would be desirable in the production
- // case too, but it fails with sudo because
- // /dev/stderr is a symlink to a pty owned by
- // root: "nginx: [emerg] open() "/dev/stderr"
- // failed (13: Permission denied)"
+ // This would be desirable when changing uid
+ // too, but it fails because /dev/stderr is a
+ // symlink to a pty owned by root: "nginx:
+ // [emerg] open() "/dev/stderr" failed (13:
+ // Permission denied)"
cmdline = append(cmdline, "--log-file", "/dev/stderr")
}
err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index daa0414a3..4ed7603d2 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -60,6 +60,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
return err
}
prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+ opts := runOptions{}
if iamroot {
postgresUser, err := user.Lookup("postgres")
if err != nil {
@@ -85,15 +86,9 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
- // We can't use "sudo -u" here because it creates an
- // intermediate process that interferes with our
- // ability to reliably kill postgres. The setuidgid
- // program just calls exec without forking, so it
- // doesn't have this problem.
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
- }
- err = super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...)
+ opts.user = "postgres"
+ }
+ err = super.RunProgram(ctx, super.tempdir, opts, prog, args...)
if err != nil {
return err
}
@@ -120,11 +115,11 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
"-k", datadir, // socket dir
"-p", super.cluster.PostgreSQL.Connection["port"],
}
+ opts := runOptions{}
if iamroot {
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
+ opts.user = "postgres"
}
- fail(super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...))
+ fail(super.RunProgram(ctx, super.tempdir, opts, prog, args...))
}()
for {
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 5e88775e5..e892d3e62 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -491,9 +491,6 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
logprefix := prog
{
- if logprefix == "setuidgid" && len(args) >= 3 {
- logprefix = args[2]
- }
innerargs := args
if logprefix == "sudo" {
for i := 0; i < len(args); i++ {
@@ -556,6 +553,15 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
cmd.Env = dedupEnv(env)
if opts.user != "" {
+ // Note: We use this approach instead of "sudo"
+ // because in certain circumstances (we are pid 1 in a
+ // docker container, and our passenger child process
+ // changes to pgid 1) the intermediate sudo process
+ // notices we have the same pgid as our child and
+ // refuses to propagate signals from us to our child,
+ // so we can't signal/shutdown our passenger/rails
+ // apps. "chpst" or "setuidgid" would work, but these
+ // few lines avoid depending on runit/daemontools.
u, err := user.Lookup(opts.user)
if err != nil {
return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 049df2564..504cdff8f 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -141,7 +141,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
"cadaver",
"curl",
"cython3",
- "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
"gettext",
commit 6cd0e19a277812f302f46693d824cc17d17ed6b1
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 4 15:42:35 2021 -0500
16306: Exit 0 when startup-then-shutdown goes as planned.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index e0e275522..373613bb3 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -108,6 +108,11 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
fmt.Fprintln(stdout, url)
if *shutdown {
super.Stop()
+ // Wait for children to exit. Don't report the
+ // ensuing "context cancelled" error, though:
+ // return nil to indicate successful startup.
+ _ = super.Wait()
+ return nil
}
}
// Wait for signal/crash + orderly shutdown
commit d71d4273d1f3d0b6381efafa649b81c6b4107cf1
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 4 15:26:46 2021 -0500
16306: Fix inability to shutdown passenger processes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index 8f6339e63..b2b8c896c 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -39,17 +39,17 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
}
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
// Generate a self-signed root certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
if err != nil {
return err
}
// Generate server key
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "server.key", "2048")
if err != nil {
return err
}
@@ -63,12 +63,12 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
return err
}
// Generate signing request
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
if err != nil {
return err
}
// Sign certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
if err != nil {
return err
}
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index d49fabe56..d14d05152 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -120,7 +120,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", nil, nil, nginx, args...))
+ fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
}()
// Choose one of the ports where Nginx should listen, and wait
// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 03464aaf7..4a605e35a 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -50,32 +50,32 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
defer passengerInstallMutex.Unlock()
var buf bytes.Buffer
- err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler")
+ err = super.RunProgram(ctx, runner.src, runOptions{output: &buf}, "gem", "list", "--details", "bundler")
if err != nil {
return err
}
for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
break
}
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "build-native-support")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "install-standalone-runtime")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "validate-install")
if err != nil && !strings.Contains(err.Error(), "exit status 2") {
// Exit code 2 indicates there were warnings (like
// "other passenger installations have been detected",
@@ -139,8 +139,14 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
"--no-install-runtime",
"--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
}
+ opts := runOptions{
+ env: append([]string{
+ "HOME=/var/www",
+ "TMPDIR=" + super.wwwtempdir,
+ }, railsEnv...),
+ }
if super.ClusterType == "production" {
- cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+ opts.user = "www-data"
} else {
// This would be desirable in the production
// case too, but it fails with sudo because
@@ -149,8 +155,7 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
// failed (13: Permission denied)"
cmdline = append(cmdline, "--log-file", "/dev/stderr")
}
- env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
- err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
+ err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
fail(err)
}()
return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index e45c4e168..daa0414a3 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -48,7 +48,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
}
buf := bytes.NewBuffer(nil)
- err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{output: buf}, "pg_config", "--bindir")
if err != nil {
return err
}
@@ -93,17 +93,17 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
args = append([]string{"postgres", prog}, args...)
prog = "setuidgid"
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...)
if err != nil {
return err
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir)
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "cp", "server.crt", "server.key", datadir)
if err != nil {
return err
}
if iamroot {
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
if err != nil {
return err
}
@@ -124,7 +124,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
args = append([]string{"postgres", prog}, args...)
prog = "setuidgid"
}
- fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...))
}()
for {
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index 1f6cb764e..bd1e94265 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -23,11 +23,11 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
if super.ClusterType == "production" {
return nil
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb")
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb")
if err != nil {
return err
}
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 77fdc98be..090e85244 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -31,7 +31,7 @@ func (runner runServiceCommand) String() string {
func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
binfile := filepath.Join(super.bindir, "arvados-server")
- err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
+ err := super.RunProgram(ctx, super.bindir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
@@ -46,7 +46,7 @@ func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
}()
}
return nil
@@ -77,7 +77,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup
return ctx.Err()
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
@@ -93,7 +93,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile))
}()
}
return nil
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 7dba9b5dc..5e88775e5 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -21,6 +21,7 @@ import (
"os/user"
"path/filepath"
"reflect"
+ "strconv"
"strings"
"sync"
"syscall"
@@ -206,13 +207,13 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
} else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat")
if err != nil {
return err
}
dirty := buf.Len() > 0
buf.Reset()
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H")
if err != nil {
return err
}
@@ -407,7 +408,7 @@ func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (
if super.ClusterType == "production" {
return binfile, nil
}
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
@@ -470,6 +471,12 @@ func (super *Supervisor) lookPath(prog string) string {
return prog
}
+type runOptions struct {
+ output io.Writer // attach stdout
+ env []string // add/replace environment variables
+ user string // run as specified user
+}
+
// RunProgram runs prog with args, using dir as working directory. If ctx is
// cancelled while the child is running, RunProgram terminates the child, waits
// for it to exit, then returns.
@@ -478,7 +485,7 @@ func (super *Supervisor) lookPath(prog string) string {
//
// Child's stdout will be written to output if non-nil, otherwise the
// boot command's stderr.
-func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error {
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
@@ -531,10 +538,10 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
}()
copiers.Add(1)
go func() {
- if output == nil {
+ if opts.output == nil {
io.Copy(logwriter, stdout)
} else {
- io.Copy(output, stdout)
+ io.Copy(opts.output, stdout)
}
copiers.Done()
}()
@@ -544,10 +551,25 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
} else {
cmd.Dir = filepath.Join(super.SourcePath, dir)
}
- env = append([]string(nil), env...)
+ env := append([]string(nil), opts.env...)
env = append(env, super.environ...)
cmd.Env = dedupEnv(env)
+ if opts.user != "" {
+ u, err := user.Lookup(opts.user)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ Credential: &syscall.Credential{
+ Uid: uint32(uid),
+ Gid: uint32(gid),
+ },
+ }
+ }
+
exited := false
defer func() { exited = true }()
go func() {
commit 2cfb41d14010e26d97df93c4cf8ad00f0ac01701
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 4 11:22:09 2021 -0500
16306: Refactor docker scripts into arvados-package command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index 1343ca77b..c85979268 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -5,120 +5,158 @@
package main
import (
+ "bytes"
"context"
- "flag"
"fmt"
"io"
+ "io/ioutil"
"os"
"os/exec"
+ "os/user"
+ "path/filepath"
+ "strings"
- "git.arvados.org/arvados.git/lib/install"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/sirupsen/logrus"
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
)
-type build struct{}
-
-func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- logger := ctxlog.New(stderr, "text", "info")
- err := (&builder{
- PackageVersion: "0.0.0",
- logger: logger,
- }).run(context.Background(), prog, args, stdin, stdout, stderr)
- if err != nil {
- logger.WithError(err).Error("failed")
- return 1
+func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion == "" {
+ var buf bytes.Buffer
+ cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty")
+ cmd.Stdout = &buf
+ cmd.Stderr = stderr
+ cmd.Dir = opts.SourceDir
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("git describe: %w", err)
+ }
+ opts.PackageVersion = strings.TrimSpace(buf.String())
+ ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion)
}
- return 0
-}
-type builder struct {
- PackageVersion string
- SourcePath string
- OutputDir string
- SkipInstall bool
- logger logrus.FieldLogger
-}
+ if opts.PackageChown == "" {
+ whoami, err := user.Current()
+ if err != nil {
+ return fmt.Errorf("user.Current: %w", err)
+ }
+ opts.PackageChown = whoami.Uid + ":" + whoami.Gid
+ }
-func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
- flags := flag.NewFlagSet("", flag.ContinueOnError)
- flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
- flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
- flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
- flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
- err := flags.Parse(args)
+ // Build in a tempdir, then move to the desired destination
+ // dir. Otherwise, errors might cause us to leave a mess:
+ // truncated files, files owned by root, etc.
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".")
if err != nil {
return err
}
- if len(flags.Args()) > 0 {
- return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ defer os.RemoveAll(tmpdir)
+
+ selfbin, err := os.Readlink("/proc/self/exe")
+ if err != nil {
+ return fmt.Errorf("readlink /proc/self/exe: %w", err)
}
- if !bldr.SkipInstall {
- exitcode := install.Command.RunCommand("arvados-server install", []string{
+ buildImageName := "arvados-package-build-" + opts.TargetOS
+ packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb"
+
+ if ok, err := dockerImageExists(ctx, buildImageName); err != nil {
+ return err
+ } else if !ok || opts.RebuildImage {
+ buildCtrName := strings.Replace(buildImageName, ":", "-", -1)
+ err = dockerRm(ctx, buildCtrName)
+ if err != nil {
+ return err
+ }
+
+ defer dockerRm(ctx, buildCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", buildCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ opts.TargetOS,
+ "/arvados-package", "_install",
+ "-eatmydata",
"-type", "package",
- "-package-version", bldr.PackageVersion,
- "-source", bldr.SourcePath,
- }, stdin, stdout, stderr)
- if exitcode != 0 {
- return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
}
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+
+ ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName)
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", tmpdir+":/pkg",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ buildImageName,
+ "eatmydata", "/arvados-package", "fpm",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ "-package-dir", "/pkg",
+ "-package-chown", opts.PackageChown,
+ )
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("gem install fpm: %w", err)
+ return fmt.Errorf("docker run: %w", err)
}
- if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
- // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
- cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- err = cmd.Run()
- if err != nil {
- return fmt.Errorf("monkeypatch fpm: %w", err)
- }
+ err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
+ if err != nil {
+ return err
}
- // Remove unneeded files. This is much faster than "fpm
- // --exclude X" because fpm copies everything into a staging
- // area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
cmd.Stdout = stdout
cmd.Stderr = stderr
+ cmd.Dir = opts.PackageDir
err = cmd.Run()
if err != nil {
- return fmt.Errorf("rm -rf [...]: %w", err)
+ return fmt.Errorf("dpkg-scanpackages: %w", err)
}
- format := "deb" // TODO: rpm
+ return nil
+}
- cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
- "--name", "arvados-server-easy",
- "--version", bldr.PackageVersion,
- "--input-type", "dir",
- "--output-type", format)
- deps, err := install.ProductionDependencies()
+func dockerRm(ctx context.Context, name string) error {
+ cli, err := client.NewEnvClient()
if err != nil {
return err
}
- for _, pkg := range deps {
- cmd.Args = append(cmd.Args, "--depends", pkg)
+ ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1})
+ if err != nil {
+ return err
}
- cmd.Args = append(cmd.Args,
- "--verbose",
- "--deb-use-file-permissions",
- "--rpm-use-file-permissions",
- "/var/lib/arvados",
- "/var/www/.gem",
- "/var/www/.passenger",
- "/var/www/.bundle",
- )
- fmt.Fprintf(stderr, "... %s\n", cmd.Args)
- cmd.Dir = bldr.OutputDir
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- return cmd.Run()
+ for _, ctr := range ctrs {
+ for _, ctrname := range ctr.Names {
+ if ctrname == "/"+name {
+ err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{})
+ if err != nil {
+ return fmt.Errorf("error removing container %s: %w", ctr.ID, err)
+ }
+ break
+ }
+ }
+ }
+ return nil
}
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 02bc16cea..9b9971e92 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -5,9 +5,16 @@
package main
import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
"os"
+ "path/filepath"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
)
var (
@@ -16,10 +23,69 @@ var (
"-version": cmd.Version,
"--version": cmd.Version,
- "build": build{},
+ "build": cmdFunc(build),
+ "fpm": cmdFunc(fpm),
+ "testinstall": cmdFunc(testinstall),
+ "_install": install.Command, // internal use
})
)
func main() {
os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
+
+type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error
+
+func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ opts, err := parseFlags(args)
+ if err != nil {
+ logger.WithError(err).Error("error parsing command line flags")
+ return 1
+ }
+ err = cf(ctx, opts, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type opts struct {
+ PackageVersion string
+ PackageDir string
+ PackageChown string
+ RebuildImage bool
+ SourceDir string
+ TargetOS string
+}
+
+func parseFlags(args []string) (opts, error) {
+ opts := opts{
+ TargetOS: "debian:10",
+ }
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
+ flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location")
+ flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)")
+ flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
+ flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
+ flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+ err := flags.Parse(args)
+ if err != nil {
+ return opts, err
+ }
+ if len(flags.Args()) > 0 {
+ return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if opts.SourceDir == "" {
+ d, err := os.Getwd()
+ if err != nil {
+ return opts, fmt.Errorf("Getwd: %w", err)
+ }
+ opts.SourceDir = d
+ }
+ opts.PackageDir = filepath.Clean(opts.PackageDir)
+ return opts, nil
+}
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/fpm.go
similarity index 53%
copy from cmd/arvados-package/build.go
copy to cmd/arvados-package/fpm.go
index 1343ca77b..a86232036 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/fpm.go
@@ -6,67 +6,37 @@ package main
import (
"context"
- "flag"
"fmt"
"io"
"os"
"os/exec"
+ "path/filepath"
"git.arvados.org/arvados.git/lib/install"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/sirupsen/logrus"
)
-type build struct{}
-
-func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- logger := ctxlog.New(stderr, "text", "info")
- err := (&builder{
- PackageVersion: "0.0.0",
- logger: logger,
- }).run(context.Background(), prog, args, stdin, stdout, stderr)
- if err != nil {
- logger.WithError(err).Error("failed")
- return 1
+func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ var chownUid, chownGid int
+ if opts.PackageChown != "" {
+ _, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid)
+ if err != nil {
+ return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err)
+ }
}
- return 0
-}
-
-type builder struct {
- PackageVersion string
- SourcePath string
- OutputDir string
- SkipInstall bool
- logger logrus.FieldLogger
-}
-func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
- flags := flag.NewFlagSet("", flag.ContinueOnError)
- flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
- flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
- flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
- flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
- err := flags.Parse(args)
- if err != nil {
- return err
- }
- if len(flags.Args()) > 0 {
- return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
- }
- if !bldr.SkipInstall {
- exitcode := install.Command.RunCommand("arvados-server install", []string{
- "-type", "package",
- "-package-version", bldr.PackageVersion,
- "-source", bldr.SourcePath,
- }, stdin, stdout, stderr)
- if exitcode != 0 {
- return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
- }
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", opts.PackageVersion,
+ "-source", opts.SourceDir,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
}
+
cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
cmd.Stdout = stdout
cmd.Stderr = stderr
- err = cmd.Run()
+ err := cmd.Run()
if err != nil {
return fmt.Errorf("gem install fpm: %w", err)
}
@@ -94,10 +64,12 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
}
format := "deb" // TODO: rpm
+ pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format)
cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+ "--package", pkgfile,
"--name", "arvados-server-easy",
- "--version", bldr.PackageVersion,
+ "--version", opts.PackageVersion,
"--input-type", "dir",
"--output-type", format)
deps, err := install.ProductionDependencies()
@@ -117,8 +89,25 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
"/var/www/.bundle",
)
fmt.Fprintf(stderr, "... %s\n", cmd.Args)
- cmd.Dir = bldr.OutputDir
+ cmd.Dir = opts.PackageDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("fpm: %w", err)
+ }
+
+ if opts.PackageChown != "" {
+ err = os.Chown(pkgfile, chownUid, chownGid)
+ if err != nil {
+ return fmt.Errorf("chown %s: %w", pkgfile, err)
+ }
+ }
+
+ cmd = exec.Command("ls", "-l", pkgfile)
cmd.Stdout = stdout
cmd.Stderr = stderr
- return cmd.Run()
+ _ = cmd.Run()
+
+ return nil
}
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
new file mode 100644
index 000000000..719258a9b
--- /dev/null
+++ b/cmd/arvados-package/install.go
@@ -0,0 +1,143 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+// sourcesfile=/tmp/sources.conf.d-arvados
+// echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+// installimage="arvados-installpackage-${osbase}"
+// if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+// echo >&2 building ${installimage}...
+// installctr=${installimage/:/-}
+// docker rm "${installctr}" || true
+// docker run -it \
+// --name "${installctr}" \
+// --tmpfs /tmp \
+// -v /tmp/pkg:/pkg:ro \
+// -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+// --env DEBIAN_FRONTEND=noninteractive \
+// "${osbase}" \
+// bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
+// docker commit "${installctr}" "${installimage}"
+// docker rm "${installctr}"
+// installctr=
+// fi
+
+func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion != "" {
+ return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
+ }
+ depsImageName := "arvados-package-deps-" + opts.TargetOS
+ depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir("", prog+".")
+ if err != nil {
+ return fmt.Errorf("TempDir: %w", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ sourcesFile := tmpdir + "/arvados-local.list"
+ err = ioutil.WriteFile(sourcesFile, []byte("deb [trusted=yes] file:///pkg ./\n"), 0644)
+ if err != nil {
+ return fmt.Errorf("Write %s: %w", sourcesFile, err)
+ }
+
+ if exists, err := dockerImageExists(ctx, depsImageName); err != nil {
+ return err
+ } else if !exists || opts.RebuildImage {
+ err = dockerRm(ctx, depsCtrName)
+ if err != nil {
+ return err
+ }
+ defer dockerRm(ctx, depsCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", depsCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ opts.TargetOS,
+ "bash", "-c", `
+set -e
+apt-get update
+apt-get install -y eatmydata
+eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql
+eatmydata apt-get remove -y arvados-server-easy
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+ }
+
+ cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ depsImageName,
+ "bash", "-c", `
+set -e
+PATH="/var/lib/arvados/bin:$PATH"
+apt-get update
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy
+apt-get -y autoremove
+/etc/init.d/postgresql start
+arvados-server init -cluster-id x1234
+exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+ return nil
+}
+
+func dockerImageExists(ctx context.Context, name string) (bool, error) {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return false, err
+ }
+ imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+ if err != nil {
+ return false, err
+ }
+ for _, img := range imgs {
+ for _, tag := range img.RepoTags {
+ if tag == name {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
commit 985ece13dd7147f153c7a59c5b3665570c5943c8
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 29 15:18:19 2020 -0500
16306: Rename arvados-dev buildpackage -> arvados-package build
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-package/build.go
similarity index 96%
rename from cmd/arvados-dev/buildpackage.go
rename to cmd/arvados-package/build.go
index a24fa47c9..1343ca77b 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-package/build.go
@@ -17,9 +17,9 @@ import (
"github.com/sirupsen/logrus"
)
-type buildPackage struct{}
+type build struct{}
-func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.New(stderr, "text", "info")
err := (&builder{
PackageVersion: "0.0.0",
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-package/build_test.go
similarity index 86%
rename from cmd/arvados-dev/buildpackage_test.go
rename to cmd/arvados-package/build_test.go
index 61bdeae82..abb54532d 100644
--- a/cmd/arvados-dev/buildpackage_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -27,13 +27,13 @@ func init() {
flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
}
-type BuildpackageSuite struct{}
+type BuildSuite struct{}
-var _ = check.Suite(&BuildpackageSuite{})
+var _ = check.Suite(&BuildSuite{})
func Test(t *testing.T) { check.TestingT(t) }
-func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
if testing.Short() {
c.Skip("skipping docker tests in short mode")
} else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
@@ -59,10 +59,10 @@ func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
cmd = exec.Command("docker", "run", "--rm",
"-v", tmpdir+"/pkg:/pkg",
- "-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+ "-v", tmpdir+"/bin/arvados-package:/arvados-package:ro",
"-v", srctree+":/usr/local/src/arvados:ro",
buildimage,
- "/arvados-dev", "buildpackage",
+ "/arvados-package", "build",
"-package-version", "0.9.99",
"-source", "/usr/local/src/arvados",
"-output-directory", "/pkg")
@@ -71,7 +71,7 @@ func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
err = cmd.Run()
c.Assert(err, check.IsNil)
- fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+ fi, err := os.Stat(tmpdir + "/pkg/arvados-server-easy_0.9.99_amd64.deb")
c.Assert(err, check.IsNil)
c.Logf("%#v", fi)
}
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-package/cmd.go
similarity index 92%
rename from cmd/arvados-dev/cmd.go
rename to cmd/arvados-package/cmd.go
index e0d5144e5..02bc16cea 100644
--- a/cmd/arvados-dev/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -16,7 +16,7 @@ var (
"-version": cmd.Version,
"--version": cmd.Version,
- "buildpackage": buildPackage{},
+ "build": build{},
})
)
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-package/docker-boot.sh
similarity index 100%
rename from cmd/arvados-dev/docker-boot.sh
rename to cmd/arvados-package/docker-boot.sh
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-package/docker-build-install.sh
similarity index 92%
rename from cmd/arvados-dev/docker-build-install.sh
rename to cmd/arvados-package/docker-build-install.sh
index 966318b94..4fdcccbce 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-package/docker-build-install.sh
@@ -51,7 +51,7 @@ osbase=${opts[os]:-debian:10}
mkdir -p /tmp/pkg
-buildimage=arvados-buildpackage-${osbase}
+buildimage=arvados-package-build-${osbase}
if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
(
echo >&2 building arvados-server...
@@ -81,19 +81,19 @@ pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
rm -v -f "${pkgfile}"
(
- echo >&2 building arvados-dev...
- cd cmd/arvados-dev
+ echo >&2 building arvados-package...
+ cd cmd/arvados-package
go install
)
echo >&2 building ${pkgfile}...
docker run --rm \
--tmpfs /tmp:exec,mode=01777 \
-v /tmp/pkg:/pkg \
- -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+ -v "${GOPATH:-${HOME}/go}"/bin/arvados-package:/arvados-package:ro \
-v "$(pwd)":/arvados:ro \
"${buildimage}" \
eatmydata \
- /arvados-dev buildpackage \
+ /arvados-package build \
-source /arvados \
-package-version "${version}" \
-output-directory /pkg
@@ -119,7 +119,7 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
-v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
--env DEBIAN_FRONTEND=noninteractive \
"${osbase}" \
- bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql'
+ bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
docker commit "${installctr}" "${installimage}"
docker rm "${installctr}"
installctr=
commit e1756a76e8c7fed496f513909bd2c295e24cf8ee
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:36:51 2020 -0500
16306: Comment out Workbench2 so health check succeeds.
To be re-enabled when supported by install/package/boot.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 8debf59be..d4d9ef47e 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -149,10 +149,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
InternalURLs:
"http://0.0.0.0:8001/": {}
ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
- Workbench2:
- InternalURLs:
- "http://0.0.0.0:8002/": {}
- ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ #Workbench2:
+ # InternalURLs:
+ # "http://0.0.0.0:8002/": {}
+ # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
Health:
InternalURLs:
"http://0.0.0.0:9007/": {}
commit 9c139cc4e2f519e2deb3617918a9199b76c33f10
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:35:39 2020 -0500
16306: Fix env var ordering.
Ensure our GOBIN takes precedence over inherited value.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4f5b251cb..049df2564 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -508,7 +508,8 @@ rm ${zip}
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
- cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+ cmd.Env = append(cmd.Env, os.Environ()...)
+ cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin")
cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
cmd.Stdout = stdout
cmd.Stderr = stderr
commit ae339cf6802741ea52fbe5cd8716306dcf99b021
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:35:26 2020 -0500
16306: Remove more unneeded files from package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index a6e0b544a..a24fa47c9 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -85,12 +85,12 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
// Remove unneeded files. This is much faster than "fpm
// --exclude X" because fpm copies everything into a staging
// area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git /var/lib/arvados/go")
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("rm -rf /var/www/.gem/ruby/*/cache/: %w", err)
+ return fmt.Errorf("rm -rf [...]: %w", err)
}
format := "deb" // TODO: rpm
commit 684b81a42cb9290bde284723603d6e4dbfd7ad35
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:35:03 2020 -0500
16306: Fix permission issues.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 20576b6b9..7dba9b5dc 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -162,7 +162,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
if err != nil {
return err
}
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 2b0faefe2..4f5b251cb 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -446,7 +446,7 @@ rm ${zip}
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
cmd := exec.Command("rsync",
- "-a", "--no-owner", "--delete-after", "--delete-excluded",
+ "-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded",
"--exclude", "/coverage",
"--exclude", "/log",
"--exclude", "/tmp",
@@ -469,7 +469,6 @@ rm ${zip}
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
} {
cmd = exec.Command(cmdline[0], cmdline[1:]...)
- cmd.Env = append([]string{}, os.Environ()...)
cmd.Dir = "/var/lib/arvados/" + dstdir
cmd.Stdout = stdout
cmd.Stderr = stderr
commit d77f9e6f8a14e14f477dfbf944914c78d0805e9f
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:33:27 2020 -0500
16306: Circumvent Rails redirect-to-https in workbench health check.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 794adabdd..a666ef8ec 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -193,6 +193,9 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
}
req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
+ // Avoid workbench1's redirect-http-to-https feature
+ req.Header.Set("X-Forwarded-Proto", "https")
+
ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout))
defer cancel()
req = req.WithContext(ctx)
commit 0137f9e3cd33fe676a862afd727563e2602e0984
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:32:47 2020 -0500
16306: Add health check endpoint to dispatcher.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 7614a143a..ae91a710e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -22,6 +22,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
@@ -164,6 +165,11 @@ func (disp *dispatcher) initialize() {
})
mux.Handler("GET", "/metrics", metricsH)
mux.Handler("GET", "/metrics.json", metricsH)
+ mux.Handler("GET", "/_health/:check", &health.Handler{
+ Token: disp.Cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": disp.CheckHealth},
+ })
disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
}
}
commit 6c1fd146c0b22b374f00ef94c961df8e1e1913a7
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:32:13 2020 -0500
16306: Use tmpfs to speed up package building.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 7b7f25ff9..966318b94 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -63,7 +63,7 @@ if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}
docker rm "${buildctr}" || true
docker run \
--name "${buildctr}" \
- -v /tmp/pkg:/pkg \
+ --tmpfs /tmp:exec,mode=01777 \
-v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
-v "$(pwd)":/arvados:ro \
"${osbase}" \
@@ -87,6 +87,7 @@ rm -v -f "${pkgfile}"
)
echo >&2 building ${pkgfile}...
docker run --rm \
+ --tmpfs /tmp:exec,mode=01777 \
-v /tmp/pkg:/pkg \
-v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
-v "$(pwd)":/arvados:ro \
@@ -113,6 +114,7 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
docker rm "${installctr}" || true
docker run -it \
--name "${installctr}" \
+ --tmpfs /tmp \
-v /tmp/pkg:/pkg:ro \
-v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
--env DEBIAN_FRONTEND=noninteractive \
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 34fe7cc6d..2b0faefe2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -224,17 +224,16 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
logger.Print("ruby " + rubyversion + " already installed")
} else {
err = inst.runBash(`
-tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
-trap "rm -r ${tmp}" ERR
-wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd ${tmp}
+tmp="$(mktemp -d)"
+trap 'rm -r "${tmp}"' ERR EXIT
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C "${tmp}" -xzf -
+cd "${tmp}/ruby-`+rubyversion+`"
./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
make -j8
make install
/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
-rm -r ${tmp}
`, stdout, stderr)
if err != nil {
return 1
commit 706c36ff5644bb675af36c15c5f30ef0adde9102
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 28 10:29:16 2020 -0500
16306: Add -eatmydata flag.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index ff514a3b6..7b7f25ff9 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -68,6 +68,7 @@ if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}
-v "$(pwd)":/arvados:ro \
"${osbase}" \
/arvados-server install \
+ -eatmydata \
-type package \
-source /arvados \
-package-version "${version}"
@@ -90,6 +91,7 @@ docker run --rm \
-v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
-v "$(pwd)":/arvados:ro \
"${buildimage}" \
+ eatmydata \
/arvados-dev buildpackage \
-source /arvados \
-package-version "${version}" \
@@ -113,8 +115,9 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
--name "${installctr}" \
-v /tmp/pkg:/pkg:ro \
-v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ --env DEBIAN_FRONTEND=noninteractive \
"${osbase}" \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+ bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql'
docker commit "${installctr}" "${installimage}"
docker rm "${installctr}"
installctr=
@@ -125,4 +128,4 @@ docker run -it --rm \
-v /tmp/pkg:/pkg:ro \
-v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
"${installimage}" \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive eatmydata apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
diff --git a/lib/install/deps.go b/lib/install/deps.go
index cc9595db6..34fe7cc6d 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -34,6 +34,7 @@ type installCommand struct {
ClusterType string
SourcePath string
PackageVersion string
+ EatMyData bool
}
func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -55,6 +56,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
+ flags.BoolVar(&inst.EatMyData, "eatmydata", false, "use eatmydata to speed up install")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -109,11 +111,23 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
}
}
+ if inst.EatMyData {
+ cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends", "eatmydata")
+ cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
pkgs := prodpkgs(osv)
if pkg {
pkgs = append(pkgs,
"dpkg-dev",
+ "eatmydata", // install it for later steps, even if we're not using it now
"rsync",
)
}
@@ -174,7 +188,11 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
default:
pkgs = append(pkgs, "libcurl3")
}
- cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
+ cmd := exec.CommandContext(ctx, "apt-get")
+ if inst.EatMyData {
+ cmd = exec.CommandContext(ctx, "eatmydata", "apt-get")
+ }
+ cmd.Args = append(cmd.Args, "install", "--yes", "--no-install-recommends")
cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
@@ -205,7 +223,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
@@ -228,7 +246,7 @@ rm -r ${tmp}
if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
logger.Print("go " + goversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
cd /tmp
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
@@ -244,7 +262,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
@@ -258,7 +276,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
logger.Print("geckodriver " + geckoversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
GD=v`+geckoversion+`
wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
@@ -272,7 +290,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
NJS=`+nodejsversion+`
wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
@@ -286,7 +304,7 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
logger.Print("gradle " + gradleversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
G=`+gradleversion+`
zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
@@ -307,7 +325,7 @@ rm ${zip}
if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
logger.Print("locale " + wantlocale + " already installed")
} else {
- err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+ err = inst.runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
if err != nil {
return 1
}
@@ -386,7 +404,7 @@ rm ${zip}
// locales. Otherwise, it might need a
// restart, so we attempt to restart it with
// systemd.
- if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+ if err = inst.runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
} else if err = waitPostgreSQLReady(); err != nil {
return 1
@@ -585,8 +603,11 @@ func waitPostgreSQLReady() error {
}
}
-func runBash(script string, stdout, stderr io.Writer) error {
+func (inst *installCommand) runBash(script string, stdout, stderr io.Writer) error {
cmd := exec.Command("bash", "-")
+ if inst.EatMyData {
+ cmd = exec.Command("eatmydata", "bash", "-")
+ }
cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
cmd.Stdout = stdout
cmd.Stderr = stderr
commit 3aaefcb3c76ff470b475d950398d01255e87712a
Merge: c59af50bc 2c8874c06
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 16:48:37 2020 -0500
16306: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc doc/install/index.html.textile.liquid
index 81d7b2159,1b27ca6ed..f16ae2dad
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@@ -20,11 -20,12 +20,13 @@@ Arvados components can be installed an
<div class="offset1">
table(table table-bordered table-condensed).
|||\5=. Appropriate for|
- ||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
+ ||_. Setup difficulty|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
+ |"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
+ |"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
- |"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
+ |"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
|"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
diff --cc lib/install/deps.go
index da45b393b,342ef03a7..cc9595db6
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@@ -124,8 -101,10 +124,9 @@@ func (inst *installCommand) RunCommand(
"bison",
"bsdmainutils",
"build-essential",
- "ca-certificates",
"cadaver",
- "cython",
+ "curl",
+ "cython3",
"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
@@@ -136,9 -120,13 +137,9 @@@
"libfuse-dev",
"libgnutls28-dev",
"libjson-perl",
- "libjson-perl",
"libpam-dev",
"libpcre3-dev",
- "libpython2.7-dev",
+ "libpq-dev",
"libreadline-dev",
"libssl-dev",
"libwww-perl",
@@@ -154,22 -142,28 +155,27 @@@
"postgresql",
"postgresql-contrib",
"python3-dev",
- "python-epydoc",
+ "python3-venv",
+ "python3-virtualenv",
"r-base",
"r-cran-testthat",
+ "r-cran-devtools",
+ "r-cran-knitr",
+ "r-cran-markdown",
+ "r-cran-roxygen2",
+ "r-cran-xml",
"sudo",
- "virtualenv",
"wget",
"xvfb",
- "zlib1g-dev",
- }
+ )
switch {
case osv.Debian && osv.Major >= 10:
- debs = append(debs, "libcurl4")
+ pkgs = append(pkgs, "libcurl4")
default:
- debs = append(debs, "libcurl3")
+ pkgs = append(pkgs, "libcurl3")
}
cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
- cmd.Args = append(cmd.Args, debs...)
+ cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
cmd.Stderr = stderr
commit c59af50bc2f7a366cd12a8dd6fc7d7e3b1c32480
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 16:15:21 2020 -0500
16306: Move nginx temp dirs into a subdir.
When running as root, Nginx changes ownership of these temp dirs to
"nobody". We also use the same TMPDIR for standalone passenger
processes, which run as www-data, so we need Nginx to leave it alone.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index cdca68dd6..a4336049f 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -11,11 +11,11 @@ http {
'[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
- client_body_temp_path "{{TMPDIR}}";
- proxy_temp_path "{{TMPDIR}}";
- fastcgi_temp_path "{{TMPDIR}}";
- uwsgi_temp_path "{{TMPDIR}}";
- scgi_temp_path "{{TMPDIR}}";
+ client_body_temp_path "{{TMPDIR}}/nginx";
+ proxy_temp_path "{{TMPDIR}}/nginx";
+ fastcgi_temp_path "{{TMPDIR}}/nginx";
+ uwsgi_temp_path "{{TMPDIR}}/nginx";
+ scgi_temp_path "{{TMPDIR}}/nginx";
upstream controller {
server {{LISTENHOST}}:{{CONTROLLERPORT}};
}
commit f1bc1872a9ead4f3cb1c23c79507870c1b0f7b53
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 16:14:55 2020 -0500
16306: Remove some unneeded deps from prod/pkg.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 38f9e888d..da45b393b 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -120,6 +120,8 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
if dev || test || pkg {
pkgs = append(pkgs,
+ "automake",
+ "bison",
"bsdmainutils",
"build-essential",
"cadaver",
@@ -144,6 +146,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
"libxslt1-dev",
"linkchecker",
"lsof",
+ "make",
"net-tools",
"pandoc",
"perl-modules",
@@ -412,14 +415,20 @@ rm ${zip}
}
}
- if pkg {
+ if prod || pkg {
// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
for dstdir, srcdir := range map[string]string{
"railsapi": "services/api",
"workbench1": "apps/workbench",
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
- cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+ cmd := exec.Command("rsync",
+ "-a", "--no-owner", "--delete-after", "--delete-excluded",
+ "--exclude", "/coverage",
+ "--exclude", "/log",
+ "--exclude", "/tmp",
+ "--exclude", "/vendor",
+ "./", "/var/lib/arvados/"+dstdir+"/")
cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
cmd.Stdout = stdout
cmd.Stderr = stderr
@@ -580,8 +589,6 @@ func runBash(script string, stdout, stderr io.Writer) error {
func prodpkgs(osv osversion) []string {
pkgs := []string{
- "automake",
- "bison",
"ca-certificates",
"curl",
"fuse",
@@ -591,7 +598,6 @@ func prodpkgs(osv osversion) []string {
"haveged",
"libcurl3-gnutls",
"libxslt1.1",
- "make",
"nginx",
"python",
"sudo",
commit 3feaf72ad2a9a7b740bb3976f351f5020129eda4
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 16:13:49 2020 -0500
16306: Monkeypatch fpm bug.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 76228324e..a6e0b544a 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -9,6 +9,7 @@ import (
"flag"
"fmt"
"io"
+ "os"
"os/exec"
"git.arvados.org/arvados.git/lib/install"
@@ -70,6 +71,17 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
return fmt.Errorf("gem install fpm: %w", err)
}
+ if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+ // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+ cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("monkeypatch fpm: %w", err)
+ }
+ }
+
// Remove unneeded files. This is much faster than "fpm
// --exclude X" because fpm copies everything into a staging
// area before looking at the --exclude args.
commit 209403e32bdd6164404240af2959de86a3c46165
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 16:13:45 2020 -0500
16306: Comment why rm instead of exclude. Delete go tools, too.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 93482e1c7..76228324e 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -69,8 +69,11 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
if err != nil {
return fmt.Errorf("gem install fpm: %w", err)
}
- // Shrink our package, remove unneeded stuff
- cmd = exec.Command("bash", "-c", "rm -rf /var/www/.gem/ruby/*/cache/ /var/www/.gem/ruby/*/bundler/gems/arvados-*/.git /var/www/.gem/ruby/*/bundler/gems/nulldb-*/.git /var/www/.gem/ruby/*/bundler/gems/themes_for_rails-*/.git")
+
+ // Remove unneeded files. This is much faster than "fpm
+ // --exclude X" because fpm copies everything into a staging
+ // area before looking at the --exclude args.
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git /var/lib/arvados/go")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
@@ -93,12 +96,9 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
cmd.Args = append(cmd.Args, "--depends", pkg)
}
cmd.Args = append(cmd.Args,
+ "--verbose",
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
- "--exclude", "var/lib/arvados/go",
- "--exclude", "tmp",
- "--exclude", "log",
- "--exclude", "coverage",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
commit 54849feb0cf30bd3d78f14f1a41e6bc464fb80d6
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 10:19:43 2020 -0500
16306: Fix Nginx not starting when running as non-root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 07ff1fc1d..d49fabe56 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -12,6 +12,7 @@ import (
"net/url"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"regexp"
@@ -100,13 +101,26 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
}
}
}
+
+ args := []string{
+ "-g", "error_log stderr info;",
+ "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
+ "-c", conffile,
+ }
+ // Nginx ignores "user www-data;" when running as a non-root
+ // user... except that it causes it to ignore our other -g
+ // options. So we still have to decide for ourselves whether
+ // it's needed.
+ if u, err := user.Current(); err != nil {
+ return fmt.Errorf("user.Current(): %w", err)
+ } else if u.Uid == "0" {
+ args = append([]string{"-g", "user www-data;"}, args...)
+ }
+
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", nil, nil, nginx,
- "-g", "error_log stderr info;",
- "-g", "user www-data; pid "+filepath.Join(super.wwwtempdir, "nginx.pid")+";",
- "-c", conffile))
+ fail(super.RunProgram(ctx, ".", nil, nil, nginx, args...))
}()
// Choose one of the ports where Nginx should listen, and wait
// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 199a93a9d..fc23eb913 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -42,7 +42,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
iamroot := false
if u, err := user.Current(); err != nil {
- return fmt.Errorf("user.Current(): %s", err)
+ return fmt.Errorf("user.Current(): %w", err)
} else if u.Uid == "0" {
iamroot = true
}
commit d5c034b4e90d3fba74e5b0a879d8af424023d7b1
Author: Tom Clegg <tom at curii.com>
Date: Tue Dec 22 10:03:30 2020 -0500
16360: Don't bypass autofillConfig in production mode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 417f09d23..138c802e1 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -143,7 +143,6 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.tempdir = "/var/lib/arvados/tmp"
super.wwwtempdir = "/var/lib/arvados/wwwtmp"
super.bindir = "/var/lib/arvados/bin"
- super.configfile = "/etc/arvados/config.yml"
} else {
super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
if err != nil {
@@ -155,28 +154,28 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
if err := os.Mkdir(super.bindir, 0755); err != nil {
return err
}
+ }
- // Fill in any missing config keys, and write the resulting
- // config in the temp dir for child services to use.
- err = super.autofillConfig(cfg)
- if err != nil {
- return err
- }
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- return err
- }
- defer conffile.Close()
- err = json.NewEncoder(conffile).Encode(cfg)
- if err != nil {
- return err
- }
- err = conffile.Close()
- if err != nil {
- return err
- }
- super.configfile = conffile.Name()
+ // Fill in any missing config keys, and write the resulting
+ // config in the temp dir for child services to use.
+ err = super.autofillConfig(cfg)
+ if err != nil {
+ return err
+ }
+ conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer conffile.Close()
+ err = json.NewEncoder(conffile).Encode(cfg)
+ if err != nil {
+ return err
}
+ err = conffile.Close()
+ if err != nil {
+ return err
+ }
+ super.configfile = conffile.Name()
super.environ = os.Environ()
super.cleanEnv([]string{"ARVADOS_"})
@@ -655,26 +654,26 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
}
}
- if cluster.SystemRootToken == "" {
- cluster.SystemRootToken = randomHexString(64)
- }
- if cluster.ManagementToken == "" {
- cluster.ManagementToken = randomHexString(64)
- }
- if cluster.API.RailsSessionSecretToken == "" {
- cluster.API.RailsSessionSecretToken = randomHexString(64)
- }
- if cluster.Collections.BlobSigningKey == "" {
- cluster.Collections.BlobSigningKey = randomHexString(64)
- }
- if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
- buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
- if err != nil {
- return err
- }
- cluster.Containers.DispatchPrivateKey = string(buf)
- }
if super.ClusterType != "production" {
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
+ if cluster.API.RailsSessionSecretToken == "" {
+ cluster.API.RailsSessionSecretToken = randomHexString(64)
+ }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
+ if cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
cluster.TLS.Insecure = true
}
if super.ClusterType == "test" {
commit 20649f2d57f8e5c8bddfb672c164b7a96f75efab
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 10 17:15:08 2020 -0500
16306: shrink package size by removing unused cache and .git directories
under /var/www/.gem/
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index c562c5685..93482e1c7 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -69,6 +69,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
if err != nil {
return fmt.Errorf("gem install fpm: %w", err)
}
+ // Shrink our package, remove unneeded stuff
+ cmd = exec.Command("bash", "-c", "rm -rf /var/www/.gem/ruby/*/cache/ /var/www/.gem/ruby/*/bundler/gems/arvados-*/.git /var/www/.gem/ruby/*/bundler/gems/nulldb-*/.git /var/www/.gem/ruby/*/bundler/gems/themes_for_rails-*/.git")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("rm -rf /var/www/.gem/ruby/*/cache/: %w", err)
+ }
format := "deb" // TODO: rpm
@@ -88,6 +96,9 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
"--exclude", "var/lib/arvados/go",
+ "--exclude", "tmp",
+ "--exclude", "log",
+ "--exclude", "coverage",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
commit 821c7273388a56ad5ae9b89a0cf2f351388e8b61
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 10 10:46:18 2020 -0500
16306: switch to shared ruby libraries.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3ff23d6e9..38f9e888d 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -201,7 +201,7 @@ tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
make -j8
make install
/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
commit da7a2e35f1c53a64a360c66db0c260a9ab2190f2
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 9 15:57:47 2020 -0500
16306: in cmd/arvados-dev/docker-build-install.sh, make sure nginx is
configured to listen on all IPs.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 78b08d7dc..ff514a3b6 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -125,4 +125,4 @@ docker run -it --rm \
-v /tmp/pkg:/pkg:ro \
-v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
"${installimage}" \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
commit 0b6013ac5520b7508474705d654f1384994d10e5
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 9 13:23:35 2020 -0500
16306: Make sure that the non-passenger nginx process runs as the
www-data user, not the default 'nobody' user. Make sure that process
uses the wwwtmp directory as its temp directory. Make sure that
arvados-boot in production mode uses /etc/arvados/config.yml.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index b11d9fd49..07ff1fc1d 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -37,7 +37,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
- "TMPDIR": super.tempdir,
+ "TMPDIR": super.wwwtempdir,
}
for _, cmpt := range []struct {
varname string
@@ -105,7 +105,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
defer super.waitShutdown.Done()
fail(super.RunProgram(ctx, ".", nil, nil, nginx,
"-g", "error_log stderr info;",
- "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
+ "-g", "user www-data; pid "+filepath.Join(super.wwwtempdir, "nginx.pid")+";",
"-c", conffile))
}()
// Choose one of the ports where Nginx should listen, and wait
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 51f5c8bfd..417f09d23 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -143,6 +143,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.tempdir = "/var/lib/arvados/tmp"
super.wwwtempdir = "/var/lib/arvados/wwwtmp"
super.bindir = "/var/lib/arvados/bin"
+ super.configfile = "/etc/arvados/config.yml"
} else {
super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
if err != nil {
@@ -154,28 +155,28 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
if err := os.Mkdir(super.bindir, 0755); err != nil {
return err
}
- }
- // Fill in any missing config keys, and write the resulting
- // config in the temp dir for child services to use.
- err = super.autofillConfig(cfg)
- if err != nil {
- return err
- }
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- return err
- }
- defer conffile.Close()
- err = json.NewEncoder(conffile).Encode(cfg)
- if err != nil {
- return err
- }
- err = conffile.Close()
- if err != nil {
- return err
+ // Fill in any missing config keys, and write the resulting
+ // config in the temp dir for child services to use.
+ err = super.autofillConfig(cfg)
+ if err != nil {
+ return err
+ }
+ conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer conffile.Close()
+ err = json.NewEncoder(conffile).Encode(cfg)
+ if err != nil {
+ return err
+ }
+ err = conffile.Close()
+ if err != nil {
+ return err
+ }
+ super.configfile = conffile.Name()
}
- super.configfile = conffile.Name()
super.environ = os.Environ()
super.cleanEnv([]string{"ARVADOS_"})
commit ab5199f71c7eaf8bf8fe2b4477353cf432faf1a7
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 9 11:07:29 2020 -0500
16306: really exclude /var/lib/arvados/go from the package, cf.
https://github.com/jordansissel/fpm/issues/900
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 125ac36b1..c562c5685 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -87,7 +87,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
cmd.Args = append(cmd.Args,
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
- "--exclude", "/var/lib/arvados/go",
+ "--exclude", "var/lib/arvados/go",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
commit f8d13408e99839f52260f889a5089126761eecb1
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 9 11:07:07 2020 -0500
16306: shut up bundle warnings about /var/www
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 00713550d..125ac36b1 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -91,6 +91,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
+ "/var/www/.bundle",
)
fmt.Fprintf(stderr, "... %s\n", cmd.Args)
cmd.Dir = bldr.OutputDir
diff --git a/lib/install/deps.go b/lib/install/deps.go
index b39515cb1..3ff23d6e9 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -428,9 +428,9 @@ rm ${zip}
return 1
}
for _, cmdline := range [][]string{
- {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
{"touch", "log/production.log"},
- {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
commit e8d1a643cdbc3a5f4c0e5c745da58d9f7e1248d8
Merge: 426103b2d 5b0505685
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Aug 21 14:51:32 2020 -0400
16306: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
commit 426103b2d2f071ab0d57b3f9aaea58a3f8455c4f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Aug 17 09:14:52 2020 -0400
16306: Add --os argument to package build scripts.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 7eca33e79..39a8a56c8 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -19,6 +19,11 @@ set -e -o pipefail
declare -A opts=()
while [[ $# -gt 0 ]]; do
case "$1" in
+ --os)
+ shift
+ opts[os]="$1"
+ shift
+ ;;
--shell)
shift
opts[shell]=1
@@ -54,7 +59,7 @@ for inject in "$@"; do
esac
done
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
installimage=arvados-installpackage-${osbase}
command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
if [[ "${opts[shell]}" ]]; then
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 63b5b5c2d..78b08d7dc 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -25,6 +25,10 @@ while [[ $# -gt 0 ]]; do
--force-installimage)
opts[force-installimage]=1
;;
+ --os)
+ opts[os]="$1"
+ shift
+ ;;
*)
echo >&2 "invalid argument '$arg'"
exit 2
@@ -43,7 +47,7 @@ cleanup() {
trap cleanup ERR EXIT
version=$(git describe --tag --dirty)
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
mkdir -p /tmp/pkg
commit bb1054c0e6a9bed14e593004f380111b2e948047
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Aug 17 09:13:45 2020 -0400
16306: Update gem command line option spelling.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index aee955ea2..00713550d 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -62,7 +62,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
}
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 481300b45..03464aaf7 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -56,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 7b22f45fd..b39515cb1 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -431,7 +431,7 @@ rm ${zip}
{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
commit f2da5f54256377e604c76b74805381b2acf38f50
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Aug 7 15:00:22 2020 -0400
16306: Add "ping" health check to keep-balance.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 65bd8d4cf..8b4ee84c7 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -9,13 +9,13 @@ import (
"flag"
"fmt"
"io"
- "net/http"
"os"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -83,7 +83,6 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
}
srv := &Server{
- Handler: http.NotFoundHandler(),
Cluster: cluster,
ArvClient: ac,
RunOptions: options,
@@ -91,6 +90,11 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
Logger: options.Logger,
Dumper: options.Dumper,
}
+ srv.Handler = &health.Handler{
+ Token: cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": srv.CheckHealth},
+ }
go srv.run()
return srv
commit f81d411d2becc23fe640366620d9568149d89bd6
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Aug 7 10:51:39 2020 -0400
16306: Set Workbench1 secret key.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/init.go b/lib/install/init.go
index ca8d4da92..8debf59be 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -178,6 +178,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
DriverParameters:
Root: /var/lib/arvados/keep
Replication: 2
+ Workbench:
+ SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
`)
if err != nil {
return 1
commit 24987125b33655306c048ab64f0ffdfb2f6a6c5b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 15:18:40 2020 -0400
16306: Fix nginx port probe.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index a7c09a722..b11d9fd49 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"path/filepath"
@@ -107,5 +108,11 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
"-c", conffile))
}()
- return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+ // Choose one of the ports where Nginx should listen, and wait
+ // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+ testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+ if testurl.Port() == "" {
+ testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+ }
+ return waitForConnect(ctx, testurl.Host)
}
commit 1c2f9259deeedaad103b80fda7f440b8e60caa3f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 14:27:51 2020 -0400
16306: Include wwwtmp dir in package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index c5596d104..7b22f45fd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -178,7 +178,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
os.Mkdir("/var/lib/arvados", 0755)
os.Mkdir("/var/lib/arvados/tmp", 0700)
- if prod {
+ if prod || pkg {
os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
u, er := user.Lookup("www-data")
if er != nil {
commit 75d050ab135619fcefecbfa32aaad4dab33e7588
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:30:08 2020 -0400
16306: Fixup nginx in arvados-boot production mode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 0b9874295..7eca33e79 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -40,11 +40,18 @@ tmpdir=$(mktemp -d)
version=$(git describe --tag --dirty)
declare -a volargs=()
-for srcdir in "$@"; do
- echo >&2 "building $srcdir..."
- (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
- cmd="$(basename "$srcdir")"
- volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+for inject in "$@"; do
+ case "$inject" in
+ nginx.conf)
+ volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
+ ;;
+ *)
+ echo >&2 "building $inject..."
+ (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+ cmd="$(basename "$inject")"
+ volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+ ;;
+ esac
done
osbase=debian:10
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index c1da7d18d..a7c09a722 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -69,7 +69,13 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
}
vars[cmpt.varname+"SSLPORT"] = port
}
- tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ var conftemplate string
+ if super.ClusterType == "production" {
+ conftemplate = "/var/lib/arvados/share/nginx.conf"
+ } else {
+ conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+ }
+ tmpl, err := ioutil.ReadFile(conftemplate)
if err != nil {
return err
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 5f92a6569..51f5c8bfd 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -252,7 +252,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
}
if super.ClusterType != "test" {
tasks = append(tasks,
- runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3f19aa1a8..c5596d104 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -486,6 +486,15 @@ rm ${zip}
return 1
}
}
+
+ // Copy assets from source tree to /var/lib/arvados/share
+ cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
}
return 0
diff --git a/lib/install/init.go b/lib/install/init.go
index 6d4f19757..ca8d4da92 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -170,6 +170,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
user: arvados
password: {{printf "%q" .PostgreSQLPassword}}
SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ TLS:
+ Insecure: true
Volumes:
{{.ClusterID}}-nyw5e-000000000000000:
Driver: Directory
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 85b4f5b37..cdca68dd6 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -16,12 +16,28 @@ http {
fastcgi_temp_path "{{TMPDIR}}";
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
+ upstream controller {
+ server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+ server_name controller ~.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream arv-git-http {
server {{LISTENHOST}}:{{GITPORT}};
}
server {
- listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
- server_name arv-git-http;
+ listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+ server_name arv-git-http git.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -36,8 +52,8 @@ http {
server {{LISTENHOST}}:{{KEEPPROXYPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name keepproxy;
+ listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+ server_name keepproxy keep.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -55,8 +71,8 @@ http {
server {{LISTENHOST}}:{{KEEPWEBPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name keep-web;
+ listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+ server_name keep-web collections.* ~\.collections\.;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -75,8 +91,8 @@ http {
server {{LISTENHOST}}:{{HEALTHPORT}};
}
server {
- listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
- server_name health;
+ listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+ server_name health health.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -91,8 +107,8 @@ http {
}
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name keep-web-dl ~.*;
+ listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+ server_name keep-web-dl download.* ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -111,8 +127,8 @@ http {
server {{LISTENHOST}}:{{WSPORT}};
}
server {
- listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
- server_name websocket;
+ listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+ server_name websocket ws.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -129,8 +145,8 @@ http {
server {{LISTENHOST}}:{{WORKBENCH1PORT}};
}
server {
- listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
- server_name workbench1;
+ listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+ server_name workbench1 workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -141,20 +157,4 @@ http {
proxy_redirect off;
}
}
- upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
- }
- server {
- listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
- server_name controller;
- ssl_certificate "{{SSLCERT}}";
- ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://controller;
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
}
commit 8891660b68c10dd9d019f1caf8b7bc0b1eb493d8
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:28:11 2020 -0400
16306: Option to start shell in docker-boot.sh.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index e8703e41f..0b9874295 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -16,6 +16,19 @@
set -e -o pipefail
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --shell)
+ shift
+ opts[shell]=1
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
cleanup() {
if [[ -n "${tmpdir}" ]]; then
rm -rf "${tmpdir}"
@@ -36,7 +49,11 @@ done
osbase=debian:10
installimage=arvados-installpackage-${osbase}
+command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
+if [[ "${opts[shell]}" ]]; then
+ command="bash -login"
+fi
docker run -it --rm \
"${volargs[@]}" \
"${installimage}" \
- bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
+ bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 3c6e177cf..63b5b5c2d 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -27,7 +27,8 @@ while [[ $# -gt 0 ]]; do
;;
*)
echo >&2 "invalid argument '$arg'"
- exit 1
+ exit 2
+ ;;
esac
done
commit 398ddf58f05416e0ce62ac8faef018a44bf8cc4c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:26:57 2020 -0400
16306: Add hostname as subjectAltName.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index f0797c2ac..8f6339e63 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io/ioutil"
"net"
+ "os"
"path/filepath"
)
@@ -27,9 +28,14 @@ func (createCertificates) String() string {
func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
var san string
if net.ParseIP(super.ListenHost) != nil {
- san = fmt.Sprintf("IP:%s", super.ListenHost)
+ san += fmt.Sprintf(",IP:%s", super.ListenHost)
} else {
- san = fmt.Sprintf("DNS:%s", super.ListenHost)
+ san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+ }
+ if hostname, err := os.Hostname(); err != nil {
+ return fmt.Errorf("hostname: %w", err)
+ } else {
+ san += ",DNS:" + hostname
}
// Generate root key
@@ -52,7 +58,7 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
if err != nil {
return err
}
- err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+ err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
if err != nil {
return err
}
commit 7ac1ea4e9bc1b399368a95649647a55c8093d649
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:25:35 2020 -0400
16306: Show logprefix like 'passenger at railsapi'
for commands like 'sudo ... bundle exec passenger ...'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 2d7697233..5f92a6569 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -483,17 +483,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "setuidgid" && len(args) >= 3 {
- logprefix = args[2]
- }
- logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
- if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
- logprefix = args[1]
- } else if logprefix == "arvados-server" && len(args) > 1 {
- logprefix = args[0]
- }
- if !strings.HasPrefix(dir, "/") {
- logprefix = dir + ": " + logprefix
+ {
+ if logprefix == "setuidgid" && len(args) >= 3 {
+ logprefix = args[2]
+ }
+ innerargs := args
+ if logprefix == "sudo" {
+ for i := 0; i < len(args); i++ {
+ if args[i] == "-u" {
+ i++
+ } else if args[i] == "-E" || strings.Contains(args[i], "=") {
+ } else {
+ logprefix = args[i]
+ innerargs = args[i+1:]
+ break
+ }
+ }
+ }
+ logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+ if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+ _, dirbase := filepath.Split(dir)
+ logprefix = innerargs[1] + "@" + dirbase
+ } else if logprefix == "arvados-server" && len(args) > 1 {
+ logprefix = args[0]
+ }
+ if !strings.HasPrefix(dir, "/") {
+ logprefix = dir + ": " + logprefix
+ }
}
cmd := exec.Command(super.lookPath(prog), args...)
commit 60801a7c485c50e3c2f518f9b3ad6f86c8783fca
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 23 17:31:39 2020 -0400
16306: Packaging dev cycle, type=production support in lib/boot.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 4b395c4ba..aee955ea2 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -85,12 +85,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
cmd.Args = append(cmd.Args, "--depends", pkg)
}
cmd.Args = append(cmd.Args,
+ "--deb-use-file-permissions",
+ "--rpm-use-file-permissions",
"--exclude", "/var/lib/arvados/go",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
)
- fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
cmd.Dir = bldr.OutputDir
cmd.Stdout = stdout
cmd.Stderr = stderr
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
new file mode 100755
index 000000000..e8703e41f
--- /dev/null
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Bring up a docker container with some locally-built commands (e.g.,
+# cmd/arvados-server) replacing the ones that came with
+# arvados-server-easy when the arvados-installpackage-* image was
+# built.
+#
+# Assumes docker-build-install.sh has already succeeded.
+#
+# Example:
+#
+# docker-boot.sh cmd/arvados-server services/keep-balance
+
+set -e -o pipefail
+
+cleanup() {
+ if [[ -n "${tmpdir}" ]]; then
+ rm -rf "${tmpdir}"
+ fi
+}
+trap cleanup ERR EXIT
+
+tmpdir=$(mktemp -d)
+version=$(git describe --tag --dirty)
+
+declare -a volargs=()
+for srcdir in "$@"; do
+ echo >&2 "building $srcdir..."
+ (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+ cmd="$(basename "$srcdir")"
+ volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+done
+
+osbase=debian:10
+installimage=arvados-installpackage-${osbase}
+docker run -it --rm \
+ "${volargs[@]}" \
+ "${installimage}" \
+ bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
new file mode 100755
index 000000000..3c6e177cf
--- /dev/null
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Build an arvados-server-easy package, then install and run it on a
+# base OS image.
+#
+# Examples:
+#
+# docker-build-install.sh --force-buildimage --force-installimage # always build fresh docker images
+#
+# docker-build-install.sh # reuse cached docker images if possible
+
+set -e -o pipefail
+
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+ arg="$1"
+ shift
+ case "$arg" in
+ --force-buildimage)
+ opts[force-buildimage]=1
+ ;;
+ --force-installimage)
+ opts[force-installimage]=1
+ ;;
+ *)
+ echo >&2 "invalid argument '$arg'"
+ exit 1
+ esac
+done
+
+cleanup() {
+ if [[ -n "${buildctr}" ]]; then
+ docker rm "${buildctr}" || true
+ fi
+ if [[ -n "${installctr}" ]]; then
+ docker rm "${installctr}" || true
+ fi
+}
+trap cleanup ERR EXIT
+
+version=$(git describe --tag --dirty)
+osbase=debian:10
+
+mkdir -p /tmp/pkg
+
+buildimage=arvados-buildpackage-${osbase}
+if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
+ (
+ echo >&2 building arvados-server...
+ cd cmd/arvados-server
+ go install
+ )
+ echo >&2 building ${buildimage}...
+ buildctr=${buildimage/:/-}
+ docker rm "${buildctr}" || true
+ docker run \
+ --name "${buildctr}" \
+ -v /tmp/pkg:/pkg \
+ -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
+ -v "$(pwd)":/arvados:ro \
+ "${osbase}" \
+ /arvados-server install \
+ -type package \
+ -source /arvados \
+ -package-version "${version}"
+ docker commit "${buildctr}" "${buildimage}"
+ docker rm "${buildctr}"
+ buildctr=
+fi
+
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+rm -v -f "${pkgfile}"
+
+(
+ echo >&2 building arvados-dev...
+ cd cmd/arvados-dev
+ go install
+)
+echo >&2 building ${pkgfile}...
+docker run --rm \
+ -v /tmp/pkg:/pkg \
+ -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+ -v "$(pwd)":/arvados:ro \
+ "${buildimage}" \
+ /arvados-dev buildpackage \
+ -source /arvados \
+ -package-version "${version}" \
+ -output-directory /pkg
+
+ls -l ${pkgfile}
+(
+ echo >&2 dpkg-scanpackages...
+ cd /tmp/pkg
+ dpkg-scanpackages . | gzip > Packages.gz
+)
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+installimage="arvados-installpackage-${osbase}"
+if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+ echo >&2 building ${installimage}...
+ installctr=${installimage/:/-}
+ docker rm "${installctr}" || true
+ docker run -it \
+ --name "${installctr}" \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ "${osbase}" \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+ docker commit "${installctr}" "${installimage}"
+ docker rm "${installctr}"
+ installctr=
+fi
+
+echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
+docker run -it --rm \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ "${installimage}" \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
deleted file mode 100755
index 072dfcf9c..000000000
--- a/cmd/arvados-dev/example.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -e -o pipefail
-
-version="${PACKAGE_VERSION:-0.9.99}"
-
-# mkdir -p /tmp/pkg
-# (
-# cd cmd/arvados-dev
-# go install
-# )
-# docker run --rm \
-# -v /tmp/pkg:/pkg \
-# -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
-# -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
-# /arvados-dev buildpackage \
-# -source /arvados \
-# -package-version "${version}" \
-# -output-directory /pkg
-pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
-# ls -l ${pkgfile}
-# (
-# cd /tmp/pkg
-# dpkg-scanpackages . | gzip > Packages.gz
-# )
-sourcesfile=/tmp/sources.conf.d-arvados
-echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-docker run -it --rm \
- -v /tmp/pkg:/pkg:ro \
- -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
- ${INSTALLIMAGE:-debian:10} \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index ff99de75c..d0aa9da94 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -34,6 +34,7 @@ var (
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "init": install.InitCommand,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
})
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 0f105d6b6..c1da7d18d 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -53,7 +53,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
} {
port, err := internalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
@@ -62,7 +62,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
port, err = externalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index f18300c4c..481300b45 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -37,6 +37,10 @@ func (runner installPassenger) String() string {
}
func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if super.ClusterType == "production" {
+ // passenger has already been installed via package
+ return nil
+ }
err := super.wait(ctx, runner.depends...)
if err != nil {
return err
@@ -52,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
@@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
type runPassenger struct {
- src string
- svc arvados.Service
- depends []supervisedTask
+ src string // path to app in source tree
+ varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+ svc arvados.Service
+ depends []supervisedTask
}
func (runner runPassenger) String() string {
@@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
if err != nil {
return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
}
+ var appdir string
+ if super.ClusterType == "production" {
+ appdir = "/var/lib/arvados/" + runner.varlibdir
+ } else {
+ appdir = runner.src
+ }
loglevel := "4"
if lvl, ok := map[string]string{
"debug": "5",
@@ -116,13 +127,30 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+ cmdline := []string{
+ "bundle", "exec",
"passenger", "start",
"-p", port,
- "--log-file", "/dev/stderr",
"--log-level", loglevel,
"--no-friendly-error-pages",
- "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+ "--disable-anonymous-telemetry",
+ "--disable-security-update-check",
+ "--no-compile-runtime",
+ "--no-install-runtime",
+ "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+ }
+ if super.ClusterType == "production" {
+ cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+ } else {
+ // This would be desirable in the production
+ // case too, but it fails with sudo because
+ // /dev/stderr is a symlink to a pty owned by
+ // root: "nginx: [emerg] open() "/dev/stderr"
+ // failed (13: Permission denied)"
+ cmdline = append(cmdline, "--log-file", "/dev/stderr")
+ }
+ env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
+ err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
fail(err)
}()
return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 34ccf04a8..199a93a9d 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -36,6 +36,10 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
+
iamroot := false
if u, err := user.Current(); err != nil {
return fmt.Errorf("user.Current(): %s", err)
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index d1cf2a870..1f07601a0 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -20,6 +20,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
if err != nil {
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
if err != nil {
return err
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 5afacfe71..77fdc98be 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string {
}
func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
- binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
- err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ binfile := filepath.Join(super.bindir, "arvados-server")
+ err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
if err != nil {
return err
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index e38a4775e..2d7697233 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -14,6 +14,7 @@ import (
"io"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"os/signal"
@@ -54,7 +55,9 @@ type Supervisor struct {
tasksReady map[string]chan bool
waitShutdown sync.WaitGroup
+ bindir string
tempdir string
+ wwwtempdir string
configfile string
environ []string // for child processes
}
@@ -131,13 +134,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
return err
}
- super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
- if err != nil {
- return err
- }
- defer os.RemoveAll(super.tempdir)
- if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
- return err
+ // Choose bin and temp dirs: /var/lib/arvados/... in
+ // production, transient tempdir otherwise.
+ if super.ClusterType == "production" {
+ // These dirs have already been created by
+ // "arvados-server install" (or by extracting a
+ // package).
+ super.tempdir = "/var/lib/arvados/tmp"
+ super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+ super.bindir = "/var/lib/arvados/bin"
+ } else {
+ super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(super.tempdir)
+ super.wwwtempdir = super.tempdir
+ super.bindir = filepath.Join(super.tempdir, "bin")
+ if err := os.Mkdir(super.bindir, 0755); err != nil {
+ return err
+ }
}
// Fill in any missing config keys, and write the resulting
@@ -166,7 +182,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+ super.prependEnv("PATH", "/var/lib/arvados/bin:")
+ if super.ClusterType != "production" {
+ super.prependEnv("PATH", super.tempdir+"/bin:")
+ }
super.cluster, err = cfg.GetCluster("")
if err != nil {
@@ -182,7 +201,9 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
"PID": os.Getpid(),
})
- if super.SourceVersion == "" {
+ if super.SourceVersion == "" && super.ClusterType == "production" {
+ // don't need SourceVersion
+ } else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
@@ -224,15 +245,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
- runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
if super.ClusterType != "test" {
tasks = append(tasks,
runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
- runGoProgram{src: "services/keep-balance"},
+ runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
super.tasksReady = map[string]chan bool{}
@@ -382,9 +403,11 @@ func dedupEnv(in []string) []string {
func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
_, basename := filepath.Split(srcpath)
- bindir := filepath.Join(super.tempdir, "bin")
- binfile := filepath.Join(bindir, basename)
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ binfile := filepath.Join(super.bindir, basename)
+ if super.ClusterType == "production" {
+ return binfile, nil
+ }
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
@@ -401,10 +424,19 @@ func (super *Supervisor) setupRubyEnv() error {
"GEM_PATH=",
})
gem := "gem"
- if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
gem = "/var/lib/arvados/bin/gem"
}
cmd := exec.Command(gem, "env", "gempath")
+ if super.ClusterType == "production" {
+ cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+ path, err := exec.LookPath("sudo")
+ if err != nil {
+ return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+ }
+ cmd.Path = path
+ }
+ cmd.Stderr = super.Stderr
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
@@ -694,11 +726,10 @@ func internalPort(svc arvados.Service) (string, error) {
return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
}
for u := range svc.InternalURLs {
- if _, p, err := net.SplitHostPort(u.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(u)
+ if p := u.Port(); p != "" {
return p, nil
- } else if u.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "ws" {
return "443", nil
} else {
return "80", nil
@@ -708,11 +739,10 @@ func internalPort(svc arvados.Service) (string, error) {
}
func externalPort(svc arvados.Service) (string, error) {
- if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(svc.ExternalURL)
+ if p := u.Port(); p != "" {
return p, nil
- } else if svc.ExternalURL.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "wss" {
return "443", nil
} else {
return "80", nil
diff --git a/lib/install/deps.go b/lib/install/deps.go
index f9b962fdd..3f19aa1a8 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
"io"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"strconv"
"strings"
@@ -176,12 +177,26 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
}
os.Mkdir("/var/lib/arvados", 0755)
+ os.Mkdir("/var/lib/arvados/tmp", 0700)
+ if prod {
+ os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+ u, er := user.Lookup("www-data")
+ if er != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+ return 1
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+ if err != nil {
+ return 1
+ }
+ }
rubyversion := "2.5.7"
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
} else {
err = runBash(`
-mkdir -p /var/lib/arvados/tmp
tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
@@ -189,7 +204,9 @@ cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
make -j8
make install
-/var/lib/arvados/bin/gem install bundler
+/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
+# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
rm -r ${tmp}
`, stdout, stderr)
if err != nil {
@@ -262,7 +279,6 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
} else {
err = runBash(`
G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
@@ -414,8 +430,7 @@ rm ${zip}
for _, cmdline := range [][]string{
{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
{"touch", "log/production.log"},
- // {"chown", "-R", "root:root", "."},
- {"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
@@ -426,6 +441,7 @@ rm ${zip}
cmd.Dir = "/var/lib/arvados/" + dstdir
cmd.Stdout = stdout
cmd.Stderr = stderr
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
err = cmd.Run()
if err != nil {
return 1
@@ -569,6 +585,7 @@ func prodpkgs(osv osversion) []string {
"make",
"nginx",
"python",
+ "sudo",
}
if osv.Debian || osv.Ubuntu {
if osv.Debian && osv.Major == 8 {
diff --git a/lib/install/init.go b/lib/install/init.go
new file mode 100644
index 000000000..6d4f19757
--- /dev/null
+++ b/lib/install/init.go
@@ -0,0 +1,265 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/user"
+ "regexp"
+ "strconv"
+ "text/template"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+ ClusterID string
+ Domain string
+ PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("Hostname(): %w", err)
+ return 1
+ }
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+ flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
+ } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+ err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+ return 1
+ }
+
+ wwwuser, err := user.Lookup("www-data")
+ if err != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+ return 1
+ }
+ wwwgid, err := strconv.Atoi(wwwuser.Gid)
+ if err != nil {
+ return 1
+ }
+ initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+ err = os.Mkdir("/var/lib/arvados/keep", 0600)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+ err = os.Mkdir("/etc/arvados", 0750)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ return 1
+ }
+ err = os.Chown("/etc/arvados", 0, wwwgid)
+ f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ tmpl, err := template.New("config").Parse(`Clusters:
+ {{.ClusterID}}:
+ Services:
+ Controller:
+ InternalURLs:
+ "http://0.0.0.0:8003/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ Websocket:
+ InternalURLs:
+ "http://0.0.0.0:8005/": {}
+ ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ Keepbalance:
+ InternalURLs:
+ "http://0.0.0.0:9005/": {}
+ GitHTTP:
+ InternalURLs:
+ "http://0.0.0.0:9001/": {}
+ ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ DispatchCloud:
+ InternalURLs:
+ "http://0.0.0.0:9006/": {}
+ Keepproxy:
+ InternalURLs:
+ "http://0.0.0.0:25108/": {}
+ ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ WebDAV:
+ InternalURLs:
+ "http://0.0.0.0:9002/": {}
+ ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ WebDAVDownload:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ Keepstore:
+ InternalURLs:
+ "http://0.0.0.0:25107/": {}
+ Composer:
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ Workbench1:
+ InternalURLs:
+ "http://0.0.0.0:8001/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+ Workbench2:
+ InternalURLs:
+ "http://0.0.0.0:8002/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ Health:
+ InternalURLs:
+ "http://0.0.0.0:9007/": {}
+ API:
+ RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
+ Collections:
+ BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+ Containers:
+ DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+ ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+ PostgreSQL:
+ Connection:
+ dbname: arvados_production
+ host: localhost
+ user: arvados
+ password: {{printf "%q" .PostgreSQLPassword}}
+ SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ Volumes:
+ {{.ClusterID}}-nyw5e-000000000000000:
+ Driver: Directory
+ DriverParameters:
+ Root: /var/lib/arvados/keep
+ Replication: 2
+`)
+ if err != nil {
+ return 1
+ }
+ err = tmpl.Execute(f, initcmd)
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ return 1
+ }
+ err = f.Close()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+ ldr := config.NewLoader(nil, logger)
+ ldr.SkipLegacy = true
+ cfg, err := ldr.Load()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+
+ err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+ if err != nil {
+ return 1
+ }
+
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ cmd.Dir = "/var/lib/arvados/railsapi"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("rake db:setup: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "initialized database")
+
+ return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+ privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+ if err != nil {
+ return "", err
+ }
+ err = privkey.Validate()
+ if err != nil {
+ return "", err
+ }
+ return string(pem.EncodeToMemory(&pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+ })), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+ b := make([]byte, chars/2)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+ for _, sql := range []string{
+ `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+ `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+ } {
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
+ }
+ }
+ return nil
+}
commit ee7d60f02525760a2480157011659e7b2210f6b8
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Jul 24 10:43:29 2020 -0400
16306: Sync bundler version to services/api/Gemfile.lock.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index ff6ead0fa..b3eacfe31 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -555,7 +555,7 @@ setup_ruby_environment() {
(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- versions=(1.11.0 1.17.3 2.0.2)
+ versions=(1.16.6 1.17.3 2.0.2)
for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 6a2c4b61f..f18300c4c 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -50,9 +50,9 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
if err != nil {
return err
}
- for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+ for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index d28823f4a..f9b962fdd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -416,7 +416,7 @@ rm ${zip}
{"touch", "log/production.log"},
// {"chown", "-R", "root:root", "."},
{"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
commit 12441486d7fb3b51d2fea9d9b1da0d8e4f3ef3e0
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Jul 1 09:42:48 2020 -0400
16306: Add "install -type package" and "buildpackage".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
new file mode 100644
index 000000000..4b395c4ba
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage.go
@@ -0,0 +1,98 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os/exec"
+
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+type buildPackage struct{}
+
+func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ err := (&builder{
+ PackageVersion: "0.0.0",
+ logger: logger,
+ }).run(context.Background(), prog, args, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type builder struct {
+ PackageVersion string
+ SourcePath string
+ OutputDir string
+ SkipInstall bool
+ logger logrus.FieldLogger
+}
+
+func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
+ flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
+ flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
+ flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
+ err := flags.Parse(args)
+ if err != nil {
+ return err
+ }
+ if len(flags.Args()) > 0 {
+ return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if !bldr.SkipInstall {
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", bldr.PackageVersion,
+ "-source", bldr.SourcePath,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ }
+ }
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
+ format := "deb" // TODO: rpm
+
+ cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+ "--name", "arvados-server-easy",
+ "--version", bldr.PackageVersion,
+ "--input-type", "dir",
+ "--output-type", format)
+ deps, err := install.ProductionDependencies()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range deps {
+ cmd.Args = append(cmd.Args, "--depends", pkg)
+ }
+ cmd.Args = append(cmd.Args,
+ "--exclude", "/var/lib/arvados/go",
+ "/var/lib/arvados",
+ "/var/www/.gem",
+ "/var/www/.passenger",
+ )
+ fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+ cmd.Dir = bldr.OutputDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ return cmd.Run()
+}
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-dev/buildpackage_test.go
new file mode 100644
index 000000000..61bdeae82
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage_test.go
@@ -0,0 +1,77 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "flag"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+ os.Args = append(os.Args, "-test.timeout=30m") // kludge
+
+ // This enables a hack to speed up repeated tests: hit "docker
+ // commit --pause {containername} checkpointtag" after the
+ // test container has downloaded/compiled some stuff, then run
+ // tests with "-test.buildimage=checkpointtag" next time to
+ // retry/resume/update from that point.
+ flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
+}
+
+type BuildpackageSuite struct{}
+
+var _ = check.Suite(&BuildpackageSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+ tmpdir := c.MkDir()
+ defer os.RemoveAll(tmpdir)
+
+ err := os.Mkdir(tmpdir+"/pkg", 0755)
+ c.Assert(err, check.IsNil)
+ err = os.Mkdir(tmpdir+"/bin", 0755)
+ c.Assert(err, check.IsNil)
+
+ cmd := exec.Command("go", "install")
+ cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ srctree, err := filepath.Abs("../..")
+ c.Assert(err, check.IsNil)
+
+ cmd = exec.Command("docker", "run", "--rm",
+ "-v", tmpdir+"/pkg:/pkg",
+ "-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+ "-v", srctree+":/usr/local/src/arvados:ro",
+ buildimage,
+ "/arvados-dev", "buildpackage",
+ "-package-version", "0.9.99",
+ "-source", "/usr/local/src/arvados",
+ "-output-directory", "/pkg")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fi)
+}
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-dev/cmd.go
new file mode 100644
index 000000000..e0d5144e5
--- /dev/null
+++ b/cmd/arvados-dev/cmd.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "os"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+)
+
+var (
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "version": cmd.Version,
+ "-version": cmd.Version,
+ "--version": cmd.Version,
+
+ "buildpackage": buildPackage{},
+ })
+)
+
+func main() {
+ os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
new file mode 100755
index 000000000..072dfcf9c
--- /dev/null
+++ b/cmd/arvados-dev/example.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e -o pipefail
+
+version="${PACKAGE_VERSION:-0.9.99}"
+
+# mkdir -p /tmp/pkg
+# (
+# cd cmd/arvados-dev
+# go install
+# )
+# docker run --rm \
+# -v /tmp/pkg:/pkg \
+# -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+# -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
+# /arvados-dev buildpackage \
+# -source /arvados \
+# -package-version "${version}" \
+# -output-directory /pkg
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+# ls -l ${pkgfile}
+# (
+# cd /tmp/pkg
+# dpkg-scanpackages . | gzip > Packages.gz
+# )
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+docker run -it --rm \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ ${INSTALLIMAGE:-debian:10} \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ba57c20c3..d28823f4a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
"io"
"os"
"os/exec"
+ "path/filepath"
"strconv"
"strings"
"syscall"
@@ -24,13 +25,17 @@ import (
"github.com/lib/pq"
)
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
const devtestDatabasePassword = "insecure_arvados_test"
-type installCommand struct{}
+type installCommand struct {
+ ClusterType string
+ SourcePath string
+ PackageVersion string
+}
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.New(stderr, "text", "info")
ctx := ctxlog.Context(context.Background(), logger)
ctx, cancel := context.WithCancel(ctx)
@@ -46,7 +51,9 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.SetOutput(stderr)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
- clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+ flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+ flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -55,18 +62,23 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
return 2
} else if *versionFlag {
return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
}
- var dev, test, prod bool
- switch *clusterType {
+ var dev, test, prod, pkg bool
+ switch inst.ClusterType {
case "development":
dev = true
case "test":
test = true
case "production":
prod = true
+ case "package":
+ pkg = true
default:
- err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+ err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
return 2
}
@@ -96,53 +108,47 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
}
}
- if dev || test {
- debs := []string{
- "bison",
+ pkgs := prodpkgs(osv)
+
+ if pkg {
+ pkgs = append(pkgs,
+ "dpkg-dev",
+ "rsync",
+ )
+ }
+
+ if dev || test || pkg {
+ pkgs = append(pkgs,
"bsdmainutils",
"build-essential",
- "ca-certificates",
"cadaver",
- "curl",
"cython",
"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
- "fuse",
"gettext",
- "git",
- "gitolite3",
- "graphviz",
- "haveged",
"iceweasel",
"libattr1-dev",
"libcrypt-ssleay-perl",
- "libcrypt-ssleay-perl",
- "libcurl3-gnutls",
- "libcurl4-openssl-dev",
"libfuse-dev",
"libgnutls28-dev",
"libjson-perl",
- "libjson-perl",
"libpam-dev",
"libpcre3-dev",
- "libpq-dev",
"libpython2.7-dev",
"libreadline-dev",
"libssl-dev",
"libwww-perl",
"libxml2-dev",
- "libxslt1.1",
+ "libxslt1-dev",
"linkchecker",
"lsof",
"net-tools",
- "nginx",
"pandoc",
"perl-modules",
"pkg-config",
"postgresql",
"postgresql-contrib",
- "python",
"python3-dev",
"python-epydoc",
"r-base",
@@ -151,16 +157,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"virtualenv",
"wget",
"xvfb",
- "zlib1g-dev",
- }
+ )
switch {
case osv.Debian && osv.Major >= 10:
- debs = append(debs, "libcurl4")
+ pkgs = append(pkgs, "libcurl4")
default:
- debs = append(debs, "libcurl3")
+ pkgs = append(pkgs, "libcurl3")
}
cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
- cmd.Args = append(cmd.Args, debs...)
+ cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
cmd.Stderr = stderr
@@ -182,7 +187,7 @@ trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+make -j8
make install
/var/lib/arvados/bin/gem install bundler
rm -r ${tmp}
@@ -206,7 +211,9 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
return 1
}
}
+ }
+ if !prod && !pkg {
pjsversion := "1.9.8"
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
@@ -389,12 +396,89 @@ rm ${zip}
}
}
+ if pkg {
+ // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+ for dstdir, srcdir := range map[string]string{
+ "railsapi": "services/api",
+ "workbench1": "apps/workbench",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ for _, cmdline := range [][]string{
+ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+ {"touch", "log/production.log"},
+ // {"chown", "-R", "root:root", "."},
+ {"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+ } {
+ cmd = exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Env = append([]string{}, os.Environ()...)
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+ // Exit code 2 indicates there were warnings (like
+ // "other passenger installations have been detected",
+ // which we can't expect to avoid) but no errors.
+ // Other non-zero exit codes (1, 9) indicate errors.
+ return 1
+ }
+ }
+
+ // Install Go programs to /var/lib/arvados/bin/
+ for _, srcdir := range []string{
+ "cmd/arvados-client",
+ "cmd/arvados-server",
+ "services/arv-git-httpd",
+ "services/crunch-dispatch-local",
+ "services/crunch-dispatch-slurm",
+ "services/health",
+ "services/keep-balance",
+ "services/keep-web",
+ "services/keepproxy",
+ "services/keepstore",
+ "services/ws",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+ cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ }
+
return 0
}
type osversion struct {
Debian bool
Ubuntu bool
+ Centos bool
Major int
}
@@ -432,6 +516,8 @@ func identifyOS() (osversion, error) {
osv.Ubuntu = true
case "debian":
osv.Debian = true
+ case "centos":
+ osv.Centos = true
default:
return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
}
@@ -466,3 +552,56 @@ func runBash(script string, stdout, stderr io.Writer) error {
cmd.Stderr = stderr
return cmd.Run()
}
+
+func prodpkgs(osv osversion) []string {
+ pkgs := []string{
+ "automake",
+ "bison",
+ "ca-certificates",
+ "curl",
+ "fuse",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "libcurl3-gnutls",
+ "libxslt1.1",
+ "make",
+ "nginx",
+ "python",
+ }
+ if osv.Debian || osv.Ubuntu {
+ if osv.Debian && osv.Major == 8 {
+ pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+ } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+ pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+ }
+ return append(pkgs,
+ "g++",
+ "libcurl4-openssl-dev", // services/api
+ "libpq-dev",
+ "libpython2.7", // services/fuse
+ "mime-support", // keep-web
+ "zlib1g-dev", // services/api
+ )
+ } else if osv.Centos {
+ return append(pkgs,
+ "fuse-libs", // services/fuse
+ "gcc",
+ "gcc-c++",
+ "libcurl-devel", // services/api
+ "mailcap", // keep-web
+ "postgresql-devel", // services/api
+ )
+ } else {
+ panic("os version not supported")
+ }
+}
+
+func ProductionDependencies() ([]string, error) {
+ osv, err := identifyOS()
+ if err != nil {
+ return nil, err
+ }
+ return prodpkgs(osv), nil
+}
commit b4d6c92791088a0b6a39a71b2b8b6ddc4b13af41
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Jun 2 11:14:10 2020 -0400
16306: Add automatic install page.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/_config.yml b/doc/_config.yml
index 7d7101f41..e38f0fa8f 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -190,6 +190,8 @@ navbar:
- install/arvados-on-kubernetes.html.textile.liquid
- install/arvados-on-kubernetes-minikube.html.textile.liquid
- install/arvados-on-kubernetes-GKE.html.textile.liquid
+ - Automatic installation:
+ - install/automatic.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
new file mode 100644
index 000000000..049bf6891
--- /dev/null
+++ b/doc/install/automatic.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example at gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# apt-get install arvados-server
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example at gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 1a41980e2..81d7b2159 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -23,6 +23,7 @@ table(table table-bordered table-condensed).
||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
|"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list