[ARVADOS] created: 2.1.0-336-g4865911a6

Git user git at public.arvados.org
Wed Jan 27 08:15:13 UTC 2021


        at  4865911a605128adb454b3280b1cf9dcd38f499e (commit)


commit 4865911a605128adb454b3280b1cf9dcd38f499e
Author: Tom Clegg <tom at curii.com>
Date:   Wed Jan 27 03:14:44 2021 -0500

    16306: Accept -package-version arg to testinstall.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index cf9cbfa6c..e826408c1 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -126,7 +126,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
 		return err
 	}
 
-	cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
+	cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages --multiversion . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	cmd.Dir = opts.PackageDir
diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index b4ad68b1d..75e8f7604 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -56,6 +56,7 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
 	cmd = exec.Command("go", "run", ".",
 		"testinstall",
 		"-package-dir", tmpdir,
+		"-package-version", "1.2.3~rc4",
 	)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 774771efb..984779950 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -6,7 +6,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -20,9 +19,6 @@ import (
 )
 
 func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
-	if opts.PackageVersion != "" {
-		return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
-	}
 	depsImageName := "arvados-package-deps-" + opts.TargetOS
 	depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
 
@@ -77,6 +73,10 @@ eatmydata apt-get remove -y arvados-server-easy
 		}
 	}
 
+	versionsuffix := ""
+	if opts.PackageVersion != "" {
+		versionsuffix = "=" + opts.PackageVersion
+	}
 	cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
 		"--tmpfs", "/tmp:exec,mode=01777",
 		"-v", opts.PackageDir+":/pkg:ro",
@@ -87,7 +87,7 @@ eatmydata apt-get remove -y arvados-server-easy
 set -e
 PATH="/var/lib/arvados/bin:$PATH"
 apt-get update
-eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy`+versionsuffix+`
 apt-get -y autoremove
 /etc/init.d/postgresql start
 arvados-server init -cluster-id x1234

commit 01e15db1f4a331508117bc841256acec8ca361de
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 25 10:45:30 2021 -0500

    16306: Fix runPassenger HOME env in non-production case.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 84f51d0a7..420393997 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -141,12 +141,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 		}
 		opts := runOptions{
 			env: append([]string{
-				"HOME=/var/www",
 				"TMPDIR=" + super.wwwtempdir,
 			}, railsEnv...),
 		}
 		if super.ClusterType == "production" {
 			opts.user = "www-data"
+			opts.env = append(opts.env, "HOME=/var/www")
 		} else {
 			// This would be desirable when changing uid
 			// too, but it fails because /dev/stderr is a

commit 18def2a271e02fd64749fe650034f50d1b659e45
Merge: 591922d85 fbc95892b
Author: Tom Clegg <tom at curii.com>
Date:   Fri Jan 22 00:58:58 2021 -0500

    16306: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>


commit 591922d85a4cbb6fc890bde7b5f4848f4c92468a
Author: Tom Clegg <tom at curii.com>
Date:   Fri Jan 22 00:53:42 2021 -0500

    16306: Remove obsolete RailsSessionSecretToken config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/install/init.go b/lib/install/init.go
index d4d9ef47e..7ae42c531 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -156,8 +156,6 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
       Health:
         InternalURLs:
           "http://0.0.0.0:9007/": {}
-    API:
-      RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
     Collections:
       BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
     Containers:

commit 8cd08f2ce640e0b1967db489d29e3761ac63f0d7
Merge: a4a17913e 4c6c49190
Author: Tom Clegg <tom at curii.com>
Date:   Thu Jan 21 16:33:40 2021 -0500

    16306: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>


commit a4a17913e299851f0b5979189d36997a62ff43e1
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jan 19 15:42:50 2021 -0500

    16306: Hide "easy install" option from the install docs.
    
    To be un-hidden when it is fully implemented and usable.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/doc/_config.yml b/doc/_config.yml
index 5386e8797..75a55b469 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -208,8 +208,6 @@ navbar:
       - install/arvados-on-kubernetes.html.textile.liquid
       - install/arvados-on-kubernetes-minikube.html.textile.liquid
       - install/arvados-on-kubernetes-GKE.html.textile.liquid
-    - Automatic installation:
-      - install/automatic.html.textile.liquid
     - Manual installation:
       - install/install-manual-prerequisites.html.textile.liquid
       - install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
index 049bf6891..79e850538 100644
--- a/doc/install/automatic.html.textile.liquid
+++ b/doc/install/automatic.html.textile.liquid
@@ -9,6 +9,10 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
+{% include 'notebox_begin' %}
+This installation method is not fully implemented, which is why this page is not yet listed in the "table of installation options":{{site.baseurl}}/install/index.html or in the left nav.
+{% include 'notebox_end' %}
+
 This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
 
 A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
@@ -24,7 +28,9 @@ You will need:
 h2. Initialize the cluster
 
 <pre>
-# apt-get install arvados-server
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# apt-get update
+# apt-get install arvados-server-easy
 # arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example at gmail.com.example
 </pre>
 
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index f16ae2dad..1b27ca6ed 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -25,7 +25,6 @@ table(table table-bordered table-condensed).
 |"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
 |"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
 |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
-|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
 |"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
 |"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
 </div>

commit edc2687671a2fec74304c6e00092034d50c8f667
Author: Tom Clegg <tom at curii.com>
Date:   Tue Jan 19 15:32:15 2021 -0500

    16306: Fixup test.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index 9f002b535..b4ad68b1d 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -32,6 +32,8 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
 	}
 	tmpdir := c.MkDir()
 	defer os.RemoveAll(tmpdir)
+	err := os.Chmod(tmpdir, 0755)
+	c.Assert(err, check.IsNil)
 
 	cmd := exec.Command("go", "run", ".",
 		"build",
@@ -41,13 +43,16 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
 	)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
+	err = cmd.Run()
 	c.Check(err, check.IsNil)
 
 	fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
 	c.Assert(err, check.IsNil)
 	c.Logf("%#v", fi)
 
+	buf, _ := exec.Command("ls", "-l", tmpdir).CombinedOutput()
+	c.Logf("%s", buf)
+
 	cmd = exec.Command("go", "run", ".",
 		"testinstall",
 		"-package-dir", tmpdir,

commit d20342c56db7e148f793a29fd79619ce2e3f0243
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 18 16:33:16 2021 -0500

    16306: Remove old bash scripts.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/docker-boot.sh b/cmd/arvados-package/docker-boot.sh
deleted file mode 100755
index 39a8a56c8..000000000
--- a/cmd/arvados-package/docker-boot.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Bring up a docker container with some locally-built commands (e.g.,
-# cmd/arvados-server) replacing the ones that came with
-# arvados-server-easy when the arvados-installpackage-* image was
-# built.
-#
-# Assumes docker-build-install.sh has already succeeded.
-#
-# Example:
-#
-#    docker-boot.sh cmd/arvados-server services/keep-balance
-
-set -e -o pipefail
-
-declare -A opts=()
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --os)
-            shift
-            opts[os]="$1"
-            shift
-            ;;
-        --shell)
-            shift
-            opts[shell]=1
-            ;;
-        *)
-            break
-            ;;
-    esac
-done
-
-cleanup() {
-    if [[ -n "${tmpdir}" ]]; then
-        rm -rf "${tmpdir}"
-    fi
-}
-trap cleanup ERR EXIT
-
-tmpdir=$(mktemp -d)
-version=$(git describe --tag --dirty)
-
-declare -a volargs=()
-for inject in "$@"; do
-    case "$inject" in
-        nginx.conf)
-            volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
-            ;;
-        *)
-            echo >&2 "building $inject..."
-            (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
-            cmd="$(basename "$inject")"
-            volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
-            ;;
-    esac
-done
-
-osbase=${opts[os]:-debian:10}
-installimage=arvados-installpackage-${osbase}
-command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
-if [[ "${opts[shell]}" ]]; then
-    command="bash -login"
-fi
-docker run -it --rm \
-       "${volargs[@]}" \
-       "${installimage}" \
-       bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-package/docker-build-install.sh b/cmd/arvados-package/docker-build-install.sh
deleted file mode 100755
index 4fdcccbce..000000000
--- a/cmd/arvados-package/docker-build-install.sh
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Build an arvados-server-easy package, then install and run it on a
-# base OS image.
-#
-# Examples:
-#
-#    docker-build-install.sh --force-buildimage --force-installimage     # always build fresh docker images
-#
-#    docker-build-install.sh                                             # reuse cached docker images if possible
-
-set -e -o pipefail
-
-declare -A opts=()
-while [[ $# -gt 0 ]]; do
-    arg="$1"
-    shift
-    case "$arg" in
-        --force-buildimage)
-            opts[force-buildimage]=1
-            ;;
-        --force-installimage)
-            opts[force-installimage]=1
-            ;;
-        --os)
-            opts[os]="$1"
-            shift
-            ;;
-        *)
-            echo >&2 "invalid argument '$arg'"
-            exit 2
-            ;;
-    esac
-done
-
-cleanup() {
-    if [[ -n "${buildctr}" ]]; then
-        docker rm "${buildctr}" || true
-    fi
-    if [[ -n "${installctr}" ]]; then
-        docker rm "${installctr}" || true
-    fi
-}
-trap cleanup ERR EXIT
-
-version=$(git describe --tag --dirty)
-osbase=${opts[os]:-debian:10}
-
-mkdir -p /tmp/pkg
-
-buildimage=arvados-package-build-${osbase}
-if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
-    (
-        echo >&2 building arvados-server...
-        cd cmd/arvados-server
-        go install
-    )
-    echo >&2 building ${buildimage}...
-    buildctr=${buildimage/:/-}
-    docker rm "${buildctr}" || true
-    docker run \
-           --name "${buildctr}" \
-           --tmpfs /tmp:exec,mode=01777 \
-           -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
-           -v "$(pwd)":/arvados:ro \
-           "${osbase}" \
-           /arvados-server install \
-           -eatmydata \
-           -type package \
-           -source /arvados \
-           -package-version "${version}"
-    docker commit "${buildctr}" "${buildimage}"
-    docker rm "${buildctr}"
-    buildctr=
-fi
-
-pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
-rm -v -f "${pkgfile}"
-
-(
-    echo >&2 building arvados-package...
-    cd cmd/arvados-package
-    go install
-)
-echo >&2 building ${pkgfile}...
-docker run --rm \
-       --tmpfs /tmp:exec,mode=01777 \
-       -v /tmp/pkg:/pkg \
-       -v "${GOPATH:-${HOME}/go}"/bin/arvados-package:/arvados-package:ro \
-       -v "$(pwd)":/arvados:ro \
-       "${buildimage}" \
-       eatmydata \
-       /arvados-package build \
-       -source /arvados \
-       -package-version "${version}" \
-       -output-directory /pkg
-
-ls -l ${pkgfile}
-(
-    echo >&2 dpkg-scanpackages...
-    cd /tmp/pkg
-    dpkg-scanpackages . | gzip > Packages.gz
-)
-sourcesfile=/tmp/sources.conf.d-arvados
-echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-
-installimage="arvados-installpackage-${osbase}"
-if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
-    echo >&2 building ${installimage}...
-    installctr=${installimage/:/-}
-    docker rm "${installctr}" || true
-    docker run -it \
-           --name "${installctr}" \
-           --tmpfs /tmp \
-           -v /tmp/pkg:/pkg:ro \
-           -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
-           --env DEBIAN_FRONTEND=noninteractive \
-           "${osbase}" \
-           bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
-    docker commit "${installctr}" "${installimage}"
-    docker rm "${installctr}"
-    installctr=
-fi
-
-echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
-docker run -it --rm \
-       -v /tmp/pkg:/pkg:ro \
-       -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
-       "${installimage}" \
-       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive eatmydata apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 719258a9b..774771efb 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -19,27 +19,6 @@ import (
 	"github.com/docker/docker/client"
 )
 
-// sourcesfile=/tmp/sources.conf.d-arvados
-// echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-
-// installimage="arvados-installpackage-${osbase}"
-// if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
-//     echo >&2 building ${installimage}...
-//     installctr=${installimage/:/-}
-//     docker rm "${installctr}" || true
-//     docker run -it \
-//            --name "${installctr}" \
-//            --tmpfs /tmp \
-//            -v /tmp/pkg:/pkg:ro \
-//            -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
-//            --env DEBIAN_FRONTEND=noninteractive \
-//            "${osbase}" \
-//            bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
-//     docker commit "${installctr}" "${installimage}"
-//     docker rm "${installctr}"
-//     installctr=
-// fi
-
 func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
 	if opts.PackageVersion != "" {
 		return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")

commit 4085665f87c82939961e55bf004ca47c03ca30ac
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 18 16:31:31 2021 -0500

    16306: Update tests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go
index abb54532d..9f002b535 100644
--- a/cmd/arvados-package/build_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -5,10 +5,8 @@
 package main
 
 import (
-	"flag"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"testing"
 
 	"gopkg.in/check.v1"
@@ -18,13 +16,6 @@ var buildimage string
 
 func init() {
 	os.Args = append(os.Args, "-test.timeout=30m") // kludge
-
-	// This enables a hack to speed up repeated tests: hit "docker
-	// commit --pause {containername} checkpointtag" after the
-	// test container has downloaded/compiled some stuff, then run
-	// tests with "-test.buildimage=checkpointtag" next time to
-	// retry/resume/update from that point.
-	flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
 }
 
 type BuildSuite struct{}
@@ -42,36 +33,30 @@ func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
 	tmpdir := c.MkDir()
 	defer os.RemoveAll(tmpdir)
 
-	err := os.Mkdir(tmpdir+"/pkg", 0755)
-	c.Assert(err, check.IsNil)
-	err = os.Mkdir(tmpdir+"/bin", 0755)
-	c.Assert(err, check.IsNil)
-
-	cmd := exec.Command("go", "install")
-	cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
-	cmd.Stdout = os.Stdout
+	cmd := exec.Command("go", "run", ".",
+		"build",
+		"-package-dir", tmpdir,
+		"-package-version", "1.2.3~rc4",
+		"-source", "../..",
+	)
+	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	c.Assert(err, check.IsNil)
+	err := cmd.Run()
+	c.Check(err, check.IsNil)
 
-	srctree, err := filepath.Abs("../..")
+	fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
 	c.Assert(err, check.IsNil)
+	c.Logf("%#v", fi)
 
-	cmd = exec.Command("docker", "run", "--rm",
-		"-v", tmpdir+"/pkg:/pkg",
-		"-v", tmpdir+"/bin/arvados-package:/arvados-package:ro",
-		"-v", srctree+":/usr/local/src/arvados:ro",
-		buildimage,
-		"/arvados-package", "build",
-		"-package-version", "0.9.99",
-		"-source", "/usr/local/src/arvados",
-		"-output-directory", "/pkg")
-	cmd.Stdout = os.Stdout
+	cmd = exec.Command("go", "run", ".",
+		"testinstall",
+		"-package-dir", tmpdir,
+	)
+	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
-	c.Assert(err, check.IsNil)
+	c.Check(err, check.IsNil)
 
-	fi, err := os.Stat(tmpdir + "/pkg/arvados-server-easy_0.9.99_amd64.deb")
-	c.Assert(err, check.IsNil)
-	c.Logf("%#v", fi)
+	err = os.RemoveAll(tmpdir)
+	c.Check(err, check.IsNil)
 }

commit 1297d71125f5a694996147406edf38bdc913396c
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 18 16:31:40 2021 -0500

    16306: Use cwd as default source tree.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 6cf3ba3c0..b0c0e9e3c 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -68,7 +68,8 @@ type opts struct {
 
 func parseFlags(args []string) (opts, error) {
 	opts := opts{
-		TargetOS: "debian:10",
+		SourceDir: ".",
+		TargetOS:  "debian:10",
 	}
 	flags := flag.NewFlagSet("", flag.ContinueOnError)
 	flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
@@ -123,5 +124,9 @@ Options:
 		opts.SourceDir = d
 	}
 	opts.PackageDir = filepath.Clean(opts.PackageDir)
+	opts.SourceDir, err = filepath.Abs(opts.SourceDir)
+	if err != nil {
+		return opts, err
+	}
 	return opts, nil
 }

commit 41305b5ac71cc9a306dc654c42c11ffcc4258a47
Merge: b979f8011 025639399
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 18 13:55:27 2021 -0500

    16306: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --cc lib/boot/supervisor.go
index e892d3e62,f2e715a76..838808df5
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@@ -244,11 -222,11 +244,11 @@@ func (super *Supervisor) run(cfg *arvad
  		runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
  		runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
  		runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
- 		runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
+ 		runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
  		installPassenger{src: "services/api"},
- 		runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
- 		installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
 -		runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
++		runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ 		installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
 -		runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 +		runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
  		seedDatabase{},
  	}
  	if super.ClusterType != "test" {
@@@ -682,26 -605,27 +682,26 @@@ func (super *Supervisor) autofillConfig
  			}
  		}
  	}
 -	if cluster.SystemRootToken == "" {
 -		cluster.SystemRootToken = randomHexString(64)
 -	}
 -	if cluster.ManagementToken == "" {
 -		cluster.ManagementToken = randomHexString(64)
 -	}
 -	if cluster.Collections.BlobSigningKey == "" {
 -		cluster.Collections.BlobSigningKey = randomHexString(64)
 -	}
 -	if cluster.Users.AnonymousUserToken == "" {
 -		cluster.Users.AnonymousUserToken = randomHexString(64)
 -	}
 -
 -	if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
 -		buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 -		if err != nil {
 -			return err
 -		}
 -		cluster.Containers.DispatchPrivateKey = string(buf)
 -	}
  	if super.ClusterType != "production" {
 +		if cluster.SystemRootToken == "" {
 +			cluster.SystemRootToken = randomHexString(64)
 +		}
 +		if cluster.ManagementToken == "" {
 +			cluster.ManagementToken = randomHexString(64)
 +		}
- 		if cluster.API.RailsSessionSecretToken == "" {
- 			cluster.API.RailsSessionSecretToken = randomHexString(64)
- 		}
 +		if cluster.Collections.BlobSigningKey == "" {
 +			cluster.Collections.BlobSigningKey = randomHexString(64)
 +		}
++		if cluster.Users.AnonymousUserToken == "" {
++			cluster.Users.AnonymousUserToken = randomHexString(64)
++		}
 +		if cluster.Containers.DispatchPrivateKey == "" {
 +			buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 +			if err != nil {
 +				return err
 +			}
 +			cluster.Containers.DispatchPrivateKey = string(buf)
 +		}
  		cluster.TLS.Insecure = true
  	}
  	if super.ClusterType == "test" {

commit b979f80113c4f8a4c13c96b4679c28788c099333
Author: Tom Clegg <tom at curii.com>
Date:   Thu Jan 7 17:24:29 2021 -0500

    16306: Command usage help.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index c85979268..cf9cbfa6c 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -108,7 +108,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
 		"-v", selfbin+":/arvados-package:ro",
 		"-v", opts.SourceDir+":/arvados:ro",
 		buildImageName,
-		"eatmydata", "/arvados-package", "fpm",
+		"eatmydata", "/arvados-package", "_fpm",
 		"-source", "/arvados",
 		"-package-version", opts.PackageVersion,
 		"-package-dir", "/pkg",
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 9b9971e92..6cf3ba3c0 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/lib/install"
@@ -24,13 +25,17 @@ var (
 		"--version": cmd.Version,
 
 		"build":       cmdFunc(build),
-		"fpm":         cmdFunc(fpm),
 		"testinstall": cmdFunc(testinstall),
+		"_fpm":        cmdFunc(fpm),    // internal use
 		"_install":    install.Command, // internal use
 	})
 )
 
 func main() {
+	if len(os.Args) < 2 || strings.HasPrefix(os.Args[1], "-") {
+		parseFlags([]string{"-help"})
+		os.Exit(2)
+	}
 	os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
 }
 
@@ -72,6 +77,37 @@ func parseFlags(args []string) (opts, error) {
 	flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
 	flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
 	flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+	flags.Usage = func() {
+		fmt.Fprint(flags.Output(), `Usage: arvados-package <subcommand> [options]
+
+Subcommands:
+	build
+		use a docker container to build a package from a checked
+		out version of the arvados source tree
+	testinstall
+		use a docker container to install a package and confirm
+		the resulting installation is functional
+	version
+		show program version
+
+Internally used subcommands:
+	_fpm
+		build a package
+	_install
+		equivalent to "arvados-server install"
+
+Automation/integration notes:
+	The first time a given machine runs "build" or "testinstall" (and
+	any time the -rebuild-image is used), new docker images are built,
+	which is quite slow. If you use on-demand VMs to run automated builds,
+	run "build" and "testinstall" once when setting up your initial VM
+	image, and be prepared to rebuild that VM image when package-building
+	slows down (this will happen when new dependencies are introduced).
+
+Options:
+`)
+		flags.PrintDefaults()
+	}
 	err := flags.Parse(args)
 	if err != nil {
 		return opts, err

commit 57c3fbfb0f75dc38be717d52f29abe5bcefea1bb
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 4 15:58:24 2021 -0500

    16306: Remove daemontools dependency.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 4a605e35a..84f51d0a7 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -148,11 +148,11 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 		if super.ClusterType == "production" {
 			opts.user = "www-data"
 		} else {
-			// This would be desirable in the production
-			// case too, but it fails with sudo because
-			// /dev/stderr is a symlink to a pty owned by
-			// root: "nginx: [emerg] open() "/dev/stderr"
-			// failed (13: Permission denied)"
+			// This would be desirable when changing uid
+			// too, but it fails because /dev/stderr is a
+			// symlink to a pty owned by root: "nginx:
+			// [emerg] open() "/dev/stderr" failed (13:
+			// Permission denied)"
 			cmdline = append(cmdline, "--log-file", "/dev/stderr")
 		}
 		err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index daa0414a3..4ed7603d2 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -60,6 +60,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 		return err
 	}
 	prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+	opts := runOptions{}
 	if iamroot {
 		postgresUser, err := user.Lookup("postgres")
 		if err != nil {
@@ -85,15 +86,9 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 		if err != nil {
 			return err
 		}
-		// We can't use "sudo -u" here because it creates an
-		// intermediate process that interferes with our
-		// ability to reliably kill postgres. The setuidgid
-		// program just calls exec without forking, so it
-		// doesn't have this problem.
-		args = append([]string{"postgres", prog}, args...)
-		prog = "setuidgid"
-	}
-	err = super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...)
+		opts.user = "postgres"
+	}
+	err = super.RunProgram(ctx, super.tempdir, opts, prog, args...)
 	if err != nil {
 		return err
 	}
@@ -120,11 +115,11 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 			"-k", datadir, // socket dir
 			"-p", super.cluster.PostgreSQL.Connection["port"],
 		}
+		opts := runOptions{}
 		if iamroot {
-			args = append([]string{"postgres", prog}, args...)
-			prog = "setuidgid"
+			opts.user = "postgres"
 		}
-		fail(super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...))
+		fail(super.RunProgram(ctx, super.tempdir, opts, prog, args...))
 	}()
 
 	for {
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 5e88775e5..e892d3e62 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -491,9 +491,6 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
 
 	logprefix := prog
 	{
-		if logprefix == "setuidgid" && len(args) >= 3 {
-			logprefix = args[2]
-		}
 		innerargs := args
 		if logprefix == "sudo" {
 			for i := 0; i < len(args); i++ {
@@ -556,6 +553,15 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
 	cmd.Env = dedupEnv(env)
 
 	if opts.user != "" {
+		// Note: We use this approach instead of "sudo"
+		// because in certain circumstances (we are pid 1 in a
+		// docker container, and our passenger child process
+		// changes to pgid 1) the intermediate sudo process
+		// notices we have the same pgid as our child and
+		// refuses to propagate signals from us to our child,
+		// so we can't signal/shutdown our passenger/rails
+		// apps. "chpst" or "setuidgid" would work, but these
+		// few lines avoid depending on runit/daemontools.
 		u, err := user.Lookup(opts.user)
 		if err != nil {
 			return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 049df2564..504cdff8f 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -141,7 +141,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 			"cadaver",
 			"curl",
 			"cython3",
-			"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
 			"default-jdk-headless",
 			"default-jre-headless",
 			"gettext",

commit 6cd0e19a277812f302f46693d824cc17d17ed6b1
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 4 15:42:35 2021 -0500

    16306: Exit 0 when startup-then-shutdown goes as planned.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index e0e275522..373613bb3 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -108,6 +108,11 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
 		fmt.Fprintln(stdout, url)
 		if *shutdown {
 			super.Stop()
+			// Wait for children to exit. Don't report the
+			// ensuing "context cancelled" error, though:
+			// return nil to indicate successful startup.
+			_ = super.Wait()
+			return nil
 		}
 	}
 	// Wait for signal/crash + orderly shutdown

commit d71d4273d1f3d0b6381efafa649b81c6b4107cf1
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 4 15:26:46 2021 -0500

    16306: Fix inability to shutdown passenger processes.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index 8f6339e63..b2b8c896c 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -39,17 +39,17 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
 	}
 
 	// Generate root key
-	err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+	err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
 	if err != nil {
 		return err
 	}
 	// Generate a self-signed root certificate
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
 	if err != nil {
 		return err
 	}
 	// Generate server key
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "server.key", "2048")
 	if err != nil {
 		return err
 	}
@@ -63,12 +63,12 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
 		return err
 	}
 	// Generate signing request
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
 	if err != nil {
 		return err
 	}
 	// Sign certificate
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index d49fabe56..d14d05152 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -120,7 +120,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 	super.waitShutdown.Add(1)
 	go func() {
 		defer super.waitShutdown.Done()
-		fail(super.RunProgram(ctx, ".", nil, nil, nginx, args...))
+		fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
 	}()
 	// Choose one of the ports where Nginx should listen, and wait
 	// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 03464aaf7..4a605e35a 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -50,32 +50,32 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 	defer passengerInstallMutex.Unlock()
 
 	var buf bytes.Buffer
-	err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler")
+	err = super.RunProgram(ctx, runner.src, runOptions{output: &buf}, "gem", "list", "--details", "bundler")
 	if err != nil {
 		return err
 	}
 	for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
 		if !strings.Contains(buf.String(), "("+version+")") {
-			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+			err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
 			if err != nil {
 				return err
 			}
 			break
 		}
 	}
-	err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+	err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
 	if err != nil {
 		return err
 	}
-	err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support")
+	err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "build-native-support")
 	if err != nil {
 		return err
 	}
-	err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime")
+	err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "install-standalone-runtime")
 	if err != nil {
 		return err
 	}
-	err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install")
+	err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "validate-install")
 	if err != nil && !strings.Contains(err.Error(), "exit status 2") {
 		// Exit code 2 indicates there were warnings (like
 		// "other passenger installations have been detected",
@@ -139,8 +139,14 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 			"--no-install-runtime",
 			"--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
 		}
+		opts := runOptions{
+			env: append([]string{
+				"HOME=/var/www",
+				"TMPDIR=" + super.wwwtempdir,
+			}, railsEnv...),
+		}
 		if super.ClusterType == "production" {
-			cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+			opts.user = "www-data"
 		} else {
 			// This would be desirable in the production
 			// case too, but it fails with sudo because
@@ -149,8 +155,7 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 			// failed (13: Permission denied)"
 			cmdline = append(cmdline, "--log-file", "/dev/stderr")
 		}
-		env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
-		err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
+		err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
 		fail(err)
 	}()
 	return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index e45c4e168..daa0414a3 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -48,7 +48,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 	}
 
 	buf := bytes.NewBuffer(nil)
-	err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{output: buf}, "pg_config", "--bindir")
 	if err != nil {
 		return err
 	}
@@ -93,17 +93,17 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 		args = append([]string{"postgres", prog}, args...)
 		prog = "setuidgid"
 	}
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...)
 	if err != nil {
 		return err
 	}
 
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir)
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, "cp", "server.crt", "server.key", datadir)
 	if err != nil {
 		return err
 	}
 	if iamroot {
-		err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+		err = super.RunProgram(ctx, super.tempdir, runOptions{}, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
 		if err != nil {
 			return err
 		}
@@ -124,7 +124,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 			args = append([]string{"postgres", prog}, args...)
 			prog = "setuidgid"
 		}
-		fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
+		fail(super.RunProgram(ctx, super.tempdir, runOptions{}, prog, args...))
 	}()
 
 	for {
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index 1f6cb764e..bd1e94265 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -23,11 +23,11 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
 	if super.ClusterType == "production" {
 		return nil
 	}
-	err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
+	err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
 	if err != nil {
 		return err
 	}
-	err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb")
+	err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb")
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 77fdc98be..090e85244 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -31,7 +31,7 @@ func (runner runServiceCommand) String() string {
 
 func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
 	binfile := filepath.Join(super.bindir, "arvados-server")
-	err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
+	err := super.RunProgram(ctx, super.bindir, runOptions{}, binfile, "-version")
 	if err != nil {
 		return err
 	}
@@ -46,7 +46,7 @@ func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super
 		super.waitShutdown.Add(1)
 		go func() {
 			defer super.waitShutdown.Done()
-			fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile))
+			fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
 		}()
 	}
 	return nil
@@ -77,7 +77,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup
 		return ctx.Err()
 	}
 
-	err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+	err = super.RunProgram(ctx, super.tempdir, runOptions{}, binfile, "-version")
 	if err != nil {
 		return err
 	}
@@ -93,7 +93,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup
 		super.waitShutdown.Add(1)
 		go func() {
 			defer super.waitShutdown.Done()
-			fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile))
+			fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile))
 		}()
 	}
 	return nil
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 7dba9b5dc..5e88775e5 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -21,6 +21,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -206,13 +207,13 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 	} else if super.SourceVersion == "" {
 		// Find current source tree version.
 		var buf bytes.Buffer
-		err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+		err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat")
 		if err != nil {
 			return err
 		}
 		dirty := buf.Len() > 0
 		buf.Reset()
-		err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+		err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H")
 		if err != nil {
 			return err
 		}
@@ -407,7 +408,7 @@ func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (
 	if super.ClusterType == "production" {
 		return binfile, nil
 	}
-	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 	return binfile, err
 }
 
@@ -470,6 +471,12 @@ func (super *Supervisor) lookPath(prog string) string {
 	return prog
 }
 
+type runOptions struct {
+	output io.Writer // attach stdout
+	env    []string  // add/replace environment variables
+	user   string    // run as specified user
+}
+
 // RunProgram runs prog with args, using dir as working directory. If ctx is
 // cancelled while the child is running, RunProgram terminates the child, waits
 // for it to exit, then returns.
@@ -478,7 +485,7 @@ func (super *Supervisor) lookPath(prog string) string {
 //
 // Child's stdout will be written to output if non-nil, otherwise the
 // boot command's stderr.
-func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error {
 	cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
 	super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 
@@ -531,10 +538,10 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
 	}()
 	copiers.Add(1)
 	go func() {
-		if output == nil {
+		if opts.output == nil {
 			io.Copy(logwriter, stdout)
 		} else {
-			io.Copy(output, stdout)
+			io.Copy(opts.output, stdout)
 		}
 		copiers.Done()
 	}()
@@ -544,10 +551,25 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
 	} else {
 		cmd.Dir = filepath.Join(super.SourcePath, dir)
 	}
-	env = append([]string(nil), env...)
+	env := append([]string(nil), opts.env...)
 	env = append(env, super.environ...)
 	cmd.Env = dedupEnv(env)
 
+	if opts.user != "" {
+		u, err := user.Lookup(opts.user)
+		if err != nil {
+			return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
+		}
+		uid, _ := strconv.Atoi(u.Uid)
+		gid, _ := strconv.Atoi(u.Gid)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Credential: &syscall.Credential{
+				Uid: uint32(uid),
+				Gid: uint32(gid),
+			},
+		}
+	}
+
 	exited := false
 	defer func() { exited = true }()
 	go func() {

commit 2cfb41d14010e26d97df93c4cf8ad00f0ac01701
Author: Tom Clegg <tom at curii.com>
Date:   Mon Jan 4 11:22:09 2021 -0500

    16306: Refactor docker scripts into arvados-package command.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index 1343ca77b..c85979268 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -5,120 +5,158 @@
 package main
 
 import (
+	"bytes"
 	"context"
-	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
 
-	"git.arvados.org/arvados.git/lib/install"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
-	"github.com/sirupsen/logrus"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
 )
 
-type build struct{}
-
-func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
-	logger := ctxlog.New(stderr, "text", "info")
-	err := (&builder{
-		PackageVersion: "0.0.0",
-		logger:         logger,
-	}).run(context.Background(), prog, args, stdin, stdout, stderr)
-	if err != nil {
-		logger.WithError(err).Error("failed")
-		return 1
+func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+	if opts.PackageVersion == "" {
+		var buf bytes.Buffer
+		cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty")
+		cmd.Stdout = &buf
+		cmd.Stderr = stderr
+		cmd.Dir = opts.SourceDir
+		err := cmd.Run()
+		if err != nil {
+			return fmt.Errorf("git describe: %w", err)
+		}
+		opts.PackageVersion = strings.TrimSpace(buf.String())
+		ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion)
 	}
-	return 0
-}
 
-type builder struct {
-	PackageVersion string
-	SourcePath     string
-	OutputDir      string
-	SkipInstall    bool
-	logger         logrus.FieldLogger
-}
+	if opts.PackageChown == "" {
+		whoami, err := user.Current()
+		if err != nil {
+			return fmt.Errorf("user.Current: %w", err)
+		}
+		opts.PackageChown = whoami.Uid + ":" + whoami.Gid
+	}
 
-func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
-	flags := flag.NewFlagSet("", flag.ContinueOnError)
-	flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
-	flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
-	flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
-	flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
-	err := flags.Parse(args)
+	// Build in a tempdir, then move to the desired destination
+	// dir. Otherwise, errors might cause us to leave a mess:
+	// truncated files, files owned by root, etc.
+	_, prog := filepath.Split(os.Args[0])
+	tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".")
 	if err != nil {
 		return err
 	}
-	if len(flags.Args()) > 0 {
-		return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+	defer os.RemoveAll(tmpdir)
+
+	selfbin, err := os.Readlink("/proc/self/exe")
+	if err != nil {
+		return fmt.Errorf("readlink /proc/self/exe: %w", err)
 	}
-	if !bldr.SkipInstall {
-		exitcode := install.Command.RunCommand("arvados-server install", []string{
+	buildImageName := "arvados-package-build-" + opts.TargetOS
+	packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb"
+
+	if ok, err := dockerImageExists(ctx, buildImageName); err != nil {
+		return err
+	} else if !ok || opts.RebuildImage {
+		buildCtrName := strings.Replace(buildImageName, ":", "-", -1)
+		err = dockerRm(ctx, buildCtrName)
+		if err != nil {
+			return err
+		}
+
+		defer dockerRm(ctx, buildCtrName)
+		cmd := exec.CommandContext(ctx, "docker", "run",
+			"--name", buildCtrName,
+			"--tmpfs", "/tmp:exec,mode=01777",
+			"-v", selfbin+":/arvados-package:ro",
+			"-v", opts.SourceDir+":/arvados:ro",
+			opts.TargetOS,
+			"/arvados-package", "_install",
+			"-eatmydata",
 			"-type", "package",
-			"-package-version", bldr.PackageVersion,
-			"-source", bldr.SourcePath,
-		}, stdin, stdout, stderr)
-		if exitcode != 0 {
-			return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+			"-source", "/arvados",
+			"-package-version", opts.PackageVersion,
+		)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("docker run: %w", err)
 		}
+
+		cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("docker commit: %w", err)
+		}
+
+		ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName)
 	}
-	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+
+	cmd := exec.CommandContext(ctx, "docker", "run",
+		"--rm",
+		"--tmpfs", "/tmp:exec,mode=01777",
+		"-v", tmpdir+":/pkg",
+		"-v", selfbin+":/arvados-package:ro",
+		"-v", opts.SourceDir+":/arvados:ro",
+		buildImageName,
+		"eatmydata", "/arvados-package", "fpm",
+		"-source", "/arvados",
+		"-package-version", opts.PackageVersion,
+		"-package-dir", "/pkg",
+		"-package-chown", opts.PackageChown,
+	)
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	err = cmd.Run()
 	if err != nil {
-		return fmt.Errorf("gem install fpm: %w", err)
+		return fmt.Errorf("docker run: %w", err)
 	}
 
-	if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
-		// Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
-		cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
-		cmd.Stdout = stdout
-		cmd.Stderr = stderr
-		err = cmd.Run()
-		if err != nil {
-			return fmt.Errorf("monkeypatch fpm: %w", err)
-		}
+	err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
+	if err != nil {
+		return err
 	}
 
-	// Remove unneeded files. This is much faster than "fpm
-	// --exclude X" because fpm copies everything into a staging
-	// area before looking at the --exclude args.
-	cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+	cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
+	cmd.Dir = opts.PackageDir
 	err = cmd.Run()
 	if err != nil {
-		return fmt.Errorf("rm -rf [...]: %w", err)
+		return fmt.Errorf("dpkg-scanpackages: %w", err)
 	}
 
-	format := "deb" // TODO: rpm
+	return nil
+}
 
-	cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
-		"--name", "arvados-server-easy",
-		"--version", bldr.PackageVersion,
-		"--input-type", "dir",
-		"--output-type", format)
-	deps, err := install.ProductionDependencies()
+func dockerRm(ctx context.Context, name string) error {
+	cli, err := client.NewEnvClient()
 	if err != nil {
 		return err
 	}
-	for _, pkg := range deps {
-		cmd.Args = append(cmd.Args, "--depends", pkg)
+	ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1})
+	if err != nil {
+		return err
 	}
-	cmd.Args = append(cmd.Args,
-		"--verbose",
-		"--deb-use-file-permissions",
-		"--rpm-use-file-permissions",
-		"/var/lib/arvados",
-		"/var/www/.gem",
-		"/var/www/.passenger",
-		"/var/www/.bundle",
-	)
-	fmt.Fprintf(stderr, "... %s\n", cmd.Args)
-	cmd.Dir = bldr.OutputDir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	return cmd.Run()
+	for _, ctr := range ctrs {
+		for _, ctrname := range ctr.Names {
+			if ctrname == "/"+name {
+				err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{})
+				if err != nil {
+					return fmt.Errorf("error removing container %s: %w", ctr.ID, err)
+				}
+				break
+			}
+		}
+	}
+	return nil
 }
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index 02bc16cea..9b9971e92 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -5,9 +5,16 @@
 package main
 
 import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 
 	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/install"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 )
 
 var (
@@ -16,10 +23,69 @@ var (
 		"-version":  cmd.Version,
 		"--version": cmd.Version,
 
-		"build": build{},
+		"build":       cmdFunc(build),
+		"fpm":         cmdFunc(fpm),
+		"testinstall": cmdFunc(testinstall),
+		"_install":    install.Command, // internal use
 	})
 )
 
 func main() {
 	os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
 }
+
+type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error
+
+func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "text", "info")
+	ctx := ctxlog.Context(context.Background(), logger)
+	opts, err := parseFlags(args)
+	if err != nil {
+		logger.WithError(err).Error("error parsing command line flags")
+		return 1
+	}
+	err = cf(ctx, opts, stdin, stdout, stderr)
+	if err != nil {
+		logger.WithError(err).Error("failed")
+		return 1
+	}
+	return 0
+}
+
+type opts struct {
+	PackageVersion string
+	PackageDir     string
+	PackageChown   string
+	RebuildImage   bool
+	SourceDir      string
+	TargetOS       string
+}
+
+func parseFlags(args []string) (opts, error) {
+	opts := opts{
+		TargetOS: "debian:10",
+	}
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
+	flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location")
+	flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)")
+	flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
+	flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
+	flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+	err := flags.Parse(args)
+	if err != nil {
+		return opts, err
+	}
+	if len(flags.Args()) > 0 {
+		return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+	}
+	if opts.SourceDir == "" {
+		d, err := os.Getwd()
+		if err != nil {
+			return opts, fmt.Errorf("Getwd: %w", err)
+		}
+		opts.SourceDir = d
+	}
+	opts.PackageDir = filepath.Clean(opts.PackageDir)
+	return opts, nil
+}
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/fpm.go
similarity index 53%
copy from cmd/arvados-package/build.go
copy to cmd/arvados-package/fpm.go
index 1343ca77b..a86232036 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/fpm.go
@@ -6,67 +6,37 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 
 	"git.arvados.org/arvados.git/lib/install"
-	"git.arvados.org/arvados.git/sdk/go/ctxlog"
-	"github.com/sirupsen/logrus"
 )
 
-type build struct{}
-
-func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
-	logger := ctxlog.New(stderr, "text", "info")
-	err := (&builder{
-		PackageVersion: "0.0.0",
-		logger:         logger,
-	}).run(context.Background(), prog, args, stdin, stdout, stderr)
-	if err != nil {
-		logger.WithError(err).Error("failed")
-		return 1
+func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+	var chownUid, chownGid int
+	if opts.PackageChown != "" {
+		_, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid)
+		if err != nil {
+			return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err)
+		}
 	}
-	return 0
-}
-
-type builder struct {
-	PackageVersion string
-	SourcePath     string
-	OutputDir      string
-	SkipInstall    bool
-	logger         logrus.FieldLogger
-}
 
-func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
-	flags := flag.NewFlagSet("", flag.ContinueOnError)
-	flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
-	flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
-	flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
-	flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
-	err := flags.Parse(args)
-	if err != nil {
-		return err
-	}
-	if len(flags.Args()) > 0 {
-		return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
-	}
-	if !bldr.SkipInstall {
-		exitcode := install.Command.RunCommand("arvados-server install", []string{
-			"-type", "package",
-			"-package-version", bldr.PackageVersion,
-			"-source", bldr.SourcePath,
-		}, stdin, stdout, stderr)
-		if exitcode != 0 {
-			return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
-		}
+	exitcode := install.Command.RunCommand("arvados-server install", []string{
+		"-type", "package",
+		"-package-version", opts.PackageVersion,
+		"-source", opts.SourceDir,
+	}, stdin, stdout, stderr)
+	if exitcode != 0 {
+		return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
 	}
+
 	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	err = cmd.Run()
+	err := cmd.Run()
 	if err != nil {
 		return fmt.Errorf("gem install fpm: %w", err)
 	}
@@ -94,10 +64,12 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 	}
 
 	format := "deb" // TODO: rpm
+	pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format)
 
 	cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+		"--package", pkgfile,
 		"--name", "arvados-server-easy",
-		"--version", bldr.PackageVersion,
+		"--version", opts.PackageVersion,
 		"--input-type", "dir",
 		"--output-type", format)
 	deps, err := install.ProductionDependencies()
@@ -117,8 +89,25 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		"/var/www/.bundle",
 	)
 	fmt.Fprintf(stderr, "... %s\n", cmd.Args)
-	cmd.Dir = bldr.OutputDir
+	cmd.Dir = opts.PackageDir
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("fpm: %w", err)
+	}
+
+	if opts.PackageChown != "" {
+		err = os.Chown(pkgfile, chownUid, chownGid)
+		if err != nil {
+			return fmt.Errorf("chown %s: %w", pkgfile, err)
+		}
+	}
+
+	cmd = exec.Command("ls", "-l", pkgfile)
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	return cmd.Run()
+	_ = cmd.Run()
+
+	return nil
 }
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
new file mode 100644
index 000000000..719258a9b
--- /dev/null
+++ b/cmd/arvados-package/install.go
@@ -0,0 +1,143 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+// sourcesfile=/tmp/sources.conf.d-arvados
+// echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+// installimage="arvados-installpackage-${osbase}"
+// if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+//     echo >&2 building ${installimage}...
+//     installctr=${installimage/:/-}
+//     docker rm "${installctr}" || true
+//     docker run -it \
+//            --name "${installctr}" \
+//            --tmpfs /tmp \
+//            -v /tmp/pkg:/pkg:ro \
+//            -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+//            --env DEBIAN_FRONTEND=noninteractive \
+//            "${osbase}" \
+//            bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
+//     docker commit "${installctr}" "${installimage}"
+//     docker rm "${installctr}"
+//     installctr=
+// fi
+
+func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+	if opts.PackageVersion != "" {
+		return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
+	}
+	depsImageName := "arvados-package-deps-" + opts.TargetOS
+	depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+
+	_, prog := filepath.Split(os.Args[0])
+	tmpdir, err := ioutil.TempDir("", prog+".")
+	if err != nil {
+		return fmt.Errorf("TempDir: %w", err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sourcesFile := tmpdir + "/arvados-local.list"
+	err = ioutil.WriteFile(sourcesFile, []byte("deb [trusted=yes] file:///pkg ./\n"), 0644)
+	if err != nil {
+		return fmt.Errorf("Write %s: %w", sourcesFile, err)
+	}
+
+	if exists, err := dockerImageExists(ctx, depsImageName); err != nil {
+		return err
+	} else if !exists || opts.RebuildImage {
+		err = dockerRm(ctx, depsCtrName)
+		if err != nil {
+			return err
+		}
+		defer dockerRm(ctx, depsCtrName)
+		cmd := exec.CommandContext(ctx, "docker", "run",
+			"--name", depsCtrName,
+			"--tmpfs", "/tmp:exec,mode=01777",
+			"-v", opts.PackageDir+":/pkg:ro",
+			"-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+			"--env", "DEBIAN_FRONTEND=noninteractive",
+			opts.TargetOS,
+			"bash", "-c", `
+set -e
+apt-get update
+apt-get install -y eatmydata
+eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql
+eatmydata apt-get remove -y arvados-server-easy
+`)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("docker run: %w", err)
+		}
+
+		cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("docker commit: %w", err)
+		}
+	}
+
+	cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+		"--tmpfs", "/tmp:exec,mode=01777",
+		"-v", opts.PackageDir+":/pkg:ro",
+		"-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+		"--env", "DEBIAN_FRONTEND=noninteractive",
+		depsImageName,
+		"bash", "-c", `
+set -e
+PATH="/var/lib/arvados/bin:$PATH"
+apt-get update
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy
+apt-get -y autoremove
+/etc/init.d/postgresql start
+arvados-server init -cluster-id x1234
+exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+`)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("docker run: %w", err)
+	}
+	return nil
+}
+
+func dockerImageExists(ctx context.Context, name string) (bool, error) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return false, err
+	}
+	imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+	if err != nil {
+		return false, err
+	}
+	for _, img := range imgs {
+		for _, tag := range img.RepoTags {
+			if tag == name {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}

commit 985ece13dd7147f153c7a59c5b3665570c5943c8
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 29 15:18:19 2020 -0500

    16306: Rename arvados-dev buildpackage -> arvados-package build
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-package/build.go
similarity index 96%
rename from cmd/arvados-dev/buildpackage.go
rename to cmd/arvados-package/build.go
index a24fa47c9..1343ca77b 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-package/build.go
@@ -17,9 +17,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type buildPackage struct{}
+type build struct{}
 
-func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	logger := ctxlog.New(stderr, "text", "info")
 	err := (&builder{
 		PackageVersion: "0.0.0",
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-package/build_test.go
similarity index 86%
rename from cmd/arvados-dev/buildpackage_test.go
rename to cmd/arvados-package/build_test.go
index 61bdeae82..abb54532d 100644
--- a/cmd/arvados-dev/buildpackage_test.go
+++ b/cmd/arvados-package/build_test.go
@@ -27,13 +27,13 @@ func init() {
 	flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
 }
 
-type BuildpackageSuite struct{}
+type BuildSuite struct{}
 
-var _ = check.Suite(&BuildpackageSuite{})
+var _ = check.Suite(&BuildSuite{})
 
 func Test(t *testing.T) { check.TestingT(t) }
 
-func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
 	if testing.Short() {
 		c.Skip("skipping docker tests in short mode")
 	} else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
@@ -59,10 +59,10 @@ func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
 
 	cmd = exec.Command("docker", "run", "--rm",
 		"-v", tmpdir+"/pkg:/pkg",
-		"-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+		"-v", tmpdir+"/bin/arvados-package:/arvados-package:ro",
 		"-v", srctree+":/usr/local/src/arvados:ro",
 		buildimage,
-		"/arvados-dev", "buildpackage",
+		"/arvados-package", "build",
 		"-package-version", "0.9.99",
 		"-source", "/usr/local/src/arvados",
 		"-output-directory", "/pkg")
@@ -71,7 +71,7 @@ func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
 	err = cmd.Run()
 	c.Assert(err, check.IsNil)
 
-	fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+	fi, err := os.Stat(tmpdir + "/pkg/arvados-server-easy_0.9.99_amd64.deb")
 	c.Assert(err, check.IsNil)
 	c.Logf("%#v", fi)
 }
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-package/cmd.go
similarity index 92%
rename from cmd/arvados-dev/cmd.go
rename to cmd/arvados-package/cmd.go
index e0d5144e5..02bc16cea 100644
--- a/cmd/arvados-dev/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -16,7 +16,7 @@ var (
 		"-version":  cmd.Version,
 		"--version": cmd.Version,
 
-		"buildpackage": buildPackage{},
+		"build": build{},
 	})
 )
 
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-package/docker-boot.sh
similarity index 100%
rename from cmd/arvados-dev/docker-boot.sh
rename to cmd/arvados-package/docker-boot.sh
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-package/docker-build-install.sh
similarity index 92%
rename from cmd/arvados-dev/docker-build-install.sh
rename to cmd/arvados-package/docker-build-install.sh
index 966318b94..4fdcccbce 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-package/docker-build-install.sh
@@ -51,7 +51,7 @@ osbase=${opts[os]:-debian:10}
 
 mkdir -p /tmp/pkg
 
-buildimage=arvados-buildpackage-${osbase}
+buildimage=arvados-package-build-${osbase}
 if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
     (
         echo >&2 building arvados-server...
@@ -81,19 +81,19 @@ pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
 rm -v -f "${pkgfile}"
 
 (
-    echo >&2 building arvados-dev...
-    cd cmd/arvados-dev
+    echo >&2 building arvados-package...
+    cd cmd/arvados-package
     go install
 )
 echo >&2 building ${pkgfile}...
 docker run --rm \
        --tmpfs /tmp:exec,mode=01777 \
        -v /tmp/pkg:/pkg \
-       -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+       -v "${GOPATH:-${HOME}/go}"/bin/arvados-package:/arvados-package:ro \
        -v "$(pwd)":/arvados:ro \
        "${buildimage}" \
        eatmydata \
-       /arvados-dev buildpackage \
+       /arvados-package build \
        -source /arvados \
        -package-version "${version}" \
        -output-directory /pkg
@@ -119,7 +119,7 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
            -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
            --env DEBIAN_FRONTEND=noninteractive \
            "${osbase}" \
-           bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql'
+           bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
     docker commit "${installctr}" "${installimage}"
     docker rm "${installctr}"
     installctr=

commit e1756a76e8c7fed496f513909bd2c295e24cf8ee
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:36:51 2020 -0500

    16306: Comment out Workbench2 so health check succeeds.
    
    To be re-enabled when supported by install/package/boot.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/install/init.go b/lib/install/init.go
index 8debf59be..d4d9ef47e 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -149,10 +149,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
         InternalURLs:
           "http://0.0.0.0:8001/": {}
         ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
-      Workbench2:
-        InternalURLs:
-          "http://0.0.0.0:8002/": {}
-        ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+      #Workbench2:
+      #  InternalURLs:
+      #    "http://0.0.0.0:8002/": {}
+      #  ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
       Health:
         InternalURLs:
           "http://0.0.0.0:9007/": {}

commit 9c139cc4e2f519e2deb3617918a9199b76c33f10
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:35:39 2020 -0500

    16306: Fix env var ordering.
    
    Ensure our GOBIN takes precedence over inherited value.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4f5b251cb..049df2564 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -508,7 +508,8 @@ rm ${zip}
 		} {
 			fmt.Fprintf(stderr, "building %s...\n", srcdir)
 			cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
-			cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+			cmd.Env = append(cmd.Env, os.Environ()...)
+			cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin")
 			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
 			cmd.Stdout = stdout
 			cmd.Stderr = stderr

commit ae339cf6802741ea52fbe5cd8716306dcf99b021
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:35:26 2020 -0500

    16306: Remove more unneeded files from package.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index a6e0b544a..a24fa47c9 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -85,12 +85,12 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 	// Remove unneeded files. This is much faster than "fpm
 	// --exclude X" because fpm copies everything into a staging
 	// area before looking at the --exclude args.
-	cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git /var/lib/arvados/go")
+	cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	err = cmd.Run()
 	if err != nil {
-		return fmt.Errorf("rm -rf /var/www/.gem/ruby/*/cache/: %w", err)
+		return fmt.Errorf("rm -rf [...]: %w", err)
 	}
 
 	format := "deb" // TODO: rpm

commit 684b81a42cb9290bde284723603d6e4dbfd7ad35
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:35:03 2020 -0500

    16306: Fix permission issues.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 20576b6b9..7dba9b5dc 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -162,7 +162,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 	if err != nil {
 		return err
 	}
-	conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+	conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
 		return err
 	}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 2b0faefe2..4f5b251cb 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -446,7 +446,7 @@ rm ${zip}
 		} {
 			fmt.Fprintf(stderr, "building %s...\n", srcdir)
 			cmd := exec.Command("rsync",
-				"-a", "--no-owner", "--delete-after", "--delete-excluded",
+				"-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded",
 				"--exclude", "/coverage",
 				"--exclude", "/log",
 				"--exclude", "/tmp",
@@ -469,7 +469,6 @@ rm ${zip}
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
 			} {
 				cmd = exec.Command(cmdline[0], cmdline[1:]...)
-				cmd.Env = append([]string{}, os.Environ()...)
 				cmd.Dir = "/var/lib/arvados/" + dstdir
 				cmd.Stdout = stdout
 				cmd.Stderr = stderr

commit d77f9e6f8a14e14f477dfbf944914c78d0805e9f
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:33:27 2020 -0500

    16306: Circumvent Rails redirect-to-https in workbench health check.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 794adabdd..a666ef8ec 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -193,6 +193,9 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
 	}
 	req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
 
+	// Avoid workbench1's redirect-http-to-https feature
+	req.Header.Set("X-Forwarded-Proto", "https")
+
 	ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout))
 	defer cancel()
 	req = req.WithContext(ctx)

commit 0137f9e3cd33fe676a862afd727563e2602e0984
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:32:47 2020 -0500

    16306: Add health check endpoint to dispatcher.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 7614a143a..ae91a710e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -22,6 +22,7 @@ import (
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 	"github.com/julienschmidt/httprouter"
 	"github.com/prometheus/client_golang/prometheus"
@@ -164,6 +165,11 @@ func (disp *dispatcher) initialize() {
 		})
 		mux.Handler("GET", "/metrics", metricsH)
 		mux.Handler("GET", "/metrics.json", metricsH)
+		mux.Handler("GET", "/_health/:check", &health.Handler{
+			Token:  disp.Cluster.ManagementToken,
+			Prefix: "/_health/",
+			Routes: health.Routes{"ping": disp.CheckHealth},
+		})
 		disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
 	}
 }

commit 6c1fd146c0b22b374f00ef94c961df8e1e1913a7
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:32:13 2020 -0500

    16306: Use tmpfs to speed up package building.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 7b7f25ff9..966318b94 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -63,7 +63,7 @@ if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}
     docker rm "${buildctr}" || true
     docker run \
            --name "${buildctr}" \
-           -v /tmp/pkg:/pkg \
+           --tmpfs /tmp:exec,mode=01777 \
            -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
            -v "$(pwd)":/arvados:ro \
            "${osbase}" \
@@ -87,6 +87,7 @@ rm -v -f "${pkgfile}"
 )
 echo >&2 building ${pkgfile}...
 docker run --rm \
+       --tmpfs /tmp:exec,mode=01777 \
        -v /tmp/pkg:/pkg \
        -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
        -v "$(pwd)":/arvados:ro \
@@ -113,6 +114,7 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
     docker rm "${installctr}" || true
     docker run -it \
            --name "${installctr}" \
+           --tmpfs /tmp \
            -v /tmp/pkg:/pkg:ro \
            -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
            --env DEBIAN_FRONTEND=noninteractive \
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 34fe7cc6d..2b0faefe2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -224,17 +224,16 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 		logger.Print("ruby " + rubyversion + " already installed")
 	} else {
 		err = inst.runBash(`
-tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
-trap "rm -r ${tmp}" ERR
-wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd ${tmp}
+tmp="$(mktemp -d)"
+trap 'rm -r "${tmp}"' ERR EXIT
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C "${tmp}" -xzf -
+cd "${tmp}/ruby-`+rubyversion+`"
 ./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
 make -j8
 make install
 /var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
 # "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
 /var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
-rm -r ${tmp}
 `, stdout, stderr)
 		if err != nil {
 			return 1

commit 706c36ff5644bb675af36c15c5f30ef0adde9102
Author: Tom Clegg <tom at curii.com>
Date:   Mon Dec 28 10:29:16 2020 -0500

    16306: Add -eatmydata flag.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index ff514a3b6..7b7f25ff9 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -68,6 +68,7 @@ if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}
            -v "$(pwd)":/arvados:ro \
            "${osbase}" \
            /arvados-server install \
+           -eatmydata \
            -type package \
            -source /arvados \
            -package-version "${version}"
@@ -90,6 +91,7 @@ docker run --rm \
        -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
        -v "$(pwd)":/arvados:ro \
        "${buildimage}" \
+       eatmydata \
        /arvados-dev buildpackage \
        -source /arvados \
        -package-version "${version}" \
@@ -113,8 +115,9 @@ if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repositor
            --name "${installctr}" \
            -v /tmp/pkg:/pkg:ro \
            -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+           --env DEBIAN_FRONTEND=noninteractive \
            "${osbase}" \
-           bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+           bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql'
     docker commit "${installctr}" "${installimage}"
     docker rm "${installctr}"
     installctr=
@@ -125,4 +128,4 @@ docker run -it --rm \
        -v /tmp/pkg:/pkg:ro \
        -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
        "${installimage}" \
-       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
+       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive eatmydata apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
diff --git a/lib/install/deps.go b/lib/install/deps.go
index cc9595db6..34fe7cc6d 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -34,6 +34,7 @@ type installCommand struct {
 	ClusterType    string
 	SourcePath     string
 	PackageVersion string
+	EatMyData      bool
 }
 
 func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -55,6 +56,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 	flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
 	flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
 	flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
+	flags.BoolVar(&inst.EatMyData, "eatmydata", false, "use eatmydata to speed up install")
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -109,11 +111,23 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 		}
 	}
 
+	if inst.EatMyData {
+		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends", "eatmydata")
+		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return 1
+		}
+	}
+
 	pkgs := prodpkgs(osv)
 
 	if pkg {
 		pkgs = append(pkgs,
 			"dpkg-dev",
+			"eatmydata", // install it for later steps, even if we're not using it now
 			"rsync",
 		)
 	}
@@ -174,7 +188,11 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 		default:
 			pkgs = append(pkgs, "libcurl3")
 		}
-		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
+		cmd := exec.CommandContext(ctx, "apt-get")
+		if inst.EatMyData {
+			cmd = exec.CommandContext(ctx, "eatmydata", "apt-get")
+		}
+		cmd.Args = append(cmd.Args, "install", "--yes", "--no-install-recommends")
 		cmd.Args = append(cmd.Args, pkgs...)
 		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
 		cmd.Stdout = stdout
@@ -205,7 +223,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 	if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
 		logger.Print("ruby " + rubyversion + " already installed")
 	} else {
-		err = runBash(`
+		err = inst.runBash(`
 tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
 trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
@@ -228,7 +246,7 @@ rm -r ${tmp}
 		if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
 			logger.Print("go " + goversion + " already installed")
 		} else {
-			err = runBash(`
+			err = inst.runBash(`
 cd /tmp
 wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
 ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
@@ -244,7 +262,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
 		if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
 			logger.Print("phantomjs " + pjsversion + " already installed")
 		} else {
-			err = runBash(`
+			err = inst.runBash(`
 PJS=phantomjs-`+pjsversion+`-linux-x86_64
 wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
 ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
@@ -258,7 +276,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
 		if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
 			logger.Print("geckodriver " + geckoversion + " already installed")
 		} else {
-			err = runBash(`
+			err = inst.runBash(`
 GD=v`+geckoversion+`
 wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
 ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
@@ -272,7 +290,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
 		if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
 			logger.Print("nodejs " + nodejsversion + " already installed")
 		} else {
-			err = runBash(`
+			err = inst.runBash(`
 NJS=`+nodejsversion+`
 wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
 ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
@@ -286,7 +304,7 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 		if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
 			logger.Print("gradle " + gradleversion + " already installed")
 		} else {
-			err = runBash(`
+			err = inst.runBash(`
 G=`+gradleversion+`
 zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
 trap "rm ${zip}" ERR
@@ -307,7 +325,7 @@ rm ${zip}
 		if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
 			logger.Print("locale " + wantlocale + " already installed")
 		} else {
-			err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+			err = inst.runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
 			if err != nil {
 				return 1
 			}
@@ -386,7 +404,7 @@ rm ${zip}
 			// locales. Otherwise, it might need a
 			// restart, so we attempt to restart it with
 			// systemd.
-			if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+			if err = inst.runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
 				logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
 			} else if err = waitPostgreSQLReady(); err != nil {
 				return 1
@@ -585,8 +603,11 @@ func waitPostgreSQLReady() error {
 	}
 }
 
-func runBash(script string, stdout, stderr io.Writer) error {
+func (inst *installCommand) runBash(script string, stdout, stderr io.Writer) error {
 	cmd := exec.Command("bash", "-")
+	if inst.EatMyData {
+		cmd = exec.Command("eatmydata", "bash", "-")
+	}
 	cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr

commit 3aaefcb3c76ff470b475d950398d01255e87712a
Merge: c59af50bc 2c8874c06
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 16:48:37 2020 -0500

    16306: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --cc doc/install/index.html.textile.liquid
index 81d7b2159,1b27ca6ed..f16ae2dad
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@@ -20,11 -20,12 +20,13 @@@ Arvados components can be installed an
  <div class="offset1">
  table(table table-bordered table-condensed).
  |||\5=. Appropriate for|
- ||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
+ ||_. Setup difficulty|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
  |"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
+ |"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
+ |"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
  |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
 +|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
- |"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
+ |"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
  |"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
  </div>
  
diff --cc lib/install/deps.go
index da45b393b,342ef03a7..cc9595db6
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@@ -124,8 -101,10 +124,9 @@@ func (inst *installCommand) RunCommand(
  			"bison",
  			"bsdmainutils",
  			"build-essential",
 -			"ca-certificates",
  			"cadaver",
- 			"cython",
+ 			"curl",
+ 			"cython3",
  			"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
  			"default-jdk-headless",
  			"default-jre-headless",
@@@ -136,9 -120,13 +137,9 @@@
  			"libfuse-dev",
  			"libgnutls28-dev",
  			"libjson-perl",
 -			"libjson-perl",
  			"libpam-dev",
  			"libpcre3-dev",
- 			"libpython2.7-dev",
+ 			"libpq-dev",
  			"libreadline-dev",
  			"libssl-dev",
  			"libwww-perl",
@@@ -154,22 -142,28 +155,27 @@@
  			"postgresql",
  			"postgresql-contrib",
  			"python3-dev",
- 			"python-epydoc",
+ 			"python3-venv",
+ 			"python3-virtualenv",
  			"r-base",
  			"r-cran-testthat",
+ 			"r-cran-devtools",
+ 			"r-cran-knitr",
+ 			"r-cran-markdown",
+ 			"r-cran-roxygen2",
+ 			"r-cran-xml",
  			"sudo",
- 			"virtualenv",
  			"wget",
  			"xvfb",
 -			"zlib1g-dev",
 -		}
 +		)
  		switch {
  		case osv.Debian && osv.Major >= 10:
 -			debs = append(debs, "libcurl4")
 +			pkgs = append(pkgs, "libcurl4")
  		default:
 -			debs = append(debs, "libcurl3")
 +			pkgs = append(pkgs, "libcurl3")
  		}
  		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
 -		cmd.Args = append(cmd.Args, debs...)
 +		cmd.Args = append(cmd.Args, pkgs...)
  		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
  		cmd.Stdout = stdout
  		cmd.Stderr = stderr

commit c59af50bc2f7a366cd12a8dd6fc7d7e3b1c32480
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 16:15:21 2020 -0500

    16306: Move nginx temp dirs into a subdir.
    
    When running as root, Nginx changes ownership of these temp dirs to
    "nobody".  We also use the same TMPDIR for standalone passenger
    processes, which run as www-data, so we need Nginx to leave it alone.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index cdca68dd6..a4336049f 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -11,11 +11,11 @@ http {
     '[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
     '"$http_referer" "$http_user_agent"';
   access_log "{{ACCESSLOG}}" customlog;
-  client_body_temp_path "{{TMPDIR}}";
-  proxy_temp_path "{{TMPDIR}}";
-  fastcgi_temp_path "{{TMPDIR}}";
-  uwsgi_temp_path "{{TMPDIR}}";
-  scgi_temp_path "{{TMPDIR}}";
+  client_body_temp_path "{{TMPDIR}}/nginx";
+  proxy_temp_path "{{TMPDIR}}/nginx";
+  fastcgi_temp_path "{{TMPDIR}}/nginx";
+  uwsgi_temp_path "{{TMPDIR}}/nginx";
+  scgi_temp_path "{{TMPDIR}}/nginx";
   upstream controller {
     server {{LISTENHOST}}:{{CONTROLLERPORT}};
   }

commit f1bc1872a9ead4f3cb1c23c79507870c1b0f7b53
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 16:14:55 2020 -0500

    16306: Remove some unneeded deps from prod/pkg.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index 38f9e888d..da45b393b 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -120,6 +120,8 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 
 	if dev || test || pkg {
 		pkgs = append(pkgs,
+			"automake",
+			"bison",
 			"bsdmainutils",
 			"build-essential",
 			"cadaver",
@@ -144,6 +146,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 			"libxslt1-dev",
 			"linkchecker",
 			"lsof",
+			"make",
 			"net-tools",
 			"pandoc",
 			"perl-modules",
@@ -412,14 +415,20 @@ rm ${zip}
 		}
 	}
 
-	if pkg {
+	if prod || pkg {
 		// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
 		for dstdir, srcdir := range map[string]string{
 			"railsapi":   "services/api",
 			"workbench1": "apps/workbench",
 		} {
 			fmt.Fprintf(stderr, "building %s...\n", srcdir)
-			cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+			cmd := exec.Command("rsync",
+				"-a", "--no-owner", "--delete-after", "--delete-excluded",
+				"--exclude", "/coverage",
+				"--exclude", "/log",
+				"--exclude", "/tmp",
+				"--exclude", "/vendor",
+				"./", "/var/lib/arvados/"+dstdir+"/")
 			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
 			cmd.Stdout = stdout
 			cmd.Stderr = stderr
@@ -580,8 +589,6 @@ func runBash(script string, stdout, stderr io.Writer) error {
 
 func prodpkgs(osv osversion) []string {
 	pkgs := []string{
-		"automake",
-		"bison",
 		"ca-certificates",
 		"curl",
 		"fuse",
@@ -591,7 +598,6 @@ func prodpkgs(osv osversion) []string {
 		"haveged",
 		"libcurl3-gnutls",
 		"libxslt1.1",
-		"make",
 		"nginx",
 		"python",
 		"sudo",

commit 3feaf72ad2a9a7b740bb3976f351f5020129eda4
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 16:13:49 2020 -0500

    16306: Monkeypatch fpm bug.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 76228324e..a6e0b544a 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -9,6 +9,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"os"
 	"os/exec"
 
 	"git.arvados.org/arvados.git/lib/install"
@@ -70,6 +71,17 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		return fmt.Errorf("gem install fpm: %w", err)
 	}
 
+	if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+		// Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+		cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("monkeypatch fpm: %w", err)
+		}
+	}
+
 	// Remove unneeded files. This is much faster than "fpm
 	// --exclude X" because fpm copies everything into a staging
 	// area before looking at the --exclude args.

commit 209403e32bdd6164404240af2959de86a3c46165
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 16:13:45 2020 -0500

    16306: Comment why rm instead of exclude. Delete go tools, too.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 93482e1c7..76228324e 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -69,8 +69,11 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 	if err != nil {
 		return fmt.Errorf("gem install fpm: %w", err)
 	}
-	// Shrink our package, remove unneeded stuff
-	cmd = exec.Command("bash", "-c", "rm -rf /var/www/.gem/ruby/*/cache/ /var/www/.gem/ruby/*/bundler/gems/arvados-*/.git /var/www/.gem/ruby/*/bundler/gems/nulldb-*/.git /var/www/.gem/ruby/*/bundler/gems/themes_for_rails-*/.git")
+
+	// Remove unneeded files. This is much faster than "fpm
+	// --exclude X" because fpm copies everything into a staging
+	// area before looking at the --exclude args.
+	cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git /var/lib/arvados/go")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	err = cmd.Run()
@@ -93,12 +96,9 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		cmd.Args = append(cmd.Args, "--depends", pkg)
 	}
 	cmd.Args = append(cmd.Args,
+		"--verbose",
 		"--deb-use-file-permissions",
 		"--rpm-use-file-permissions",
-		"--exclude", "var/lib/arvados/go",
-		"--exclude", "tmp",
-		"--exclude", "log",
-		"--exclude", "coverage",
 		"/var/lib/arvados",
 		"/var/www/.gem",
 		"/var/www/.passenger",

commit 54849feb0cf30bd3d78f14f1a41e6bc464fb80d6
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 10:19:43 2020 -0500

    16306: Fix Nginx not starting when running as non-root.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 07ff1fc1d..d49fabe56 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"regexp"
 
@@ -100,13 +101,26 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 			}
 		}
 	}
+
+	args := []string{
+		"-g", "error_log stderr info;",
+		"-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
+		"-c", conffile,
+	}
+	// Nginx ignores "user www-data;" when running as a non-root
+	// user... except that it causes it to ignore our other -g
+	// options. So we still have to decide for ourselves whether
+	// it's needed.
+	if u, err := user.Current(); err != nil {
+		return fmt.Errorf("user.Current(): %w", err)
+	} else if u.Uid == "0" {
+		args = append([]string{"-g", "user www-data;"}, args...)
+	}
+
 	super.waitShutdown.Add(1)
 	go func() {
 		defer super.waitShutdown.Done()
-		fail(super.RunProgram(ctx, ".", nil, nil, nginx,
-			"-g", "error_log stderr info;",
-			"-g", "user www-data; pid "+filepath.Join(super.wwwtempdir, "nginx.pid")+";",
-			"-c", conffile))
+		fail(super.RunProgram(ctx, ".", nil, nil, nginx, args...))
 	}()
 	// Choose one of the ports where Nginx should listen, and wait
 	// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 199a93a9d..fc23eb913 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -42,7 +42,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 
 	iamroot := false
 	if u, err := user.Current(); err != nil {
-		return fmt.Errorf("user.Current(): %s", err)
+		return fmt.Errorf("user.Current(): %w", err)
 	} else if u.Uid == "0" {
 		iamroot = true
 	}

commit d5c034b4e90d3fba74e5b0a879d8af424023d7b1
Author: Tom Clegg <tom at curii.com>
Date:   Tue Dec 22 10:03:30 2020 -0500

    16360: Don't bypass autofillConfig in production mode.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 417f09d23..138c802e1 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -143,7 +143,6 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		super.tempdir = "/var/lib/arvados/tmp"
 		super.wwwtempdir = "/var/lib/arvados/wwwtmp"
 		super.bindir = "/var/lib/arvados/bin"
-		super.configfile = "/etc/arvados/config.yml"
 	} else {
 		super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
 		if err != nil {
@@ -155,28 +154,28 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		if err := os.Mkdir(super.bindir, 0755); err != nil {
 			return err
 		}
+	}
 
-		// Fill in any missing config keys, and write the resulting
-		// config in the temp dir for child services to use.
-		err = super.autofillConfig(cfg)
-		if err != nil {
-			return err
-		}
-		conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
-		if err != nil {
-			return err
-		}
-		defer conffile.Close()
-		err = json.NewEncoder(conffile).Encode(cfg)
-		if err != nil {
-			return err
-		}
-		err = conffile.Close()
-		if err != nil {
-			return err
-		}
-		super.configfile = conffile.Name()
+	// Fill in any missing config keys, and write the resulting
+	// config in the temp dir for child services to use.
+	err = super.autofillConfig(cfg)
+	if err != nil {
+		return err
+	}
+	conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer conffile.Close()
+	err = json.NewEncoder(conffile).Encode(cfg)
+	if err != nil {
+		return err
 	}
+	err = conffile.Close()
+	if err != nil {
+		return err
+	}
+	super.configfile = conffile.Name()
 
 	super.environ = os.Environ()
 	super.cleanEnv([]string{"ARVADOS_"})
@@ -655,26 +654,26 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 			}
 		}
 	}
-	if cluster.SystemRootToken == "" {
-		cluster.SystemRootToken = randomHexString(64)
-	}
-	if cluster.ManagementToken == "" {
-		cluster.ManagementToken = randomHexString(64)
-	}
-	if cluster.API.RailsSessionSecretToken == "" {
-		cluster.API.RailsSessionSecretToken = randomHexString(64)
-	}
-	if cluster.Collections.BlobSigningKey == "" {
-		cluster.Collections.BlobSigningKey = randomHexString(64)
-	}
-	if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
-		buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
-		if err != nil {
-			return err
-		}
-		cluster.Containers.DispatchPrivateKey = string(buf)
-	}
 	if super.ClusterType != "production" {
+		if cluster.SystemRootToken == "" {
+			cluster.SystemRootToken = randomHexString(64)
+		}
+		if cluster.ManagementToken == "" {
+			cluster.ManagementToken = randomHexString(64)
+		}
+		if cluster.API.RailsSessionSecretToken == "" {
+			cluster.API.RailsSessionSecretToken = randomHexString(64)
+		}
+		if cluster.Collections.BlobSigningKey == "" {
+			cluster.Collections.BlobSigningKey = randomHexString(64)
+		}
+		if cluster.Containers.DispatchPrivateKey == "" {
+			buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+			if err != nil {
+				return err
+			}
+			cluster.Containers.DispatchPrivateKey = string(buf)
+		}
 		cluster.TLS.Insecure = true
 	}
 	if super.ClusterType == "test" {

commit 20649f2d57f8e5c8bddfb672c164b7a96f75efab
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Nov 10 17:15:08 2020 -0500

    16306: shrink package size by removing unused cache and .git directories
           under /var/www/.gem/
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index c562c5685..93482e1c7 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -69,6 +69,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 	if err != nil {
 		return fmt.Errorf("gem install fpm: %w", err)
 	}
+	// Shrink our package, remove unneeded stuff
+	cmd = exec.Command("bash", "-c", "rm -rf /var/www/.gem/ruby/*/cache/ /var/www/.gem/ruby/*/bundler/gems/arvados-*/.git /var/www/.gem/ruby/*/bundler/gems/nulldb-*/.git /var/www/.gem/ruby/*/bundler/gems/themes_for_rails-*/.git")
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("rm -rf /var/www/.gem/ruby/*/cache/: %w", err)
+	}
 
 	format := "deb" // TODO: rpm
 
@@ -88,6 +96,9 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		"--deb-use-file-permissions",
 		"--rpm-use-file-permissions",
 		"--exclude", "var/lib/arvados/go",
+		"--exclude", "tmp",
+		"--exclude", "log",
+		"--exclude", "coverage",
 		"/var/lib/arvados",
 		"/var/www/.gem",
 		"/var/www/.passenger",

commit 821c7273388a56ad5ae9b89a0cf2f351388e8b61
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Nov 10 10:46:18 2020 -0500

    16306: switch to shared ruby libraries.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3ff23d6e9..38f9e888d 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -201,7 +201,7 @@ tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
 trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
 cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
 make -j8
 make install
 /var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc

commit da7a2e35f1c53a64a360c66db0c260a9ab2190f2
Author: Ward Vandewege <ward at curii.com>
Date:   Mon Nov 9 15:57:47 2020 -0500

    16306: in cmd/arvados-dev/docker-build-install.sh, make sure nginx is
           configured to listen on all IPs.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 78b08d7dc..ff514a3b6 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -125,4 +125,4 @@ docker run -it --rm \
        -v /tmp/pkg:/pkg:ro \
        -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
        "${installimage}" \
-       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
+       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'

commit 0b6013ac5520b7508474705d654f1384994d10e5
Author: Ward Vandewege <ward at curii.com>
Date:   Mon Nov 9 13:23:35 2020 -0500

    16306: Make sure that the non-passenger nginx process runs as the
    www-data user, not the default 'nobody' user. Make sure that process
    uses the wwwtmp directory as its temp directory. Make sure that
    arvados-boot in production mode uses /etc/arvados/config.yml.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index b11d9fd49..07ff1fc1d 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -37,7 +37,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 		"SSLKEY":     filepath.Join(super.tempdir, "server.key"),
 		"ACCESSLOG":  filepath.Join(super.tempdir, "nginx_access.log"),
 		"ERRORLOG":   filepath.Join(super.tempdir, "nginx_error.log"),
-		"TMPDIR":     super.tempdir,
+		"TMPDIR":     super.wwwtempdir,
 	}
 	for _, cmpt := range []struct {
 		varname string
@@ -105,7 +105,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 		defer super.waitShutdown.Done()
 		fail(super.RunProgram(ctx, ".", nil, nil, nginx,
 			"-g", "error_log stderr info;",
-			"-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
+			"-g", "user www-data; pid "+filepath.Join(super.wwwtempdir, "nginx.pid")+";",
 			"-c", conffile))
 	}()
 	// Choose one of the ports where Nginx should listen, and wait
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 51f5c8bfd..417f09d23 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -143,6 +143,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		super.tempdir = "/var/lib/arvados/tmp"
 		super.wwwtempdir = "/var/lib/arvados/wwwtmp"
 		super.bindir = "/var/lib/arvados/bin"
+		super.configfile = "/etc/arvados/config.yml"
 	} else {
 		super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
 		if err != nil {
@@ -154,28 +155,28 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		if err := os.Mkdir(super.bindir, 0755); err != nil {
 			return err
 		}
-	}
 
-	// Fill in any missing config keys, and write the resulting
-	// config in the temp dir for child services to use.
-	err = super.autofillConfig(cfg)
-	if err != nil {
-		return err
-	}
-	conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	defer conffile.Close()
-	err = json.NewEncoder(conffile).Encode(cfg)
-	if err != nil {
-		return err
-	}
-	err = conffile.Close()
-	if err != nil {
-		return err
+		// Fill in any missing config keys, and write the resulting
+		// config in the temp dir for child services to use.
+		err = super.autofillConfig(cfg)
+		if err != nil {
+			return err
+		}
+		conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		defer conffile.Close()
+		err = json.NewEncoder(conffile).Encode(cfg)
+		if err != nil {
+			return err
+		}
+		err = conffile.Close()
+		if err != nil {
+			return err
+		}
+		super.configfile = conffile.Name()
 	}
-	super.configfile = conffile.Name()
 
 	super.environ = os.Environ()
 	super.cleanEnv([]string{"ARVADOS_"})

commit ab5199f71c7eaf8bf8fe2b4477353cf432faf1a7
Author: Ward Vandewege <ward at curii.com>
Date:   Mon Nov 9 11:07:29 2020 -0500

    16306: really exclude /var/lib/arvados/go from the package, cf.
           https://github.com/jordansissel/fpm/issues/900
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 125ac36b1..c562c5685 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -87,7 +87,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 	cmd.Args = append(cmd.Args,
 		"--deb-use-file-permissions",
 		"--rpm-use-file-permissions",
-		"--exclude", "/var/lib/arvados/go",
+		"--exclude", "var/lib/arvados/go",
 		"/var/lib/arvados",
 		"/var/www/.gem",
 		"/var/www/.passenger",

commit f8d13408e99839f52260f889a5089126761eecb1
Author: Ward Vandewege <ward at curii.com>
Date:   Mon Nov 9 11:07:07 2020 -0500

    16306: shut up bundle warnings about /var/www
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 00713550d..125ac36b1 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -91,6 +91,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		"/var/lib/arvados",
 		"/var/www/.gem",
 		"/var/www/.passenger",
+		"/var/www/.bundle",
 	)
 	fmt.Fprintf(stderr, "... %s\n", cmd.Args)
 	cmd.Dir = bldr.OutputDir
diff --git a/lib/install/deps.go b/lib/install/deps.go
index b39515cb1..3ff23d6e9 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -428,9 +428,9 @@ rm ${zip}
 				return 1
 			}
 			for _, cmdline := range [][]string{
-				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
 				{"touch", "log/production.log"},
-				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},

commit e8d1a643cdbc3a5f4c0e5c745da58d9f7e1248d8
Merge: 426103b2d 5b0505685
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Aug 21 14:51:32 2020 -0400

    16306: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>


commit 426103b2d2f071ab0d57b3f9aaea58a3f8455c4f
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Mon Aug 17 09:14:52 2020 -0400

    16306: Add --os argument to package build scripts.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 7eca33e79..39a8a56c8 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -19,6 +19,11 @@ set -e -o pipefail
 declare -A opts=()
 while [[ $# -gt 0 ]]; do
     case "$1" in
+        --os)
+            shift
+            opts[os]="$1"
+            shift
+            ;;
         --shell)
             shift
             opts[shell]=1
@@ -54,7 +59,7 @@ for inject in "$@"; do
     esac
 done
 
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
 installimage=arvados-installpackage-${osbase}
 command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
 if [[ "${opts[shell]}" ]]; then
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 63b5b5c2d..78b08d7dc 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -25,6 +25,10 @@ while [[ $# -gt 0 ]]; do
         --force-installimage)
             opts[force-installimage]=1
             ;;
+        --os)
+            opts[os]="$1"
+            shift
+            ;;
         *)
             echo >&2 "invalid argument '$arg'"
             exit 2
@@ -43,7 +47,7 @@ cleanup() {
 trap cleanup ERR EXIT
 
 version=$(git describe --tag --dirty)
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
 
 mkdir -p /tmp/pkg
 

commit bb1054c0e6a9bed14e593004f380111b2e948047
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Mon Aug 17 09:13:45 2020 -0400

    16306: Update gem command line option spelling.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index aee955ea2..00713550d 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -62,7 +62,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 			return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
 		}
 	}
-	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	err = cmd.Run()
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 481300b45..03464aaf7 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -56,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 	}
 	for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
 		if !strings.Contains(buf.String(), "("+version+")") {
-			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
 			if err != nil {
 				return err
 			}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 7b22f45fd..b39515cb1 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -431,7 +431,7 @@ rm ${zip}
 				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
 				{"touch", "log/production.log"},
 				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
-				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},

commit f2da5f54256377e604c76b74805381b2acf38f50
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Aug 7 15:00:22 2020 -0400

    16306: Add "ping" health check to keep-balance.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 65bd8d4cf..8b4ee84c7 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -9,13 +9,13 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/service"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/health"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
@@ -83,7 +83,6 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
 			}
 
 			srv := &Server{
-				Handler:    http.NotFoundHandler(),
 				Cluster:    cluster,
 				ArvClient:  ac,
 				RunOptions: options,
@@ -91,6 +90,11 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
 				Logger:     options.Logger,
 				Dumper:     options.Dumper,
 			}
+			srv.Handler = &health.Handler{
+				Token:  cluster.ManagementToken,
+				Prefix: "/_health/",
+				Routes: health.Routes{"ping": srv.CheckHealth},
+			}
 
 			go srv.run()
 			return srv

commit f81d411d2becc23fe640366620d9568149d89bd6
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Aug 7 10:51:39 2020 -0400

    16306: Set Workbench1 secret key.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/install/init.go b/lib/install/init.go
index ca8d4da92..8debf59be 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -178,6 +178,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
         DriverParameters:
           Root: /var/lib/arvados/keep
         Replication: 2
+    Workbench:
+      SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
 `)
 	if err != nil {
 		return 1

commit 24987125b33655306c048ab64f0ffdfb2f6a6c5b
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 15:18:40 2020 -0400

    16306: Fix nginx port probe.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index a7c09a722..b11d9fd49 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -107,5 +108,11 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 			"-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
 			"-c", conffile))
 	}()
-	return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+	// Choose one of the ports where Nginx should listen, and wait
+	// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+	testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+	if testurl.Port() == "" {
+		testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+	}
+	return waitForConnect(ctx, testurl.Host)
 }

commit 1c2f9259deeedaad103b80fda7f440b8e60caa3f
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 14:27:51 2020 -0400

    16306: Include wwwtmp dir in package.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index c5596d104..7b22f45fd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -178,7 +178,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 
 	os.Mkdir("/var/lib/arvados", 0755)
 	os.Mkdir("/var/lib/arvados/tmp", 0700)
-	if prod {
+	if prod || pkg {
 		os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
 		u, er := user.Lookup("www-data")
 		if er != nil {

commit 75d050ab135619fcefecbfa32aaad4dab33e7588
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 11:30:08 2020 -0400

    16306: Fixup nginx in arvados-boot production mode.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 0b9874295..7eca33e79 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -40,11 +40,18 @@ tmpdir=$(mktemp -d)
 version=$(git describe --tag --dirty)
 
 declare -a volargs=()
-for srcdir in "$@"; do
-    echo >&2 "building $srcdir..."
-    (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
-    cmd="$(basename "$srcdir")"
-    volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+for inject in "$@"; do
+    case "$inject" in
+        nginx.conf)
+            volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
+            ;;
+        *)
+            echo >&2 "building $inject..."
+            (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+            cmd="$(basename "$inject")"
+            volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+            ;;
+    esac
 done
 
 osbase=debian:10
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index c1da7d18d..a7c09a722 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -69,7 +69,13 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 		}
 		vars[cmpt.varname+"SSLPORT"] = port
 	}
-	tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+	var conftemplate string
+	if super.ClusterType == "production" {
+		conftemplate = "/var/lib/arvados/share/nginx.conf"
+	} else {
+		conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+	}
+	tmpl, err := ioutil.ReadFile(conftemplate)
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 5f92a6569..51f5c8bfd 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -252,7 +252,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 	}
 	if super.ClusterType != "test" {
 		tasks = append(tasks,
-			runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
+			runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
 			runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
 		)
 	}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3f19aa1a8..c5596d104 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -486,6 +486,15 @@ rm ${zip}
 				return 1
 			}
 		}
+
+		// Copy assets from source tree to /var/lib/arvados/share
+		cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return 1
+		}
 	}
 
 	return 0
diff --git a/lib/install/init.go b/lib/install/init.go
index 6d4f19757..ca8d4da92 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -170,6 +170,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
         user: arvados
         password: {{printf "%q" .PostgreSQLPassword}}
     SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+    TLS:
+      Insecure: true
     Volumes:
       {{.ClusterID}}-nyw5e-000000000000000:
         Driver: Directory
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 85b4f5b37..cdca68dd6 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -16,12 +16,28 @@ http {
   fastcgi_temp_path "{{TMPDIR}}";
   uwsgi_temp_path "{{TMPDIR}}";
   scgi_temp_path "{{TMPDIR}}";
+  upstream controller {
+    server {{LISTENHOST}}:{{CONTROLLERPORT}};
+  }
+  server {
+    listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+    server_name controller ~.*;
+    ssl_certificate "{{SSLCERT}}";
+    ssl_certificate_key "{{SSLKEY}}";
+    location  / {
+      proxy_pass http://controller;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto https;
+      proxy_redirect off;
+    }
+  }
   upstream arv-git-http {
     server {{LISTENHOST}}:{{GITPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
-    server_name arv-git-http;
+    listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+    server_name arv-git-http git.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -36,8 +52,8 @@ http {
     server {{LISTENHOST}}:{{KEEPPROXYPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
-    server_name keepproxy;
+    listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+    server_name keepproxy keep.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -55,8 +71,8 @@ http {
     server {{LISTENHOST}}:{{KEEPWEBPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
-    server_name keep-web;
+    listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+    server_name keep-web collections.* ~\.collections\.;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -75,8 +91,8 @@ http {
     server {{LISTENHOST}}:{{HEALTHPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
-    server_name health;
+    listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+    server_name health health.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -91,8 +107,8 @@ http {
     }
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
-    server_name keep-web-dl ~.*;
+    listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+    server_name keep-web-dl download.* ~.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -111,8 +127,8 @@ http {
     server {{LISTENHOST}}:{{WSPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
-    server_name websocket;
+    listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+    server_name websocket ws.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -129,8 +145,8 @@ http {
     server {{LISTENHOST}}:{{WORKBENCH1PORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
-    server_name workbench1;
+    listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+    server_name workbench1 workbench.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -141,20 +157,4 @@ http {
       proxy_redirect off;
     }
   }
-  upstream controller {
-    server {{LISTENHOST}}:{{CONTROLLERPORT}};
-  }
-  server {
-    listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
-    server_name controller;
-    ssl_certificate "{{SSLCERT}}";
-    ssl_certificate_key "{{SSLKEY}}";
-    location  / {
-      proxy_pass http://controller;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-      proxy_redirect off;
-    }
-  }
 }

commit 8891660b68c10dd9d019f1caf8b7bc0b1eb493d8
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 11:28:11 2020 -0400

    16306: Option to start shell in docker-boot.sh.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index e8703e41f..0b9874295 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -16,6 +16,19 @@
 
 set -e -o pipefail
 
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --shell)
+            shift
+            opts[shell]=1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
 cleanup() {
     if [[ -n "${tmpdir}" ]]; then
         rm -rf "${tmpdir}"
@@ -36,7 +49,11 @@ done
 
 osbase=debian:10
 installimage=arvados-installpackage-${osbase}
+command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
+if [[ "${opts[shell]}" ]]; then
+    command="bash -login"
+fi
 docker run -it --rm \
        "${volargs[@]}" \
        "${installimage}" \
-       bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
+       bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 3c6e177cf..63b5b5c2d 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -27,7 +27,8 @@ while [[ $# -gt 0 ]]; do
             ;;
         *)
             echo >&2 "invalid argument '$arg'"
-            exit 1
+            exit 2
+            ;;
     esac
 done
 

commit 398ddf58f05416e0ce62ac8faef018a44bf8cc4c
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 11:26:57 2020 -0400

    16306: Add hostname as subjectAltName.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index f0797c2ac..8f6339e63 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
 	"path/filepath"
 )
 
@@ -27,9 +28,14 @@ func (createCertificates) String() string {
 func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
 	var san string
 	if net.ParseIP(super.ListenHost) != nil {
-		san = fmt.Sprintf("IP:%s", super.ListenHost)
+		san += fmt.Sprintf(",IP:%s", super.ListenHost)
 	} else {
-		san = fmt.Sprintf("DNS:%s", super.ListenHost)
+		san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+	}
+	if hostname, err := os.Hostname(); err != nil {
+		return fmt.Errorf("hostname: %w", err)
+	} else {
+		san += ",DNS:" + hostname
 	}
 
 	// Generate root key
@@ -52,7 +58,7 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+	err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
 	if err != nil {
 		return err
 	}

commit 7ac1ea4e9bc1b399368a95649647a55c8093d649
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 30 11:25:35 2020 -0400

    16306: Show logprefix like 'passenger at railsapi'
    
    for commands like 'sudo ... bundle exec passenger ...'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 2d7697233..5f92a6569 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -483,17 +483,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
 	super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 
 	logprefix := prog
-	if logprefix == "setuidgid" && len(args) >= 3 {
-		logprefix = args[2]
-	}
-	logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
-	if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
-		logprefix = args[1]
-	} else if logprefix == "arvados-server" && len(args) > 1 {
-		logprefix = args[0]
-	}
-	if !strings.HasPrefix(dir, "/") {
-		logprefix = dir + ": " + logprefix
+	{
+		if logprefix == "setuidgid" && len(args) >= 3 {
+			logprefix = args[2]
+		}
+		innerargs := args
+		if logprefix == "sudo" {
+			for i := 0; i < len(args); i++ {
+				if args[i] == "-u" {
+					i++
+				} else if args[i] == "-E" || strings.Contains(args[i], "=") {
+				} else {
+					logprefix = args[i]
+					innerargs = args[i+1:]
+					break
+				}
+			}
+		}
+		logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+		logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+		if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+			_, dirbase := filepath.Split(dir)
+			logprefix = innerargs[1] + "@" + dirbase
+		} else if logprefix == "arvados-server" && len(args) > 1 {
+			logprefix = args[0]
+		}
+		if !strings.HasPrefix(dir, "/") {
+			logprefix = dir + ": " + logprefix
+		}
 	}
 
 	cmd := exec.Command(super.lookPath(prog), args...)

commit 60801a7c485c50e3c2f518f9b3ad6f86c8783fca
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jul 23 17:31:39 2020 -0400

    16306: Packaging dev cycle, type=production support in lib/boot.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 4b395c4ba..aee955ea2 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -85,12 +85,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
 		cmd.Args = append(cmd.Args, "--depends", pkg)
 	}
 	cmd.Args = append(cmd.Args,
+		"--deb-use-file-permissions",
+		"--rpm-use-file-permissions",
 		"--exclude", "/var/lib/arvados/go",
 		"/var/lib/arvados",
 		"/var/www/.gem",
 		"/var/www/.passenger",
 	)
-	fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+	fmt.Fprintf(stderr, "... %s\n", cmd.Args)
 	cmd.Dir = bldr.OutputDir
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
new file mode 100755
index 000000000..e8703e41f
--- /dev/null
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Bring up a docker container with some locally-built commands (e.g.,
+# cmd/arvados-server) replacing the ones that came with
+# arvados-server-easy when the arvados-installpackage-* image was
+# built.
+#
+# Assumes docker-build-install.sh has already succeeded.
+#
+# Example:
+#
+#    docker-boot.sh cmd/arvados-server services/keep-balance
+
+set -e -o pipefail
+
+cleanup() {
+    if [[ -n "${tmpdir}" ]]; then
+        rm -rf "${tmpdir}"
+    fi
+}
+trap cleanup ERR EXIT
+
+tmpdir=$(mktemp -d)
+version=$(git describe --tag --dirty)
+
+declare -a volargs=()
+for srcdir in "$@"; do
+    echo >&2 "building $srcdir..."
+    (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+    cmd="$(basename "$srcdir")"
+    volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+done
+
+osbase=debian:10
+installimage=arvados-installpackage-${osbase}
+docker run -it --rm \
+       "${volargs[@]}" \
+       "${installimage}" \
+       bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
new file mode 100755
index 000000000..3c6e177cf
--- /dev/null
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Build an arvados-server-easy package, then install and run it on a
+# base OS image.
+#
+# Examples:
+#
+#    docker-build-install.sh --force-buildimage --force-installimage     # always build fresh docker images
+#
+#    docker-build-install.sh                                             # reuse cached docker images if possible
+
+set -e -o pipefail
+
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+    arg="$1"
+    shift
+    case "$arg" in
+        --force-buildimage)
+            opts[force-buildimage]=1
+            ;;
+        --force-installimage)
+            opts[force-installimage]=1
+            ;;
+        *)
+            echo >&2 "invalid argument '$arg'"
+            exit 1
+    esac
+done
+
+cleanup() {
+    if [[ -n "${buildctr}" ]]; then
+        docker rm "${buildctr}" || true
+    fi
+    if [[ -n "${installctr}" ]]; then
+        docker rm "${installctr}" || true
+    fi
+}
+trap cleanup ERR EXIT
+
+version=$(git describe --tag --dirty)
+osbase=debian:10
+
+mkdir -p /tmp/pkg
+
+buildimage=arvados-buildpackage-${osbase}
+if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
+    (
+        echo >&2 building arvados-server...
+        cd cmd/arvados-server
+        go install
+    )
+    echo >&2 building ${buildimage}...
+    buildctr=${buildimage/:/-}
+    docker rm "${buildctr}" || true
+    docker run \
+           --name "${buildctr}" \
+           -v /tmp/pkg:/pkg \
+           -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
+           -v "$(pwd)":/arvados:ro \
+           "${osbase}" \
+           /arvados-server install \
+           -type package \
+           -source /arvados \
+           -package-version "${version}"
+    docker commit "${buildctr}" "${buildimage}"
+    docker rm "${buildctr}"
+    buildctr=
+fi
+
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+rm -v -f "${pkgfile}"
+
+(
+    echo >&2 building arvados-dev...
+    cd cmd/arvados-dev
+    go install
+)
+echo >&2 building ${pkgfile}...
+docker run --rm \
+       -v /tmp/pkg:/pkg \
+       -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+       -v "$(pwd)":/arvados:ro \
+       "${buildimage}" \
+       /arvados-dev buildpackage \
+       -source /arvados \
+       -package-version "${version}" \
+       -output-directory /pkg
+
+ls -l ${pkgfile}
+(
+    echo >&2 dpkg-scanpackages...
+    cd /tmp/pkg
+    dpkg-scanpackages . | gzip > Packages.gz
+)
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+installimage="arvados-installpackage-${osbase}"
+if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+    echo >&2 building ${installimage}...
+    installctr=${installimage/:/-}
+    docker rm "${installctr}" || true
+    docker run -it \
+           --name "${installctr}" \
+           -v /tmp/pkg:/pkg:ro \
+           -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+           "${osbase}" \
+           bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+    docker commit "${installctr}" "${installimage}"
+    docker rm "${installctr}"
+    installctr=
+fi
+
+echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
+docker run -it --rm \
+       -v /tmp/pkg:/pkg:ro \
+       -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+       "${installimage}" \
+       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
deleted file mode 100755
index 072dfcf9c..000000000
--- a/cmd/arvados-dev/example.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -e -o pipefail
-
-version="${PACKAGE_VERSION:-0.9.99}"
-
-# mkdir -p /tmp/pkg
-# (
-#     cd cmd/arvados-dev
-#     go install
-# )
-# docker run --rm \
-#        -v /tmp/pkg:/pkg \
-#        -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
-#        -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
-#        /arvados-dev buildpackage \
-#        -source /arvados \
-#        -package-version "${version}" \
-#        -output-directory /pkg
-pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
-# ls -l ${pkgfile}
-# (
-#     cd /tmp/pkg
-#     dpkg-scanpackages . | gzip > Packages.gz
-# )
-sourcesfile=/tmp/sources.conf.d-arvados
-echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-docker run -it --rm \
-       -v /tmp/pkg:/pkg:ro \
-       -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
-       ${INSTALLIMAGE:-debian:10} \
-       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index ff99de75c..d0aa9da94 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -34,6 +34,7 @@ var (
 		"crunch-run":         crunchrun.Command,
 		"dispatch-cloud":     dispatchcloud.Command,
 		"install":            install.Command,
+		"init":               install.InitCommand,
 		"recover-collection": recovercollection.Command,
 		"ws":                 ws.Command,
 	})
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 0f105d6b6..c1da7d18d 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -53,7 +53,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 	} {
 		port, err := internalPort(cmpt.svc)
 		if err != nil {
-			return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+			return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
 		}
 		if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
 			return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
@@ -62,7 +62,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 
 		port, err = externalPort(cmpt.svc)
 		if err != nil {
-			return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+			return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
 		}
 		if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
 			return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index f18300c4c..481300b45 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -37,6 +37,10 @@ func (runner installPassenger) String() string {
 }
 
 func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+	if super.ClusterType == "production" {
+		// passenger has already been installed via package
+		return nil
+	}
 	err := super.wait(ctx, runner.depends...)
 	if err != nil {
 		return err
@@ -52,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 	}
 	for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
 		if !strings.Contains(buf.String(), "("+version+")") {
-			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
 			if err != nil {
 				return err
 			}
@@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 }
 
 type runPassenger struct {
-	src     string
-	svc     arvados.Service
-	depends []supervisedTask
+	src       string // path to app in source tree
+	varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+	svc       arvados.Service
+	depends   []supervisedTask
 }
 
 func (runner runPassenger) String() string {
@@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 	if err != nil {
 		return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
 	}
+	var appdir string
+	if super.ClusterType == "production" {
+		appdir = "/var/lib/arvados/" + runner.varlibdir
+	} else {
+		appdir = runner.src
+	}
 	loglevel := "4"
 	if lvl, ok := map[string]string{
 		"debug":   "5",
@@ -116,13 +127,30 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 	super.waitShutdown.Add(1)
 	go func() {
 		defer super.waitShutdown.Done()
-		err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+		cmdline := []string{
+			"bundle", "exec",
 			"passenger", "start",
 			"-p", port,
-			"--log-file", "/dev/stderr",
 			"--log-level", loglevel,
 			"--no-friendly-error-pages",
-			"--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+			"--disable-anonymous-telemetry",
+			"--disable-security-update-check",
+			"--no-compile-runtime",
+			"--no-install-runtime",
+			"--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+		}
+		if super.ClusterType == "production" {
+			cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+		} else {
+			// This would be desirable in the production
+			// case too, but it fails with sudo because
+			// /dev/stderr is a symlink to a pty owned by
+			// root: "nginx: [emerg] open() "/dev/stderr"
+			// failed (13: Permission denied)"
+			cmdline = append(cmdline, "--log-file", "/dev/stderr")
+		}
+		env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
+		err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
 		fail(err)
 	}()
 	return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 34ccf04a8..199a93a9d 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -36,6 +36,10 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 		return err
 	}
 
+	if super.ClusterType == "production" {
+		return nil
+	}
+
 	iamroot := false
 	if u, err := user.Current(); err != nil {
 		return fmt.Errorf("user.Current(): %s", err)
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index d1cf2a870..1f07601a0 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -20,6 +20,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
 	if err != nil {
 		return err
 	}
+	if super.ClusterType == "production" {
+		return nil
+	}
 	err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
 	if err != nil {
 		return err
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 5afacfe71..77fdc98be 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string {
 }
 
 func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
-	binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
-	err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+	binfile := filepath.Join(super.bindir, "arvados-server")
+	err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index e38a4775e..2d7697233 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -14,6 +14,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -54,7 +55,9 @@ type Supervisor struct {
 	tasksReady    map[string]chan bool
 	waitShutdown  sync.WaitGroup
 
+	bindir     string
 	tempdir    string
+	wwwtempdir string
 	configfile string
 	environ    []string // for child processes
 }
@@ -131,13 +134,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		return err
 	}
 
-	super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(super.tempdir)
-	if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
-		return err
+	// Choose bin and temp dirs: /var/lib/arvados/... in
+	// production, transient tempdir otherwise.
+	if super.ClusterType == "production" {
+		// These dirs have already been created by
+		// "arvados-server install" (or by extracting a
+		// package).
+		super.tempdir = "/var/lib/arvados/tmp"
+		super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+		super.bindir = "/var/lib/arvados/bin"
+	} else {
+		super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(super.tempdir)
+		super.wwwtempdir = super.tempdir
+		super.bindir = filepath.Join(super.tempdir, "bin")
+		if err := os.Mkdir(super.bindir, 0755); err != nil {
+			return err
+		}
 	}
 
 	// Fill in any missing config keys, and write the resulting
@@ -166,7 +182,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 	super.setEnv("ARVADOS_CONFIG", super.configfile)
 	super.setEnv("RAILS_ENV", super.ClusterType)
 	super.setEnv("TMPDIR", super.tempdir)
-	super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+	super.prependEnv("PATH", "/var/lib/arvados/bin:")
+	if super.ClusterType != "production" {
+		super.prependEnv("PATH", super.tempdir+"/bin:")
+	}
 
 	super.cluster, err = cfg.GetCluster("")
 	if err != nil {
@@ -182,7 +201,9 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		"PID": os.Getpid(),
 	})
 
-	if super.SourceVersion == "" {
+	if super.SourceVersion == "" && super.ClusterType == "production" {
+		// don't need SourceVersion
+	} else if super.SourceVersion == "" {
 		// Find current source tree version.
 		var buf bytes.Buffer
 		err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
@@ -224,15 +245,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 		runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 		installPassenger{src: "services/api"},
-		runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
+		runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 		installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
-		runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+		runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 		seedDatabase{},
 	}
 	if super.ClusterType != "test" {
 		tasks = append(tasks,
 			runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
-			runGoProgram{src: "services/keep-balance"},
+			runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
 		)
 	}
 	super.tasksReady = map[string]chan bool{}
@@ -382,9 +403,11 @@ func dedupEnv(in []string) []string {
 
 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 	_, basename := filepath.Split(srcpath)
-	bindir := filepath.Join(super.tempdir, "bin")
-	binfile := filepath.Join(bindir, basename)
-	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+	binfile := filepath.Join(super.bindir, basename)
+	if super.ClusterType == "production" {
+		return binfile, nil
+	}
+	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 	return binfile, err
 }
 
@@ -401,10 +424,19 @@ func (super *Supervisor) setupRubyEnv() error {
 			"GEM_PATH=",
 		})
 		gem := "gem"
-		if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+		if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
 			gem = "/var/lib/arvados/bin/gem"
 		}
 		cmd := exec.Command(gem, "env", "gempath")
+		if super.ClusterType == "production" {
+			cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+			path, err := exec.LookPath("sudo")
+			if err != nil {
+				return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+			}
+			cmd.Path = path
+		}
+		cmd.Stderr = super.Stderr
 		cmd.Env = super.environ
 		buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 		if err != nil || len(buf) == 0 {
@@ -694,11 +726,10 @@ func internalPort(svc arvados.Service) (string, error) {
 		return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
 	}
 	for u := range svc.InternalURLs {
-		if _, p, err := net.SplitHostPort(u.Host); err != nil {
-			return "", err
-		} else if p != "" {
+		u := url.URL(u)
+		if p := u.Port(); p != "" {
 			return p, nil
-		} else if u.Scheme == "https" {
+		} else if u.Scheme == "https" || u.Scheme == "ws" {
 			return "443", nil
 		} else {
 			return "80", nil
@@ -708,11 +739,10 @@ func internalPort(svc arvados.Service) (string, error) {
 }
 
 func externalPort(svc arvados.Service) (string, error) {
-	if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
-		return "", err
-	} else if p != "" {
+	u := url.URL(svc.ExternalURL)
+	if p := u.Port(); p != "" {
 		return p, nil
-	} else if svc.ExternalURL.Scheme == "https" {
+	} else if u.Scheme == "https" || u.Scheme == "wss" {
 		return "443", nil
 	} else {
 		return "80", nil
diff --git a/lib/install/deps.go b/lib/install/deps.go
index f9b962fdd..3f19aa1a8 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -176,12 +177,26 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 	}
 
 	os.Mkdir("/var/lib/arvados", 0755)
+	os.Mkdir("/var/lib/arvados/tmp", 0700)
+	if prod {
+		os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+		u, er := user.Lookup("www-data")
+		if er != nil {
+			err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+			return 1
+		}
+		uid, _ := strconv.Atoi(u.Uid)
+		gid, _ := strconv.Atoi(u.Gid)
+		err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+		if err != nil {
+			return 1
+		}
+	}
 	rubyversion := "2.5.7"
 	if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
 		logger.Print("ruby " + rubyversion + " already installed")
 	} else {
 		err = runBash(`
-mkdir -p /var/lib/arvados/tmp
 tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
 trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
@@ -189,7 +204,9 @@ cd ${tmp}
 ./configure --disable-install-doc --prefix /var/lib/arvados
 make -j8
 make install
-/var/lib/arvados/bin/gem install bundler
+/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
+# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
 rm -r ${tmp}
 `, stdout, stderr)
 		if err != nil {
@@ -262,7 +279,6 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 		} else {
 			err = runBash(`
 G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
 zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
 trap "rm ${zip}" ERR
 wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
@@ -414,8 +430,7 @@ rm ${zip}
 			for _, cmdline := range [][]string{
 				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
 				{"touch", "log/production.log"},
-				// {"chown", "-R", "root:root", "."},
-				{"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
@@ -426,6 +441,7 @@ rm ${zip}
 				cmd.Dir = "/var/lib/arvados/" + dstdir
 				cmd.Stdout = stdout
 				cmd.Stderr = stderr
+				fmt.Fprintf(stderr, "... %s\n", cmd.Args)
 				err = cmd.Run()
 				if err != nil {
 					return 1
@@ -569,6 +585,7 @@ func prodpkgs(osv osversion) []string {
 		"make",
 		"nginx",
 		"python",
+		"sudo",
 	}
 	if osv.Debian || osv.Ubuntu {
 		if osv.Debian && osv.Major == 8 {
diff --git a/lib/install/init.go b/lib/install/init.go
new file mode 100644
index 000000000..6d4f19757
--- /dev/null
+++ b/lib/install/init.go
@@ -0,0 +1,265 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strconv"
+	"text/template"
+
+	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+	ClusterID          string
+	Domain             string
+	PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "text", "info")
+	ctx := ctxlog.Context(context.Background(), logger)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var err error
+	defer func() {
+		if err != nil {
+			logger.WithError(err).Info("exiting")
+		}
+	}()
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		err = fmt.Errorf("Hostname(): %w", err)
+		return 1
+	}
+
+	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+	flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+	flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+	err = flags.Parse(args)
+	if err == flag.ErrHelp {
+		err = nil
+		return 0
+	} else if err != nil {
+		return 2
+	} else if *versionFlag {
+		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+	} else if len(flags.Args()) > 0 {
+		err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+		return 2
+	} else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+		err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+		return 1
+	}
+
+	wwwuser, err := user.Lookup("www-data")
+	if err != nil {
+		err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+		return 1
+	}
+	wwwgid, err := strconv.Atoi(wwwuser.Gid)
+	if err != nil {
+		return 1
+	}
+	initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+	err = os.Mkdir("/var/lib/arvados/keep", 0600)
+	if err != nil && !os.IsExist(err) {
+		err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+	err = os.Mkdir("/etc/arvados", 0750)
+	if err != nil && !os.IsExist(err) {
+		err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+		return 1
+	}
+	err = os.Chown("/etc/arvados", 0, wwwgid)
+	f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	if err != nil {
+		err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+		return 1
+	}
+	tmpl, err := template.New("config").Parse(`Clusters:
+  {{.ClusterID}}:
+    Services:
+      Controller:
+        InternalURLs:
+          "http://0.0.0.0:8003/": {}
+        ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+      RailsAPI:
+        InternalURLs:
+          "http://0.0.0.0:8004/": {}
+      Websocket:
+        InternalURLs:
+          "http://0.0.0.0:8005/": {}
+        ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+      Keepbalance:
+        InternalURLs:
+          "http://0.0.0.0:9005/": {}
+      GitHTTP:
+        InternalURLs:
+          "http://0.0.0.0:9001/": {}
+        ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+      DispatchCloud:
+        InternalURLs:
+          "http://0.0.0.0:9006/": {}
+      Keepproxy:
+        InternalURLs:
+          "http://0.0.0.0:25108/": {}
+        ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+      WebDAV:
+        InternalURLs:
+          "http://0.0.0.0:9002/": {}
+        ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+      WebDAVDownload:
+        InternalURLs:
+          "http://0.0.0.0:8004/": {}
+        ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+      Keepstore:
+        InternalURLs:
+          "http://0.0.0.0:25107/": {}
+      Composer:
+        ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+      Workbench1:
+        InternalURLs:
+          "http://0.0.0.0:8001/": {}
+        ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+      Workbench2:
+        InternalURLs:
+          "http://0.0.0.0:8002/": {}
+        ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+      Health:
+        InternalURLs:
+          "http://0.0.0.0:9007/": {}
+    API:
+      RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
+    Collections:
+      BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+    Containers:
+      DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+    ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+    PostgreSQL:
+      Connection:
+        dbname: arvados_production
+        host: localhost
+        user: arvados
+        password: {{printf "%q" .PostgreSQLPassword}}
+    SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+    Volumes:
+      {{.ClusterID}}-nyw5e-000000000000000:
+        Driver: Directory
+        DriverParameters:
+          Root: /var/lib/arvados/keep
+        Replication: 2
+`)
+	if err != nil {
+		return 1
+	}
+	err = tmpl.Execute(f, initcmd)
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+		return 1
+	}
+	err = f.Close()
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+	ldr := config.NewLoader(nil, logger)
+	ldr.SkipLegacy = true
+	cfg, err := ldr.Load()
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+		return 1
+	}
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return 1
+	}
+
+	err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+	if err != nil {
+		return 1
+	}
+
+	cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+	cmd.Dir = "/var/lib/arvados/railsapi"
+	cmd.Stdout = stderr
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		err = fmt.Errorf("rake db:setup: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "initialized database")
+
+	return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+	privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return "", err
+	}
+	err = privkey.Validate()
+	if err != nil {
+		return "", err
+	}
+	return string(pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+	})), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+	b := make([]byte, chars/2)
+	_, err := rand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+	for _, sql := range []string{
+		`CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+		`CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+		`CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+	} {
+		cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+		cmd.Stdout = stderr
+		cmd.Stderr = stderr
+		err := cmd.Run()
+		if err != nil {
+			return fmt.Errorf("error setting up arvados user/database: %w", err)
+		}
+	}
+	return nil
+}

commit ee7d60f02525760a2480157011659e7b2210f6b8
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jul 24 10:43:29 2020 -0400

    16306: Sync bundler version to services/api/Gemfile.lock.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index ff6ead0fa..b3eacfe31 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -555,7 +555,7 @@ setup_ruby_environment() {
         (
             export HOME=$GEMHOME
             bundlers="$(gem list --details bundler)"
-            versions=(1.11.0 1.17.3 2.0.2)
+            versions=(1.16.6 1.17.3 2.0.2)
             for v in ${versions[@]}; do
                 if ! echo "$bundlers" | fgrep -q "($v)"; then
                     gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 6a2c4b61f..f18300c4c 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -50,9 +50,9 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 	if err != nil {
 		return err
 	}
-	for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+	for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
 		if !strings.Contains(buf.String(), "("+version+")") {
-			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
 			if err != nil {
 				return err
 			}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index d28823f4a..f9b962fdd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -416,7 +416,7 @@ rm ${zip}
 				{"touch", "log/production.log"},
 				// {"chown", "-R", "root:root", "."},
 				{"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
-				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
 				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},

commit 12441486d7fb3b51d2fea9d9b1da0d8e4f3ef3e0
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Wed Jul 1 09:42:48 2020 -0400

    16306: Add "install -type package" and "buildpackage".
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
new file mode 100644
index 000000000..4b395c4ba
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage.go
@@ -0,0 +1,98 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os/exec"
+
+	"git.arvados.org/arvados.git/lib/install"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+type buildPackage struct{}
+
+func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "text", "info")
+	err := (&builder{
+		PackageVersion: "0.0.0",
+		logger:         logger,
+	}).run(context.Background(), prog, args, stdin, stdout, stderr)
+	if err != nil {
+		logger.WithError(err).Error("failed")
+		return 1
+	}
+	return 0
+}
+
+type builder struct {
+	PackageVersion string
+	SourcePath     string
+	OutputDir      string
+	SkipInstall    bool
+	logger         logrus.FieldLogger
+}
+
+func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
+	flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
+	flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
+	flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
+	err := flags.Parse(args)
+	if err != nil {
+		return err
+	}
+	if len(flags.Args()) > 0 {
+		return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+	}
+	if !bldr.SkipInstall {
+		exitcode := install.Command.RunCommand("arvados-server install", []string{
+			"-type", "package",
+			"-package-version", bldr.PackageVersion,
+			"-source", bldr.SourcePath,
+		}, stdin, stdout, stderr)
+		if exitcode != 0 {
+			return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+		}
+	}
+	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("gem install fpm: %w", err)
+	}
+
+	format := "deb" // TODO: rpm
+
+	cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+		"--name", "arvados-server-easy",
+		"--version", bldr.PackageVersion,
+		"--input-type", "dir",
+		"--output-type", format)
+	deps, err := install.ProductionDependencies()
+	if err != nil {
+		return err
+	}
+	for _, pkg := range deps {
+		cmd.Args = append(cmd.Args, "--depends", pkg)
+	}
+	cmd.Args = append(cmd.Args,
+		"--exclude", "/var/lib/arvados/go",
+		"/var/lib/arvados",
+		"/var/www/.gem",
+		"/var/www/.passenger",
+	)
+	fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+	cmd.Dir = bldr.OutputDir
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	return cmd.Run()
+}
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-dev/buildpackage_test.go
new file mode 100644
index 000000000..61bdeae82
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage_test.go
@@ -0,0 +1,77 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+	os.Args = append(os.Args, "-test.timeout=30m") // kludge
+
+	// This enables a hack to speed up repeated tests: hit "docker
+	// commit --pause {containername} checkpointtag" after the
+	// test container has downloaded/compiled some stuff, then run
+	// tests with "-test.buildimage=checkpointtag" next time to
+	// retry/resume/update from that point.
+	flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
+}
+
+type BuildpackageSuite struct{}
+
+var _ = check.Suite(&BuildpackageSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+	if testing.Short() {
+		c.Skip("skipping docker tests in short mode")
+	} else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+		c.Skip("skipping docker tests because docker is not available")
+	}
+	tmpdir := c.MkDir()
+	defer os.RemoveAll(tmpdir)
+
+	err := os.Mkdir(tmpdir+"/pkg", 0755)
+	c.Assert(err, check.IsNil)
+	err = os.Mkdir(tmpdir+"/bin", 0755)
+	c.Assert(err, check.IsNil)
+
+	cmd := exec.Command("go", "install")
+	cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+
+	srctree, err := filepath.Abs("../..")
+	c.Assert(err, check.IsNil)
+
+	cmd = exec.Command("docker", "run", "--rm",
+		"-v", tmpdir+"/pkg:/pkg",
+		"-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+		"-v", srctree+":/usr/local/src/arvados:ro",
+		buildimage,
+		"/arvados-dev", "buildpackage",
+		"-package-version", "0.9.99",
+		"-source", "/usr/local/src/arvados",
+		"-output-directory", "/pkg")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+
+	fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+	c.Assert(err, check.IsNil)
+	c.Logf("%#v", fi)
+}
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-dev/cmd.go
new file mode 100644
index 000000000..e0d5144e5
--- /dev/null
+++ b/cmd/arvados-dev/cmd.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"os"
+
+	"git.arvados.org/arvados.git/lib/cmd"
+)
+
+var (
+	handler = cmd.Multi(map[string]cmd.Handler{
+		"version":   cmd.Version,
+		"-version":  cmd.Version,
+		"--version": cmd.Version,
+
+		"buildpackage": buildPackage{},
+	})
+)
+
+func main() {
+	os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
new file mode 100755
index 000000000..072dfcf9c
--- /dev/null
+++ b/cmd/arvados-dev/example.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e -o pipefail
+
+version="${PACKAGE_VERSION:-0.9.99}"
+
+# mkdir -p /tmp/pkg
+# (
+#     cd cmd/arvados-dev
+#     go install
+# )
+# docker run --rm \
+#        -v /tmp/pkg:/pkg \
+#        -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+#        -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
+#        /arvados-dev buildpackage \
+#        -source /arvados \
+#        -package-version "${version}" \
+#        -output-directory /pkg
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+# ls -l ${pkgfile}
+# (
+#     cd /tmp/pkg
+#     dpkg-scanpackages . | gzip > Packages.gz
+# )
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+docker run -it --rm \
+       -v /tmp/pkg:/pkg:ro \
+       -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+       ${INSTALLIMAGE:-debian:10} \
+       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ba57c20c3..d28823f4a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -24,13 +25,17 @@ import (
 	"github.com/lib/pq"
 )
 
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
 
 const devtestDatabasePassword = "insecure_arvados_test"
 
-type installCommand struct{}
+type installCommand struct {
+	ClusterType    string
+	SourcePath     string
+	PackageVersion string
+}
 
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	logger := ctxlog.New(stderr, "text", "info")
 	ctx := ctxlog.Context(context.Background(), logger)
 	ctx, cancel := context.WithCancel(ctx)
@@ -46,7 +51,9 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
 	flags.SetOutput(stderr)
 	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-	clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+	flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+	flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+	flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -55,18 +62,23 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		return 2
 	} else if *versionFlag {
 		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+	} else if len(flags.Args()) > 0 {
+		err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+		return 2
 	}
 
-	var dev, test, prod bool
-	switch *clusterType {
+	var dev, test, prod, pkg bool
+	switch inst.ClusterType {
 	case "development":
 		dev = true
 	case "test":
 		test = true
 	case "production":
 		prod = true
+	case "package":
+		pkg = true
 	default:
-		err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+		err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
 		return 2
 	}
 
@@ -96,53 +108,47 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		}
 	}
 
-	if dev || test {
-		debs := []string{
-			"bison",
+	pkgs := prodpkgs(osv)
+
+	if pkg {
+		pkgs = append(pkgs,
+			"dpkg-dev",
+			"rsync",
+		)
+	}
+
+	if dev || test || pkg {
+		pkgs = append(pkgs,
 			"bsdmainutils",
 			"build-essential",
-			"ca-certificates",
 			"cadaver",
-			"curl",
 			"cython",
 			"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
 			"default-jdk-headless",
 			"default-jre-headless",
-			"fuse",
 			"gettext",
-			"git",
-			"gitolite3",
-			"graphviz",
-			"haveged",
 			"iceweasel",
 			"libattr1-dev",
 			"libcrypt-ssleay-perl",
-			"libcrypt-ssleay-perl",
-			"libcurl3-gnutls",
-			"libcurl4-openssl-dev",
 			"libfuse-dev",
 			"libgnutls28-dev",
 			"libjson-perl",
-			"libjson-perl",
 			"libpam-dev",
 			"libpcre3-dev",
-			"libpq-dev",
 			"libpython2.7-dev",
 			"libreadline-dev",
 			"libssl-dev",
 			"libwww-perl",
 			"libxml2-dev",
-			"libxslt1.1",
+			"libxslt1-dev",
 			"linkchecker",
 			"lsof",
 			"net-tools",
-			"nginx",
 			"pandoc",
 			"perl-modules",
 			"pkg-config",
 			"postgresql",
 			"postgresql-contrib",
-			"python",
 			"python3-dev",
 			"python-epydoc",
 			"r-base",
@@ -151,16 +157,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 			"virtualenv",
 			"wget",
 			"xvfb",
-			"zlib1g-dev",
-		}
+		)
 		switch {
 		case osv.Debian && osv.Major >= 10:
-			debs = append(debs, "libcurl4")
+			pkgs = append(pkgs, "libcurl4")
 		default:
-			debs = append(debs, "libcurl3")
+			pkgs = append(pkgs, "libcurl3")
 		}
 		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
-		cmd.Args = append(cmd.Args, debs...)
+		cmd.Args = append(cmd.Args, pkgs...)
 		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr
@@ -182,7 +187,7 @@ trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
 cd ${tmp}
 ./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+make -j8
 make install
 /var/lib/arvados/bin/gem install bundler
 rm -r ${tmp}
@@ -206,7 +211,9 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
 				return 1
 			}
 		}
+	}
 
+	if !prod && !pkg {
 		pjsversion := "1.9.8"
 		if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
 			logger.Print("phantomjs " + pjsversion + " already installed")
@@ -389,12 +396,89 @@ rm ${zip}
 		}
 	}
 
+	if pkg {
+		// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+		for dstdir, srcdir := range map[string]string{
+			"railsapi":   "services/api",
+			"workbench1": "apps/workbench",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+			for _, cmdline := range [][]string{
+				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+				{"touch", "log/production.log"},
+				// {"chown", "-R", "root:root", "."},
+				{"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+			} {
+				cmd = exec.Command(cmdline[0], cmdline[1:]...)
+				cmd.Env = append([]string{}, os.Environ()...)
+				cmd.Dir = "/var/lib/arvados/" + dstdir
+				cmd.Stdout = stdout
+				cmd.Stderr = stderr
+				err = cmd.Run()
+				if err != nil {
+					return 1
+				}
+			}
+			cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+			cmd.Dir = "/var/lib/arvados/" + dstdir
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+				// Exit code 2 indicates there were warnings (like
+				// "other passenger installations have been detected",
+				// which we can't expect to avoid) but no errors.
+				// Other non-zero exit codes (1, 9) indicate errors.
+				return 1
+			}
+		}
+
+		// Install Go programs to /var/lib/arvados/bin/
+		for _, srcdir := range []string{
+			"cmd/arvados-client",
+			"cmd/arvados-server",
+			"services/arv-git-httpd",
+			"services/crunch-dispatch-local",
+			"services/crunch-dispatch-slurm",
+			"services/health",
+			"services/keep-balance",
+			"services/keep-web",
+			"services/keepproxy",
+			"services/keepstore",
+			"services/ws",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+			cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+		}
+	}
+
 	return 0
 }
 
 type osversion struct {
 	Debian bool
 	Ubuntu bool
+	Centos bool
 	Major  int
 }
 
@@ -432,6 +516,8 @@ func identifyOS() (osversion, error) {
 		osv.Ubuntu = true
 	case "debian":
 		osv.Debian = true
+	case "centos":
+		osv.Centos = true
 	default:
 		return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
 	}
@@ -466,3 +552,56 @@ func runBash(script string, stdout, stderr io.Writer) error {
 	cmd.Stderr = stderr
 	return cmd.Run()
 }
+
+func prodpkgs(osv osversion) []string {
+	pkgs := []string{
+		"automake",
+		"bison",
+		"ca-certificates",
+		"curl",
+		"fuse",
+		"git",
+		"gitolite3",
+		"graphviz",
+		"haveged",
+		"libcurl3-gnutls",
+		"libxslt1.1",
+		"make",
+		"nginx",
+		"python",
+	}
+	if osv.Debian || osv.Ubuntu {
+		if osv.Debian && osv.Major == 8 {
+			pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+		} else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+			pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+		}
+		return append(pkgs,
+			"g++",
+			"libcurl4-openssl-dev", // services/api
+			"libpq-dev",
+			"libpython2.7", // services/fuse
+			"mime-support", // keep-web
+			"zlib1g-dev",   // services/api
+		)
+	} else if osv.Centos {
+		return append(pkgs,
+			"fuse-libs", // services/fuse
+			"gcc",
+			"gcc-c++",
+			"libcurl-devel",    // services/api
+			"mailcap",          // keep-web
+			"postgresql-devel", // services/api
+		)
+	} else {
+		panic("os version not supported")
+	}
+}
+
+func ProductionDependencies() ([]string, error) {
+	osv, err := identifyOS()
+	if err != nil {
+		return nil, err
+	}
+	return prodpkgs(osv), nil
+}

commit b4d6c92791088a0b6a39a71b2b8b6ddc4b13af41
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Tue Jun 2 11:14:10 2020 -0400

    16306: Add automatic install page.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/doc/_config.yml b/doc/_config.yml
index 7d7101f41..e38f0fa8f 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -190,6 +190,8 @@ navbar:
       - install/arvados-on-kubernetes.html.textile.liquid
       - install/arvados-on-kubernetes-minikube.html.textile.liquid
       - install/arvados-on-kubernetes-GKE.html.textile.liquid
+    - Automatic installation:
+      - install/automatic.html.textile.liquid
     - Manual installation:
       - install/install-manual-prerequisites.html.textile.liquid
       - install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
new file mode 100644
index 000000000..049bf6891
--- /dev/null
+++ b/doc/install/automatic.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example at gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# apt-get install arvados-server
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example at gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 1a41980e2..81d7b2159 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -23,6 +23,7 @@ table(table table-bordered table-condensed).
 ||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
 |"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
 |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
 |"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
 |"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
 </div>

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list