[ARVADOS] created: c78289e5956a55c7540c2a3f6a543f16b1eec7c2
Git user
git at public.curoverse.com
Tue Feb 28 15:55:47 EST 2017
at c78289e5956a55c7540c2a3f6a543f16b1eec7c2 (commit)
commit c78289e5956a55c7540c2a3f6a543f16b1eec7c2
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 28 15:52:18 2017 -0500
11183: Add "arvados-admin setup"
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index d4e77c9..36bba41 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -2,7 +2,7 @@ FROM debian:jessie
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN apt-get update && apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1204/Dockerfile b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
index daeabc9..3a03b9a 100644
--- a/build/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
@@ -2,7 +2,7 @@ FROM ubuntu:precise
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential unzip
+RUN apt-get update && apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential unzip && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index aa92ad2..13e9bc6 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -1,8 +1,8 @@
FROM ubuntu:trusty
-MAINTAINER Brett Smith <brett at curoverse.com>
+MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
+RUN apt-get update && apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index fec55e6..d85e8b2 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -2,7 +2,7 @@ FROM ubuntu:xenial
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
+RUN apt-get update && apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
diff --git a/build/run-build-packages-all-targets.sh b/build/run-build-packages-all-targets.sh
index a4dd9a6..2d16147 100755
--- a/build/run-build-packages-all-targets.sh
+++ b/build/run-build-packages-all-targets.sh
@@ -38,7 +38,7 @@ fi
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,only-build: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -66,6 +66,9 @@ while [ $# -gt 0 ]; do
--test-packages)
TEST_PACKAGES="--test-packages"
;;
+ --only-build)
+ ONLY_BUILD="$1 $2"; shift
+ ;;
--only-test)
ONLY_TEST="$1 $2"; shift
;;
@@ -84,7 +87,7 @@ cd $(dirname $0)
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST $ONLY_BUILD ; then
true
else
FINAL_EXITCODE=$?
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 685ca51..78adfbb 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -129,6 +129,7 @@ popd
if test -z "$packages" ; then
packages="arvados-api-server
+ arvados-admin
arvados-docker-cleaner
arvados-git-httpd
arvados-node-manager
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 37e963b..389dc7b 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -340,6 +340,8 @@ package_go_binary sdk/go/crunchrunner crunchrunner \
"Crunchrunner executes a command inside a container and uploads the output"
package_go_binary services/arv-git-httpd arvados-git-httpd \
"Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary cmd/arvados-admin arvados-admin \
+ "Arvados cluster administration tool"
package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
"Dispatch Crunch containers on the local system"
package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
diff --git a/build/run-library.sh b/build/run-library.sh
index a13470b..8a3fb39 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -112,6 +112,8 @@ package_go_binary() {
fi
fi
+ go generate || return 1
+
cd $WORKSPACE/packages/$TARGET
test_package_presence $prog $version go
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 44f9a30..4e20048 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -101,6 +101,10 @@ tools/crunchstat-summary
tools/keep-exercise
tools/keep-rsync
tools/keep-block-check
+lib/agent
+lib/crunchstat
+lib/setup
+cmd/arvados-admin
(*) apps/workbench is shorthand for apps/workbench_units +
apps/workbench_functionals + apps/workbench_integration
@@ -195,6 +199,9 @@ sanity_checks() {
echo -n 'gitolite: '
which gitolite \
|| fatal "No gitolite. Try: apt-get install gitolite3"
+ echo -n 'docker-compose: '
+ which docker-compose \
+ || fatal "No docker-compose. Try: sudo curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` --output /usr/local/bin/docker-compose && sudo chmod +x /usr/local/bin/docker-compose"
}
rotate_logfile() {
@@ -544,12 +551,14 @@ do_test_once() {
then
covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ gopkgpath="git.curoverse.com/arvados.git/$1"
# We do "go get -t" here to catch compilation errors
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
# compilation errors.
go get -t "git.curoverse.com/arvados.git/$1" && \
cd "$WORKSPACE/$1" && \
+ go generate && \
[[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
if [[ -n "${testargs[$1]}" ]]
then
@@ -559,7 +568,7 @@ do_test_once() {
else
# The above form gets verbose even when testargs is
# empty, so use this form in such cases:
- go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+ go test ${short:+-short} ${coverflags[@]} .
fi
result=${result:-$?}
if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
@@ -591,6 +600,9 @@ do_test_once() {
else
"test_$1"
fi
+ if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then
+ cd "${WORKSPACE}/${1}" && npm test || result=1
+ fi
result=${result:-$?}
checkexit $result "$1 tests"
title "End of $1 tests (`timer`)"
@@ -608,9 +620,15 @@ do_install() {
do_install_once() {
title "Running $1 install"
timer_reset
+ cd "${WORKSPACE}/${1}" || return 1
+ if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then
+ npm install || return 1
+ fi
if [[ "$2" == "go" ]]
then
- go get -t "git.curoverse.com/arvados.git/$1"
+ go get -d -t "git.curoverse.com/arvados.git/$1" \
+ && go generate \
+ && go get "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
# $3 can name a path directory for us to use, including trailing
@@ -625,8 +643,7 @@ do_install_once() {
# install" ensures that the dependencies are met, the second "pip
# install" ensures that we've actually installed the local package
# we just built.
- cd "$WORKSPACE/$1" \
- && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
&& "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
&& "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
@@ -776,7 +793,9 @@ gostuff=(
sdk/go/streamer
sdk/go/crunchrunner
sdk/go/stats
+ lib/agent
lib/crunchstat
+ lib/setup
services/arv-git-httpd
services/crunchstat
services/keep-web
@@ -788,9 +807,11 @@ gostuff=(
services/crunch-dispatch-slurm
services/crunch-run
services/ws
+ services/boot
tools/keep-block-check
tools/keep-exercise
tools/keep-rsync
+ cmd/arvados-admin
)
for g in "${gostuff[@]}"
do
diff --git a/cmd/arvados-admin/.gitignore b/cmd/arvados-admin/.gitignore
new file mode 100644
index 0000000..87fbbf4
--- /dev/null
+++ b/cmd/arvados-admin/.gitignore
@@ -0,0 +1 @@
+arvados-admin
diff --git a/cmd/arvados-admin/main.go b/cmd/arvados-admin/main.go
new file mode 100644
index 0000000..de532cb
--- /dev/null
+++ b/cmd/arvados-admin/main.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+
+ "git.curoverse.com/arvados.git/cmd"
+ "git.curoverse.com/arvados.git/lib/agent"
+ "git.curoverse.com/arvados.git/lib/setup"
+)
+
+var cmds = map[string]cmd.Command{
+ "agent": agent.Command(),
+ "setup": setup.Command(),
+}
+
+func main() {
+ err := cmd.Dispatch(cmds, os.Args[0], os.Args[1:])
+ if err != nil {
+ if err != flag.ErrHelp {
+ fmt.Fprintf(os.Stderr, "%s\n", err)
+ }
+ os.Exit(1)
+ }
+}
diff --git a/cmd/arvados-admin/setup_debian8_test.go b/cmd/arvados-admin/setup_debian8_test.go
new file mode 100644
index 0000000..671f8c7
--- /dev/null
+++ b/cmd/arvados-admin/setup_debian8_test.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+ "log"
+ "net"
+ "os"
+ "os/exec"
+ "testing"
+)
+
+func TestSetupDebian8(t *testing.T) {
+ cwd, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ ln, err := net.Listen("tcp", ":")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, port, err := net.SplitHostPort(ln.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ln.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+ log.Printf("Publishing consul webgui at %v", ln.Addr())
+ for _, cmdline := range [][]string{
+ {"go", "build"},
+ {"docker", "build", "--tag=arvados-admin-debian8-test", "test-debian8"},
+ {"docker", "run", "--rm", "--publish=" + port + ":18500", "--cap-add=IPC_LOCK", "--cap-add=SYS_ADMIN", "--volume=/sys/fs/cgroup", "--volume=" + cwd + "/arvados-admin:/usr/bin/arvados-admin:ro", "--volume=/var/cache/arvados:/var/cache/arvados:ro", "arvados-admin-debian8-test"},
+ } {
+ cmd := exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+}
diff --git a/cmd/arvados-admin/setup_docker_compose_test.go b/cmd/arvados-admin/setup_docker_compose_test.go
new file mode 100644
index 0000000..e88dc16
--- /dev/null
+++ b/cmd/arvados-admin/setup_docker_compose_test.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+ "os"
+ "os/exec"
+ "testing"
+)
+
+func TestSetupDockerCompose(t *testing.T) {
+ for _, cmdline := range [][]string{
+ {"go", "build"},
+ {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "down", "-v"},
+ {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "up"},
+ } {
+ cmd := exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+}
diff --git a/cmd/arvados-admin/test-debian8/Dockerfile b/cmd/arvados-admin/test-debian8/Dockerfile
new file mode 100644
index 0000000..645aec2
--- /dev/null
+++ b/cmd/arvados-admin/test-debian8/Dockerfile
@@ -0,0 +1,14 @@
+FROM debian:8
+RUN apt-get update
+
+# preload (but don't install) packages arvados-boot might decide to install
+RUN DEBIAN_FRONTEND=noninteractive apt-get -dy install --no-install-recommends ca-certificates locales nginx postgresql runit
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends runit locales
+
+RUN ["bash", "-c", "echo en_US.utf8 UTF-8 | tee -a /etc/locale.gen && locale-gen -a && \
+ (echo LANG=en_US.UTF-8; echo LC_ALL=en_US.UTF-8) > /etc/default/locale"]
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends ca-certificates locales nginx postgresql runit
+
+CMD ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && arvados-admin setup"]
diff --git a/cmd/arvados-admin/test-docker-compose/agent.yml b/cmd/arvados-admin/test-docker-compose/agent.yml
new file mode 100644
index 0000000..46f4de7
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/agent.yml
@@ -0,0 +1,4 @@
+ControlHosts:
+ - sys0
+ - sys1
+ - sys2
diff --git a/cmd/arvados-admin/test-docker-compose/docker-compose.yml b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
new file mode 100644
index 0000000..34e7cd4
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
@@ -0,0 +1,40 @@
+version: '2'
+services:
+ sys0:
+ build: ../test-debian8
+ cap_add:
+ - IPC_LOCK
+ - SYS_ADMIN
+ volumes:
+ - ../arvados-admin:/usr/bin/arvados-admin:ro
+ - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+ - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+ - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true -init-vault=true && wait"]
+ sys1:
+ build: ../test-debian8
+ cap_add:
+ - IPC_LOCK
+ - SYS_ADMIN
+ volumes:
+ - ../arvados-admin:/usr/bin/arvados-admin:ro
+ - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+ - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+ - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
+ sys2:
+ build: ../test-debian8
+ cap_add:
+ - IPC_LOCK
+ - SYS_ADMIN
+ volumes:
+ - ../arvados-admin:/usr/bin/arvados-admin:ro
+ - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+ - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+ - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
+volumes:
+ vault:
diff --git a/cmd/arvados-admin/test-docker-compose/encrypt-key.txt b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt
new file mode 100644
index 0000000..507ff36
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt
@@ -0,0 +1 @@
+qigR/fVUccR07/J56MsloA==
diff --git a/cmd/arvados-admin/test-docker-compose/master-token.txt b/cmd/arvados-admin/test-docker-compose/master-token.txt
new file mode 100644
index 0000000..f12bf2c
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/master-token.txt
@@ -0,0 +1 @@
+2f79a06949ba76666308f5c821f234c9c038664df2b8662b587b9500ef4853a1
\ No newline at end of file
diff --git a/cmd/dispatch.go b/cmd/dispatch.go
new file mode 100644
index 0000000..2fc2a70
--- /dev/null
+++ b/cmd/dispatch.go
@@ -0,0 +1,59 @@
+package cmd
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "sort"
+
+ "git.curoverse.com/arvados.git/sdk/go/config"
+)
+
+// A Command is a subcommand that can be invoked by Dispatch.
+type Command interface {
+ DefaultConfigFile() string
+ ParseFlags([]string) error
+ Run() error
+}
+
+// Dispatch parses flags from args, chooses an entry in cmds using the
+// next argument after the parsed flags, loads the command's
+// configuration file if it exists, passes any additional flags to the
+// command's ParseFlags method, and -- if all of those steps complete
+// without errors -- runs the command.
+func Dispatch(cmds map[string]Command, prog string, args []string) error {
+ fs := flag.NewFlagSet(prog, flag.ContinueOnError)
+ err := fs.Parse(args)
+ if err != nil {
+ return err
+ }
+
+ subcmd := fs.Arg(0)
+ cmd, ok := cmds[subcmd]
+ if !ok {
+ if subcmd != "" && subcmd != "help" {
+ return fmt.Errorf("unrecognized subcommand %q", subcmd)
+ }
+ var subcmds []string
+ for s := range cmds {
+ subcmds = append(subcmds, s)
+ }
+ sort.Sort(sort.StringSlice(subcmds))
+ return fmt.Errorf("available subcommands: %q", subcmds)
+ }
+
+ err = config.LoadFile(cmd, cmd.DefaultConfigFile())
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ if fs.NArg() > 1 {
+ args = fs.Args()[1:]
+ } else {
+ args = nil
+ }
+ err = cmd.ParseFlags(args)
+ if err != nil {
+ return err
+ }
+ return cmd.Run()
+}
diff --git a/lib/agent/agent.go b/lib/agent/agent.go
new file mode 100644
index 0000000..52a3d83
--- /dev/null
+++ b/lib/agent/agent.go
@@ -0,0 +1,107 @@
+package agent
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strings"
+)
+
+type Agent struct {
+ // 5 alphanumeric chars. Must be either xx*, yy*, zz*, or
+ // globally unique.
+ ClusterID string
+
+ // "runit" or "systemd"
+ DaemonSupervisor string
+
+ // Hostnames or IP addresses of control hosts. Use at least 3
+ // in production. System functions only when a majority are
+ // alive.
+ ControlHosts []string
+ Ports PortsConfig
+ DataDir string
+ UsrDir string
+ RunitSvDir string
+
+ ArvadosAptRepo RepoConfig
+
+ // Unseal the vault automatically at startup
+ Unseal bool
+}
+
+type PortsConfig struct {
+ ConsulDNS int
+ ConsulHTTP int
+ ConsulHTTPS int
+ ConsulRPC int
+ ConsulSerfLAN int
+ ConsulSerfWAN int
+ ConsulServer int
+ NomadHTTP int
+ NomadRPC int
+ NomadSerf int
+ VaultServer int
+}
+
+type RepoConfig struct {
+ Enabled bool
+ URL string
+ Release string
+}
+
+func Command() *Agent {
+ var repoConf RepoConfig
+ if rel, err := ioutil.ReadFile("/etc/os-release"); err == nil {
+ rel := string(rel)
+ for _, try := range []string{"jessie", "precise", "xenial"} {
+ if !strings.Contains(rel, try) {
+ continue
+ }
+ repoConf = RepoConfig{
+ Enabled: true,
+ URL: "http://apt.arvados.org/",
+ Release: try,
+ }
+ break
+ }
+ }
+ ds := "runit"
+ if _, err := os.Stat("/run/systemd/system"); err == nil {
+ ds = "systemd"
+ }
+ return &Agent{
+ ClusterID: "zzzzz",
+ DaemonSupervisor: ds,
+ ArvadosAptRepo: repoConf,
+ ControlHosts: []string{"127.0.0.1"},
+ Ports: PortsConfig{
+ ConsulDNS: 18600,
+ ConsulHTTP: 18500,
+ ConsulHTTPS: -1,
+ ConsulRPC: 18400,
+ ConsulSerfLAN: 18301,
+ ConsulSerfWAN: 18302,
+ ConsulServer: 18300,
+ NomadHTTP: 14646,
+ NomadRPC: 14647,
+ NomadSerf: 14648,
+ VaultServer: 18200,
+ },
+ DataDir: "/var/lib/arvados",
+ UsrDir: "/usr/local/arvados",
+ RunitSvDir: "/etc/sv",
+ }
+}
+
+func (*Agent) ParseFlags(args []string) error {
+ return nil
+}
+
+func (a *Agent) Run() error {
+ return fmt.Errorf("not implemented: %T.Run()", a)
+}
+
+func (*Agent) DefaultConfigFile() string {
+ return "/etc/arvados/agent/agent.yml"
+}
diff --git a/lib/setup/check.go b/lib/setup/check.go
new file mode 100644
index 0000000..2b12c1f
--- /dev/null
+++ b/lib/setup/check.go
@@ -0,0 +1,12 @@
+package setup
+
+import "time"
+
+func waitCheck(timeout time.Duration, check func() error) error {
+ deadline := time.Now().Add(timeout)
+ var err error
+ for err = check(); err != nil && !time.Now().After(deadline); err = check() {
+ time.Sleep(time.Second)
+ }
+ return err
+}
diff --git a/lib/setup/command.go b/lib/setup/command.go
new file mode 100644
index 0000000..79c28b7
--- /dev/null
+++ b/lib/setup/command.go
@@ -0,0 +1,26 @@
+package setup
+
+import (
+ "os"
+ "os/exec"
+)
+
+func command(prog string, args ...string) *exec.Cmd {
+ cmd := exec.Command(prog, args...)
+ cmd.Stderr = os.Stderr
+ cmd.Stdout = os.Stderr
+ return cmd
+}
+
+func runStatusCmd(prog string, args ...string) (bool, error) {
+ cmd := command(prog, args...)
+ err := cmd.Run()
+ switch err.(type) {
+ case *exec.ExitError:
+ return false, nil
+ case nil:
+ return true, nil
+ default:
+ return false, err
+ }
+}
diff --git a/lib/setup/consul.go b/lib/setup/consul.go
new file mode 100644
index 0000000..9cc2f29
--- /dev/null
+++ b/lib/setup/consul.go
@@ -0,0 +1,167 @@
+package setup
+
+import (
+ "crypto/rand"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/hashicorp/consul/api"
+)
+
+func (s *Setup) installConsul() error {
+ prog := path.Join(s.UsrDir, "bin", "consul")
+ err := (&download{
+ URL: "https://releases.hashicorp.com/consul/0.7.5/consul_0.7.5_linux_amd64.zip",
+ Dest: prog,
+ Size: 36003713,
+ Mode: 0755,
+ PreloadDir: s.PreloadDir,
+ }).install()
+ if err != nil {
+ return err
+ }
+
+ if err := s.consulInit(); err != nil {
+ return err
+ }
+ if s.consulCheck() == nil {
+ return nil
+ }
+
+ dataDir := path.Join(s.DataDir, "consul")
+ if err := os.MkdirAll(dataDir, 0700); err != nil {
+ return err
+ }
+
+ cf := path.Join(s.DataDir, "consul-config.json")
+ {
+ c := map[string]interface{}{
+ "acl_agent_token": s.masterToken,
+ "acl_datacenter": s.ClusterID,
+ "acl_default_policy": "deny",
+ "acl_enforce_version_8": true,
+ "acl_master_token": s.masterToken,
+ "bootstrap_expect": len(s.ControlHosts),
+ "client_addr": "0.0.0.0",
+ "data_dir": dataDir,
+ "datacenter": s.ClusterID,
+ "encrypt": s.encryptKey,
+ "server": true,
+ "ui": true,
+ "ports": map[string]int{
+ "dns": s.Ports.ConsulDNS,
+ "http": s.Ports.ConsulHTTP,
+ "https": s.Ports.ConsulHTTPS,
+ "rpc": s.Ports.ConsulRPC,
+ "serf_lan": s.Ports.ConsulSerfLAN,
+ "serf_wan": s.Ports.ConsulSerfWAN,
+ "server": s.Ports.ConsulServer,
+ },
+ }
+ err = atomicWriteJSON(cf, c, 0600)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = s.installService(daemon{
+ name: "arvados-consul",
+ prog: prog,
+ args: []string{"agent", "-config-file=" + cf},
+ noRegister: true,
+ })
+ if err != nil {
+ return err
+ }
+ if err = waitCheck(20*time.Second, s.consulCheck); err != nil {
+ return err
+ }
+ if len(s.ControlHosts) > 1 {
+ args := []string{"join"}
+ args = append(args, fmt.Sprintf("-rpc-addr=127.0.0.1:%d", s.Ports.ConsulRPC))
+ args = append(args, s.ControlHosts...)
+ cmd := exec.Command(prog, args...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("consul join: %s", err)
+ }
+ }
+ return nil
+}
+
+var consulCfg = api.DefaultConfig()
+
+func (s *Setup) consulMaster() (*api.Client, error) {
+ masterToken, err := ioutil.ReadFile(path.Join(s.DataDir, "master-token.txt"))
+ if err != nil {
+ return nil, err
+ }
+ ccfg := api.DefaultConfig()
+ ccfg.Address = fmt.Sprintf("127.0.0.1:%d", s.Ports.ConsulHTTP)
+ ccfg.Datacenter = s.ClusterID
+ ccfg.Token = string(masterToken)
+ return api.NewClient(ccfg)
+}
+
+func (s *Setup) consulInit() error {
+ prog := path.Join(s.UsrDir, "bin", "consul")
+ keyPath := path.Join(s.DataDir, "encrypt-key.txt")
+ key, err := ioutil.ReadFile(keyPath)
+ if os.IsNotExist(err) {
+ key, err = exec.Command(prog, "keygen").CombinedOutput()
+ if err != nil {
+ return err
+ }
+ err = atomicWriteFile(keyPath, key, 0400)
+ }
+ if err != nil {
+ return err
+ }
+ s.encryptKey = strings.TrimSpace(string(key))
+
+ tokPath := path.Join(s.DataDir, "master-token.txt")
+ if tok, err := ioutil.ReadFile(tokPath); err != nil {
+ s.masterToken = generateToken()
+ err = atomicWriteFile(tokPath, []byte(s.masterToken), 0600)
+ if err != nil {
+ return err
+ }
+ } else {
+ s.masterToken = string(tok)
+ }
+ return nil
+}
+
+func (s *Setup) consulCheck() error {
+ consul, err := s.consulMaster()
+ if err != nil {
+ return err
+ }
+ _, err = consul.Catalog().Datacenters()
+ return err
+}
+
+// OnlyNode returns true if this is the only consul node.
+func (s *Setup) OnlyNode() (bool, error) {
+ c, err := s.consulMaster()
+ if err != nil {
+ return false, err
+ }
+ nodes, _, err := c.Catalog().Nodes(nil)
+ return len(nodes) == 1, err
+}
+
+func generateToken() string {
+ var r [16]byte
+ if _, err := rand.Read(r[:]); err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", r)
+}
diff --git a/lib/setup/daemon.go b/lib/setup/daemon.go
new file mode 100644
index 0000000..3985259
--- /dev/null
+++ b/lib/setup/daemon.go
@@ -0,0 +1,75 @@
+package setup
+
+import (
+ "log"
+ "math/rand"
+ "os"
+
+ "github.com/hashicorp/consul/api"
+)
+
+type daemon struct {
+ name string
+ prog string // program to run (absolute path) -- if blank, use name
+ args []string
+ noRegister bool
+}
+
+func (s *Setup) installService(d daemon) error {
+ if d.prog == "" {
+ d.prog = d.name
+ }
+ if _, err := os.Stat(d.prog); err != nil {
+ return err
+ }
+ sup := s.superviseDaemon(d)
+ if ok, err := sup.Running(); err != nil {
+ return err
+ } else if !ok {
+ if err := sup.Start(); err != nil {
+ return err
+ }
+ }
+ if d.noRegister {
+ return nil
+ }
+ consul, err := s.consulMaster()
+ if err != nil {
+ return err
+ }
+ agent := consul.Agent()
+ svcs, err := agent.Services()
+ if err != nil {
+ return err
+ }
+ if svc, ok := svcs[d.name]; ok {
+ log.Printf("%q is registered: %#v", d.name, svc)
+ return nil
+ }
+ return agent.ServiceRegister(&api.AgentServiceRegistration{
+ ID: d.name,
+ Name: d.name,
+ Port: availablePort(),
+ })
+}
+
+type supervisor interface {
+ Running() (bool, error)
+ Start() error
+}
+
+func (s *Setup) superviseDaemon(d daemon) supervisor {
+ switch s.DaemonSupervisor {
+ case "runit":
+ return &runitService{daemon: d, etcsv: s.RunitSvDir}
+ case "systemd":
+ return &systemdSupervisor{daemon: d}
+ default:
+ log.Fatalf("unknown DaemonSupervisor %q", s.DaemonSupervisor)
+ return nil
+ }
+}
+
+func availablePort() int {
+ return rand.Intn(10000) + 20000
+}
diff --git a/lib/setup/download.go b/lib/setup/download.go
new file mode 100644
index 0000000..550f6cb
--- /dev/null
+++ b/lib/setup/download.go
@@ -0,0 +1,130 @@
+package setup
+
+import (
+ "archive/zip"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+)
+
+type download struct {
+ URL string
+ Dest string
+ Size int64
+ Mode os.FileMode
+ Hash string
+ PreloadDir string
+}
+
+func (d *download) install() error {
+ fi, err := os.Stat(d.Dest)
+ if os.IsNotExist(err) {
+ // fall through to fix
+ } else if err != nil {
+ return err
+ } else if d.Size > 0 && fi.Size() != d.Size {
+ err = fmt.Errorf("Size mismatch: %q is %d bytes, expected %d", d.Dest, fi.Size(), d.Size)
+ } else if d.Mode > 0 && fi.Mode() != d.Mode {
+ err = fmt.Errorf("Mode mismatch: %q is %s, expected %s", d.Dest, fi.Mode(), d.Mode)
+ } else {
+ return nil
+ }
+
+ out, err := ioutil.TempFile(path.Dir(d.Dest), path.Base(d.Dest))
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if out != nil {
+ os.Remove(out.Name())
+ out.Close()
+ }
+ }()
+
+ var size int64
+ {
+ got := false
+ if d.PreloadDir != "" {
+ fn := path.Join(d.PreloadDir, path.Base(d.URL))
+ f, err := os.Open(fn)
+ defer f.Close()
+ if err == nil {
+ size, err = io.Copy(out, f)
+ if err != nil {
+ return err
+ }
+ got = true
+ }
+ }
+ if !got {
+ resp, err := http.Get(d.URL)
+ if err != nil {
+ return err
+ }
+ size, err = io.Copy(out, resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ if strings.HasSuffix(d.URL, ".zip") && !strings.HasSuffix(d.Dest, ".zip") {
+ r, err := zip.NewReader(out, size)
+ if err != nil {
+ return err
+ }
+ defer os.Remove(out.Name())
+ out = nil
+
+ found := false
+ for _, f := range r.File {
+ if !strings.HasSuffix(d.Dest, "/"+f.Name) {
+ continue
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer rc.Close()
+
+ out, err = ioutil.TempFile(path.Dir(d.Dest), path.Base(d.Dest))
+ if err != nil {
+ return err
+ }
+
+ size, err = io.Copy(out, rc)
+ if err != nil {
+ return err
+ }
+ found = true
+ break
+ }
+ if !found {
+ return fmt.Errorf("File not found in archive")
+ }
+ }
+
+ if d.Size > 0 && d.Size != size {
+ return fmt.Errorf("Size mismatch: got %d bytes, expected %d", size, d.Size)
+ } else if d.Size == 0 {
+ log.Printf("%v: size was %d", d, size)
+ }
+ if err = out.Close(); err != nil {
+ return err
+ }
+ if err = os.Chmod(out.Name(), d.Mode); err != nil {
+ return err
+ }
+ err = os.Rename(out.Name(), d.Dest)
+ if err == nil {
+ // skip deferred os.Remove(out.Name())
+ out = nil
+ }
+ return err
+}
diff --git a/lib/setup/os_package.go b/lib/setup/os_package.go
new file mode 100644
index 0000000..106945b
--- /dev/null
+++ b/lib/setup/os_package.go
@@ -0,0 +1,57 @@
+package setup
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+)
+
+type osPackage struct {
+ Debian string
+ RedHat string
+}
+
+var (
+ osPackageMutex sync.Mutex
+ osPackageDidUpdate bool
+)
+
+func (pkg *osPackage) install() error {
+ osPackageMutex.Lock()
+ defer osPackageMutex.Unlock()
+
+ if _, err := os.Stat("/var/lib/dpkg/info/" + pkg.Debian + ".list"); err == nil {
+ return nil
+ }
+ if !osPackageDidUpdate {
+ d, err := os.Open("/var/lib/apt/lists")
+ if err != nil {
+ return err
+ }
+ defer d.Close()
+ if files, err := d.Readdir(4); len(files) < 4 || err != nil {
+ err = pkg.aptGet("update")
+ if err != nil {
+ return err
+ }
+ osPackageDidUpdate = true
+ }
+ }
+ return pkg.aptGet("install", "-y", "--no-install-recommends", pkg.Debian)
+}
+
+func (*osPackage) aptGet(args ...string) error {
+ cmd := command("apt-get", args...)
+ for _, kv := range os.Environ() {
+ if !strings.HasPrefix(kv, "DEBIAN_FRONTEND=") {
+ cmd.Env = append(cmd.Env, kv)
+ }
+ }
+ cmd.Env = append(cmd.Env, "DEBIAN_FRONTEND=noninteractive")
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("%s: %s", cmd.Args, err)
+ }
+ return nil
+}
diff --git a/lib/setup/runit.go b/lib/setup/runit.go
new file mode 100644
index 0000000..3e9f347
--- /dev/null
+++ b/lib/setup/runit.go
@@ -0,0 +1,41 @@
+package setup
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "path"
+)
+
+func (s *Setup) installRunit() error {
+ if s.DaemonSupervisor != "runit" {
+ return nil
+ }
+ return (&osPackage{Debian: "runit"}).install()
+}
+
+type runitService struct {
+ daemon
+ etcsv string
+}
+
+func (r *runitService) Start() error {
+ script := &bytes.Buffer{}
+ fmt.Fprintf(script, "#!/bin/sh\n\nexec %q", r.prog)
+ for _, arg := range r.args {
+ fmt.Fprintf(script, " %q", arg)
+ }
+ fmt.Fprintf(script, " 2>&1\n")
+ return atomicWriteFile(path.Join(r.svdir(), "run"), script.Bytes(), 0755)
+}
+
+func (r *runitService) Running() (bool, error) {
+ if _, err := os.Stat(r.svdir()); err != nil && os.IsNotExist(err) {
+ return false, nil
+ }
+ return runStatusCmd("sv", "stat", r.svdir())
+}
+
+func (r *runitService) svdir() string {
+ return path.Join(r.etcsv, r.name)
+}
diff --git a/lib/setup/setup.go b/lib/setup/setup.go
new file mode 100644
index 0000000..379ccb2
--- /dev/null
+++ b/lib/setup/setup.go
@@ -0,0 +1,79 @@
+package setup
+
+import (
+ "flag"
+ "fmt"
+ "log"
+ "os"
+
+ "git.curoverse.com/arvados.git/lib/agent"
+ "git.curoverse.com/arvados.git/sdk/go/config"
+ vaultAPI "github.com/hashicorp/vault/api"
+)
+
+func Command() *Setup {
+ hostname, err := os.Hostname()
+ if err != nil {
+ log.Fatalf("hostname: %s", err)
+ }
+
+ return &Setup{
+ Agent: agent.Command(),
+ LANHost: hostname,
+ PreloadDir: "/var/cache/arvados",
+ }
+}
+
+type Setup struct {
+ *agent.Agent
+ InitVault bool
+ LANHost string
+ PreloadDir string
+
+ encryptKey string
+ masterToken string
+ vaultCfg *vaultAPI.Config
+}
+
+func (s *Setup) ParseFlags(args []string) error {
+ fs := flag.NewFlagSet("setup", flag.ContinueOnError)
+ fs.StringVar(&s.ClusterID, "cluster-id", s.ClusterID, "five-character cluster ID")
+ fs.BoolVar(&s.InitVault, "init-vault", s.InitVault, "initialize the vault if needed")
+ fs.BoolVar(&s.Unseal, "unseal", s.Unseal, "unseal the vault automatically")
+ return fs.Parse(args)
+}
+
+func (s *Setup) Run() error {
+ err := config.LoadFile(s, s.DefaultConfigFile())
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ for _, f := range []func() error{
+ s.makeDirs,
+ (&osPackage{Debian: "ca-certificates"}).install,
+ (&osPackage{Debian: "nginx"}).install,
+ s.installRunit,
+ s.installConsul,
+ s.installVault,
+ } {
+ err := f()
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *Setup) makeDirs() error {
+ for _, path := range []string{s.DataDir, s.UsrDir, s.UsrDir + "/bin"} {
+ if fi, err := os.Stat(path); err != nil {
+ err = os.MkdirAll(path, 0755)
+ if err != nil {
+ return err
+ }
+ } else if !fi.IsDir() {
+ return fmt.Errorf("%s: is not a directory", path)
+ }
+ }
+ return nil
+}
diff --git a/lib/setup/systemd.go b/lib/setup/systemd.go
new file mode 100644
index 0000000..bc3502f
--- /dev/null
+++ b/lib/setup/systemd.go
@@ -0,0 +1,20 @@
+package setup
+
+import "fmt"
+
+type systemdSupervisor struct {
+ daemon
+}
+
+func (ss *systemdSupervisor) Start() error {
+ cmd := command("systemd-run", append([]string{"--unit=arvados-" + ss.name, ss.prog}, ss.args...)...)
+ err := cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("systemd-run: %s", err)
+ }
+ return err
+}
+
+func (ss *systemdSupervisor) Running() (bool, error) {
+ return runStatusCmd("systemctl", "status", "arvados-"+ss.name)
+}
diff --git a/lib/setup/vault.go b/lib/setup/vault.go
new file mode 100644
index 0000000..8443764
--- /dev/null
+++ b/lib/setup/vault.go
@@ -0,0 +1,244 @@
+package setup
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ consulAPI "github.com/hashicorp/consul/api"
+ vaultAPI "github.com/hashicorp/vault/api"
+)
+
+func (s *Setup) installVault() error {
+ if err := s.consulInit(); err != nil {
+ return err
+ }
+ if err := s.vaultInit(); err != nil {
+ return err
+ }
+ if s.vaultCheck() == nil {
+ return nil
+ }
+
+ log.Printf("download & install vault")
+ bin := s.UsrDir + "/bin/vault"
+ err := (&download{
+ URL: "https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_linux_amd64.zip",
+ Dest: bin,
+ Size: 52518022,
+ Mode: 0755,
+ PreloadDir: s.PreloadDir,
+ }).install()
+ if err != nil {
+ return err
+ }
+
+ haAddr := fmt.Sprintf("http://%s:%d", s.LANHost, s.Ports.VaultServer)
+
+ cfgPath := path.Join(s.DataDir, "vault.hcl")
+ err = atomicWriteFile(cfgPath, []byte(fmt.Sprintf(`
+ cluster_name = %q
+ backend "consul" {
+ address = "127.0.0.1:%d"
+ redirect_addr = %q
+ cluster_addr = %q
+ path = %q
+ token = %q
+ }
+ listener "tcp" {
+ address = %q
+ tls_disable = 1
+ }
+ `,
+ s.ClusterID,
+ s.Ports.ConsulHTTP,
+ haAddr,
+ haAddr,
+ "vault-"+s.ClusterID+"/",
+ s.masterToken,
+ fmt.Sprintf("%s:%d", s.LANHost, s.Ports.VaultServer),
+ )), 0600)
+ if err != nil {
+ return err
+ }
+
+ args := []string{"server", "-config=" + cfgPath}
+ err = s.installService(daemon{
+ name: "arvados-vault",
+ prog: bin,
+ args: args,
+ noRegister: true,
+ })
+ if err != nil {
+ return err
+ }
+
+ if !s.Unseal {
+ return nil
+ }
+
+ if err := s.vaultBootstrap(); err != nil {
+ return err
+ }
+ return waitCheck(30*time.Second, s.vaultCheck)
+}
+
+func (s *Setup) vaultBootstrap() error {
+ var vault *vaultAPI.Client
+ var initialized bool
+ resp := &vaultAPI.InitResponse{}
+ if err := waitCheck(time.Minute, func() error {
+ var err error
+ vault, err = s.vaultClient()
+ if err != nil {
+ return err
+ }
+ initialized, err = vault.Sys().InitStatus()
+ if err != nil {
+ return err
+ } else if s.InitVault {
+ return nil
+ }
+ _, err = os.Stat(path.Join(s.DataDir, "vault", "mgmt-token.txt"))
+ if err != nil {
+ log.Print("vault is not initialized, waiting")
+ return fmt.Errorf("vault is not initialized")
+ }
+ return nil
+ }); err != nil {
+ return err
+ } else if !initialized && s.InitVault {
+ resp, err = vault.Sys().Init(&vaultAPI.InitRequest{
+ SecretShares: 5,
+ SecretThreshold: 3,
+ })
+ if err != nil {
+ return fmt.Errorf("vault-init: %s", err)
+ }
+ atomicWriteJSON(path.Join(s.DataDir, "vault", "keys.json"), resp, 0400)
+ atomicWriteFile(path.Join(s.DataDir, "vault", "root-token.txt"), []byte(resp.RootToken), 0400)
+ } else {
+ j, err := ioutil.ReadFile(path.Join(s.DataDir, "vault", "keys.json"))
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(j, resp)
+ if err != nil {
+ return err
+ }
+ }
+ vault.SetToken(resp.RootToken)
+
+ ok := false
+ for _, key := range resp.Keys {
+ resp, err := vault.Sys().Unseal(key)
+ if err != nil {
+ log.Printf("error: unseal: %s", err)
+ continue
+ }
+ if !resp.Sealed {
+ log.Printf("unseal successful")
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ return fmt.Errorf("vault unseal failed!")
+ }
+
+ if s.InitVault {
+ // Use master token to create a management token
+ master, err := s.consulMaster()
+ if err != nil {
+ return err
+ }
+ mgmtToken, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
+ if err != nil {
+ return err
+ }
+
+ // Mount+configure consul backend
+ alreadyMounted := false
+ if err = waitCheck(30*time.Second, func() error {
+ // Typically this first fails "500 node not active but
+ // active node not found" but then succeeds.
+ err := vault.Sys().Mount("consul", &vaultAPI.MountInput{Type: "consul"})
+ if err != nil && strings.Contains(err.Error(), "existing mount at consul") {
+ alreadyMounted = true
+ err = nil
+ }
+ return err
+ }); err != nil {
+ return err
+ }
+ _, err = vault.Logical().Write("consul/config/access", map[string]interface{}{
+ "address": fmt.Sprintf("127.0.0.1:%d", s.Ports.ConsulHTTP),
+ "token": string(mgmtToken),
+ })
+ if err != nil {
+ return err
+ }
+
+ // Create a role
+ _, err = vault.Logical().Write("consul/roles/write-all", map[string]interface{}{
+ "policy": base64.StdEncoding.EncodeToString([]byte(`key "" { policy = "write" }`)),
+ })
+ if err != nil {
+ return err
+ }
+
+ // Write mgmtToken after bootstrapping is done. If
+ // other nodes share our vault data dir, this is their
+ // signal to try unseal.
+ if err = atomicWriteFile(path.Join(s.DataDir, "vault", "mgmt-token.txt"), []byte(mgmtToken), 0400); err != nil {
+ return err
+ }
+ }
+
+ // Test: generate a new token with the write-all role
+ secret, err := vault.Logical().Read("consul/creds/write-all")
+ if err != nil {
+ return err
+ }
+ token, ok := secret.Data["token"].(string)
+ if !ok {
+ return fmt.Errorf("secret token broken?? %+v", secret)
+ }
+ log.Printf("Vault supplied token with lease duration %s (renewable=%v): %q", time.Duration(secret.LeaseDuration)*time.Second, secret.Renewable, token)
+
+ return nil
+}
+
+func (s *Setup) vaultInit() error {
+ s.vaultCfg = vaultAPI.DefaultConfig()
+ s.vaultCfg.Address = fmt.Sprintf("http://%s:%d", s.LANHost, s.Ports.VaultServer)
+ return nil
+}
+
+func (s *Setup) vaultClient() (*vaultAPI.Client, error) {
+ return vaultAPI.NewClient(s.vaultCfg)
+}
+
+func (s *Setup) vaultCheck() error {
+ vault, err := s.vaultClient()
+ if err != nil {
+ return err
+ }
+ token, err := ioutil.ReadFile(path.Join(s.DataDir, "vault", "root-token.txt"))
+ if err != nil {
+ return err
+ }
+ vault.SetToken(string(token))
+ if init, err := vault.Sys().InitStatus(); err != nil {
+ return err
+ } else if !init {
+ return fmt.Errorf("vault is not initialized")
+ }
+ return nil
+}
diff --git a/lib/setup/write_file.go b/lib/setup/write_file.go
new file mode 100644
index 0000000..b50079c
--- /dev/null
+++ b/lib/setup/write_file.go
@@ -0,0 +1,49 @@
+package setup
+
+import (
+ "encoding/json"
+ "io/ioutil"
+ "os"
+ "path"
+)
+
+func atomicWriteFile(name string, data []byte, mode os.FileMode) error {
+ if err := os.MkdirAll(path.Dir(name), 0755); err != nil {
+ return err
+ }
+ tmp, err := ioutil.TempFile(path.Dir(name), path.Base(name)+"~")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if tmp != nil {
+ os.Remove(tmp.Name())
+ }
+ }()
+ _, err = tmp.Write(data)
+ if err != nil {
+ return err
+ }
+ err = tmp.Close()
+ if err != nil {
+ return err
+ }
+ err = os.Chmod(tmp.Name(), mode)
+ if err != nil {
+ return err
+ }
+ err = os.Rename(tmp.Name(), name)
+ if err != nil {
+ return err
+ }
+ tmp = nil
+ return nil
+}
+
+func atomicWriteJSON(name string, data interface{}, mode os.FileMode) error {
+ j, err := json.MarshalIndent(data, "", " ")
+ if err != nil {
+ return err
+ }
+ return atomicWriteFile(name, j, mode)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list