[ARVADOS] updated: ceb95004bcc5653ae7f496657a68c6d12cbfe5e3
Git user
git at public.curoverse.com
Mon Feb 27 04:01:47 EST 2017
Summary of changes:
.../app/views/work_units/_show_child.html.erb | 4 +-
build/run-tests.sh | 14 +-
cmd/arvados-admin/setup_docker_compose_test.go | 2 +-
.../test-docker-compose/docker-compose.yml | 11 +-
doc/README.textile | 1 +
doc/_config.yml | 1 +
doc/api/methods.html.textile.liquid | 4 +-
.../methods/authorized_keys.html.textile.liquid | 2 +-
.../methods/container_requests.html.textile.liquid | 15 +-
.../install-dispatch.html.textile.liquid | 2 +-
.../install-keep-balance.html.textile.liquid | 166 ++++++++++++++
doc/install/install-keepproxy.html.textile.liquid | 10 +-
doc/install/install-keepstore.html.textile.liquid | 1 +
doc/user/topics/arv-docker.html.textile.liquid | 2 +-
doc/user/topics/arv-run.html.textile.liquid | 2 +-
lib/setup/consul.go | 84 ++++---
lib/setup/setup.go | 14 ++
lib/setup/vault.go | 244 +++++++++++++++++++++
sdk/cli/bin/crunch-job | 5 +-
sdk/cli/test/binstub_output_coll_owner/python | 4 +
sdk/cli/test/test_crunch-job.rb | 9 +
sdk/cwl/arvados_cwl/__init__.py | 4 +
sdk/go/arvadostest/fixtures.go | 2 +
sdk/go/dispatch/dispatch.go | 42 +++-
sdk/go/dispatch/dispatch_test.go | 43 ++++
sdk/go/dispatch/throttle_test.go | 6 +-
sdk/python/arvados/arvfile.py | 25 ++-
sdk/python/arvados/collection.py | 50 +++--
sdk/python/tests/run_test_server.py | 9 +
services/api/app/models/arvados_model.rb | 6 +-
services/api/app/models/container.rb | 53 ++---
services/api/app/models/container_request.rb | 8 +-
services/api/app/models/node.rb | 4 +-
services/api/app/models/workflow.rb | 4 +
services/api/config/initializers/time_format.rb | 6 +
..._and_workflow_def_in_full_text_search_index.rb} | 17 +-
services/api/db/structure.sql | 138 ++++++------
services/api/lib/crunch_dispatch.rb | 81 ++++---
services/api/lib/has_uuid.rb | 4 +-
services/api/lib/sweep_trashed_collections.rb | 3 +
services/api/test/fixtures/collections.yml | 30 +++
services/api/test/unit/collection_test.rb | 16 ++
services/api/test/unit/container_test.rb | 41 +++-
services/api/test/unit/crunch_dispatch_test.rb | 20 ++
services/api/test/unit/fail_jobs_test.rb | 6 +-
services/api/test/unit/link_test.rb | 24 +-
services/api/test/unit/log_test.rb | 4 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 22 ++
services/crunch-dispatch-slurm/squeue.go | 25 ++-
services/crunch-run/crunchrun.go | 14 +-
services/fuse/arvados_fuse/__init__.py | 9 +-
services/fuse/arvados_fuse/command.py | 6 +
.../arvnodeman/computenode/dispatch/__init__.py | 2 +
.../arvnodeman/computenode/dispatch/slurm.py | 18 +-
services/nodemanager/arvnodeman/daemon.py | 6 +-
services/nodemanager/arvnodeman/jobqueue.py | 20 +-
services/nodemanager/arvnodeman/launcher.py | 2 +-
services/nodemanager/arvnodeman/nodelist.py | 42 +++-
.../nodemanager/tests/test_computenode_dispatch.py | 8 +-
.../tests/test_computenode_dispatch_slurm.py | 15 +-
services/nodemanager/tests/test_jobqueue.py | 19 +-
services/nodemanager/tests/test_nodelist.py | 45 +++-
62 files changed, 1192 insertions(+), 304 deletions(-)
create mode 100644 doc/install/install-keep-balance.html.textile.liquid
create mode 100644 lib/setup/vault.go
create mode 100755 sdk/cli/test/binstub_output_coll_owner/python
create mode 100644 sdk/go/dispatch/dispatch_test.go
copy services/api/db/migrate/{20161213172944_full_text_search_indexes.rb => 20170216170823_no_cr_mounts_and_workflow_def_in_full_text_search_index.rb} (55%)
via ceb95004bcc5653ae7f496657a68c6d12cbfe5e3 (commit)
via 307e7af02a89f35e768b1cce368593eb64cdcb3d (commit)
via a4512ef71ae5cb208ad987af0a1114d0c645dcb8 (commit)
via e47bb9c1d07dc67d0a3609a494a91c32ecbe6d85 (commit)
via 17f4a03d71d8d6130f796d61fa49b8480bf555b6 (commit)
via dccfd637a120e10166a4d9f6b75381c69121f24c (commit)
via 353eb4f4c4fb52e7f2a1c9aaad93e9d6bf0088f4 (commit)
via 9b2aa42213a7d333bbe93e040c2d152a70e9b5af (commit)
via cca78a112d6a2aa4f76c1956cfe8ec2d43a68759 (commit)
via b86728d23fa94fa80460be443e3e963376bc47d9 (commit)
via 00d95f0f330799a816214f3f329f8d8cf7e2a447 (commit)
via c702045c2d8ab4ce332b279018ed3128a688be6c (commit)
via 1a06dafaa1eb33b62abbbf0ef193b3bbc496255c (commit)
via a04b28a606d52b066a00d4e960bf351df5cb9c6f (commit)
via a6e4bcccfd646843ba35c9a79367f9bbd7124f8d (commit)
via 6029ffdc433d69f7fe139b0e23673fa12715e413 (commit)
via a6c7c8db01d37534622763dd385019e9dad17181 (commit)
via e2f2a9dd7d533fb3863926825628ca6171fef716 (commit)
via 19dcf1ed20214b8349c76037ba8679db07bf44fc (commit)
via 1e6a756a10a1c0a77aeea5041844ba3a572bdd70 (commit)
via 2a30eaccec6497cce95413dbae1fcddbdfdd728f (commit)
via aedb62c77a43e10dbeddd0a0fb428f42c4b61ce4 (commit)
via fe85406fbaec9952cba8350711366d95e05f869f (commit)
via 04fb9a1407a154f0d86d67b2bd2e35c07a6a9989 (commit)
via 25af6c40181b95f13ec3e9e366c53cb50868d065 (commit)
via 3a31db02591a2f57d51b98ba9add7d835c5c6c26 (commit)
via 0c529ed05805507b4d2c903b9587e9b61cec5ee6 (commit)
via d8f1b7e8105b2c879956a60ebdcf91bbca8bcbe3 (commit)
via 11f5e94cfe77fd61c4220679eceb4a09d3c4139a (commit)
via ced8b355e88199516443aeb7504ebaf57f2df82c (commit)
via d2c9ca9783955cef851d33639cae36044941abb3 (commit)
via 423d27d0d7439e95dd8ef6b1dbfe890055cc0fa9 (commit)
via fc54d2d34ecf61619b7f8c0a92e01c6094be968b (commit)
via 6159bebd68231a80514463f8b019eba055596a26 (commit)
via 802af81e13dd11a7f2d9796a2ada8faf3b722477 (commit)
via 1732df243e8ddc1eab0eca157e4b83bd8079f774 (commit)
via 2b34839cdf95291a7356554e05e50b9ced177dd6 (commit)
via 14304c7af0b0dd7bc9345b6c5aeb61a3bdc1d3b0 (commit)
via 8b75947ee3f99b87eec443763653ca6ae3eb21e1 (commit)
via 4bd54c4c42f13434b84410b7917a8cd208d613d7 (commit)
via 4f0e07d462b7860bb10686c27fac16970220377f (commit)
via 4cd89bd1767bece226c412ae7c9ea37669e8706b (commit)
via 84bd6883a5bab065c88d4ed495b8c03ab7ba5f97 (commit)
via d069de03a99dc58fd38f241435fcbaac84e9f63a (commit)
via d0ba6eaac1387ad817de1c2df2d7f4f00800aaa5 (commit)
via b1fffbeb4e06d0ec36c41c2fd9a0f23871f081c5 (commit)
via 484370cbfe47b04e1d4222dd4a7606171c87a324 (commit)
via f29e958f50a914504f971d344c93ee7297d77fbb (commit)
via 85887cd7fed798345e340480062b8ffcf3cf053a (commit)
via f6071ef7bc7f6b7308c202e330cabd4ca111aadd (commit)
via 3fa4a2b6138e3e9e468dd885a743ca38f08f0755 (commit)
via ed1acd6fd780467ba69998e76e28fda61beedc0f (commit)
via 24b1aecb485b170c6e127251201a6bac87d7860a (commit)
via d5ddfd9d876a75327795793544d105051f2a306e (commit)
via 3a6966e5997ed5de342947759042ec5584f770c6 (commit)
via f6bdb550ec87fd38f528f5eb67925d6bcf5af22f (commit)
via 21598295f38998d8028aaa117f192de6b5758808 (commit)
via 425f9a753e4390215b95b794785efd38bbc5f42d (commit)
via be33e85dd22b898e3a8f27ba8b42d9faef6e8516 (commit)
via 73c2bd55dc6e13891c6d1be63fc0613728bd929a (commit)
via 72873affa7f249faa16d5d21200e935d27aea911 (commit)
via dc086bed8661d0e7579df7a240f7e67c86946a59 (commit)
via 197dd5583c13fb6cddba8d74848df20ba57ed924 (commit)
via 2d80bb7d83bda777181542afcb0b7293cb53eefd (commit)
via 1aa3f43606ba5e31633c063851d25b8fab4b93e2 (commit)
via 0403a0c816df1edea311b9197147fd254d131712 (commit)
via fb60dbc9b79cfa8f34a10909ea0fd2a51b600ce6 (commit)
via 314f3c34442e0daffbea4ed62adea9e722673810 (commit)
via 15c1329834e02814fb5b510bf13f1ea2a97ee028 (commit)
via 264ffa31bae106bb6c36643e13186289b6cd0e18 (commit)
via 0e6c1a287933d8e55508c2457b7ce31a2bb5a965 (commit)
via 908959eafe8a9925cb2c204d6511095f702c1667 (commit)
via 83c8f1685d812c31d8bd568f3c2ac1edcd120aed (commit)
via f14ce11321e919cc39b878fe9f7847e1a9bb0de3 (commit)
via bf9af31f49954d949317475bdcdc0694d247f82d (commit)
via 4440f049a94bd3570271f8f0b2461d7f3f3c4582 (commit)
from cea3337f1d104150d3314e43bc1c07eef0851bc5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ceb95004bcc5653ae7f496657a68c6d12cbfe5e3
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 27 03:59:22 2017 -0500
more vault
diff --git a/cmd/arvados-admin/setup_docker_compose_test.go b/cmd/arvados-admin/setup_docker_compose_test.go
index d949604..e88dc16 100644
--- a/cmd/arvados-admin/setup_docker_compose_test.go
+++ b/cmd/arvados-admin/setup_docker_compose_test.go
@@ -9,7 +9,7 @@ import (
func TestSetupDockerCompose(t *testing.T) {
for _, cmdline := range [][]string{
{"go", "build"},
- {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "down"},
+ {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "down", "-v"},
{"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "up"},
} {
cmd := exec.Command(cmdline[0], cmdline[1:]...)
diff --git a/lib/setup/setup.go b/lib/setup/setup.go
index 0170c96..379ccb2 100644
--- a/lib/setup/setup.go
+++ b/lib/setup/setup.go
@@ -3,6 +3,7 @@ package setup
import (
"flag"
"fmt"
+ "log"
"os"
"git.curoverse.com/arvados.git/lib/agent"
@@ -11,8 +12,14 @@ import (
)
func Command() *Setup {
+ hostname, err := os.Hostname()
+ if err != nil {
+ log.Fatalf("hostname: %s", err)
+ }
+
return &Setup{
Agent: agent.Command(),
+ LANHost: hostname,
PreloadDir: "/var/cache/arvados",
}
}
@@ -20,6 +27,7 @@ func Command() *Setup {
type Setup struct {
*agent.Agent
InitVault bool
+ LANHost string
PreloadDir string
encryptKey string
diff --git a/lib/setup/vault.go b/lib/setup/vault.go
index c2bfc88..8443764 100644
--- a/lib/setup/vault.go
+++ b/lib/setup/vault.go
@@ -7,7 +7,6 @@ import (
"io/ioutil"
"log"
"os"
- "os/exec"
"path"
"strings"
"time"
@@ -40,12 +39,7 @@ func (s *Setup) installVault() error {
return err
}
- buf, err := exec.Command("hostname").Output()
- if err != nil {
- return err
- }
- host := strings.TrimSpace(string(buf))
- haAddr := fmt.Sprintf("http://%s:%d", host, s.Ports.VaultServer)
+ haAddr := fmt.Sprintf("http://%s:%d", s.LANHost, s.Ports.VaultServer)
cfgPath := path.Join(s.DataDir, "vault.hcl")
err = atomicWriteFile(cfgPath, []byte(fmt.Sprintf(`
@@ -68,7 +62,7 @@ func (s *Setup) installVault() error {
haAddr,
"vault-"+s.ClusterID+"/",
s.masterToken,
- fmt.Sprintf("%s:%d", host, s.Ports.VaultServer),
+ fmt.Sprintf("%s:%d", s.LANHost, s.Ports.VaultServer),
)), 0600)
if err != nil {
return err
@@ -97,7 +91,7 @@ func (s *Setup) installVault() error {
func (s *Setup) vaultBootstrap() error {
var vault *vaultAPI.Client
- var init bool
+ var initialized bool
resp := &vaultAPI.InitResponse{}
if err := waitCheck(time.Minute, func() error {
var err error
@@ -105,13 +99,13 @@ func (s *Setup) vaultBootstrap() error {
if err != nil {
return err
}
- init, err = vault.Sys().InitStatus()
+ initialized, err = vault.Sys().InitStatus()
if err != nil {
return err
} else if s.InitVault {
return nil
}
- _, err = os.Stat(path.Join(s.DataDir, "vault", "keys.json"))
+ _, err = os.Stat(path.Join(s.DataDir, "vault", "mgmt-token.txt"))
if err != nil {
log.Print("vault is not initialized, waiting")
return fmt.Errorf("vault is not initialized")
@@ -119,7 +113,7 @@ func (s *Setup) vaultBootstrap() error {
return nil
}); err != nil {
return err
- } else if !init {
+ } else if !initialized && s.InitVault {
resp, err = vault.Sys().Init(&vaultAPI.InitRequest{
SecretShares: 5,
SecretThreshold: 3,
@@ -158,44 +152,56 @@ func (s *Setup) vaultBootstrap() error {
return fmt.Errorf("vault unseal failed!")
}
- // Use master token to create a management token
- master, err := s.consulMaster()
- if err != nil {
- return err
- }
- mgmtToken, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
- if err != nil {
- return err
- }
- if err = atomicWriteFile(path.Join(s.DataDir, "vault", "mgmt-token.txt"), []byte(mgmtToken), 0400); err != nil {
- return err
- }
+ if s.InitVault {
+ // Use master token to create a management token
+ master, err := s.consulMaster()
+ if err != nil {
+ return err
+ }
+ mgmtToken, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
+ if err != nil {
+ return err
+ }
- // Mount+configure consul backend
- if err = waitCheck(30*time.Second, func() error {
- // Typically this first fails "500 node not active but
- // active node not found" but then succeeds.
- return vault.Sys().Mount("consul", &vaultAPI.MountInput{Type: "consul"})
- }); err != nil {
- return err
- }
- _, err = vault.Logical().Write("consul/config/access", map[string]interface{}{
- "address": fmt.Sprintf("127.0.0.1:%d", s.Ports.ConsulHTTP),
- "token": string(mgmtToken),
- })
- if err != nil {
- return err
- }
+ // Mount+configure consul backend
+ alreadyMounted := false
+ if err = waitCheck(30*time.Second, func() error {
+ // Typically this first fails "500 node not active but
+ // active node not found" but then succeeds.
+ err := vault.Sys().Mount("consul", &vaultAPI.MountInput{Type: "consul"})
+ if err != nil && strings.Contains(err.Error(), "existing mount at consul") {
+ alreadyMounted = true
+ err = nil
+ }
+ return err
+ }); err != nil {
+ return err
+ }
+ _, err = vault.Logical().Write("consul/config/access", map[string]interface{}{
+ "address": fmt.Sprintf("127.0.0.1:%d", s.Ports.ConsulHTTP),
+ "token": string(mgmtToken),
+ })
+ if err != nil {
+ return err
+ }
- // Create a role
- _, err = vault.Logical().Write("consul/roles/write-all", map[string]interface{}{
- "policy": base64.StdEncoding.EncodeToString([]byte(`key "" { policy = "write" }`)),
- })
- if err != nil {
- return err
+ // Create a role
+ _, err = vault.Logical().Write("consul/roles/write-all", map[string]interface{}{
+ "policy": base64.StdEncoding.EncodeToString([]byte(`key "" { policy = "write" }`)),
+ })
+ if err != nil {
+ return err
+ }
+
+ // Write mgmtToken after bootstrapping is done. If
+ // other nodes share our vault data dir, this is their
+ // signal to try unseal.
+ if err = atomicWriteFile(path.Join(s.DataDir, "vault", "mgmt-token.txt"), []byte(mgmtToken), 0400); err != nil {
+ return err
+ }
}
- // Generate a new token with the write-all role
+ // Test: generate a new token with the write-all role
secret, err := vault.Logical().Read("consul/creds/write-all")
if err != nil {
return err
@@ -205,16 +211,17 @@ func (s *Setup) vaultBootstrap() error {
return fmt.Errorf("secret token broken?? %+v", secret)
}
log.Printf("Vault supplied token with lease duration %s (renewable=%v): %q", time.Duration(secret.LeaseDuration)*time.Second, secret.Renewable, token)
+
return nil
}
func (s *Setup) vaultInit() error {
s.vaultCfg = vaultAPI.DefaultConfig()
+ s.vaultCfg.Address = fmt.Sprintf("http://%s:%d", s.LANHost, s.Ports.VaultServer)
return nil
}
func (s *Setup) vaultClient() (*vaultAPI.Client, error) {
- s.vaultCfg.Address = fmt.Sprintf("http://0.0.0.0:%d", s.Ports.VaultServer)
return vaultAPI.NewClient(s.vaultCfg)
}
commit 307e7af02a89f35e768b1cce368593eb64cdcb3d
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Feb 26 05:12:59 2017 -0500
vault
diff --git a/cmd/arvados-admin/test-docker-compose/docker-compose.yml b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
index 66378e5..34e7cd4 100644
--- a/cmd/arvados-admin/test-docker-compose/docker-compose.yml
+++ b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
@@ -10,7 +10,8 @@ services:
- ./agent.yml:/etc/arvados/agent/agent.yml:ro
- ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
- ./master-token.txt:/var/lib/arvados/master-token.txt:ro
- command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true -init-vault=true && wait"]
sys1:
build: ../test-debian8
cap_add:
@@ -21,7 +22,8 @@ services:
- ./agent.yml:/etc/arvados/agent/agent.yml:ro
- ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
- ./master-token.txt:/var/lib/arvados/master-token.txt:ro
- command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"]
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
sys2:
build: ../test-debian8
cap_add:
@@ -32,4 +34,7 @@ services:
- ./agent.yml:/etc/arvados/agent/agent.yml:ro
- ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
- ./master-token.txt:/var/lib/arvados/master-token.txt:ro
- command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"]
+ - vault:/var/lib/arvados/vault
+ command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
+volumes:
+ vault:
diff --git a/lib/setup/consul.go b/lib/setup/consul.go
index b1682b2..9cc2f29 100644
--- a/lib/setup/consul.go
+++ b/lib/setup/consul.go
@@ -14,7 +14,7 @@ import (
)
func (s *Setup) installConsul() error {
- prog := s.UsrDir + "/bin/consul"
+ prog := path.Join(s.UsrDir, "bin", "consul")
err := (&download{
URL: "https://releases.hashicorp.com/consul/0.7.5/consul_0.7.5_linux_amd64.zip",
Dest: prog,
@@ -25,49 +25,34 @@ func (s *Setup) installConsul() error {
if err != nil {
return err
}
- dataDir := path.Join(s.DataDir, "consul")
- if err := os.MkdirAll(dataDir, 0700); err != nil {
- return err
- }
- keyPath := path.Join(s.DataDir, "encrypt-key.txt")
- key, err := ioutil.ReadFile(keyPath)
- if os.IsNotExist(err) {
- key, err = exec.Command(prog, "keygen").CombinedOutput()
- if err != nil {
- return err
- }
- err = atomicWriteFile(keyPath, key, 0400)
- }
- if err != nil {
+ if err := s.consulInit(); err != nil {
return err
}
- encryptKey := strings.TrimSpace(string(key))
+ if s.consulCheck() == nil {
+ return nil
+ }
- tokPath := path.Join(s.DataDir, "master-token.txt")
- if tok, err := ioutil.ReadFile(tokPath); err != nil {
- s.masterToken = generateToken()
- err = atomicWriteFile(tokPath, []byte(s.masterToken), 0600)
- if err != nil {
- return err
- }
- } else {
- s.masterToken = string(tok)
+ dataDir := path.Join(s.DataDir, "consul")
+ if err := os.MkdirAll(dataDir, 0700); err != nil {
+ return err
}
cf := path.Join(s.DataDir, "consul-config.json")
{
c := map[string]interface{}{
- "acl_datacenter": s.ClusterID,
- "acl_default_policy": "deny",
- "acl_master_token": s.masterToken,
- "bootstrap_expect": len(s.ControlHosts),
- "client_addr": "0.0.0.0",
- "data_dir": dataDir,
- "datacenter": s.ClusterID,
- "encrypt": encryptKey,
- "server": true,
- "ui": true,
+ "acl_agent_token": s.masterToken,
+ "acl_datacenter": s.ClusterID,
+ "acl_default_policy": "deny",
+ "acl_enforce_version_8": true,
+ "acl_master_token": s.masterToken,
+ "bootstrap_expect": len(s.ControlHosts),
+ "client_addr": "0.0.0.0",
+ "data_dir": dataDir,
+ "datacenter": s.ClusterID,
+ "encrypt": s.encryptKey,
+ "server": true,
+ "ui": true,
"ports": map[string]int{
"dns": s.Ports.ConsulDNS,
"http": s.Ports.ConsulHTTP,
@@ -125,6 +110,35 @@ func (s *Setup) consulMaster() (*api.Client, error) {
return api.NewClient(ccfg)
}
+func (s *Setup) consulInit() error {
+ prog := path.Join(s.UsrDir, "bin", "consul")
+ keyPath := path.Join(s.DataDir, "encrypt-key.txt")
+ key, err := ioutil.ReadFile(keyPath)
+ if os.IsNotExist(err) {
+ key, err = exec.Command(prog, "keygen").CombinedOutput()
+ if err != nil {
+ return err
+ }
+ err = atomicWriteFile(keyPath, key, 0400)
+ }
+ if err != nil {
+ return err
+ }
+ s.encryptKey = strings.TrimSpace(string(key))
+
+ tokPath := path.Join(s.DataDir, "master-token.txt")
+ if tok, err := ioutil.ReadFile(tokPath); err != nil {
+ s.masterToken = generateToken()
+ err = atomicWriteFile(tokPath, []byte(s.masterToken), 0600)
+ if err != nil {
+ return err
+ }
+ } else {
+ s.masterToken = string(tok)
+ }
+ return nil
+}
+
func (s *Setup) consulCheck() error {
consul, err := s.consulMaster()
if err != nil {
diff --git a/lib/setup/setup.go b/lib/setup/setup.go
index 935db1a..0170c96 100644
--- a/lib/setup/setup.go
+++ b/lib/setup/setup.go
@@ -7,6 +7,7 @@ import (
"git.curoverse.com/arvados.git/lib/agent"
"git.curoverse.com/arvados.git/sdk/go/config"
+ vaultAPI "github.com/hashicorp/vault/api"
)
func Command() *Setup {
@@ -18,14 +19,18 @@ func Command() *Setup {
type Setup struct {
*agent.Agent
+ InitVault bool
PreloadDir string
+ encryptKey string
masterToken string
+ vaultCfg *vaultAPI.Config
}
func (s *Setup) ParseFlags(args []string) error {
fs := flag.NewFlagSet("setup", flag.ContinueOnError)
fs.StringVar(&s.ClusterID, "cluster-id", s.ClusterID, "five-character cluster ID")
+ fs.BoolVar(&s.InitVault, "init-vault", s.InitVault, "initialize the vault if needed")
fs.BoolVar(&s.Unseal, "unseal", s.Unseal, "unseal the vault automatically")
return fs.Parse(args)
}
@@ -41,6 +46,7 @@ func (s *Setup) Run() error {
(&osPackage{Debian: "nginx"}).install,
s.installRunit,
s.installConsul,
+ s.installVault,
} {
err := f()
if err != nil {
diff --git a/lib/setup/vault.go b/lib/setup/vault.go
new file mode 100644
index 0000000..c2bfc88
--- /dev/null
+++ b/lib/setup/vault.go
@@ -0,0 +1,237 @@
+package setup
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+ "time"
+
+ consulAPI "github.com/hashicorp/consul/api"
+ vaultAPI "github.com/hashicorp/vault/api"
+)
+
+func (s *Setup) installVault() error {
+ if err := s.consulInit(); err != nil {
+ return err
+ }
+ if err := s.vaultInit(); err != nil {
+ return err
+ }
+ if s.vaultCheck() == nil {
+ return nil
+ }
+
+ log.Printf("download & install vault")
+ bin := s.UsrDir + "/bin/vault"
+ err := (&download{
+ URL: "https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_linux_amd64.zip",
+ Dest: bin,
+ Size: 52518022,
+ Mode: 0755,
+ PreloadDir: s.PreloadDir,
+ }).install()
+ if err != nil {
+ return err
+ }
+
+ buf, err := exec.Command("hostname").Output()
+ if err != nil {
+ return err
+ }
+ host := strings.TrimSpace(string(buf))
+ haAddr := fmt.Sprintf("http://%s:%d", host, s.Ports.VaultServer)
+
+ cfgPath := path.Join(s.DataDir, "vault.hcl")
+ err = atomicWriteFile(cfgPath, []byte(fmt.Sprintf(`
+ cluster_name = %q
+ backend "consul" {
+ address = "127.0.0.1:%d"
+ redirect_addr = %q
+ cluster_addr = %q
+ path = %q
+ token = %q
+ }
+ listener "tcp" {
+ address = %q
+ tls_disable = 1
+ }
+ `,
+ s.ClusterID,
+ s.Ports.ConsulHTTP,
+ haAddr,
+ haAddr,
+ "vault-"+s.ClusterID+"/",
+ s.masterToken,
+ fmt.Sprintf("%s:%d", host, s.Ports.VaultServer),
+ )), 0600)
+ if err != nil {
+ return err
+ }
+
+ args := []string{"server", "-config=" + cfgPath}
+ err = s.installService(daemon{
+ name: "arvados-vault",
+ prog: bin,
+ args: args,
+ noRegister: true,
+ })
+ if err != nil {
+ return err
+ }
+
+ if !s.Unseal {
+ return nil
+ }
+
+ if err := s.vaultBootstrap(); err != nil {
+ return err
+ }
+ return waitCheck(30*time.Second, s.vaultCheck)
+}
+
+func (s *Setup) vaultBootstrap() error {
+ var vault *vaultAPI.Client
+ var init bool
+ resp := &vaultAPI.InitResponse{}
+ if err := waitCheck(time.Minute, func() error {
+ var err error
+ vault, err = s.vaultClient()
+ if err != nil {
+ return err
+ }
+ init, err = vault.Sys().InitStatus()
+ if err != nil {
+ return err
+ } else if s.InitVault {
+ return nil
+ }
+ _, err = os.Stat(path.Join(s.DataDir, "vault", "keys.json"))
+ if err != nil {
+ log.Print("vault is not initialized, waiting")
+ return fmt.Errorf("vault is not initialized")
+ }
+ return nil
+ }); err != nil {
+ return err
+ } else if !init {
+ resp, err = vault.Sys().Init(&vaultAPI.InitRequest{
+ SecretShares: 5,
+ SecretThreshold: 3,
+ })
+ if err != nil {
+ return fmt.Errorf("vault-init: %s", err)
+ }
+ atomicWriteJSON(path.Join(s.DataDir, "vault", "keys.json"), resp, 0400)
+ atomicWriteFile(path.Join(s.DataDir, "vault", "root-token.txt"), []byte(resp.RootToken), 0400)
+ } else {
+ j, err := ioutil.ReadFile(path.Join(s.DataDir, "vault", "keys.json"))
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(j, resp)
+ if err != nil {
+ return err
+ }
+ }
+ vault.SetToken(resp.RootToken)
+
+ ok := false
+ for _, key := range resp.Keys {
+ resp, err := vault.Sys().Unseal(key)
+ if err != nil {
+ log.Printf("error: unseal: %s", err)
+ continue
+ }
+ if !resp.Sealed {
+ log.Printf("unseal successful")
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ return fmt.Errorf("vault unseal failed!")
+ }
+
+ // Use master token to create a management token
+ master, err := s.consulMaster()
+ if err != nil {
+ return err
+ }
+ mgmtToken, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
+ if err != nil {
+ return err
+ }
+ if err = atomicWriteFile(path.Join(s.DataDir, "vault", "mgmt-token.txt"), []byte(mgmtToken), 0400); err != nil {
+ return err
+ }
+
+ // Mount+configure consul backend
+ if err = waitCheck(30*time.Second, func() error {
+ // Typically this first fails "500 node not active but
+ // active node not found" but then succeeds.
+ return vault.Sys().Mount("consul", &vaultAPI.MountInput{Type: "consul"})
+ }); err != nil {
+ return err
+ }
+ _, err = vault.Logical().Write("consul/config/access", map[string]interface{}{
+ "address": fmt.Sprintf("127.0.0.1:%d", s.Ports.ConsulHTTP),
+ "token": string(mgmtToken),
+ })
+ if err != nil {
+ return err
+ }
+
+ // Create a role
+ _, err = vault.Logical().Write("consul/roles/write-all", map[string]interface{}{
+ "policy": base64.StdEncoding.EncodeToString([]byte(`key "" { policy = "write" }`)),
+ })
+ if err != nil {
+ return err
+ }
+
+ // Generate a new token with the write-all role
+ secret, err := vault.Logical().Read("consul/creds/write-all")
+ if err != nil {
+ return err
+ }
+ token, ok := secret.Data["token"].(string)
+ if !ok {
+ return fmt.Errorf("secret token broken?? %+v", secret)
+ }
+ log.Printf("Vault supplied token with lease duration %s (renewable=%v): %q", time.Duration(secret.LeaseDuration)*time.Second, secret.Renewable, token)
+ return nil
+}
+
+func (s *Setup) vaultInit() error {
+ s.vaultCfg = vaultAPI.DefaultConfig()
+ return nil
+}
+
+func (s *Setup) vaultClient() (*vaultAPI.Client, error) {
+ s.vaultCfg.Address = fmt.Sprintf("http://0.0.0.0:%d", s.Ports.VaultServer)
+ return vaultAPI.NewClient(s.vaultCfg)
+}
+
+func (s *Setup) vaultCheck() error {
+ vault, err := s.vaultClient()
+ if err != nil {
+ return err
+ }
+ token, err := ioutil.ReadFile(path.Join(s.DataDir, "vault", "root-token.txt"))
+ if err != nil {
+ return err
+ }
+ vault.SetToken(string(token))
+ if init, err := vault.Sys().InitStatus(); err != nil {
+ return err
+ } else if !init {
+ return fmt.Errorf("vault is not initialized")
+ }
+ return nil
+}
commit a4512ef71ae5cb208ad987af0a1114d0c645dcb8
Merge: cea3337 e47bb9c
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Feb 25 19:35:18 2017 -0500
Merge branch 'master' into deploy-agent
Conflicts:
build/run-tests.sh
diff --cc build/run-tests.sh
index 3cbc6b3,44f9a30..759555e
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -553,11 -548,10 +553,11 @@@ do_test_once()
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
# compilation errors.
- cd "$GOPATH/src/${gopkgpath}" || return 1
- go get -t . || return 1
- go generate || return 1
- gofmt -e -d . | egrep . && result=1
- if [[ -n "${testargs[$1]}" ]]
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ cd "$WORKSPACE/$1" && \
++ go generate && \
+ [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
+ if [[ -n "${testargs[$1]}" ]]
then
# "go test -check.vv giturl" doesn't work, but this
# does:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list