[ARVADOS] created: 2.1.0-280-g35b422b07
Git user
git at public.arvados.org
Thu Jan 14 21:19:57 UTC 2021
at 35b422b074dd06dfea090a65b6072b09600302ff (commit)
commit 35b422b074dd06dfea090a65b6072b09600302ff
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 14 16:16:20 2021 -0500
17170: Use exec() to eliminate intermediate arvados-client process.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index 40f40291f..a63089d17 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -58,26 +58,19 @@ Options:
return 2
}
sshargs = append([]string{
+ "ssh",
"-o", "ProxyCommand " + selfbin + " connect-ssh -detach-keys=" + shellescape(*detachKeys) + " " + shellescape(target),
"-o", "StrictHostKeyChecking no",
target},
sshargs...)
- cmd := exec.Command("ssh", sshargs...)
- cmd.Stdin = stdin
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- err = cmd.Run()
- if err == nil {
- return 0
- } else if exiterr, ok := err.(*exec.ExitError); !ok {
- fmt.Fprintln(stderr, err)
- return 1
- } else if status, ok := exiterr.Sys().(syscall.WaitStatus); !ok {
+ sshbin, err := exec.LookPath("ssh")
+ if err != nil {
fmt.Fprintln(stderr, err)
return 1
- } else {
- return status.ExitStatus()
}
+ err = syscall.Exec(sshbin, sshargs, os.Environ())
+ fmt.Fprintf(stderr, "exec(%q) failed: %s\n", sshbin, err)
+ return 1
}
// connectSSHCommand connects stdin/stdout to a container's gateway
commit 452fbd0bf14678f1ceb8e60a8864693e062b89b6
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 14 15:40:07 2021 -0500
17170: Improve error messages.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index f255bff84..807995b3c 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -55,12 +55,17 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
return
}
- ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID})
- if err != nil {
+ switch ctr.State {
+ case arvados.ContainerStateQueued, arvados.ContainerStateLocked:
+ err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusServiceUnavailable)
return
- }
- if ctr.GatewayAddress == "" || ctr.State != arvados.ContainerStateRunning {
- err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusBadGateway)
+ case arvados.ContainerStateRunning:
+ if ctr.GatewayAddress == "" {
+ err = httpserver.ErrorWithStatus(errors.New("container is running but gateway is not available"), http.StatusServiceUnavailable)
+ return
+ }
+ default:
+ err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusGone)
return
}
// crunch-run uses a self-signed / unverifiable TLS
@@ -101,6 +106,7 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
},
})
if err != nil {
+ err = httpserver.ErrorWithStatus(err, http.StatusBadGateway)
return
}
if respondAuth == "" {
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index 0d869ca7f..3764a8a43 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -194,7 +194,7 @@ func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
go ssh.DiscardRequests(reqs)
for newch := range newchans {
if newch.ChannelType() != "session" {
- newch.Reject(ssh.UnknownChannelType, "unknown channel type")
+ newch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unsupported channel type %q", newch.ChannelType()))
continue
}
ch, reqs, err := newch.Accept()
commit 8c283c6262809df6f9db1aa176a1a8a5e95a717b
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 14 15:39:34 2021 -0500
17170: Check container is readable before checking write permission.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index ff3e3de5b..f255bff84 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -34,6 +34,11 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
if err != nil {
return
}
+ ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID})
+ if err != nil {
+ return
+ }
+
ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
crs, err := conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
if err != nil {
commit 879bd007e5133c3124cb91a0fb660dfda9cf4ace
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 14 15:38:38 2021 -0500
17170: Fix -help output.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index 1ed6dc279..40f40291f 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -27,7 +27,7 @@ func (shellCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
f := flag.NewFlagSet(prog, flag.ContinueOnError)
f.SetOutput(stderr)
f.Usage = func() {
- fmt.Print(stderr, prog+`: open an interactive shell on a running container.
+ fmt.Fprint(stderr, prog+`: open an interactive shell on a running container.
Usage: `+prog+` [options] [username@]container-uuid [ssh-options] [remote-command [args...]]
@@ -38,8 +38,7 @@ Options:
detachKeys := f.String("detach-keys", "ctrl-],ctrl-]", "set detach key sequence, as in docker-attach(1)")
err := f.Parse(args)
if err != nil {
- fmt.Println(stderr, err)
- f.Usage()
+ fmt.Fprintln(stderr, err)
return 2
}
@@ -103,7 +102,6 @@ Options:
detachKeys := f.String("detach-keys", "", "set detach key sequence, as in docker-attach(1)")
if err := f.Parse(args); err != nil {
fmt.Fprintln(stderr, err)
- f.Usage()
return 2
} else if f.NArg() != 1 {
f.Usage()
commit a80122e544ddf72fe31d02398d914089b300d1c9
Merge: f2c6e467c 025639399
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 14 11:08:46 2021 -0500
17170: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit f2c6e467c4cfd079b00070ac4f50b551b6ee3bdf
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 23:22:51 2021 -0500
17170: Fix closing pty/tty when nil.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index 8b2fe4144..0d869ca7f 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -204,8 +204,6 @@ func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
}
var pty0, tty0 *os.File
go func() {
- defer pty0.Close()
- defer tty0.Close()
// Where to send errors/messages for the
// client to see
logw := io.Writer(ch.Stderr())
@@ -280,6 +278,8 @@ func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
break
}
+ defer p.Close()
+ defer t.Close()
pty0, tty0 = p, t
ok = true
var payload struct {
commit 8439ad9df4a6d9b28bbed985bf87599f2d1b3820
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 23:22:40 2021 -0500
17170: Fixup error display.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 5fecf662f..26f41e128 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -343,12 +343,12 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
body, _ := ioutil.ReadAll(resp.Body)
var message string
var errDoc httpserver.ErrorResponse
- if err := json.Unmarshal(body, &errDoc); err != nil {
+ if err := json.Unmarshal(body, &errDoc); err == nil {
message = strings.Join(errDoc.Errors, "; ")
} else {
message = fmt.Sprintf("%q", body)
}
- err = fmt.Errorf("server did not provide a tunnel: %q (HTTP %d)", message, resp.StatusCode)
+ err = fmt.Errorf("server did not provide a tunnel: %s (HTTP %d)", message, resp.StatusCode)
return
}
if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
commit 5d05dd3ef4822dfb3b476777ba1aacaafed8278d
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 17:18:08 2021 -0500
17170: "arvados-client shell" accepts container request UUID.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index b5fde6cf1..1ed6dc279 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -125,18 +125,24 @@ Options:
func(context.Context) ([]string, error) {
return []string{os.Getenv("ARVADOS_API_TOKEN")}, nil
})
- // if strings.Contains(targetUUID, "-xvhdp-") {
- // cr, err := rpcconn.ContainerRequestGet(context.TODO(), arvados.GetOptions{UUID: targetUUID})
- // if err != nil {
- // fmt.Fprintln(stderr, err)
- // return 1
- // }
- // if cr.ContainerUUID == "" {
- // fmt.Fprintf(stderr, "no container assigned, container request state is %s\n", strings.ToLower(cr.State))
- // return 1
- // }
- // targetUUID = cr.ContainerUUID
- // }
+ if strings.Contains(targetUUID, "-xvhdp-") {
+ crs, err := rpcconn.ContainerRequestList(context.TODO(), arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", targetUUID}}})
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ if len(crs.Items) < 1 {
+ fmt.Fprintf(stderr, "container request %q not found\n", targetUUID)
+ return 1
+ }
+ cr := crs.Items[0]
+ if cr.ContainerUUID == "" {
+ fmt.Fprintf(stderr, "no container assigned, container request state is %s\n", strings.ToLower(string(cr.State)))
+ return 1
+ }
+ targetUUID = cr.ContainerUUID
+ fmt.Fprintln(stderr, "connecting to container", targetUUID)
+ }
sshconn, err := rpcconn.ContainerSSH(context.TODO(), arvados.ContainerSSHOptions{
UUID: targetUUID,
DetachKeys: *detachKeys,
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index a9a638759..5fecf662f 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -361,6 +361,13 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
return
}
+func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
+ ep := arvados.EndpointContainerRequestList
+ var resp arvados.ContainerRequestList
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
func (conn *Conn) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
ep := arvados.EndpointSpecimenCreate
var resp arvados.Specimen
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index e8baa01b4..4675906e7 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -46,6 +46,7 @@ var (
EndpointContainerLock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
EndpointContainerUnlock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
EndpointContainerSSH = APIEndpoint{"GET", "arvados/v1/connect/{uuid}/ssh", ""} // move to /containers after #17014 fixes routing
+ EndpointContainerRequestList = APIEndpoint{"GET", "arvados/v1/container_requests", ""}
EndpointUserActivate = APIEndpoint{"POST", "arvados/v1/users/{uuid}/activate", ""}
EndpointUserCreate = APIEndpoint{"POST", "arvados/v1/users", "user"}
EndpointUserCurrent = APIEndpoint{"GET", "arvados/v1/users/current", ""}
commit bdd9b68e133bb3b3ef7914893ad15f311ce2de24
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 17:17:50 2021 -0500
17170: Improve error messages.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 48e67e274..a9a638759 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -313,6 +313,7 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
if err != nil {
+ err = fmt.Errorf("tls.Dial: %w", err)
return
}
u.RawQuery = url.Values{
@@ -334,12 +335,20 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
bufw.Flush()
resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
if err != nil {
+ err = fmt.Errorf("http.ReadResponse: %w", err)
return
}
if resp.StatusCode != http.StatusSwitchingProtocols {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
- err = fmt.Errorf("server did not provide a tunnel: %d %q", resp.StatusCode, body)
+ var message string
+ var errDoc httpserver.ErrorResponse
+ if err := json.Unmarshal(body, &errDoc); err != nil {
+ message = strings.Join(errDoc.Errors, "; ")
+ } else {
+ message = fmt.Sprintf("%q", body)
+ }
+ err = fmt.Errorf("server did not provide a tunnel: %q (HTTP %d)", message, resp.StatusCode)
return
}
if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
commit 0a6c5d8a8758df0395e40b3cf1d125e59d6c88dd
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 17:06:57 2021 -0500
17170: Don't reuse transport that might have http2 enabled.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 7dd89452b..48e67e274 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -298,8 +298,9 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
// hostname or ::1 or 1::1
addr = net.JoinHostPort(addr, "https")
}
- netconn, err := tls.Dial("tcp", addr, conn.httpClient.Transport.(*http.Transport).TLSClientConfig)
+ netconn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: conn.httpClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify})
if err != nil {
+ err = fmt.Errorf("tls.Dial: %w", err)
return
}
defer func() {
commit 5c34cfad641b76ffaa43be8234af1da9d5736ec2
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 17:06:11 2021 -0500
17170: Dry up close-connection-on-error.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 2accfd8f2..7dd89452b 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -302,12 +302,16 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
if err != nil {
return
}
+ defer func() {
+ if err != nil {
+ netconn.Close()
+ }
+ }()
bufr := bufio.NewReader(netconn)
bufw := bufio.NewWriter(netconn)
u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
if err != nil {
- netconn.Close()
return
}
u.RawQuery = url.Values{
@@ -316,11 +320,9 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
}.Encode()
tokens, err := conn.tokenProvider(ctx)
if err != nil {
- netconn.Close()
return
} else if len(tokens) < 1 {
err = httpserver.ErrorWithStatus(errors.New("unauthorized"), http.StatusUnauthorized)
- netconn.Close()
return
}
bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
@@ -331,20 +333,17 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
bufw.Flush()
resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
if err != nil {
- netconn.Close()
return
}
if resp.StatusCode != http.StatusSwitchingProtocols {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
err = fmt.Errorf("server did not provide a tunnel: %d %q", resp.StatusCode, body)
- netconn.Close()
return
}
if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
err = fmt.Errorf("bad response from server: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
- netconn.Close()
return
}
sshconn.Conn = netconn
commit 03141ccc3015a04f2f171b5a532a9dfd57b8bcd6
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 17:03:40 2021 -0500
17170: Allow shell only if this user submitted all associated CRs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index f6eaea6d9..ff3e3de5b 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -18,6 +18,7 @@ import (
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
@@ -29,6 +30,26 @@ import (
// If the returned error is nil, the caller is responsible for closing
// sshconn.Conn.
func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+ user, err := conn.railsProxy.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ return
+ }
+ ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
+ crs, err := conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
+ if err != nil {
+ return
+ }
+ for _, cr := range crs.Items {
+ if cr.ModifiedByUserUUID != user.UUID {
+ err = httpserver.ErrorWithStatus(errors.New("permission denied: container is associated with requests submitted by other users"), http.StatusForbidden)
+ return
+ }
+ }
+ if crs.ItemsAvailable != len(crs.Items) {
+ err = httpserver.ErrorWithStatus(errors.New("incomplete response while checking permission"), http.StatusInternalServerError)
+ return
+ }
+
ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID})
if err != nil {
return
commit a2c539196d3b76b466039dfd00e7595ec9dd6abc
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 00:13:48 2021 -0500
17170: Test arvados-client shell command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index d15b9d7b5..b5fde6cf1 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -143,7 +143,7 @@ Options:
LoginUsername: loginUsername,
})
if err != nil {
- fmt.Fprintln(stderr, err)
+ fmt.Fprintln(stderr, "error setting up tunnel:", err)
return 1
}
defer sshconn.Conn.Close()
diff --git a/cmd/arvados-client/container_gateway_test.go b/cmd/arvados-client/container_gateway_test.go
new file mode 100644
index 000000000..deff7b170
--- /dev/null
+++ b/cmd/arvados-client/container_gateway_test.go
@@ -0,0 +1,26 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "os"
+ "os/exec"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+func (s *ClientSuite) TestShellGatewayNotAvailable(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ cmd := exec.Command("go", "run", ".", "shell", arvadostest.QueuedContainerUUID, "-o", "controlpath=none", "echo", "ok")
+ cmd.Env = append(cmd.Env, os.Environ()...)
+ cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2)
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ c.Check(cmd.Run(), check.NotNil)
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*gateway is not available, container is queued.*`)
+}
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 9b2695b55..2accfd8f2 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -337,13 +337,13 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
if resp.StatusCode != http.StatusSwitchingProtocols {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
- err = fmt.Errorf("tunnel connection failed: %d %q", resp.StatusCode, body)
+ err = fmt.Errorf("server did not provide a tunnel: %d %q", resp.StatusCode, body)
netconn.Close()
return
}
if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
- err = fmt.Errorf("bad response: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
+ err = fmt.Errorf("bad response from server: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
netconn.Close()
return
}
commit 4cc07d6ab4ae16bd6bc69b2c8022414d66ee4113
Author: Tom Clegg <tom at curii.com>
Date: Wed Jan 13 00:10:09 2021 -0500
17170: Fix dial when API host var is host:port.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 75603bb59..9b2695b55 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -293,7 +293,12 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
// a running container. If the returned error is nil, the caller is
// responsible for closing sshconn.Conn.
func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
- netconn, err := tls.Dial("tcp", net.JoinHostPort(conn.baseURL.Host, "https"), nil)
+ addr := conn.baseURL.Host
+ if strings.Index(addr, ":") < 1 || (strings.Contains(addr, "::") && addr[0] != '[') {
+ // hostname or ::1 or 1::1
+ addr = net.JoinHostPort(addr, "https")
+ }
+ netconn, err := tls.Dial("tcp", addr, conn.httpClient.Transport.(*http.Transport).TLSClientConfig)
if err != nil {
return
}
commit 6be5fe90dc9a5452b432176bfa83cba83070f8cf
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 12 23:22:46 2021 -0500
17170: Test controller->crunch-run tunnel.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
new file mode 100644
index 000000000..336d5b634
--- /dev/null
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -0,0 +1,123 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/crunchrun"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ContainerGatewaySuite{})
+
+type ContainerGatewaySuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ ctx context.Context
+ ctrUUID string
+ gw *crunchrun.Gateway
+}
+
+func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ s.ctrUUID = arvadostest.QueuedContainerUUID
+
+ h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
+ fmt.Fprint(h, s.ctrUUID)
+ authKey := fmt.Sprintf("%x", h.Sum(nil))
+
+ s.gw = &crunchrun.Gateway{
+ DockerContainerID: new(string),
+ ContainerUUID: s.ctrUUID,
+ AuthSecret: authKey,
+ Address: "localhost:0",
+ Log: ctxlog.TestLogger(c),
+ }
+ c.Assert(s.gw.Start(), check.IsNil)
+ rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "state": arvados.ContainerStateLocked}})
+ c.Assert(err, check.IsNil)
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "state": arvados.ContainerStateRunning,
+ "gateway_address": s.gw.Address}})
+ c.Assert(err, check.IsNil)
+}
+
+func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
+ c.Logf("connecting to %s", s.gw.Address)
+ sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ c.Assert(err, check.IsNil)
+ c.Assert(sshconn.Conn, check.NotNil)
+ defer sshconn.Conn.Close()
+
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+
+ // Receive text banner
+ buf := make([]byte, 12)
+ _, err := io.ReadFull(sshconn.Conn, buf)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
+
+ // Send text banner
+ _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
+ c.Check(err, check.IsNil)
+
+ // Receive binary
+ _, err = io.ReadFull(sshconn.Conn, buf[:4])
+ c.Check(err, check.IsNil)
+ c.Check(buf[:4], check.DeepEquals, []byte{0, 0, 1, 0xfc})
+
+ // If we can get this far into an SSH handshake...
+ c.Log("success, tunnel is working")
+ }()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ c.Fail()
+ }
+}
+
+func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
+ c.Log("trying with no token")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{})
+ _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ c.Check(err, check.ErrorMatches, `.* 401 .*`)
+
+ c.Log("trying with anonymous token")
+ ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AnonymousToken}})
+ _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ c.Check(err, check.ErrorMatches, `.* 404 .*`)
+}
commit fed1d9f5e7a442abb6b2e86114e8c3d05cff5329
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 12 21:57:40 2021 -0500
17170: Fixup gateway auth secret.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index dd296c569..f6eaea6d9 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -63,7 +63,7 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
return errors.New("no certificate received, cannot compute authorization header")
}
h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
- fmt.Fprint(h, "%s", opts.UUID)
+ fmt.Fprint(h, opts.UUID)
authKey := fmt.Sprintf("%x", h.Sum(nil))
h = hmac.New(sha256.New, []byte(authKey))
h.Write(rawCerts[0])
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index e092e7ada..6a74280ca 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -996,7 +996,7 @@ func (wp *Pool) waitUntilLoaded() {
func (wp *Pool) gatewayAuthSecret(uuid string) string {
h := hmac.New(sha256.New, []byte(wp.systemRootToken))
- fmt.Fprint(h, "%s", uuid)
+ fmt.Fprint(h, uuid)
return fmt.Sprintf("%x", h.Sum(nil))
}
commit 2c3418d59cd27806322e07519ef9483c2cb433c2
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 12 21:03:40 2021 -0500
17170: Use TLS for controller->crunch-run traffic.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index 31d44e5e0..dd296c569 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -9,9 +9,10 @@ import (
"context"
"crypto/hmac"
"crypto/sha256"
+ "crypto/tls"
+ "crypto/x509"
"errors"
"fmt"
- "net"
"net/http"
"net/url"
"strings"
@@ -36,20 +37,53 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusBadGateway)
return
}
- netconn, err := net.Dial("tcp", ctr.GatewayAddress)
+ // crunch-run uses a self-signed / unverifiable TLS
+ // certificate, so we use the following scheme to ensure we're
+ // not talking to a MITM.
+ //
+ // 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
+ // this will be the same ctrKey that a-d-c supplied to
+ // crunch-run in the GatewayAuthSecret env var.
+ //
+ // 2. Compute requestAuth = HMAC-SHA256(ctrKey,serverCert) and
+ // send it to crunch-run as the X-Arvados-Authorization
+ // header, proving that we know ctrKey. (Note a MITM cannot
+ // replay the proof to a real crunch-run server, because the
+ // real crunch-run server would have a different cert.)
+ //
+ // 3. Compute respondAuth = HMAC-SHA256(ctrKey,requestAuth)
+ // and ensure the server returns it in the
+ // X-Arvados-Authorization-Response header, proving that the
+ // server knows ctrKey.
+ var requestAuth, respondAuth string
+ netconn, err := tls.Dial("tcp", ctr.GatewayAddress, &tls.Config{
+ InsecureSkipVerify: true,
+ VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ if len(rawCerts) == 0 {
+ return errors.New("no certificate received, cannot compute authorization header")
+ }
+ h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
+ fmt.Fprint(h, "%s", opts.UUID)
+ authKey := fmt.Sprintf("%x", h.Sum(nil))
+ h = hmac.New(sha256.New, []byte(authKey))
+ h.Write(rawCerts[0])
+ requestAuth = fmt.Sprintf("%x", h.Sum(nil))
+ h.Reset()
+ h.Write([]byte(requestAuth))
+ respondAuth = fmt.Sprintf("%x", h.Sum(nil))
+ return nil
+ },
+ })
if err != nil {
return
}
+ if respondAuth == "" {
+ err = httpserver.ErrorWithStatus(errors.New("BUG: no respondAuth"), http.StatusInternalServerError)
+ return
+ }
bufr := bufio.NewReader(netconn)
bufw := bufio.NewWriter(netconn)
- // Note this auth header does not protect from replay/mitm
- // attacks (TODO: use TLS for that). It only authenticates us
- // to crunch-run.
- h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
- fmt.Fprint(h, "%s", opts.UUID)
- auth := fmt.Sprintf("%x", h.Sum(nil))
-
u := url.URL{
Scheme: "http",
Host: ctr.GatewayAddress,
@@ -59,14 +93,19 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
bufw.WriteString("Host: " + u.Host + "\r\n")
bufw.WriteString("Upgrade: ssh\r\n")
bufw.WriteString("X-Arvados-Target-Uuid: " + opts.UUID + "\r\n")
- bufw.WriteString("X-Arvados-Authorization: " + auth + "\r\n")
+ bufw.WriteString("X-Arvados-Authorization: " + requestAuth + "\r\n")
bufw.WriteString("X-Arvados-Detach-Keys: " + opts.DetachKeys + "\r\n")
bufw.WriteString("X-Arvados-Login-Username: " + opts.LoginUsername + "\r\n")
bufw.WriteString("\r\n")
bufw.Flush()
resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
if err != nil {
- err = fmt.Errorf("error reading http response from gateway: %w", err)
+ err = httpserver.ErrorWithStatus(fmt.Errorf("error reading http response from gateway: %w", err), http.StatusBadGateway)
+ netconn.Close()
+ return
+ }
+ if resp.Header.Get("X-Arvados-Authorization-Response") != respondAuth {
+ err = httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
netconn.Close()
return
}
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index 92c11383f..8b2fe4144 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -5,8 +5,11 @@
package crunchrun
import (
+ "crypto/hmac"
"crypto/rand"
"crypto/rsa"
+ "crypto/sha256"
+ "crypto/tls"
"fmt"
"io"
"net"
@@ -16,17 +19,32 @@ import (
"sync"
"syscall"
+ "git.arvados.org/arvados.git/lib/selfsigned"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/creack/pty"
"github.com/google/shlex"
"golang.org/x/crypto/ssh"
)
+type Gateway struct {
+ DockerContainerID *string
+ ContainerUUID string
+ Address string // listen host:port; if port=0, Start() will change it to the selected port
+ AuthSecret string
+ Log interface {
+ Printf(fmt string, args ...interface{})
+ }
+
+ sshConfig ssh.ServerConfig
+ requestAuth string
+ respondAuth string
+}
+
// startGatewayServer starts an http server that allows authenticated
// clients to open an interactive "docker exec" session and (in
// future) connect to tcp ports inside the docker container.
-func (runner *ContainerRunner) startGatewayServer() error {
- runner.gatewaySSHConfig = &ssh.ServerConfig{
+func (gw *Gateway) Start() error {
+ gw.sshConfig = ssh.ServerConfig{
NoClientAuth: true,
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
if c.User() == "_" {
@@ -47,7 +65,7 @@ func (runner *ContainerRunner) startGatewayServer() error {
}
},
}
- pvt, err := rsa.GenerateKey(rand.Reader, 4096)
+ pvt, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
@@ -59,20 +77,34 @@ func (runner *ContainerRunner) startGatewayServer() error {
if err != nil {
return err
}
- runner.gatewaySSHConfig.AddHostKey(signer)
+ gw.sshConfig.AddHostKey(signer)
- // GatewayAddress (provided by arvados-dispatch-cloud) is
+ // Address (typically provided by arvados-dispatch-cloud) is
// HOST:PORT where HOST is our IP address or hostname as seen
// from arvados-controller, and PORT is either the desired
- // port where we should run our gateway server, or "0" if
- // we should choose an available port.
- host, port, err := net.SplitHostPort(os.Getenv("GatewayAddress"))
+ // port where we should run our gateway server, or "0" if we
+ // should choose an available port.
+ host, port, err := net.SplitHostPort(gw.Address)
if err != nil {
return err
}
+ cert, err := selfsigned.CertGenerator{}.Generate()
+ if err != nil {
+ return err
+ }
+ h := hmac.New(sha256.New, []byte(gw.AuthSecret))
+ h.Write(cert.Certificate[0])
+ gw.requestAuth = fmt.Sprintf("%x", h.Sum(nil))
+ h.Reset()
+ h.Write([]byte(gw.requestAuth))
+ gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
+
srv := &httpserver.Server{
Server: http.Server{
- Handler: http.HandlerFunc(runner.handleSSH),
+ Handler: http.HandlerFunc(gw.handleSSH),
+ TLSConfig: &tls.Config{
+ Certificates: []tls.Certificate{cert},
+ },
},
Addr: ":" + port,
}
@@ -90,7 +122,7 @@ func (runner *ContainerRunner) startGatewayServer() error {
// gateway_address to "HOST:PORT" where HOST is our
// external hostname/IP as provided by arvados-dispatch-cloud,
// and PORT is the port number we ended up listening on.
- runner.gatewayAddress = net.JoinHostPort(host, port)
+ gw.Address = net.JoinHostPort(host, port)
return nil
}
@@ -104,15 +136,15 @@ func (runner *ContainerRunner) startGatewayServer() error {
// Connection: upgrade
// Upgrade: ssh
// X-Arvados-Target-Uuid: uuid of container
-// X-Arvados-Authorization: must match GatewayAuthSecret provided by
-// a-d-c (this prevents other containers and shell nodes from
-// connecting directly)
+// X-Arvados-Authorization: must match
+// hmac(AuthSecret,certfingerprint) (this prevents other containers
+// and shell nodes from connecting directly)
//
// Optional header:
//
// X-Arvados-Detach-Keys: argument to "docker attach --detach-keys",
// e.g., "ctrl-p,ctrl-q"
-func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Request) {
+func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
// In future we'll handle browser traffic too, but for now the
// only traffic we expect is an SSH tunnel from
// (*lib/controller/localdb.Conn)ContainerSSH()
@@ -120,11 +152,11 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
http.Error(w, "path not found", http.StatusNotFound)
return
}
- if want := req.Header.Get("X-Arvados-Target-Uuid"); want != runner.Container.UUID {
- http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, runner.Container.UUID), http.StatusBadGateway)
+ if want := req.Header.Get("X-Arvados-Target-Uuid"); want != gw.ContainerUUID {
+ http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, gw.ContainerUUID), http.StatusBadGateway)
return
}
- if req.Header.Get("X-Arvados-Authorization") != runner.gatewayAuthSecret {
+ if req.Header.Get("X-Arvados-Authorization") != gw.requestAuth {
http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
return
}
@@ -146,15 +178,16 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
defer netconn.Close()
w.Header().Set("Connection", "upgrade")
w.Header().Set("Upgrade", "ssh")
+ w.Header().Set("X-Arvados-Authorization-Response", gw.respondAuth)
netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
w.Header().Write(netconn)
netconn.Write([]byte("\r\n"))
ctx := req.Context()
- conn, newchans, reqs, err := ssh.NewServerConn(netconn, runner.gatewaySSHConfig)
+ conn, newchans, reqs, err := ssh.NewServerConn(netconn, &gw.sshConfig)
if err != nil {
- runner.CrunchLog.Printf("ssh.NewServerConn: %s", err)
+ gw.Log.Printf("ssh.NewServerConn: %s", err)
return
}
defer conn.Close()
@@ -166,7 +199,7 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
}
ch, reqs, err := newch.Accept()
if err != nil {
- runner.CrunchLog.Printf("accept channel: %s", err)
+ gw.Log.Printf("accept channel: %s", err)
return
}
var pty0, tty0 *os.File
@@ -217,7 +250,7 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
// Send our own debug messages to tty as well.
logw = tty0
}
- cmd.Args = append(cmd.Args, runner.ContainerID)
+ cmd.Args = append(cmd.Args, *gw.DockerContainerID)
cmd.Args = append(cmd.Args, execargs...)
cmd.SysProcAttr = &syscall.SysProcAttr{
Setctty: tty0 != nil,
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index b252e0dce..f28593fe0 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -33,7 +33,6 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"git.arvados.org/arvados.git/sdk/go/manifest"
- "golang.org/x/crypto/ssh"
"golang.org/x/net/context"
dockertypes "github.com/docker/docker/api/types"
@@ -180,9 +179,7 @@ type ContainerRunner struct {
containerWatchdogInterval time.Duration
- gatewayAddress string
- gatewaySSHConfig *ssh.ServerConfig
- gatewayAuthSecret string
+ gateway Gateway
}
// setupSignals sets up signal handling to gracefully terminate the underlying
@@ -1474,7 +1471,7 @@ func (runner *ContainerRunner) UpdateContainerRunning() error {
return ErrCancelled
}
return runner.DispatcherArvClient.Update("containers", runner.Container.UUID,
- arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running", "gateway_address": runner.gatewayAddress}}, nil)
+ arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running", "gateway_address": runner.gateway.Address}}, nil)
}
// ContainerToken returns the api_token the container (and any
@@ -1873,9 +1870,15 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
return 1
}
- cr.gatewayAuthSecret = os.Getenv("GatewayAuthSecret")
+ cr.gateway = Gateway{
+ Address: os.Getenv("GatewayAddress"),
+ AuthSecret: os.Getenv("GatewayAuthSecret"),
+ ContainerUUID: containerID,
+ DockerContainerID: &cr.ContainerID,
+ Log: cr.CrunchLog,
+ }
os.Unsetenv("GatewayAuthSecret")
- err = cr.startGatewayServer()
+ err = cr.gateway.Start()
if err != nil {
log.Printf("error starting gateway server: %s", err)
return 1
diff --git a/lib/selfsigned/cert.go b/lib/selfsigned/cert.go
new file mode 100644
index 000000000..ae521dd17
--- /dev/null
+++ b/lib/selfsigned/cert.go
@@ -0,0 +1,76 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package selfsigned
+
+import (
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "fmt"
+ "math/big"
+ "net"
+ "time"
+)
+
+type CertGenerator struct {
+ Bits int
+ Hosts []string
+ IsCA bool
+}
+
+func (gen CertGenerator) Generate() (cert tls.Certificate, err error) {
+ keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment
+ if gen.IsCA {
+ keyUsage |= x509.KeyUsageCertSign
+ }
+ notBefore := time.Now()
+ notAfter := time.Now().Add(time.Hour * 24 * 365)
+ snMax := new(big.Int).Lsh(big.NewInt(1), 128)
+ sn, err := rand.Int(rand.Reader, snMax)
+ if err != nil {
+ err = fmt.Errorf("Failed to generate serial number: %w", err)
+ return
+ }
+ template := x509.Certificate{
+ SerialNumber: sn,
+ Subject: pkix.Name{
+ Organization: []string{"N/A"},
+ },
+ NotBefore: notBefore,
+ NotAfter: notAfter,
+ KeyUsage: keyUsage,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ BasicConstraintsValid: true,
+ IsCA: gen.IsCA,
+ }
+ for _, h := range gen.Hosts {
+ if ip := net.ParseIP(h); ip != nil {
+ template.IPAddresses = append(template.IPAddresses, ip)
+ } else {
+ template.DNSNames = append(template.DNSNames, h)
+ }
+ }
+ bits := gen.Bits
+ if bits == 0 {
+ bits = 4096
+ }
+ priv, err := rsa.GenerateKey(rand.Reader, bits)
+ if err != nil {
+ err = fmt.Errorf("error generating key: %w", err)
+ return
+ }
+ certder, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+ if err != nil {
+ err = fmt.Errorf("error creating certificate: %w", err)
+ return
+ }
+ cert = tls.Certificate{
+ Certificate: [][]byte{certder},
+ PrivateKey: priv,
+ }
+ return
+}
diff --git a/lib/selfsigned/cert_test.go b/lib/selfsigned/cert_test.go
new file mode 100644
index 000000000..16ed8bd91
--- /dev/null
+++ b/lib/selfsigned/cert_test.go
@@ -0,0 +1,26 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package selfsigned
+
+import (
+ "testing"
+)
+
+func TestCert(t *testing.T) {
+ cert, err := CertGenerator{Bits: 1024, Hosts: []string{"localhost"}, IsCA: false}.Generate()
+ if err != nil {
+ t.Error(err)
+ }
+ if len(cert.Certificate) < 1 {
+ t.Error("no certificate!")
+ }
+ cert, err = CertGenerator{Bits: 2048, Hosts: []string{"localhost"}, IsCA: true}.Generate()
+ if err != nil {
+ t.Error(err)
+ }
+ if len(cert.Certificate) < 1 {
+ t.Error("no certificate!")
+ }
+}
commit 0a39317d5ec49ad439833df0a70965394cafb6e8
Author: Tom Clegg <tom at curii.com>
Date: Tue Jan 12 00:30:45 2021 -0500
17170: Specify login username. Improve logging.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index 46fa6932b..d15b9d7b5 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -29,7 +29,7 @@ func (shellCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
f.Usage = func() {
fmt.Print(stderr, prog+`: open an interactive shell on a running container.
-Usage: `+prog+` [options] container-uuid [ssh-options] [remote-command [args...]]
+Usage: `+prog+` [options] [username@]container-uuid [ssh-options] [remote-command [args...]]
Options:
`)
@@ -47,7 +47,10 @@ Options:
f.Usage()
return 2
}
- targetUUID := f.Args()[0]
+ target := f.Args()[0]
+ if !strings.Contains(target, "@") {
+ target = "root@" + target
+ }
sshargs := f.Args()[1:]
selfbin, err := os.Readlink("/proc/self/exe")
@@ -56,9 +59,9 @@ Options:
return 2
}
sshargs = append([]string{
- "-o", "ProxyCommand " + selfbin + " connect-ssh -detach-keys='" + strings.Replace(*detachKeys, "'", "'\\''", -1) + "' " + targetUUID,
+ "-o", "ProxyCommand " + selfbin + " connect-ssh -detach-keys=" + shellescape(*detachKeys) + " " + shellescape(target),
"-o", "StrictHostKeyChecking no",
- "root@" + targetUUID},
+ target},
sshargs...)
cmd := exec.Command("ssh", sshargs...)
cmd.Stdin = stdin
@@ -91,7 +94,7 @@ func (connectSSHCommand) RunCommand(prog string, args []string, stdin io.Reader,
f.Usage = func() {
fmt.Fprint(stderr, prog+`: connect to the gateway service for a running container.
-Usage: `+prog+` [options] container-uuid
+Usage: `+prog+` [options] [username@]container-uuid
Options:
`)
@@ -107,6 +110,11 @@ Options:
return 2
}
targetUUID := f.Args()[0]
+ loginUsername := "root"
+ if i := strings.Index(targetUUID, "@"); i >= 0 {
+ loginUsername = targetUUID[:i]
+ targetUUID = targetUUID[i+1:]
+ }
insecure := os.Getenv("ARVADOS_API_HOST_INSECURE")
rpcconn := rpc.NewConn("",
&url.URL{
@@ -130,8 +138,9 @@ Options:
// targetUUID = cr.ContainerUUID
// }
sshconn, err := rpcconn.ContainerSSH(context.TODO(), arvados.ContainerSSHOptions{
- UUID: targetUUID,
- DetachKeys: *detachKeys,
+ UUID: targetUUID,
+ DetachKeys: *detachKeys,
+ LoginUsername: loginUsername,
})
if err != nil {
fmt.Fprintln(stderr, err)
@@ -157,3 +166,7 @@ Options:
<-ctx.Done()
return 0
}
+
+func shellescape(s string) string {
+ return "'" + strings.Replace(s, "'", "'\\''", -1) + "'"
+}
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index 01b4065f9..31d44e5e0 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -17,6 +17,7 @@ import (
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
@@ -60,6 +61,7 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
bufw.WriteString("X-Arvados-Target-Uuid: " + opts.UUID + "\r\n")
bufw.WriteString("X-Arvados-Authorization: " + auth + "\r\n")
bufw.WriteString("X-Arvados-Detach-Keys: " + opts.DetachKeys + "\r\n")
+ bufw.WriteString("X-Arvados-Login-Username: " + opts.LoginUsername + "\r\n")
bufw.WriteString("\r\n")
bufw.Flush()
resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
@@ -76,5 +78,6 @@ func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOpt
}
sshconn.Conn = netconn
sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+ sshconn.Logger = ctxlog.FromContext(ctx)
return
}
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index e80542e3e..75603bb59 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -305,7 +305,10 @@ func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSH
netconn.Close()
return
}
- u.RawQuery = url.Values{"detach_keys": {options.DetachKeys}}.Encode()
+ u.RawQuery = url.Values{
+ "detach_keys": {options.DetachKeys},
+ "login_username": {options.LoginUsername},
+ }.Encode()
tokens, err := conn.tokenProvider(ctx)
if err != nil {
netconn.Close()
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index e55868ac3..92c11383f 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -29,21 +29,21 @@ func (runner *ContainerRunner) startGatewayServer() error {
runner.gatewaySSHConfig = &ssh.ServerConfig{
NoClientAuth: true,
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
- if c.User() == "root" {
+ if c.User() == "_" {
return nil, nil
} else {
- return nil, fmt.Errorf("unimplemented: cannot log in as non-root user %q", c.User())
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
},
PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
- if c.User() == "root" {
+ if c.User() == "_" {
return &ssh.Permissions{
Extensions: map[string]string{
"pubkey-fp": ssh.FingerprintSHA256(pubKey),
},
}, nil
} else {
- return nil, fmt.Errorf("unimplemented: cannot log in as non-root user %q", c.User())
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
},
}
@@ -129,6 +129,10 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
return
}
detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
+ username := req.Header.Get("X-Arvados-Login-Username")
+ if username == "" {
+ username = "root"
+ }
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
@@ -196,7 +200,7 @@ func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Reques
execargs = []string{"/bin/bash", "-login"}
}
go func() {
- cmd := exec.CommandContext(ctx, "docker", "exec", "-i", "--detach-keys="+detachKeys)
+ cmd := exec.CommandContext(ctx, "docker", "exec", "-i", "--detach-keys="+detachKeys, "--user="+username)
cmd.Stdin = ch
cmd.Stdout = ch
cmd.Stderr = ch.Stderr()
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index fa4471804..e8baa01b4 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -9,6 +9,8 @@ import (
"context"
"encoding/json"
"net"
+
+ "github.com/sirupsen/logrus"
)
type APIEndpoint struct {
@@ -64,13 +66,15 @@ var (
)
type ContainerSSHOptions struct {
- UUID string `json:"uuid"`
- DetachKeys string `json:"detach_keys"`
+ UUID string `json:"uuid"`
+ DetachKeys string `json:"detach_keys"`
+ LoginUsername string `json:"login_username"`
}
type ContainerSSHConnection struct {
- Conn net.Conn `json:"-"`
- Bufrw *bufio.ReadWriter `json:"-"`
+ Conn net.Conn `json:"-"`
+ Bufrw *bufio.ReadWriter `json:"-"`
+ Logger logrus.FieldLogger `json:"-"`
}
type GetOptions struct {
diff --git a/sdk/go/arvados/container_gateway.go b/sdk/go/arvados/container_gateway.go
index 07f8c0793..00c98d572 100644
--- a/sdk/go/arvados/container_gateway.go
+++ b/sdk/go/arvados/container_gateway.go
@@ -8,8 +8,10 @@ import (
"context"
"io"
"net/http"
+ "sync"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
)
func (sshconn ContainerSSHConnection) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -28,26 +30,45 @@ func (sshconn ContainerSSHConnection) ServeHTTP(w http.ResponseWriter, req *http
}
defer conn.Close()
+ var bytesIn, bytesOut int64
+ var wg sync.WaitGroup
ctx, cancel := context.WithCancel(context.Background())
+ wg.Add(1)
go func() {
+ defer wg.Done()
defer cancel()
- _, err := io.CopyN(conn, sshconn.Bufrw, int64(sshconn.Bufrw.Reader.Buffered()))
+ n, err := io.CopyN(conn, sshconn.Bufrw, int64(sshconn.Bufrw.Reader.Buffered()))
+ bytesOut += n
if err == nil {
- _, err = io.Copy(conn, sshconn.Conn)
+ n, err = io.Copy(conn, sshconn.Conn)
+ bytesOut += n
}
if err != nil {
ctxlog.FromContext(req.Context()).WithError(err).Error("error copying downstream")
}
}()
+ wg.Add(1)
go func() {
+ defer wg.Done()
defer cancel()
- _, err := io.CopyN(sshconn.Conn, bufrw, int64(bufrw.Reader.Buffered()))
+ n, err := io.CopyN(sshconn.Conn, bufrw, int64(bufrw.Reader.Buffered()))
+ bytesIn += n
if err == nil {
- _, err = io.Copy(sshconn.Conn, conn)
+ n, err = io.Copy(sshconn.Conn, conn)
+ bytesIn += n
}
if err != nil {
ctxlog.FromContext(req.Context()).WithError(err).Error("error copying upstream")
}
}()
<-ctx.Done()
+ if sshconn.Logger != nil {
+ go func() {
+ wg.Wait()
+ sshconn.Logger.WithFields(logrus.Fields{
+ "bytesIn": bytesIn,
+ "bytesOut": bytesOut,
+ }).Info("closed connection")
+ }()
+ }
}
commit 56e130608f8977d20b21c54f6ab8973d71e045a0
Author: Tom Clegg <tom at curii.com>
Date: Mon Jan 11 15:53:28 2021 -0500
17170: Add "arvados-client shell" subcommand and backend support.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index 47fcd5ad7..aefcce79a 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -57,6 +57,8 @@ var (
"mount": mount.Command,
"deduplication-report": deduplicationreport.Command,
"costanalyzer": costanalyzer.Command,
+ "shell": shellCommand{},
+ "connect-ssh": connectSSHCommand{},
})
)
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
new file mode 100644
index 000000000..46fa6932b
--- /dev/null
+++ b/cmd/arvados-client/container_gateway.go
@@ -0,0 +1,159 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "net/url"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// shellCommand connects the terminal to an interactive shell on a
+// running container.
+type shellCommand struct{}
+
+func (shellCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ f := flag.NewFlagSet(prog, flag.ContinueOnError)
+ f.SetOutput(stderr)
+ f.Usage = func() {
+ fmt.Print(stderr, prog+`: open an interactive shell on a running container.
+
+Usage: `+prog+` [options] container-uuid [ssh-options] [remote-command [args...]]
+
+Options:
+`)
+ f.PrintDefaults()
+ }
+ detachKeys := f.String("detach-keys", "ctrl-],ctrl-]", "set detach key sequence, as in docker-attach(1)")
+ err := f.Parse(args)
+ if err != nil {
+ fmt.Println(stderr, err)
+ f.Usage()
+ return 2
+ }
+
+ if f.NArg() < 1 {
+ f.Usage()
+ return 2
+ }
+ targetUUID := f.Args()[0]
+ sshargs := f.Args()[1:]
+
+ selfbin, err := os.Readlink("/proc/self/exe")
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+ sshargs = append([]string{
+ "-o", "ProxyCommand " + selfbin + " connect-ssh -detach-keys='" + strings.Replace(*detachKeys, "'", "'\\''", -1) + "' " + targetUUID,
+ "-o", "StrictHostKeyChecking no",
+ "root@" + targetUUID},
+ sshargs...)
+ cmd := exec.Command("ssh", sshargs...)
+ cmd.Stdin = stdin
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err == nil {
+ return 0
+ } else if exiterr, ok := err.(*exec.ExitError); !ok {
+ fmt.Fprintln(stderr, err)
+ return 1
+ } else if status, ok := exiterr.Sys().(syscall.WaitStatus); !ok {
+ fmt.Fprintln(stderr, err)
+ return 1
+ } else {
+ return status.ExitStatus()
+ }
+}
+
+// connectSSHCommand connects stdin/stdout to a container's gateway
+// server (see lib/crunchrun/ssh.go).
+//
+// It is intended to be invoked with OpenSSH client's ProxyCommand
+// config.
+type connectSSHCommand struct{}
+
+func (connectSSHCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ f := flag.NewFlagSet(prog, flag.ContinueOnError)
+ f.SetOutput(stderr)
+ f.Usage = func() {
+ fmt.Fprint(stderr, prog+`: connect to the gateway service for a running container.
+
+Usage: `+prog+` [options] container-uuid
+
+Options:
+`)
+ f.PrintDefaults()
+ }
+ detachKeys := f.String("detach-keys", "", "set detach key sequence, as in docker-attach(1)")
+ if err := f.Parse(args); err != nil {
+ fmt.Fprintln(stderr, err)
+ f.Usage()
+ return 2
+ } else if f.NArg() != 1 {
+ f.Usage()
+ return 2
+ }
+ targetUUID := f.Args()[0]
+ insecure := os.Getenv("ARVADOS_API_HOST_INSECURE")
+ rpcconn := rpc.NewConn("",
+ &url.URL{
+ Scheme: "https",
+ Host: os.Getenv("ARVADOS_API_HOST"),
+ },
+ insecure == "1" || insecure == "yes" || insecure == "true",
+ func(context.Context) ([]string, error) {
+ return []string{os.Getenv("ARVADOS_API_TOKEN")}, nil
+ })
+ // if strings.Contains(targetUUID, "-xvhdp-") {
+ // cr, err := rpcconn.ContainerRequestGet(context.TODO(), arvados.GetOptions{UUID: targetUUID})
+ // if err != nil {
+ // fmt.Fprintln(stderr, err)
+ // return 1
+ // }
+ // if cr.ContainerUUID == "" {
+ // fmt.Fprintf(stderr, "no container assigned, container request state is %s\n", strings.ToLower(cr.State))
+ // return 1
+ // }
+ // targetUUID = cr.ContainerUUID
+ // }
+ sshconn, err := rpcconn.ContainerSSH(context.TODO(), arvados.ContainerSSHOptions{
+ UUID: targetUUID,
+ DetachKeys: *detachKeys,
+ })
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ defer sshconn.Conn.Close()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ defer cancel()
+ _, err := io.Copy(stdout, sshconn.Conn)
+ if err != nil && ctx.Err() == nil {
+ fmt.Fprintf(stderr, "receive: %v\n", err)
+ }
+ }()
+ go func() {
+ defer cancel()
+ _, err := io.Copy(sshconn.Conn, stdin)
+ if err != nil && ctx.Err() == nil {
+ fmt.Fprintf(stderr, "send: %v\n", err)
+ }
+ }()
+ <-ctx.Done()
+ return 0
+}
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index ca55be53e..7d0353c9e 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -153,11 +153,13 @@ server {
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
+ proxy_set_header Host $http_host;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
proxy_set_header X-External-Client $external_client;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header X-Real-IP $remote_addr;
}
}
diff --git a/go.mod b/go.mod
index 262978d91..88dcb86c7 100644
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,7 @@ require (
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
github.com/coreos/go-oidc v2.1.0+incompatible
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
+ github.com/creack/pty v1.1.7
github.com/dnaeon/go-vcr v1.0.1 // indirect
github.com/docker/distribution v2.6.0-rc.1.0.20180105232752-277ed486c948+incompatible // indirect
github.com/docker/docker v1.4.2-0.20180109013817-94b8a116fbf1
@@ -33,6 +34,7 @@ require (
github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
github.com/go-ldap/ldap v3.0.3+incompatible
github.com/gogo/protobuf v1.1.1
+ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/mux v1.6.1-0.20180107155708-5bbbb5b2b572
github.com/hashicorp/golang-lru v0.5.1
diff --git a/go.sum b/go.sum
index 85d205112..91b5689eb 100644
--- a/go.sum
+++ b/go.sum
@@ -72,6 +72,8 @@ github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7 h1:e3u8KWFMR3irlDo1Z/tL8Hsz1MJmCLkSoX5AZRMKZkg=
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -134,6 +136,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 130368124..a32382ce2 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -336,6 +336,10 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
}
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
+ return conn.chooseBackend(options.UUID).ContainerSSH(ctx, options)
+}
+
func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
return conn.generated_SpecimenList(ctx, options)
}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 6669e020f..7847be0a4 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -25,6 +25,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/jmoiron/sqlx"
+
// sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
@@ -100,6 +101,7 @@ func (h *Handler) setup() {
mux.Handle("/arvados/v1/collections/", rtr)
mux.Handle("/arvados/v1/users", rtr)
mux.Handle("/arvados/v1/users/", rtr)
+ mux.Handle("/arvados/v1/connect/", rtr)
mux.Handle("/login", rtr)
mux.Handle("/logout", rtr)
}
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
new file mode 100644
index 000000000..01b4065f9
--- /dev/null
+++ b/lib/controller/localdb/container_gateway.go
@@ -0,0 +1,80 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "bufio"
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
+)
+
+// ContainerSSH returns a connection to the SSH server in the
+// appropriate crunch-run process on the worker node where the
+// specified container is running.
+//
+// If the returned error is nil, the caller is responsible for closing
+// sshconn.Conn.
+func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+ ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID})
+ if err != nil {
+ return
+ }
+ if ctr.GatewayAddress == "" || ctr.State != arvados.ContainerStateRunning {
+ err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusBadGateway)
+ return
+ }
+ netconn, err := net.Dial("tcp", ctr.GatewayAddress)
+ if err != nil {
+ return
+ }
+ bufr := bufio.NewReader(netconn)
+ bufw := bufio.NewWriter(netconn)
+
+ // Note this auth header does not protect from replay/mitm
+ // attacks (TODO: use TLS for that). It only authenticates us
+ // to crunch-run.
+ h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
+ fmt.Fprint(h, "%s", opts.UUID)
+ auth := fmt.Sprintf("%x", h.Sum(nil))
+
+ u := url.URL{
+ Scheme: "http",
+ Host: ctr.GatewayAddress,
+ Path: "/ssh",
+ }
+ bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
+ bufw.WriteString("Host: " + u.Host + "\r\n")
+ bufw.WriteString("Upgrade: ssh\r\n")
+ bufw.WriteString("X-Arvados-Target-Uuid: " + opts.UUID + "\r\n")
+ bufw.WriteString("X-Arvados-Authorization: " + auth + "\r\n")
+ bufw.WriteString("X-Arvados-Detach-Keys: " + opts.DetachKeys + "\r\n")
+ bufw.WriteString("\r\n")
+ bufw.Flush()
+ resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
+ if err != nil {
+ err = fmt.Errorf("error reading http response from gateway: %w", err)
+ netconn.Close()
+ return
+ }
+ if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
+ strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
+ err = httpserver.ErrorWithStatus(errors.New("bad upgrade"), http.StatusBadGateway)
+ netconn.Close()
+ return
+ }
+ sshconn.Conn = netconn
+ sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+ return
+}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 294452434..a09b66ced 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -186,6 +186,13 @@ func (rtr *router) addRoutes() {
return rtr.backend.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
},
},
+ {
+ arvados.EndpointContainerSSH,
+ func() interface{} { return &arvados.ContainerSSHOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.ContainerSSH(ctx, *opts.(*arvados.ContainerSSHOptions))
+ },
+ },
{
arvados.EndpointSpecimenCreate,
func() interface{} { return &arvados.CreateOptions{} },
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index cd98b6471..e80542e3e 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -5,6 +5,7 @@
package rpc
import (
+ "bufio"
"bytes"
"context"
"crypto/tls"
@@ -12,6 +13,7 @@ import (
"errors"
"fmt"
"io"
+ "io/ioutil"
"net"
"net/http"
"net/url"
@@ -21,6 +23,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
)
type TokenProvider func(context.Context) ([]string, error)
@@ -286,6 +289,61 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
return resp, err
}
+// ContainerSSH returns a connection to the out-of-band SSH server for
+// a running container. If the returned error is nil, the caller is
+// responsible for closing sshconn.Conn.
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+ netconn, err := tls.Dial("tcp", net.JoinHostPort(conn.baseURL.Host, "https"), nil)
+ if err != nil {
+ return
+ }
+ bufr := bufio.NewReader(netconn)
+ bufw := bufio.NewWriter(netconn)
+
+ u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
+ if err != nil {
+ netconn.Close()
+ return
+ }
+ u.RawQuery = url.Values{"detach_keys": {options.DetachKeys}}.Encode()
+ tokens, err := conn.tokenProvider(ctx)
+ if err != nil {
+ netconn.Close()
+ return
+ } else if len(tokens) < 1 {
+ err = httpserver.ErrorWithStatus(errors.New("unauthorized"), http.StatusUnauthorized)
+ netconn.Close()
+ return
+ }
+ bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
+ bufw.WriteString("Authorization: Bearer " + tokens[0] + "\r\n")
+ bufw.WriteString("Host: " + u.Host + "\r\n")
+ bufw.WriteString("Upgrade: ssh\r\n")
+ bufw.WriteString("\r\n")
+ bufw.Flush()
+ resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
+ if err != nil {
+ netconn.Close()
+ return
+ }
+ if resp.StatusCode != http.StatusSwitchingProtocols {
+ defer resp.Body.Close()
+ body, _ := ioutil.ReadAll(resp.Body)
+ err = fmt.Errorf("tunnel connection failed: %d %q", resp.StatusCode, body)
+ netconn.Close()
+ return
+ }
+ if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
+ strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
+ err = fmt.Errorf("bad response: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
+ netconn.Close()
+ return
+ }
+ sshconn.Conn = netconn
+ sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+ return
+}
+
func (conn *Conn) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
ep := arvados.EndpointSpecimenCreate
var resp arvados.Specimen
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
new file mode 100644
index 000000000..e55868ac3
--- /dev/null
+++ b/lib/crunchrun/container_gateway.go
@@ -0,0 +1,292 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "crypto/rand"
+ "crypto/rsa"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "sync"
+ "syscall"
+
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
+ "github.com/creack/pty"
+ "github.com/google/shlex"
+ "golang.org/x/crypto/ssh"
+)
+
+// startGatewayServer starts an http server that allows authenticated
+// clients to open an interactive "docker exec" session and (in
+// future) connect to tcp ports inside the docker container.
+func (runner *ContainerRunner) startGatewayServer() error {
+ runner.gatewaySSHConfig = &ssh.ServerConfig{
+ NoClientAuth: true,
+ PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+ if c.User() == "root" {
+ return nil, nil
+ } else {
+ return nil, fmt.Errorf("unimplemented: cannot log in as non-root user %q", c.User())
+ }
+ },
+ PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
+ if c.User() == "root" {
+ return &ssh.Permissions{
+ Extensions: map[string]string{
+ "pubkey-fp": ssh.FingerprintSHA256(pubKey),
+ },
+ }, nil
+ } else {
+ return nil, fmt.Errorf("unimplemented: cannot log in as non-root user %q", c.User())
+ }
+ },
+ }
+ pvt, err := rsa.GenerateKey(rand.Reader, 4096)
+ if err != nil {
+ return err
+ }
+ err = pvt.Validate()
+ if err != nil {
+ return err
+ }
+ signer, err := ssh.NewSignerFromKey(pvt)
+ if err != nil {
+ return err
+ }
+ runner.gatewaySSHConfig.AddHostKey(signer)
+
+ // GatewayAddress (provided by arvados-dispatch-cloud) is
+ // HOST:PORT where HOST is our IP address or hostname as seen
+ // from arvados-controller, and PORT is either the desired
+ // port where we should run our gateway server, or "0" if
+ // we should choose an available port.
+ host, port, err := net.SplitHostPort(os.Getenv("GatewayAddress"))
+ if err != nil {
+ return err
+ }
+ srv := &httpserver.Server{
+ Server: http.Server{
+ Handler: http.HandlerFunc(runner.handleSSH),
+ },
+ Addr: ":" + port,
+ }
+ err = srv.Start()
+ if err != nil {
+ return err
+ }
+ // Get the port number we are listening on (the port might be
+ // "0" or a port name, in which case this will be different).
+ _, port, err = net.SplitHostPort(srv.Addr)
+ if err != nil {
+ return err
+ }
+ // When changing state to Running, we will set
+ // gateway_address to "HOST:PORT" where HOST is our
+ // external hostname/IP as provided by arvados-dispatch-cloud,
+ // and PORT is the port number we ended up listening on.
+ runner.gatewayAddress = net.JoinHostPort(host, port)
+ return nil
+}
+
+// handleSSH connects to an SSH server that runs commands as root in
+// the container. The tunnel itself can only be created by an
+// authenticated caller, so the SSH server itself is wide open (any
+// password or key will be accepted).
+//
+// Requests must have path "/ssh" and the following headers:
+//
+// Connection: upgrade
+// Upgrade: ssh
+// X-Arvados-Target-Uuid: uuid of container
+// X-Arvados-Authorization: must match GatewayAuthSecret provided by
+// a-d-c (this prevents other containers and shell nodes from
+// connecting directly)
+//
+// Optional header:
+//
+// X-Arvados-Detach-Keys: argument to "docker attach --detach-keys",
+// e.g., "ctrl-p,ctrl-q"
+func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Request) {
+ // In future we'll handle browser traffic too, but for now the
+ // only traffic we expect is an SSH tunnel from
+ // (*lib/controller/localdb.Conn)ContainerSSH()
+ if req.Method != "GET" || req.Header.Get("Upgrade") != "ssh" {
+ http.Error(w, "path not found", http.StatusNotFound)
+ return
+ }
+ if want := req.Header.Get("X-Arvados-Target-Uuid"); want != runner.Container.UUID {
+ http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, runner.Container.UUID), http.StatusBadGateway)
+ return
+ }
+ if req.Header.Get("X-Arvados-Authorization") != runner.gatewayAuthSecret {
+ http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
+ return
+ }
+ detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
+ return
+ }
+ netconn, _, err := hj.Hijack()
+ if !ok {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer netconn.Close()
+ w.Header().Set("Connection", "upgrade")
+ w.Header().Set("Upgrade", "ssh")
+ netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
+ w.Header().Write(netconn)
+ netconn.Write([]byte("\r\n"))
+
+ ctx := req.Context()
+
+ conn, newchans, reqs, err := ssh.NewServerConn(netconn, runner.gatewaySSHConfig)
+ if err != nil {
+ runner.CrunchLog.Printf("ssh.NewServerConn: %s", err)
+ return
+ }
+ defer conn.Close()
+ go ssh.DiscardRequests(reqs)
+ for newch := range newchans {
+ if newch.ChannelType() != "session" {
+ newch.Reject(ssh.UnknownChannelType, "unknown channel type")
+ continue
+ }
+ ch, reqs, err := newch.Accept()
+ if err != nil {
+ runner.CrunchLog.Printf("accept channel: %s", err)
+ return
+ }
+ var pty0, tty0 *os.File
+ go func() {
+ defer pty0.Close()
+ defer tty0.Close()
+ // Where to send errors/messages for the
+ // client to see
+ logw := io.Writer(ch.Stderr())
+ // How to end lines when sending
+ // errors/messages to the client (changes to
+ // \r\n when using a pty)
+ eol := "\n"
+ // Env vars to add to child process
+ termEnv := []string(nil)
+ for req := range reqs {
+ ok := false
+ switch req.Type {
+ case "shell", "exec":
+ ok = true
+ var payload struct {
+ Command string
+ }
+ ssh.Unmarshal(req.Payload, &payload)
+ execargs, err := shlex.Split(payload.Command)
+ if err != nil {
+ fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
+ return
+ }
+ if len(execargs) == 0 {
+ execargs = []string{"/bin/bash", "-login"}
+ }
+ go func() {
+ cmd := exec.CommandContext(ctx, "docker", "exec", "-i", "--detach-keys="+detachKeys)
+ cmd.Stdin = ch
+ cmd.Stdout = ch
+ cmd.Stderr = ch.Stderr()
+ if tty0 != nil {
+ cmd.Args = append(cmd.Args, "-t")
+ cmd.Stdin = tty0
+ cmd.Stdout = tty0
+ cmd.Stderr = tty0
+ var wg sync.WaitGroup
+ defer wg.Wait()
+ wg.Add(2)
+ go func() { io.Copy(ch, pty0); wg.Done() }()
+ go func() { io.Copy(pty0, ch); wg.Done() }()
+ // Send our own debug messages to tty as well.
+ logw = tty0
+ }
+ cmd.Args = append(cmd.Args, runner.ContainerID)
+ cmd.Args = append(cmd.Args, execargs...)
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ Setctty: tty0 != nil,
+ Setsid: true,
+ }
+ cmd.Env = append(os.Environ(), termEnv...)
+ err := cmd.Run()
+ errClose := ch.CloseWrite()
+ var resp struct {
+ Status uint32
+ }
+ if err, ok := err.(*exec.ExitError); ok {
+ if status, ok := err.Sys().(syscall.WaitStatus); ok {
+ resp.Status = uint32(status.ExitStatus())
+ }
+ }
+ if resp.Status == 0 && (err != nil || errClose != nil) {
+ resp.Status = 1
+ }
+ ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
+ ch.Close()
+ }()
+ case "pty-req":
+ eol = "\r\n"
+ p, t, err := pty.Open()
+ if err != nil {
+ fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
+ break
+ }
+ pty0, tty0 = p, t
+ ok = true
+ var payload struct {
+ Term string
+ Cols uint32
+ Rows uint32
+ X uint32
+ Y uint32
+ }
+ ssh.Unmarshal(req.Payload, &payload)
+ termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
+ err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
+ if err != nil {
+ fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
+ }
+ case "window-change":
+ var payload struct {
+ Cols uint32
+ Rows uint32
+ X uint32
+ Y uint32
+ }
+ ssh.Unmarshal(req.Payload, &payload)
+ err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
+ if err != nil {
+ fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
+ break
+ }
+ ok = true
+ case "env":
+ // TODO: implement "env"
+ // requests by setting env
+ // vars in the docker-exec
+ // command (not docker-exec's
+ // own environment, which
+ // would be a gaping security
+ // hole).
+ default:
+ // fmt.Fprintf(logw, "declining %q req"+eol, req.Type)
+ }
+ if req.WantReply {
+ req.Reply(ok, nil)
+ }
+ }
+ }()
+ }
+}
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 341938354..b252e0dce 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -33,6 +33,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"git.arvados.org/arvados.git/sdk/go/manifest"
+ "golang.org/x/crypto/ssh"
"golang.org/x/net/context"
dockertypes "github.com/docker/docker/api/types"
@@ -178,6 +179,10 @@ type ContainerRunner struct {
arvMountLog *ThrottledLogger
containerWatchdogInterval time.Duration
+
+ gatewayAddress string
+ gatewaySSHConfig *ssh.ServerConfig
+ gatewayAuthSecret string
}
// setupSignals sets up signal handling to gracefully terminate the underlying
@@ -1469,7 +1474,7 @@ func (runner *ContainerRunner) UpdateContainerRunning() error {
return ErrCancelled
}
return runner.DispatcherArvClient.Update("containers", runner.Container.UUID,
- arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running"}}, nil)
+ arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running", "gateway_address": runner.gatewayAddress}}, nil)
}
// ContainerToken returns the api_token the container (and any
@@ -1868,6 +1873,14 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
return 1
}
+ cr.gatewayAuthSecret = os.Getenv("GatewayAuthSecret")
+ os.Unsetenv("GatewayAuthSecret")
+ err = cr.startGatewayServer()
+ if err != nil {
+ log.Printf("error starting gateway server: %s", err)
+ return 1
+ }
+
parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
if tmperr != nil {
log.Printf("%s: %v", containerID, tmperr)
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index a25ed6015..e092e7ada 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -5,8 +5,10 @@
package worker
import (
+ "crypto/hmac"
"crypto/md5"
"crypto/rand"
+ "crypto/sha256"
"errors"
"fmt"
"io"
@@ -116,6 +118,7 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe
timeoutTERM: duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM),
timeoutSignal: duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal),
timeoutStaleRunLock: duration(cluster.Containers.CloudVMs.TimeoutStaleRunLock, defaultTimeoutStaleRunLock),
+ systemRootToken: cluster.SystemRootToken,
installPublicKey: installPublicKey,
tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix,
stop: make(chan bool),
@@ -154,6 +157,7 @@ type Pool struct {
timeoutTERM time.Duration
timeoutSignal time.Duration
timeoutStaleRunLock time.Duration
+ systemRootToken string
installPublicKey ssh.PublicKey
tagKeyPrefix string
@@ -990,6 +994,12 @@ func (wp *Pool) waitUntilLoaded() {
}
}
+func (wp *Pool) gatewayAuthSecret(uuid string) string {
+ h := hmac.New(sha256.New, []byte(wp.systemRootToken))
+ fmt.Fprint(h, "%s", uuid)
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
+
// Return a random string of n hexadecimal digits (n*4 random bits). n
// must be even.
func randomHex(n int) string {
diff --git a/lib/dispatchcloud/worker/runner.go b/lib/dispatchcloud/worker/runner.go
index 475212134..0fd99aeee 100644
--- a/lib/dispatchcloud/worker/runner.go
+++ b/lib/dispatchcloud/worker/runner.go
@@ -8,6 +8,7 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "net"
"syscall"
"time"
@@ -48,6 +49,8 @@ func newRemoteRunner(uuid string, wkr *worker) *remoteRunner {
"ARVADOS_API_HOST": wkr.wp.arvClient.APIHost,
"ARVADOS_API_TOKEN": wkr.wp.arvClient.AuthToken,
"InstanceType": instJSON.String(),
+ "GatewayAddress": net.JoinHostPort(wkr.instance.Address(), "0"),
+ "GatewayAuthSecret": wkr.wp.gatewayAuthSecret(uuid),
}
if wkr.wp.arvClient.Insecure {
env["ARVADOS_API_HOST_INSECURE"] = "1"
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 5a2cfb880..fa4471804 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -5,8 +5,10 @@
package arvados
import (
+ "bufio"
"context"
"encoding/json"
+ "net"
)
type APIEndpoint struct {
@@ -41,6 +43,7 @@ var (
EndpointContainerDelete = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
EndpointContainerLock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
EndpointContainerUnlock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
+ EndpointContainerSSH = APIEndpoint{"GET", "arvados/v1/connect/{uuid}/ssh", ""} // move to /containers after #17014 fixes routing
EndpointUserActivate = APIEndpoint{"POST", "arvados/v1/users/{uuid}/activate", ""}
EndpointUserCreate = APIEndpoint{"POST", "arvados/v1/users", "user"}
EndpointUserCurrent = APIEndpoint{"GET", "arvados/v1/users/current", ""}
@@ -60,6 +63,16 @@ var (
EndpointAPIClientAuthorizationCurrent = APIEndpoint{"GET", "arvados/v1/api_client_authorizations/current", ""}
)
+type ContainerSSHOptions struct {
+ UUID string `json:"uuid"`
+ DetachKeys string `json:"detach_keys"`
+}
+
+type ContainerSSHConnection struct {
+ Conn net.Conn `json:"-"`
+ Bufrw *bufio.ReadWriter `json:"-"`
+}
+
type GetOptions struct {
UUID string `json:"uuid,omitempty"`
Select []string `json:"select"`
@@ -175,6 +188,7 @@ type API interface {
ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
ContainerLock(ctx context.Context, options GetOptions) (Container, error)
ContainerUnlock(ctx context.Context, options GetOptions) (Container, error)
+ ContainerSSH(ctx context.Context, options ContainerSSHOptions) (ContainerSSHConnection, error)
SpecimenCreate(ctx context.Context, options CreateOptions) (Specimen, error)
SpecimenUpdate(ctx context.Context, options UpdateOptions) (Specimen, error)
SpecimenGet(ctx context.Context, options GetOptions) (Specimen, error)
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 265944e81..4d1511b41 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -30,6 +30,7 @@ type Container struct {
RuntimeStatus map[string]interface{} `json:"runtime_status"`
StartedAt *time.Time `json:"started_at"` // nil if not yet started
FinishedAt *time.Time `json:"finished_at"` // nil if not yet finished
+ GatewayAddress string `json:"gateway_address"`
}
// ContainerRequest is an arvados#container_request resource.
diff --git a/sdk/go/arvados/container_gateway.go b/sdk/go/arvados/container_gateway.go
new file mode 100644
index 000000000..07f8c0793
--- /dev/null
+++ b/sdk/go/arvados/container_gateway.go
@@ -0,0 +1,53 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "context"
+ "io"
+ "net/http"
+
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+func (sshconn ContainerSSHConnection) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Connection", "upgrade")
+ w.Header().Set("Upgrade", "ssh")
+ w.WriteHeader(http.StatusSwitchingProtocols)
+ conn, bufrw, err := hj.Hijack()
+ if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Error("error hijacking ResponseWriter")
+ return
+ }
+ defer conn.Close()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ defer cancel()
+ _, err := io.CopyN(conn, sshconn.Bufrw, int64(sshconn.Bufrw.Reader.Buffered()))
+ if err == nil {
+ _, err = io.Copy(conn, sshconn.Conn)
+ }
+ if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Error("error copying downstream")
+ }
+ }()
+ go func() {
+ defer cancel()
+ _, err := io.CopyN(sshconn.Conn, bufrw, int64(bufrw.Reader.Buffered()))
+ if err == nil {
+ _, err = io.Copy(sshconn.Conn, conn)
+ }
+ if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Error("error copying upstream")
+ }
+ }()
+ <-ctx.Done()
+}
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 5833c2251..a4da593ec 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -76,6 +76,7 @@ class Container < ArvadosModel
t.add :runtime_user_uuid
t.add :runtime_auth_scopes
t.add :lock_count
+ t.add :gateway_address
end
# Supported states for a container
@@ -478,7 +479,7 @@ class Container < ArvadosModel
when Running
permitted.push :priority, *progress_attrs
if self.state_changed?
- permitted.push :started_at
+ permitted.push :started_at, :gateway_address
end
when Complete
diff --git a/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb b/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb
new file mode 100644
index 000000000..8683b5190
--- /dev/null
+++ b/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class AddGatewayAddressToContainers < ActiveRecord::Migration[5.2]
+ def change
+ add_column :containers, :gateway_address, :string
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 12a28c6c7..249ec67ac 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -521,7 +521,8 @@ CREATE TABLE public.containers (
runtime_user_uuid text,
runtime_auth_scopes jsonb,
runtime_token text,
- lock_count integer DEFAULT 0 NOT NULL
+ lock_count integer DEFAULT 0 NOT NULL,
+ gateway_address character varying
);
@@ -3187,6 +3188,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20200914203202'),
('20201103170213'),
('20201105190435'),
-('20201202174753');
+('20201202174753'),
+('20210108033940');
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list