[ARVADOS] updated: 2.1.0-334-g77c8223f5
Git user
git at public.arvados.org
Thu Jan 28 17:30:53 UTC 2021
Summary of changes:
.../container_requests_controller_test.rb | 1 -
build/package-build-dockerfiles/Makefile | 4 +-
build/package-build-dockerfiles/centos7/Dockerfile | 4 +-
.../package-build-dockerfiles/debian10/Dockerfile | 4 +-
.../ubuntu1604/Dockerfile | 4 +-
.../ubuntu1804/Dockerfile | 4 +-
.../ubuntu2004/Dockerfile | 4 +-
build/run-library.sh | 4 +
build/run-tests.sh | 2 +-
doc/_config.yml | 1 +
doc/_includes/_install_compute_docker.liquid | 7 +-
doc/api/keep-web-urls.html.textile.liquid | 10 ++
.../methods/container_requests.html.textile.liquid | 2 +-
.../keep-components-overview.html.textile.liquid | 61 +++++++++
.../install-dispatch-cloud.html.textile.liquid | 4 +
doc/install/install-keep-web.html.textile.liquid | 13 +-
.../getting_started/setup-cli.html.textile.liquid | 20 +++
lib/boot/supervisor.go | 2 +
lib/cloud/ec2/ec2.go | 21 ++-
lib/config/config.default.yml | 12 +-
lib/config/generated_config.go | 12 +-
lib/controller/cmd.go | 1 +
lib/controller/federation/conn.go | 62 +++++++++
lib/controller/federation/generate.go | 2 +-
lib/controller/federation/generated.go | 41 ++++++
lib/controller/federation_test.go | 137 +++++++++++--------
lib/controller/handler.go | 2 +
lib/controller/handler_test.go | 2 +
lib/controller/integration_test.go | 152 ++++++++++++++++++++-
lib/controller/localdb/conn.go | 9 +-
lib/controller/localdb/login.go | 31 +++--
lib/controller/localdb/login_ldap.go | 6 +-
lib/controller/localdb/login_ldap_test.go | 4 +-
lib/controller/localdb/login_oidc.go | 20 +--
lib/controller/localdb/login_oidc_test.go | 60 ++++++++
lib/controller/localdb/login_pam.go | 6 +-
lib/controller/localdb/login_pam_test.go | 4 +-
lib/controller/localdb/login_testuser.go | 6 +-
lib/controller/localdb/login_testuser_test.go | 4 +-
lib/controller/router/response.go | 76 +++++++----
lib/controller/router/router.go | 35 +++++
lib/controller/rpc/conn.go | 56 +++++++-
lib/crunchrun/crunchrun.go | 6 +-
lib/crunchrun/crunchrun_test.go | 6 +-
lib/install/deps.go | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 11 ++
sdk/cwl/arvados_cwl/executor.py | 7 +-
sdk/go/arvados/api.go | 9 ++
sdk/go/arvados/container.go | 5 +-
sdk/go/arvadostest/api.go | 20 +++
.../app/controllers/user_sessions_controller.rb | 35 +++--
services/api/app/models/arvados_model.rb | 34 +++++
services/api/app/models/container.rb | 10 +-
services/api/app/models/container_request.rb | 25 ++--
services/api/test/fixtures/container_requests.yml | 47 +++++--
.../v1/container_requests_controller_test.rb | 25 +++-
.../functional/user_sessions_controller_test.rb | 25 ++++
services/api/test/unit/container_request_test.rb | 47 ++-----
services/api/test/unit/container_test.rb | 14 +-
services/keep-web/s3_test.go | 1 +
60 files changed, 972 insertions(+), 269 deletions(-)
create mode 100644 doc/architecture/keep-components-overview.html.textile.liquid
create mode 100644 doc/user/getting_started/setup-cli.html.textile.liquid
via 77c8223f5ddd64cff2b08d0857749644c474946f (commit)
via 7885ae2c39bd2e7a38943fe9a56463fb4349a5ac (commit)
via 5a1a3d656c6d2d92edaed19c735a9b1e1eb39f71 (commit)
via 6470f7ce527c2cc44e25de402bd0418c821d13a8 (commit)
via fc3531079e960359afb1039a5fd24a059a813baa (commit)
via 7631343e1bfafddca31b1ababbaa63c8a9aea1bb (commit)
via 969441a091ce3aa1eb7a9525d3ab85f24fbd8fdd (commit)
via f0d3eae5fc05aaad38a2998627c59637e3ef606c (commit)
via bec7d0354140620311407073f08573b5d1c037d5 (commit)
via 75efbc85be494c802f3c6822875cb64b9a34de6b (commit)
via 714cc9a889786a062641f3627d557667cc71c337 (commit)
via 5e846bd0291b1eba00b294373fde76bf95a9e091 (commit)
via bff33bf988ea57ef226d108c574a68053be287ce (commit)
via 7c99875efea3ec5415427063cf79ddc44c58f55f (commit)
via 8cbdc1b04e64e2d5e394e86aba699b361bcdc24a (commit)
via 80dbda890bf58bda79654cf4cebdfbc2b07d6b1f (commit)
via 4248a4ebd23813e9bdcd68547ae03ff7d6082463 (commit)
via fbc95892b4b8cce3cba9ae024c252bd31146c714 (commit)
via a6fd7801f084f280cbf668f8a1f6bf9d9bbd0def (commit)
via 41dec55a4b007a79c58f77d36ad0940b5354b453 (commit)
via 4c6c49190b5a8949120d822e053657f64146df70 (commit)
via 3576206ef265d0040bcc93899b9885f16b5919e6 (commit)
via bd535934937eb0863bd8eaec0b62ab81bc8e4700 (commit)
via 5727f64521ea7222422dd48e48793a0fe10253f4 (commit)
via 882bfd2383da2f9d3cbd2b5f258e727e385a4fbb (commit)
via 3ec51dcf456b2afe02857089895a261653abddd4 (commit)
via 8bed34a9d78506caff52f550d5e47224207ac73c (commit)
via dec7dc55c641f70c5966afc2661deff25dc2cf6d (commit)
via 11e02a3a1148f5f47de17b0fb0d33cb4042820b7 (commit)
via 4c30d75e647f42318fd0069613b3ed4f82c70ea0 (commit)
via efac197a128851bd5e894267b3b7a75268182f94 (commit)
via f64f557db0bfe6f33d434853a94ee5cff7e69a5d (commit)
from 0945afa5523fb45f827750e4d1700df4ff222295 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 77c8223f5ddd64cff2b08d0857749644c474946f
Merge: 0945afa55 7885ae2c3
Author: Tom Clegg <tom at curii.com>
Date: Thu Jan 28 12:04:52 2021 -0500
17170: Merge branch 'master'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc lib/controller/federation/conn.go
index a32382ce2,00523c782..b86266d67
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@@ -336,10 -336,68 +336,72 @@@ func (conn *Conn) ContainerUnlock(ctx c
return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
}
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
+ return conn.chooseBackend(options.UUID).ContainerSSH(ctx, options)
+}
+
+ func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
+ return conn.generated_ContainerRequestList(ctx, options)
+ }
+
+ func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ be := conn.chooseBackend(options.ClusterID)
+ if be == conn.local {
+ return be.ContainerRequestCreate(ctx, options)
+ }
+ if _, ok := options.Attrs["runtime_token"]; !ok {
+ // If runtime_token is not set, create a new token
+ aca, err := conn.local.APIClientAuthorizationCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ // This should probably be StatusUnauthorized
+ // (need to update test in
+ // lib/controller/federation_test.go):
+ // When RoR is out of the picture this should be:
+ // return arvados.ContainerRequest{}, httpErrorf(http.StatusUnauthorized, "%w", err)
+ return arvados.ContainerRequest{}, httpErrorf(http.StatusForbidden, "%s", "invalid API token")
+ }
+ user, err := conn.local.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ if len(aca.Scopes) == 0 || aca.Scopes[0] != "all" {
+ return arvados.ContainerRequest{}, httpErrorf(http.StatusForbidden, "token scope is not [all]")
+ }
+ if strings.HasPrefix(aca.UUID, conn.cluster.ClusterID) {
+ // Local user, submitting to a remote cluster.
+ // Create a new time-limited token.
+ local, ok := conn.local.(*localdb.Conn)
+ if !ok {
+ return arvados.ContainerRequest{}, httpErrorf(http.StatusInternalServerError, "bug: local backend is a %T, not a *localdb.Conn", conn.local)
+ }
+ aca, err = local.CreateAPIClientAuthorization(ctx, conn.cluster.SystemRootToken, rpc.UserSessionAuthInfo{UserUUID: user.UUID,
+ ExpiresAt: time.Now().UTC().Add(conn.cluster.Collections.BlobSigningTTL.Duration())})
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ options.Attrs["runtime_token"] = aca.TokenV2()
+ } else {
+ // Remote user. Container request will use the
+ // current token, minus the trailing portion
+ // (optional container uuid).
+ options.Attrs["runtime_token"] = aca.TokenV2()
+ }
+ }
+ return be.ContainerRequestCreate(ctx, options)
+ }
+
+ func (conn *Conn) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ return conn.chooseBackend(options.UUID).ContainerRequestUpdate(ctx, options)
+ }
+
+ func (conn *Conn) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ return conn.chooseBackend(options.UUID).ContainerRequestGet(ctx, options)
+ }
+
+ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.ContainerRequest, error) {
+ return conn.chooseBackend(options.UUID).ContainerRequestDelete(ctx, options)
+ }
+
func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
return conn.generated_SpecimenList(ctx, options)
}
diff --cc lib/controller/handler.go
index 7847be0a4,b04757ac3..5f6fb192e
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -101,7 -100,8 +101,9 @@@ func (h *Handler) setup()
mux.Handle("/arvados/v1/collections/", rtr)
mux.Handle("/arvados/v1/users", rtr)
mux.Handle("/arvados/v1/users/", rtr)
+ mux.Handle("/arvados/v1/connect/", rtr)
+ mux.Handle("/arvados/v1/container_requests", rtr)
+ mux.Handle("/arvados/v1/container_requests/", rtr)
mux.Handle("/login", rtr)
mux.Handle("/logout", rtr)
}
diff --cc lib/controller/rpc/conn.go
index c9c0ac308,d9d24260b..3a19f4ab5
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@@ -23,9 -21,10 +23,11 @@@ import
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
)
+ const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
type TokenProvider func(context.Context) ([]string, error)
func PassthroughTokenProvider(ctx context.Context) ([]string, error) {
@@@ -289,82 -302,27 +305,103 @@@ func (conn *Conn) ContainerUnlock(ctx c
return resp, err
}
+// ContainerSSH returns a connection to the out-of-band SSH server for
+// a running container. If the returned error is nil, the caller is
+// responsible for closing sshconn.Conn.
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+ addr := conn.baseURL.Host
+ if strings.Index(addr, ":") < 1 || (strings.Contains(addr, "::") && addr[0] != '[') {
+ // hostname or ::1 or 1::1
+ addr = net.JoinHostPort(addr, "https")
+ }
+ insecure := false
+ if tlsconf := conn.httpClient.Transport.(*http.Transport).TLSClientConfig; tlsconf != nil && tlsconf.InsecureSkipVerify {
+ insecure = true
+ }
+ netconn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: insecure})
+ if err != nil {
+ err = fmt.Errorf("tls.Dial: %w", err)
+ return
+ }
+ defer func() {
+ if err != nil {
+ netconn.Close()
+ }
+ }()
+ bufr := bufio.NewReader(netconn)
+ bufw := bufio.NewWriter(netconn)
+
+ u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
+ if err != nil {
+ err = fmt.Errorf("tls.Dial: %w", err)
+ return
+ }
+ u.RawQuery = url.Values{
+ "detach_keys": {options.DetachKeys},
+ "login_username": {options.LoginUsername},
+ }.Encode()
+ tokens, err := conn.tokenProvider(ctx)
+ if err != nil {
+ return
+ } else if len(tokens) < 1 {
+ err = httpserver.ErrorWithStatus(errors.New("unauthorized"), http.StatusUnauthorized)
+ return
+ }
+ bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
+ bufw.WriteString("Authorization: Bearer " + tokens[0] + "\r\n")
+ bufw.WriteString("Host: " + u.Host + "\r\n")
+ bufw.WriteString("Upgrade: ssh\r\n")
+ bufw.WriteString("\r\n")
+ bufw.Flush()
+ resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
+ if err != nil {
+ err = fmt.Errorf("http.ReadResponse: %w", err)
+ return
+ }
+ if resp.StatusCode != http.StatusSwitchingProtocols {
+ defer resp.Body.Close()
+ body, _ := ioutil.ReadAll(resp.Body)
+ var message string
+ var errDoc httpserver.ErrorResponse
+ if err := json.Unmarshal(body, &errDoc); err == nil {
+ message = strings.Join(errDoc.Errors, "; ")
+ } else {
+ message = fmt.Sprintf("%q", body)
+ }
+ err = fmt.Errorf("server did not provide a tunnel: %s (HTTP %d)", message, resp.StatusCode)
+ return
+ }
+ if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
+ strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
+ err = fmt.Errorf("bad response from server: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
+ return
+ }
+ sshconn.Conn = netconn
+ sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+ return
+}
+
+ func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ ep := arvados.EndpointContainerRequestCreate
+ var resp arvados.ContainerRequest
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+ }
+
+ func (conn *Conn) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ ep := arvados.EndpointContainerRequestUpdate
+ var resp arvados.ContainerRequest
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+ }
+
+ func (conn *Conn) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ ep := arvados.EndpointContainerRequestGet
+ var resp arvados.ContainerRequest
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+ }
+
func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
ep := arvados.EndpointContainerRequestList
var resp arvados.ContainerRequestList
diff --cc sdk/go/arvados/api.go
index 4675906e7,a11872971..37a3e007b
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@@ -45,8 -41,11 +45,12 @@@ var
EndpointContainerDelete = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
EndpointContainerLock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
EndpointContainerUnlock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
+ EndpointContainerSSH = APIEndpoint{"GET", "arvados/v1/connect/{uuid}/ssh", ""} // move to /containers after #17014 fixes routing
+ EndpointContainerRequestCreate = APIEndpoint{"POST", "arvados/v1/container_requests", "container_request"}
+ EndpointContainerRequestUpdate = APIEndpoint{"PATCH", "arvados/v1/container_requests/{uuid}", "container_request"}
+ EndpointContainerRequestGet = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}", ""}
EndpointContainerRequestList = APIEndpoint{"GET", "arvados/v1/container_requests", ""}
+ EndpointContainerRequestDelete = APIEndpoint{"DELETE", "arvados/v1/container_requests/{uuid}", ""}
EndpointUserActivate = APIEndpoint{"POST", "arvados/v1/users/{uuid}/activate", ""}
EndpointUserCreate = APIEndpoint{"POST", "arvados/v1/users", "user"}
EndpointUserCurrent = APIEndpoint{"GET", "arvados/v1/users/current", ""}
@@@ -193,7 -180,11 +197,12 @@@ type API interface
ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
ContainerLock(ctx context.Context, options GetOptions) (Container, error)
ContainerUnlock(ctx context.Context, options GetOptions) (Container, error)
+ ContainerSSH(ctx context.Context, options ContainerSSHOptions) (ContainerSSHConnection, error)
+ ContainerRequestCreate(ctx context.Context, options CreateOptions) (ContainerRequest, error)
+ ContainerRequestUpdate(ctx context.Context, options UpdateOptions) (ContainerRequest, error)
+ ContainerRequestGet(ctx context.Context, options GetOptions) (ContainerRequest, error)
+ ContainerRequestList(ctx context.Context, options ListOptions) (ContainerRequestList, error)
+ ContainerRequestDelete(ctx context.Context, options DeleteOptions) (ContainerRequest, error)
SpecimenCreate(ctx context.Context, options CreateOptions) (Specimen, error)
SpecimenUpdate(ctx context.Context, options UpdateOptions) (Specimen, error)
SpecimenGet(ctx context.Context, options GetOptions) (Specimen, error)
diff --cc sdk/go/arvadostest/api.go
index 2b7854947,df3e46feb..930eabf27
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@@ -105,10 -105,26 +105,30 @@@ func (as *APIStub) ContainerUnlock(ctx
as.appendCall(ctx, as.ContainerUnlock, options)
return arvados.Container{}, as.Error
}
+func (as *APIStub) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
+ as.appendCall(ctx, as.ContainerSSH, options)
+ return arvados.ContainerSSHConnection{}, as.Error
+}
+ func (as *APIStub) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ as.appendCall(ctx, as.ContainerRequestCreate, options)
+ return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ as.appendCall(ctx, as.ContainerRequestUpdate, options)
+ return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ as.appendCall(ctx, as.ContainerRequestGet, options)
+ return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
+ as.appendCall(ctx, as.ContainerRequestList, options)
+ return arvados.ContainerRequestList{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.ContainerRequest, error) {
+ as.appendCall(ctx, as.ContainerRequestDelete, options)
+ return arvados.ContainerRequest{}, as.Error
+ }
func (as *APIStub) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
as.appendCall(ctx, as.SpecimenCreate, options)
return arvados.Specimen{}, as.Error
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list