[ARVADOS] updated: 2.1.0-334-g77c8223f5

Git user git at public.arvados.org
Thu Jan 28 17:30:53 UTC 2021


Summary of changes:
 .../container_requests_controller_test.rb          |   1 -
 build/package-build-dockerfiles/Makefile           |   4 +-
 build/package-build-dockerfiles/centos7/Dockerfile |   4 +-
 .../package-build-dockerfiles/debian10/Dockerfile  |   4 +-
 .../ubuntu1604/Dockerfile                          |   4 +-
 .../ubuntu1804/Dockerfile                          |   4 +-
 .../ubuntu2004/Dockerfile                          |   4 +-
 build/run-library.sh                               |   4 +
 build/run-tests.sh                                 |   2 +-
 doc/_config.yml                                    |   1 +
 doc/_includes/_install_compute_docker.liquid       |   7 +-
 doc/api/keep-web-urls.html.textile.liquid          |  10 ++
 .../methods/container_requests.html.textile.liquid |   2 +-
 .../keep-components-overview.html.textile.liquid   |  61 +++++++++
 .../install-dispatch-cloud.html.textile.liquid     |   4 +
 doc/install/install-keep-web.html.textile.liquid   |  13 +-
 .../getting_started/setup-cli.html.textile.liquid  |  20 +++
 lib/boot/supervisor.go                             |   2 +
 lib/cloud/ec2/ec2.go                               |  21 ++-
 lib/config/config.default.yml                      |  12 +-
 lib/config/generated_config.go                     |  12 +-
 lib/controller/cmd.go                              |   1 +
 lib/controller/federation/conn.go                  |  62 +++++++++
 lib/controller/federation/generate.go              |   2 +-
 lib/controller/federation/generated.go             |  41 ++++++
 lib/controller/federation_test.go                  | 137 +++++++++++--------
 lib/controller/handler.go                          |   2 +
 lib/controller/handler_test.go                     |   2 +
 lib/controller/integration_test.go                 | 152 ++++++++++++++++++++-
 lib/controller/localdb/conn.go                     |   9 +-
 lib/controller/localdb/login.go                    |  31 +++--
 lib/controller/localdb/login_ldap.go               |   6 +-
 lib/controller/localdb/login_ldap_test.go          |   4 +-
 lib/controller/localdb/login_oidc.go               |  20 +--
 lib/controller/localdb/login_oidc_test.go          |  60 ++++++++
 lib/controller/localdb/login_pam.go                |   6 +-
 lib/controller/localdb/login_pam_test.go           |   4 +-
 lib/controller/localdb/login_testuser.go           |   6 +-
 lib/controller/localdb/login_testuser_test.go      |   4 +-
 lib/controller/router/response.go                  |  76 +++++++----
 lib/controller/router/router.go                    |  35 +++++
 lib/controller/rpc/conn.go                         |  56 +++++++-
 lib/crunchrun/crunchrun.go                         |   6 +-
 lib/crunchrun/crunchrun_test.go                    |   6 +-
 lib/install/deps.go                                |   2 +-
 sdk/cwl/arvados_cwl/arvcontainer.py                |  11 ++
 sdk/cwl/arvados_cwl/executor.py                    |   7 +-
 sdk/go/arvados/api.go                              |   9 ++
 sdk/go/arvados/container.go                        |   5 +-
 sdk/go/arvadostest/api.go                          |  20 +++
 .../app/controllers/user_sessions_controller.rb    |  35 +++--
 services/api/app/models/arvados_model.rb           |  34 +++++
 services/api/app/models/container.rb               |  10 +-
 services/api/app/models/container_request.rb       |  25 ++--
 services/api/test/fixtures/container_requests.yml  |  47 +++++--
 .../v1/container_requests_controller_test.rb       |  25 +++-
 .../functional/user_sessions_controller_test.rb    |  25 ++++
 services/api/test/unit/container_request_test.rb   |  47 ++-----
 services/api/test/unit/container_test.rb           |  14 +-
 services/keep-web/s3_test.go                       |   1 +
 60 files changed, 972 insertions(+), 269 deletions(-)
 create mode 100644 doc/architecture/keep-components-overview.html.textile.liquid
 create mode 100644 doc/user/getting_started/setup-cli.html.textile.liquid

       via  77c8223f5ddd64cff2b08d0857749644c474946f (commit)
       via  7885ae2c39bd2e7a38943fe9a56463fb4349a5ac (commit)
       via  5a1a3d656c6d2d92edaed19c735a9b1e1eb39f71 (commit)
       via  6470f7ce527c2cc44e25de402bd0418c821d13a8 (commit)
       via  fc3531079e960359afb1039a5fd24a059a813baa (commit)
       via  7631343e1bfafddca31b1ababbaa63c8a9aea1bb (commit)
       via  969441a091ce3aa1eb7a9525d3ab85f24fbd8fdd (commit)
       via  f0d3eae5fc05aaad38a2998627c59637e3ef606c (commit)
       via  bec7d0354140620311407073f08573b5d1c037d5 (commit)
       via  75efbc85be494c802f3c6822875cb64b9a34de6b (commit)
       via  714cc9a889786a062641f3627d557667cc71c337 (commit)
       via  5e846bd0291b1eba00b294373fde76bf95a9e091 (commit)
       via  bff33bf988ea57ef226d108c574a68053be287ce (commit)
       via  7c99875efea3ec5415427063cf79ddc44c58f55f (commit)
       via  8cbdc1b04e64e2d5e394e86aba699b361bcdc24a (commit)
       via  80dbda890bf58bda79654cf4cebdfbc2b07d6b1f (commit)
       via  4248a4ebd23813e9bdcd68547ae03ff7d6082463 (commit)
       via  fbc95892b4b8cce3cba9ae024c252bd31146c714 (commit)
       via  a6fd7801f084f280cbf668f8a1f6bf9d9bbd0def (commit)
       via  41dec55a4b007a79c58f77d36ad0940b5354b453 (commit)
       via  4c6c49190b5a8949120d822e053657f64146df70 (commit)
       via  3576206ef265d0040bcc93899b9885f16b5919e6 (commit)
       via  bd535934937eb0863bd8eaec0b62ab81bc8e4700 (commit)
       via  5727f64521ea7222422dd48e48793a0fe10253f4 (commit)
       via  882bfd2383da2f9d3cbd2b5f258e727e385a4fbb (commit)
       via  3ec51dcf456b2afe02857089895a261653abddd4 (commit)
       via  8bed34a9d78506caff52f550d5e47224207ac73c (commit)
       via  dec7dc55c641f70c5966afc2661deff25dc2cf6d (commit)
       via  11e02a3a1148f5f47de17b0fb0d33cb4042820b7 (commit)
       via  4c30d75e647f42318fd0069613b3ed4f82c70ea0 (commit)
       via  efac197a128851bd5e894267b3b7a75268182f94 (commit)
       via  f64f557db0bfe6f33d434853a94ee5cff7e69a5d (commit)
      from  0945afa5523fb45f827750e4d1700df4ff222295 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 77c8223f5ddd64cff2b08d0857749644c474946f
Merge: 0945afa55 7885ae2c3
Author: Tom Clegg <tom at curii.com>
Date:   Thu Jan 28 12:04:52 2021 -0500

    17170: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --cc lib/controller/federation/conn.go
index a32382ce2,00523c782..b86266d67
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@@ -336,10 -336,68 +336,72 @@@ func (conn *Conn) ContainerUnlock(ctx c
  	return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
  }
  
 +func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
 +	return conn.chooseBackend(options.UUID).ContainerSSH(ctx, options)
 +}
 +
+ func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
+ 	return conn.generated_ContainerRequestList(ctx, options)
+ }
+ 
+ func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ 	be := conn.chooseBackend(options.ClusterID)
+ 	if be == conn.local {
+ 		return be.ContainerRequestCreate(ctx, options)
+ 	}
+ 	if _, ok := options.Attrs["runtime_token"]; !ok {
+ 		// If runtime_token is not set, create a new token
+ 		aca, err := conn.local.APIClientAuthorizationCurrent(ctx, arvados.GetOptions{})
+ 		if err != nil {
+ 			// This should probably be StatusUnauthorized
+ 			// (need to update test in
+ 			// lib/controller/federation_test.go):
+ 			// When RoR is out of the picture this should be:
+ 			// return arvados.ContainerRequest{}, httpErrorf(http.StatusUnauthorized, "%w", err)
+ 			return arvados.ContainerRequest{}, httpErrorf(http.StatusForbidden, "%s", "invalid API token")
+ 		}
+ 		user, err := conn.local.UserGetCurrent(ctx, arvados.GetOptions{})
+ 		if err != nil {
+ 			return arvados.ContainerRequest{}, err
+ 		}
+ 		if len(aca.Scopes) == 0 || aca.Scopes[0] != "all" {
+ 			return arvados.ContainerRequest{}, httpErrorf(http.StatusForbidden, "token scope is not [all]")
+ 		}
+ 		if strings.HasPrefix(aca.UUID, conn.cluster.ClusterID) {
+ 			// Local user, submitting to a remote cluster.
+ 			// Create a new time-limited token.
+ 			local, ok := conn.local.(*localdb.Conn)
+ 			if !ok {
+ 				return arvados.ContainerRequest{}, httpErrorf(http.StatusInternalServerError, "bug: local backend is a %T, not a *localdb.Conn", conn.local)
+ 			}
+ 			aca, err = local.CreateAPIClientAuthorization(ctx, conn.cluster.SystemRootToken, rpc.UserSessionAuthInfo{UserUUID: user.UUID,
+ 				ExpiresAt: time.Now().UTC().Add(conn.cluster.Collections.BlobSigningTTL.Duration())})
+ 			if err != nil {
+ 				return arvados.ContainerRequest{}, err
+ 			}
+ 			options.Attrs["runtime_token"] = aca.TokenV2()
+ 		} else {
+ 			// Remote user. Container request will use the
+ 			// current token, minus the trailing portion
+ 			// (optional container uuid).
+ 			options.Attrs["runtime_token"] = aca.TokenV2()
+ 		}
+ 	}
+ 	return be.ContainerRequestCreate(ctx, options)
+ }
+ 
+ func (conn *Conn) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ 	return conn.chooseBackend(options.UUID).ContainerRequestUpdate(ctx, options)
+ }
+ 
+ func (conn *Conn) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ 	return conn.chooseBackend(options.UUID).ContainerRequestGet(ctx, options)
+ }
+ 
+ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.ContainerRequest, error) {
+ 	return conn.chooseBackend(options.UUID).ContainerRequestDelete(ctx, options)
+ }
+ 
  func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
  	return conn.generated_SpecimenList(ctx, options)
  }
diff --cc lib/controller/handler.go
index 7847be0a4,b04757ac3..5f6fb192e
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -101,7 -100,8 +101,9 @@@ func (h *Handler) setup() 
  		mux.Handle("/arvados/v1/collections/", rtr)
  		mux.Handle("/arvados/v1/users", rtr)
  		mux.Handle("/arvados/v1/users/", rtr)
 +		mux.Handle("/arvados/v1/connect/", rtr)
+ 		mux.Handle("/arvados/v1/container_requests", rtr)
+ 		mux.Handle("/arvados/v1/container_requests/", rtr)
  		mux.Handle("/login", rtr)
  		mux.Handle("/logout", rtr)
  	}
diff --cc lib/controller/rpc/conn.go
index c9c0ac308,d9d24260b..3a19f4ab5
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@@ -23,9 -21,10 +23,11 @@@ import 
  
  	"git.arvados.org/arvados.git/sdk/go/arvados"
  	"git.arvados.org/arvados.git/sdk/go/auth"
 +	"git.arvados.org/arvados.git/sdk/go/httpserver"
  )
  
+ const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+ 
  type TokenProvider func(context.Context) ([]string, error)
  
  func PassthroughTokenProvider(ctx context.Context) ([]string, error) {
@@@ -289,82 -302,27 +305,103 @@@ func (conn *Conn) ContainerUnlock(ctx c
  	return resp, err
  }
  
 +// ContainerSSH returns a connection to the out-of-band SSH server for
 +// a running container. If the returned error is nil, the caller is
 +// responsible for closing sshconn.Conn.
 +func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
 +	addr := conn.baseURL.Host
 +	if strings.Index(addr, ":") < 1 || (strings.Contains(addr, "::") && addr[0] != '[') {
 +		// hostname or ::1 or 1::1
 +		addr = net.JoinHostPort(addr, "https")
 +	}
 +	insecure := false
 +	if tlsconf := conn.httpClient.Transport.(*http.Transport).TLSClientConfig; tlsconf != nil && tlsconf.InsecureSkipVerify {
 +		insecure = true
 +	}
 +	netconn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: insecure})
 +	if err != nil {
 +		err = fmt.Errorf("tls.Dial: %w", err)
 +		return
 +	}
 +	defer func() {
 +		if err != nil {
 +			netconn.Close()
 +		}
 +	}()
 +	bufr := bufio.NewReader(netconn)
 +	bufw := bufio.NewWriter(netconn)
 +
 +	u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
 +	if err != nil {
 +		err = fmt.Errorf("tls.Dial: %w", err)
 +		return
 +	}
 +	u.RawQuery = url.Values{
 +		"detach_keys":    {options.DetachKeys},
 +		"login_username": {options.LoginUsername},
 +	}.Encode()
 +	tokens, err := conn.tokenProvider(ctx)
 +	if err != nil {
 +		return
 +	} else if len(tokens) < 1 {
 +		err = httpserver.ErrorWithStatus(errors.New("unauthorized"), http.StatusUnauthorized)
 +		return
 +	}
 +	bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
 +	bufw.WriteString("Authorization: Bearer " + tokens[0] + "\r\n")
 +	bufw.WriteString("Host: " + u.Host + "\r\n")
 +	bufw.WriteString("Upgrade: ssh\r\n")
 +	bufw.WriteString("\r\n")
 +	bufw.Flush()
 +	resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
 +	if err != nil {
 +		err = fmt.Errorf("http.ReadResponse: %w", err)
 +		return
 +	}
 +	if resp.StatusCode != http.StatusSwitchingProtocols {
 +		defer resp.Body.Close()
 +		body, _ := ioutil.ReadAll(resp.Body)
 +		var message string
 +		var errDoc httpserver.ErrorResponse
 +		if err := json.Unmarshal(body, &errDoc); err == nil {
 +			message = strings.Join(errDoc.Errors, "; ")
 +		} else {
 +			message = fmt.Sprintf("%q", body)
 +		}
 +		err = fmt.Errorf("server did not provide a tunnel: %s (HTTP %d)", message, resp.StatusCode)
 +		return
 +	}
 +	if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
 +		strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
 +		err = fmt.Errorf("bad response from server: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
 +		return
 +	}
 +	sshconn.Conn = netconn
 +	sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
 +	return
 +}
 +
+ func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ 	ep := arvados.EndpointContainerRequestCreate
+ 	var resp arvados.ContainerRequest
+ 	err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ 	return resp, err
+ }
+ 
+ func (conn *Conn) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ 	ep := arvados.EndpointContainerRequestUpdate
+ 	var resp arvados.ContainerRequest
+ 	err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ 	return resp, err
+ }
+ 
+ func (conn *Conn) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ 	ep := arvados.EndpointContainerRequestGet
+ 	var resp arvados.ContainerRequest
+ 	err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ 	return resp, err
+ }
+ 
  func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
  	ep := arvados.EndpointContainerRequestList
  	var resp arvados.ContainerRequestList
diff --cc sdk/go/arvados/api.go
index 4675906e7,a11872971..37a3e007b
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@@ -45,8 -41,11 +45,12 @@@ var 
  	EndpointContainerDelete               = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
  	EndpointContainerLock                 = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
  	EndpointContainerUnlock               = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
 +	EndpointContainerSSH                  = APIEndpoint{"GET", "arvados/v1/connect/{uuid}/ssh", ""} // move to /containers after #17014 fixes routing
+ 	EndpointContainerRequestCreate        = APIEndpoint{"POST", "arvados/v1/container_requests", "container_request"}
+ 	EndpointContainerRequestUpdate        = APIEndpoint{"PATCH", "arvados/v1/container_requests/{uuid}", "container_request"}
+ 	EndpointContainerRequestGet           = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}", ""}
  	EndpointContainerRequestList          = APIEndpoint{"GET", "arvados/v1/container_requests", ""}
+ 	EndpointContainerRequestDelete        = APIEndpoint{"DELETE", "arvados/v1/container_requests/{uuid}", ""}
  	EndpointUserActivate                  = APIEndpoint{"POST", "arvados/v1/users/{uuid}/activate", ""}
  	EndpointUserCreate                    = APIEndpoint{"POST", "arvados/v1/users", "user"}
  	EndpointUserCurrent                   = APIEndpoint{"GET", "arvados/v1/users/current", ""}
@@@ -193,7 -180,11 +197,12 @@@ type API interface 
  	ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
  	ContainerLock(ctx context.Context, options GetOptions) (Container, error)
  	ContainerUnlock(ctx context.Context, options GetOptions) (Container, error)
 +	ContainerSSH(ctx context.Context, options ContainerSSHOptions) (ContainerSSHConnection, error)
+ 	ContainerRequestCreate(ctx context.Context, options CreateOptions) (ContainerRequest, error)
+ 	ContainerRequestUpdate(ctx context.Context, options UpdateOptions) (ContainerRequest, error)
+ 	ContainerRequestGet(ctx context.Context, options GetOptions) (ContainerRequest, error)
+ 	ContainerRequestList(ctx context.Context, options ListOptions) (ContainerRequestList, error)
+ 	ContainerRequestDelete(ctx context.Context, options DeleteOptions) (ContainerRequest, error)
  	SpecimenCreate(ctx context.Context, options CreateOptions) (Specimen, error)
  	SpecimenUpdate(ctx context.Context, options UpdateOptions) (Specimen, error)
  	SpecimenGet(ctx context.Context, options GetOptions) (Specimen, error)
diff --cc sdk/go/arvadostest/api.go
index 2b7854947,df3e46feb..930eabf27
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@@ -105,10 -105,26 +105,30 @@@ func (as *APIStub) ContainerUnlock(ctx 
  	as.appendCall(ctx, as.ContainerUnlock, options)
  	return arvados.Container{}, as.Error
  }
 +func (as *APIStub) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
 +	as.appendCall(ctx, as.ContainerSSH, options)
 +	return arvados.ContainerSSHConnection{}, as.Error
 +}
+ func (as *APIStub) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ 	as.appendCall(ctx, as.ContainerRequestCreate, options)
+ 	return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ 	as.appendCall(ctx, as.ContainerRequestUpdate, options)
+ 	return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestGet(ctx context.Context, options arvados.GetOptions) (arvados.ContainerRequest, error) {
+ 	as.appendCall(ctx, as.ContainerRequestGet, options)
+ 	return arvados.ContainerRequest{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
+ 	as.appendCall(ctx, as.ContainerRequestList, options)
+ 	return arvados.ContainerRequestList{}, as.Error
+ }
+ func (as *APIStub) ContainerRequestDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.ContainerRequest, error) {
+ 	as.appendCall(ctx, as.ContainerRequestDelete, options)
+ 	return arvados.ContainerRequest{}, as.Error
+ }
  func (as *APIStub) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
  	as.appendCall(ctx, as.SpecimenCreate, options)
  	return arvados.Specimen{}, as.Error

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list