[arvados] updated: 2.6.0-73-gcaa54be9b
git repository hosting
git at public.arvados.org
Fri Apr 21 00:15:43 UTC 2023
Summary of changes:
apps/workbench/Gemfile.lock | 10 +-
cmd/arvados-client/container_gateway.go | 112 +-
cmd/arvados-client/container_gateway_test.go | 2 +-
doc/Rakefile | 6 +-
doc/_config.yml | 1 +
doc/admin/upgrading.html.textile.liquid | 22 +-
doc/api/methods.html.textile.liquid | 2 +-
.../methods/container_requests.html.textile.liquid | 31 +-
doc/api/methods/containers.html.textile.liquid | 23 -
doc/gen_api_method_docs.py | 130 -
doc/gen_api_schema_docs.py | 79 -
...configure-s3-object-storage.html.textile.liquid | 9 -
doc/install/salt-multi-host.html.textile.liquid | 37 +-
doc/install/salt-single-host.html.textile.liquid | 20 +
doc/install/setup-login.html.textile.liquid | 40 +-
doc/install/workbench.html.textile.liquid | 90 +
go.mod | 18 +-
go.sum | 600 +-
lib/config/config.default.yml | 5 +-
lib/controller/federation/conn.go | 8 +-
lib/controller/handler.go | 143 +
lib/controller/handler_test.go | 214 +-
lib/controller/localdb/container.go | 4 +-
lib/controller/localdb/container_gateway.go | 89 +-
lib/controller/localdb/container_gateway_test.go | 112 +-
lib/controller/localdb/container_test.go | 93 +-
lib/controller/localdb/login.go | 45 +-
lib/controller/localdb/login_oidc_test.go | 2 +-
lib/controller/localdb/login_test.go | 75 +
lib/controller/router/router.go | 14 +-
lib/controller/router/router_test.go | 60 +-
lib/controller/rpc/conn.go | 33 +-
lib/crunchrun/container_gateway.go | 17 +-
lib/dispatchcloud/worker/worker.go | 30 +-
lib/dispatchcloud/worker/worker_test.go | 33 +
sdk/R/DESCRIPTION | 2 +-
sdk/R/R/Collection.R | 2 +-
sdk/R/tests/testthat/fakes/FakeRESTService.R | 6 +-
sdk/R/tests/testthat/test-Collection.R | 6 +
sdk/go/arvados/api.go | 4 +-
sdk/go/arvados/config.go | 1 -
sdk/go/arvados/log.go | 2 +-
sdk/go/arvadostest/api.go | 32 +-
sdk/go/arvadostest/proxy.go | 7 +
sdk/python/MANIFEST.in | 4 +-
sdk/python/arvados-v1-discovery.json | 11395 +++++++++
sdk/python/discovery2pydoc.py | 372 +
sdk/python/setup.py | 73 +-
services/api/Gemfile.lock | 10 +-
.../arvados/v1/container_requests_controller.rb | 16 +
.../arvados/v1/containers_controller.rb | 14 +-
.../controllers/arvados/v1/schema_controller.rb | 798 +-
.../api/app/controllers/database_controller.rb | 2 +-
services/api/app/models/api_client.rb | 10 +-
services/api/app/models/arvados_model.rb | 2 +-
.../config/initializers/schema_discovery_cache.rb | 9 -
.../20221219165512_dedup_permission_links.rb | 56 +-
services/api/lib/current_api_client.rb | 106 +-
services/api/lib/record_filters.rb | 33 +-
.../api/test/functional/arvados/v1/filters_test.rb | 35 +
.../arvados/v1/schema_controller_test.rb | 1 -
.../test/integration/discovery_document_test.rb | 58 +
services/api/test/integration/remote_user_test.rb | 1 -
services/api/test/unit/api_client_test.rb | 27 +
services/keep-balance/balance.go | 46 +-
services/keep-balance/balance_test.go | 29 +
services/keep-web/handler.go | 6 +-
services/keep-web/handler_test.go | 114 +
services/keep-web/s3.go | 52 +-
services/keepstore/s3_volume.go | 1084 -
services/keepstore/s3_volume_test.go | 594 -
services/keepstore/s3aws_volume.go | 33 +-
services/ws/event.go | 2 +-
services/ws/event_source.go | 2 +-
services/ws/event_source_test.go | 4 +-
services/ws/session_v0.go | 4 +-
services/ws/session_v0_test.go | 6 +-
.../aws/dashboards/arvados_overview.json | 1708 ++
.../aws/dashboards/node-exporter-full_rev30.json | 23200 +++++++++++++++++++
.../aws/dashboards/postgresql_exporter.json | 1826 ++
.../multi_host/aws/pillars/arvados.sls | 4 +-
.../multi_host/aws/pillars/grafana.sls | 30 +
...n.sls => letsencrypt_grafana_configuration.sls} | 4 +-
...ls => letsencrypt_prometheus_configuration.sls} | 4 +-
...uration.sls => nginx_grafana_configuration.sls} | 38 +-
...tion.sls => nginx_prometheus_configuration.sls} | 36 +-
.../multi_host/aws/pillars/postgresql.sls | 11 +-
.../aws/pillars/prometheus_node_exporter.sls | 17 +
.../aws/pillars/prometheus_pg_exporter.sls | 14 +
.../multi_host/aws/pillars/prometheus_server.sls | 89 +
.../multi_host/aws/states/grafana_admin_user.sls | 13 +
.../multi_host/aws/states/grafana_dashboards.sls | 48 +
.../multi_host/aws/states/grafana_datasource.sls | 28 +
.../multi_host/aws/states/host_entries.sls | 6 -
.../aws/states/nginx_prometheus_configuration.sls | 22 +
.../aws/states/prometheus_pg_exporter.sls | 82 +
tools/salt-install/installer.sh | 107 +-
.../local.params.example.multiple_hosts | 105 +-
...l.params.example.single_host_multiple_hostnames | 46 +-
...ocal.params.example.single_host_single_hostname | 66 +-
tools/salt-install/provision.sh | 103 +-
.../salt-install/terraform/aws/services/locals.tf | 3 +-
tools/salt-install/terraform/aws/services/main.tf | 6 +-
.../salt-install/terraform/aws/services/outputs.tf | 4 +-
tools/salt-install/terraform/aws/vpc/locals.tf | 13 +-
tools/salt-install/terraform/aws/vpc/main.tf | 46 +-
tools/salt-install/terraform/aws/vpc/outputs.tf | 16 +-
107 files changed, 41369 insertions(+), 3595 deletions(-)
delete mode 100755 doc/gen_api_method_docs.py
delete mode 100755 doc/gen_api_schema_docs.py
create mode 100644 doc/install/workbench.html.textile.liquid
create mode 100644 lib/controller/localdb/login_test.go
create mode 100644 sdk/python/arvados-v1-discovery.json
create mode 100755 sdk/python/discovery2pydoc.py
delete mode 100644 services/api/config/initializers/schema_discovery_cache.rb
create mode 100644 services/api/test/integration/discovery_document_test.rb
delete mode 100644 services/keepstore/s3_volume.go
delete mode 100644 services/keepstore/s3_volume_test.go
create mode 100644 tools/salt-install/config_examples/multi_host/aws/dashboards/arvados_overview.json
create mode 100644 tools/salt-install/config_examples/multi_host/aws/dashboards/node-exporter-full_rev30.json
create mode 100644 tools/salt-install/config_examples/multi_host/aws/dashboards/postgresql_exporter.json
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/grafana.sls
copy tools/salt-install/config_examples/multi_host/aws/pillars/{letsencrypt_websocket_configuration.sls => letsencrypt_grafana_configuration.sls} (65%)
copy tools/salt-install/config_examples/multi_host/aws/pillars/{letsencrypt_websocket_configuration.sls => letsencrypt_prometheus_configuration.sls} (64%)
copy tools/salt-install/config_examples/multi_host/aws/pillars/{nginx_download_configuration.sls => nginx_grafana_configuration.sls} (65%)
copy tools/salt-install/config_examples/multi_host/aws/pillars/{nginx_keepproxy_configuration.sls => nginx_prometheus_configuration.sls} (64%)
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_node_exporter.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_pg_exporter.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_server.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/grafana_admin_user.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/grafana_dashboards.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/grafana_datasource.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/nginx_prometheus_configuration.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/prometheus_pg_exporter.sls
via caa54be9b2984ce2d08dcc6ad377e7f0be339dae (commit)
via 92bf76b1077282126b8a1c446458c6618a36e5d5 (commit)
via 24d6a59e1b524f2af9490b593c3280e550411622 (commit)
via d02f5b352a9c4d21f072ddecda1bf2cfb5577442 (commit)
via 5cc1710b57f98905469225c68d975ad2e3e7e56d (commit)
via 6b4d02513c17b3de39578f7b3658ea178206fdd4 (commit)
via d75c91ace9d4f9c7ab000a31425ce0f784c47e9f (commit)
via afe739890e0cfe781600e3a58bd5386f1f2e3ac7 (commit)
via ddccc6d8f1e04c8708b99dd2d0d3ee4b9b70528f (commit)
via 02ae62dcdc5d5cd9231b32b407bd525e74633003 (commit)
via 7ea5b1b2b78ceaa326d8f21ce1d08df0be1d9cda (commit)
via 499240fcfa25c10bc22277b7d382dcbc31436cbe (commit)
via 22dfbd2ea818464019f605b4103124d58c7355bd (commit)
via 57792708500261a817e6957e65c80c7f798a36e9 (commit)
via 347c7205cde8e9e969975ea79dca4135e9f2ce5a (commit)
via 272ce633588e6e6881d3e27bef4ef6de12555cfe (commit)
via 1cb3683f67cd4fc29e9db10bee23745405380ce0 (commit)
via a22c685d97ceae848267907529406310276a5e39 (commit)
via 933e687424e67f3a3e3c064016abd295b49c5f98 (commit)
via deb01192f78dfe9875c7297a7e57a926086bee47 (commit)
via f5a90a8dd208596aa8ce8cfd3086bd367b84b147 (commit)
via bce5560c00bc5cb6301da0ddd82f3af8a4b24778 (commit)
via 045abe070900c3aa8769ae8f7991ec96f6c7280a (commit)
via e0287beae4381ac82334a6c3859884bcbadfb285 (commit)
via aa369aa06f977a6a3c114e11e472a0ddeb2a07c9 (commit)
via 571e94931049a757bed2971084f7d6bf7cf6cb6e (commit)
via b3c7d9bbf3d8c341e35f887434b85208481b11a5 (commit)
via b25ced299202637987056a485191fc51e65c712f (commit)
via 30951519f83d2e874f0928bc22c08db2864d163c (commit)
via df51f23dc102d118ed23866bb71ef88498d95dc2 (commit)
via 313c0242e956ee4d97cfc9c7080daeaf42475164 (commit)
via a6775c49206d5ce301ef29c5561ba8a9c01007be (commit)
via 2689eb208af65c9a645c89d46744bbd17395710c (commit)
via ddf2c60df3880ac3e3c5424424ad51c094e3759f (commit)
via 7afaea0c59c4a2da152de652870f9b0e457f74cc (commit)
via cefc46389792e233ee5db00da2a22a68e54e0a69 (commit)
via d63d6ba645b8e34cf9d248bd0407fab0b9ef6534 (commit)
via 5f0b4881b7522a952fda5be96c7041a6519d485d (commit)
via c8492303cf40c55f121720dfd0407822aa7d074e (commit)
via 1ce29e3dd0fbaaf76a662373eaf44fe488386e0e (commit)
via a56ec3ec5a275b443f39787863b133d13651d074 (commit)
via 598a1fdcfe7bcbbcd654e4be3232bf7042b5c6f2 (commit)
via ee9e02d91f35aa2f3dedbb1248c9fc91fa9a78c3 (commit)
via 280906487df75b8350db9138287e0a8d84d8c8bd (commit)
via 1cb23beff15a30af65d7c4bea189dea186424c1d (commit)
via 7ea0bf89df6f3400a1c96f8838a2a7a1e3706e48 (commit)
via 2690f3d6ca7700cc24b555ee926403235ebd9342 (commit)
via fce5461328b4875dc963ba78d60a64663300b72b (commit)
via 430ba41e3d979b82630c7aa41f00014439a7bb48 (commit)
via 3f8deee8bca244601503ec0434bbb80f0886e370 (commit)
via eeb809db27ffccd6d0c827aa0dc006443a88a754 (commit)
via 06d8ba5b386ebffe6c86762fb7f799325407a6c1 (commit)
via 903995cd2fdfd506ee8cac2e4eae0c04f56a2b2e (commit)
via 15582540d26d52d72bd363fb75842f3f1019fb20 (commit)
via 15ff8739b8ae8c89976a315da0cdfcdc2e302aa5 (commit)
via 09a0297320055e02a9928344dd12482940fa3e89 (commit)
via 8b979548c070a93eda006b45f99e256b8a61fb8d (commit)
via b21a5b18225f1e333f83b5ec9a79b5ac937567d5 (commit)
via 4109c502bc67c82479ddaf95f1754e0769c012d4 (commit)
via 6e4f551d713dafa59e2117ac156703c3774bc811 (commit)
via 3e0786c27f85e715ac3d624a397ad84ae938384d (commit)
via 7e20ccadead4c215b530fc4634014c6c3d16a51d (commit)
via d9fdb9630e56f3ccdaee6acd8b1ca4cdbdf11b0a (commit)
via e54204ff452ec8778a1f62c19e3241dea8d310af (commit)
via 25ded0f5c0b44dfcc97f945331487abf07a91feb (commit)
via 78910a657391081eb46b2b7a84d985b7daea6389 (commit)
via 28da710485559dac72a8b2380dce1973a5e2893a (commit)
via 8b42f8d30a103b22b39c8cd2d407ecbbddd008b6 (commit)
via dd08650c44e691a3828b6febb121e821de2c8019 (commit)
via 72ce6ed68a7af7de8f4afbee6b7cfb73763a5c3f (commit)
via 98969e546c909ac2ee4256934b5339080598d252 (commit)
via c9ba0f292c708d30a182343fec460aa5c48b4af3 (commit)
via b517f68ab03879edb3cec475bd1988c2e5fe96bd (commit)
via b2d19d835285b579785b7cde16ea6bd880ecc361 (commit)
via 94fefe0eee024d4a56f0b6a296ca2ce04e38e464 (commit)
via e0c3829f04a3f72410dafd17e14dd07331111fe3 (commit)
via 41935b6bc7e8bec90e284082c169479e8f02e4cd (commit)
via 3add838714e2b62091881506b90ce915b4f68501 (commit)
via ec733bd6285dcb6a7ab029ba02fe6ddbb64da8d9 (commit)
via 908d141b6564f90c2ed9e0e6c9d7a4397a528c9f (commit)
via a263abb4c84bf639b5169749a992454d0948d3d3 (commit)
via 2bc1519df3f1995c852cf9bfba6e85ebd0f33c84 (commit)
via 340caa63a2ef01224c1b69db7aa63da8ec20696b (commit)
via e2dfad1cee9bfad7c148429925ca6cea651cd419 (commit)
via 247fd765bc1fbf559572d6e5590893c9190f086a (commit)
via 4e26d30a7cb1b788c117f983e133e303c08c780f (commit)
via 682cdfb3298d5184c7999251c48aae661bca2797 (commit)
via f3d9d18903250ea879a2dee2f156463de49089f0 (commit)
via bf20e6cd15964a70857d796838160482a568125f (commit)
via 2c2308f4161cfbb3eecfbf87e8decdfe3e128f6d (commit)
via 5f1d2739d5633ab2b40acd3f413f41dfdbb7e696 (commit)
via 5720f268fb6d6995042dd689ae760770fa3cf54e (commit)
via b433aff7b75bf571cf8376cdf0a7237abd405fe8 (commit)
via be9687cc230d12b9ddc9246840861aaeae8749ea (commit)
via 7aeaedbd8009c596bfc159432bb7b1f09c19ed72 (commit)
via 187979032338beb7656e56ab21944d53fd7f4569 (commit)
via d9bab7e493575da3dd4265bff72d3d6b08c63fdb (commit)
via bdde690c4479ae294707a65f4f4d259427611d70 (commit)
via 9d031e1c35662393daf7611a8fc81f3c3c22623c (commit)
via 47592afcdea474cd6c8900544a57e86292e12e59 (commit)
via fd63f537dfc8399330f16c47599ea2e9947972ee (commit)
via a437b5c897b7630dc7f18955242b8cd2d20b682e (commit)
via 9680a4a50200ade72af3d3e35d43775efefb9f94 (commit)
via 2d3dc22dac4d83aa444a61f717a83aee7b17cf6d (commit)
via 9f4f850c2818cbf5eea45e30d27b13c20bd2be0c (commit)
via ce5ad31aac392b38889fc04640fcd06474cebeb6 (commit)
via 85c625c40ad873d0efac33f8a63c1ee256185e36 (commit)
via fe1c718ce87a72b84f59215decc613d4d5010dce (commit)
via 9c98d96b29d25199a325b488ec771072697dc5bc (commit)
via 7c47ef8311a4cd801fe22b3428a991285b5442a2 (commit)
via 2179ac7db0b30640db18e03660cbfe26b19c9d77 (commit)
via c16b905d89fb7c2b9ed8f91ca299f08874b58dab (commit)
via b8d80c01a51ce3d271f838231b90e70bf4460d0e (commit)
via dae3da4d70bfba901f2147775dd6c07be61416b2 (commit)
via 1abd17dde53982ed058abf770072123f61ed13af (commit)
via dceebc9953147b08b68a36330e213f7280851450 (commit)
via 04826743ea647f67fef414761a4ca2536523226b (commit)
via 4334ff2b99833bb31c1228cececfbcab451f5511 (commit)
from 63c57d1b8604f27f0224ed76a5b8f048f982a74f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit caa54be9b2984ce2d08dcc6ad377e7f0be339dae
Merge: 24d6a59e1 92bf76b10
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 20 20:15:21 2023 -0400
18790: Merge branch '20319-container-request-logs' into 18790-log-client
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit 24d6a59e1b524f2af9490b593c3280e550411622
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 20 16:06:48 2023 -0400
18790: Move to container_requests/*/log endpoint.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index 3b90a334e..e4d1141a0 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -66,7 +66,7 @@ func (lc *logsCommand) tailf(crUUID string, stdout, stderr io.Writer, pollInterv
defer cancel()
rpcconn := rpcFromEnv()
- err = lc.checkAPISupport(ctx, crUUID)
+ err := lc.checkAPISupport(ctx, crUUID)
if err != nil {
return err
}
@@ -74,46 +74,38 @@ func (lc *logsCommand) tailf(crUUID string, stdout, stderr io.Writer, pollInterv
var (
// files to display
watching = []string{"crunch-run.txt", "stderr.txt"}
- // last known container UUID
- cUUID = ""
// fnm => file offset of next byte to display
mark = map[string]int64{}
// fnm => current size of file reported by api
size = map[string]int64{}
- // exit after fetching next log chunk
- containerFinished = false
// has anything worked? (if so, retry after errors)
anySuccess = false
- // container UUID that we most recently displayed in a
- // "connected, polling" message (if any)
- reportedConnection = ""
+ // container UUID whose logs we are displaying
+ displayingUUID = ""
+ // container UUID we last showed in a "connected,
+ // polling" message
+ reportedUUID = ""
)
+ cr, err := rpcconn.ContainerRequestGet(ctx, arvados.GetOptions{UUID: crUUID, Select: []string{"uuid", "container_uuid", "state"}})
+ if err != nil {
+ return fmt.Errorf("error retrieving %s: %w", crUUID, err)
+ }
+ displayingUUID = cr.ContainerUUID
poll:
for delay := pollInterval; ; time.Sleep(delay) {
- if cUUID == "" {
- // This is our first iteration, or the
- // container we were watching has exited.
- ctr, err := rpcconn.ContainerRequestGet(ctx, arvados.GetOptions{UUID: crUUID})
- if err != nil {
- return err
- }
- if newUUID == ctrUUID {
- // no further attempts
- return nil
- }
- ctrUUID = newUUID
- containerFinished = false
- delay = 0
- continue
+ if cr.ContainerUUID == "" {
+ return fmt.Errorf("%s has no assigned container (state is %s)", crUUID, cr.State)
+ }
+ if delay < pollInterval {
+ delay = pollInterval
}
-
// When .../container_requests/{uuid}/log_events is
// implemented, we'll wait here for the next
// server-sent event to tell us some updated file
// sizes. For now, we poll.
for _, fnm := range watching {
- currentsize, _, err := lc.copyRange(ctx, crUUID, cUUID, fnm, "-0", nil)
+ currentsize, _, err := lc.copyRange(ctx, cr.UUID, displayingUUID, fnm, "-0", nil)
if err != nil {
if !anySuccess {
return err
@@ -122,9 +114,9 @@ poll:
delay = pollInterval
continue poll
}
- if reportedConnection != ctrUUID {
- reportedConnection = ctrUUID
- fmt.Fprintln(stderr, "connected, polling for log data from container", ctrUUID)
+ if reportedUUID != displayingUUID {
+ fmt.Fprintln(stderr, "connected, polling for log data from container", displayingUUID)
+ reportedUUID = displayingUUID
}
size[fnm] = currentsize
if oldsize, seen := mark[fnm]; !seen && currentsize > 10000 {
@@ -135,9 +127,7 @@ poll:
// Log collection must have been
// emptied and reset.
fmt.Fprintln(stderr, "--- log restarted ---")
- for fnm := range mark {
- delete(mark, fnm)
- }
+ mark = map[string]int64{}
delay = pollInterval
continue poll
}
@@ -146,7 +136,7 @@ poll:
for _, fnm := range watching {
if size[fnm] > mark[fnm] {
newData[fnm] = &bytes.Buffer{}
- _, n, err := lc.copyRange(ctx, ctrUUID, fnm, fmt.Sprintf("%d-", mark[fnm]), newData[fnm])
+ _, n, err := lc.copyRange(ctx, cr.UUID, displayingUUID, fnm, fmt.Sprintf("%d-", mark[fnm]), newData[fnm])
if err != nil {
fmt.Fprintln(stderr, err)
}
@@ -157,15 +147,32 @@ poll:
}
}
checkState := lc.display(stdout, stderr, watching, newData)
- if len(newData) > 0 {
+ if displayingUUID != cr.ContainerUUID {
+ // A different container had already been
+ // assigned when we started fetching the
+ // latest batch of logs. We can now safely
+ // start displaying logs from the new
+ // container, without missing any of the
+ // previous container's logs.
+ displayingUUID = cr.ContainerUUID
+ delay = 0
+ continue
+ } else if cr.State == arvados.ContainerRequestStateFinal {
+ break
+ } else if len(newData) > 0 {
delay = pollInterval
- }
- if len(newData) == 0 || checkState {
+ } else {
delay = delay * 2
if delay > pollInterval*5 {
delay = pollInterval * 5
}
- ctr, err := rpcconn.ContainerGet(ctx, arvados.GetOptions{UUID: ctrUUID, Select: []string{"state"}})
+ checkState = true
+ }
+ if checkState {
+ cr, err = rpcconn.ContainerRequestGet(ctx, arvados.GetOptions{UUID: crUUID, Select: []string{"uuid", "container_uuid", "state"}})
+ if err != nil {
+ return fmt.Errorf("error retrieving %s: %w", crUUID, err)
+ }
if err != nil {
if !anySuccess {
return err
@@ -174,10 +181,6 @@ poll:
delay = pollInterval
continue
}
- if ctr.State == arvados.ContainerStateCancelled || ctr.State == arvados.ContainerStateComplete {
- cUUID = ""
- delay = 0
- }
}
}
return nil
@@ -187,7 +190,7 @@ func (lc *logsCommand) srcURL(crUUID, cUUID, fnm string) string {
u := url.URL{
Scheme: "https",
Host: lc.ac.APIHost,
- Path: "/arvados/v1/containers/" + crUUID + "/log/" + cUUID + "/" + fnm,
+ Path: "/arvados/v1/container_requests/" + crUUID + "/log/" + cUUID + "/" + fnm,
}
return u.String()
}
@@ -228,10 +231,10 @@ func (lc *logsCommand) checkAPISupport(ctx context.Context, crUUID string) error
// Return values are current file size, bytes copied, error.
//
// If the file does not exist, return values are 0, 0, nil.
-func (lc *logsCommand) copyRange(ctx context.Context, uuid, fnm, byterange string, out io.Writer) (int64, int64, error) {
+func (lc *logsCommand) copyRange(ctx context.Context, crUUID, cUUID, fnm, byterange string, out io.Writer) (int64, int64, error) {
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(20*time.Second))
defer cancel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, lc.srcURL(uuid, fnm), nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, lc.srcURL(crUUID, cUUID, fnm), nil)
if err != nil {
return 0, 0, err
}
diff --git a/cmd/arvados-client/container_gateway_test.go b/cmd/arvados-client/container_gateway_test.go
index 7c2c7d121..25c794e83 100644
--- a/cmd/arvados-client/container_gateway_test.go
+++ b/cmd/arvados-client/container_gateway_test.go
@@ -182,7 +182,7 @@ func (s *ClientSuite) TestShellGateway(c *check.C) {
wg.Wait()
}
-func (s *ClientSuite) TestContainerLog(c *check.C) {
+func (s *ClientSuite) TestContainerRequestLog(c *check.C) {
arvadostest.StartKeep(2, true)
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(30*time.Second))
defer cancel()
commit d02f5b352a9c4d21f072ddecda1bf2cfb5577442
Merge: d75c91ace 5cc1710b5
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 20 10:46:12 2023 -0400
18790: Merge branch '20319-container-request-logs' into 18790-log-client
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit d75c91ace9d4f9c7ab000a31425ce0f784c47e9f
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 20 09:26:52 2023 -0400
18790: Move to container_requests/*/log endpoint.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
index 9436959af..3b90a334e 100644
--- a/cmd/arvados-client/container_gateway.go
+++ b/cmd/arvados-client/container_gateway.go
@@ -61,19 +61,12 @@ func (lc logsCommand) RunCommand(prog string, args []string, stdin io.Reader, st
return 0
}
-func (lc *logsCommand) tailf(target string, stdout, stderr io.Writer, pollInterval time.Duration) error {
+func (lc *logsCommand) tailf(crUUID string, stdout, stderr io.Writer, pollInterval time.Duration) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rpcconn := rpcFromEnv()
- ctrUUID, err := resolveToContainerUUID(rpcconn, target)
- if err != nil {
- return err
- }
- if ctrUUID != target {
- fmt.Fprintln(stderr, "target container UUID is", ctrUUID)
- }
- err = lc.checkAPISupport(ctx, ctrUUID)
+ err = lc.checkAPISupport(ctx, crUUID)
if err != nil {
return err
}
@@ -81,6 +74,8 @@ func (lc *logsCommand) tailf(target string, stdout, stderr io.Writer, pollInterv
var (
// files to display
watching = []string{"crunch-run.txt", "stderr.txt"}
+ // last known container UUID
+ cUUID = ""
// fnm => file offset of next byte to display
mark = map[string]int64{}
// fnm => current size of file reported by api
@@ -96,12 +91,29 @@ func (lc *logsCommand) tailf(target string, stdout, stderr io.Writer, pollInterv
poll:
for delay := pollInterval; ; time.Sleep(delay) {
- // When /arvados/v1/containers/{uuid}/log_events is
+ if cUUID == "" {
+ // This is our first iteration, or the
+ // container we were watching has exited.
+ ctr, err := rpcconn.ContainerRequestGet(ctx, arvados.GetOptions{UUID: crUUID})
+ if err != nil {
+ return err
+ }
+ if newUUID == ctrUUID {
+ // no further attempts
+ return nil
+ }
+ ctrUUID = newUUID
+ containerFinished = false
+ delay = 0
+ continue
+ }
+
+ // When .../container_requests/{uuid}/log_events is
// implemented, we'll wait here for the next
// server-sent event to tell us some updated file
// sizes. For now, we poll.
for _, fnm := range watching {
- currentsize, _, err := lc.copyRange(ctx, ctrUUID, fnm, "-0", nil)
+ currentsize, _, err := lc.copyRange(ctx, crUUID, cUUID, fnm, "-0", nil)
if err != nil {
if !anySuccess {
return err
@@ -145,29 +157,6 @@ poll:
}
}
checkState := lc.display(stdout, stderr, watching, newData)
- if containerFinished {
- // If the caller specified a container request
- // UUID and the container we were watching has
- // been replaced by a new one, start watching
- // logs from the new one. Otherwise, we're
- // done.
- if target == ctrUUID {
- // caller specified container UUID
- return nil
- }
- newUUID, err := resolveToContainerUUID(rpcconn, target)
- if err != nil {
- return err
- }
- if newUUID == ctrUUID {
- // no further attempts
- return nil
- }
- ctrUUID = newUUID
- containerFinished = false
- delay = 0
- continue
- }
if len(newData) > 0 {
delay = pollInterval
}
@@ -186,7 +175,7 @@ poll:
continue
}
if ctr.State == arvados.ContainerStateCancelled || ctr.State == arvados.ContainerStateComplete {
- containerFinished = true
+ cUUID = ""
delay = 0
}
}
@@ -194,28 +183,28 @@ poll:
return nil
}
-func (lc *logsCommand) srcURL(uuid, fnm string) string {
+func (lc *logsCommand) srcURL(crUUID, cUUID, fnm string) string {
u := url.URL{
Scheme: "https",
Host: lc.ac.APIHost,
- Path: "/arvados/v1/containers/" + uuid + "/log/" + fnm,
+ Path: "/arvados/v1/containers/" + crUUID + "/log/" + cUUID + "/" + fnm,
}
return u.String()
}
// Check whether the API is new enough to support the
-// .../containers/{uuid}/log endpoint.
+// .../container_requests/{uuid}/log/ endpoint.
//
-// Older versions return 200 for an OPTIONS request at the .../logs/
+// Older versions return 200 for an OPTIONS request at the .../log/
// API endpoint, but the response header does not have a "Dav" header.
//
// Note an error response with no "Dav" header is not taken to
// indicate lack of API support. It may come from a new server that
// has a configuration or networking problem.
-func (lc *logsCommand) checkAPISupport(ctx context.Context, uuid string) error {
+func (lc *logsCommand) checkAPISupport(ctx context.Context, crUUID string) error {
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(20*time.Second))
defer cancel()
- req, err := http.NewRequestWithContext(ctx, "OPTIONS", lc.srcURL(uuid, ""), nil)
+ req, err := http.NewRequestWithContext(ctx, "OPTIONS", strings.TrimSuffix(lc.srcURL(crUUID, "", ""), "/"), nil)
if err != nil {
return err
}
commit afe739890e0cfe781600e3a58bd5386f1f2e3ac7
Merge: 63c57d1b8 ddccc6d8f
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 20 00:23:50 2023 -0400
18790: Merge branch '20319-container-request-logs' into 18790-log-client
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc doc/admin/upgrading.html.textile.liquid
index 0855575c2,96966c942..8e60e3f37
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@@ -32,13 -36,9 +36,13 @@@ h2(#v2_6_0). v2.6.0 (2023-04-06
"previous: Upgrading to 2.5.0":#v2_5_0
+h3. WebDAV InternalURLs must be reachable from controller nodes
+
+Ensure your internal keep-web service addresses are listed in the @Services.WebDAV.InternalURLs@ section of your configuration file, and reachable from controller processes, as noted on the "updated install page":{{site.baseurl}}/admin/config-urls.html.
+
h3. Slow migration on upgrade
- Important! This upgrade includes a database schema update changing the integer id column in each table from 32-bit to 64-bit. Because it touches every row in the table, on moderate to large sized installations this may be very slow (on the order of hours). Plan for the arvados-api-server package upgrade to take longer than usual.
+ Important! This upgrade includes a database schema update changing the integer id column in each table from 32-bit to 64-bit. Because it touches every row in the table, on moderate to large sized installations *this may be very slow* (on the order of hours). Plan for the arvados-api-server package upgrade to take longer than usual.
h3. Default request concurrency, new limit on log requests
commit ddccc6d8f1e04c8708b99dd2d0d3ee4b9b70528f
Author: Tom Clegg <tom at curii.com>
Date: Wed Apr 19 22:45:16 2023 -0400
20319: Move /containers/*/log to /container_requests/*/log.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 268b9eefb..430c189a2 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -385,10 +385,6 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
}
-func (conn *Conn) ContainerLog(ctx context.Context, options arvados.ContainerLogOptions) (http.Handler, error) {
- return conn.chooseBackend(options.UUID).ContainerLog(ctx, options)
-}
-
func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ConnectionResponse, error) {
return conn.chooseBackend(options.UUID).ContainerSSH(ctx, options)
}
@@ -459,6 +455,10 @@ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.De
return conn.chooseBackend(options.UUID).ContainerRequestDelete(ctx, options)
}
+func (conn *Conn) ContainerRequestLog(ctx context.Context, options arvados.ContainerLogOptions) (http.Handler, error) {
+ return conn.chooseBackend(options.UUID).ContainerRequestLog(ctx, options)
+}
+
func (conn *Conn) GroupCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Group, error) {
return conn.chooseBackend(options.ClusterID).GroupCreate(ctx, options)
}
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
index 59592ece9..11d139663 100644
--- a/lib/controller/localdb/container_gateway.go
+++ b/lib/controller/localdb/container_gateway.go
@@ -40,22 +40,45 @@ var (
forceInternalURLForTest *arvados.URL
)
-// ContainerLog returns a WebDAV handler that reads logs from the
-// indicated container. It works by proxying the request to
+// ContainerRequestLog returns a WebDAV handler that reads logs from
+// the indicated container request. It works by proxying the incoming
+// HTTP request to
//
-// - the container gateway, if the container is running
+// - the container gateway, if there is an associated container that
+// is running
//
-// - a different controller process, if the container is running and
-// the gateway is accessible through a tunnel to a different
+// - a different controller process, if there is a running container
+// whose gateway is accessible through a tunnel to a different
// controller process
//
// - keep-web, if saved logs exist and there is no gateway (or the
-// container is finished)
+// associated container is finished)
//
// - an empty-collection stub, if there is no gateway and no saved
// log
-func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
- ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "log"}})
+//
+// For an incoming request
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+//
+// The upstream request may be to {c_uuid}'s container gateway
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+// X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}
+// X-Webdav-Source: /log
+//
+// ...or the upstream request may be to keep-web (where {cr_log_uuid}
+// is the container request log collection UUID)
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+// Host: {cr_log_uuid}.internal
+// X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log
+// X-Arvados-Container-Uuid: {c_uuid}
+//
+// ...or the request may be handled locally using an empty-collection
+// stub.
+func (conn *Conn) ContainerRequestLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
+ cr, err := conn.railsProxy.ContainerRequestGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "container_uuid", "log_uuid"}})
if err != nil {
if se := httpserver.HTTPStatusError(nil); errors.As(err, &se) && se.HTTPStatus() == http.StatusUnauthorized {
// Hint to WebDAV client that we accept HTTP basic auth.
@@ -66,10 +89,29 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
}
return nil, err
}
+ ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: cr.ContainerUUID, Select: []string{"uuid", "state", "gateway_address"}})
+ if err != nil {
+ return nil, err
+ }
+ // .../log/{ctr.UUID} is a directory where the currently
+ // assigned container's log data [will] appear (as opposed to
+ // previous attempts in .../log/{previous_ctr_uuid}). Requests
+ // that are outside that directory, and requests on a
+ // non-running container, are proxied to keep-web instead of
+ // going through the container gateway system.
+ //
+ // Side note: a depth>1 directory tree listing starting at
+ // .../{cr_uuid}/log will only include subdirectories for
+ // finished containers, i.e., will not include a subdirectory
+ // with log data for a current (unfinished) container UUID.
+ // In order to access live logs, a client must look up the
+ // container_uuid field of the container request record, and
+ // explicitly request a path under .../{cr_uuid}/log/{c_uuid}.
if ctr.GatewayAddress == "" ||
- (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) {
+ (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) ||
+ !(opts.Path == "/"+ctr.UUID || strings.HasPrefix(opts.Path, "/"+ctr.UUID+"/")) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
+ conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
}), nil
}
dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
@@ -78,7 +120,7 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
}
if arpc != nil {
opts.NoForward = true
- return arpc.ContainerLog(ctx, opts)
+ return arpc.ContainerRequestLog(ctx, opts)
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(ctx)
@@ -119,7 +161,9 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
// for net/http to work with.
r.URL.Scheme = "https"
r.URL.Host = "0.0.0.0:0"
- r.Header.Set("X-Arvados-Container-Gateway-Uuid", opts.UUID)
+ r.Header.Set("X-Arvados-Container-Gateway-Uuid", ctr.UUID)
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+ctr.UUID)
+ r.Header.Set("X-Webdav-Source", "/log")
proxyReq = r
},
ModifyResponse: func(resp *http.Response) error {
@@ -144,7 +188,7 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
// after we decided (above) the log was not final.
// In that case we should proxy to keep-web.
ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{
- UUID: opts.UUID,
+ UUID: ctr.UUID,
Select: []string{"uuid", "state", "gateway_address", "log"},
})
if err != nil {
@@ -154,7 +198,7 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
// No race, proxyErr was the best we can do
httpserver.Error(w, "proxy error: "+proxyErr.Error(), http.StatusServiceUnavailable)
} else {
- conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
+ conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
}
}), nil
}
@@ -163,12 +207,12 @@ func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOpt
// log content by proxying to one of the configured keep-web servers.
//
// It tries to choose a keep-web server that is running on this host.
-func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions, ctr arvados.Container, w http.ResponseWriter, r *http.Request) {
- if ctr.Log == "" {
+func (conn *Conn) serveContainerRequestLogViaKeepWeb(opts arvados.ContainerLogOptions, cr arvados.ContainerRequest, w http.ResponseWriter, r *http.Request) {
+ if cr.LogUUID == "" {
// Special case: if no log data exists yet, we serve
// an empty collection by ourselves instead of
// proxying to keep-web.
- conn.serveEmptyDir("/arvados/v1/containers/"+ctr.UUID+"/log", w, r)
+ conn.serveEmptyDir("/arvados/v1/container_requests/"+cr.UUID+"/log", w, r)
return
}
myURL, _ := service.URLFromContext(r.Context())
@@ -196,7 +240,7 @@ func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions,
r.URL.Host = webdavBase.Host
// Outgoing Host header specifies the
// collection ID.
- r.Host = strings.Replace(ctr.Log, "+", "-", -1) + ".internal"
+ r.Host = cr.LogUUID + ".internal"
// We already checked permission on the
// container, so we can use a root token here
// instead of counting on the "access to log
@@ -209,7 +253,14 @@ func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions,
// headers refer to the same paths) so we tell
// keep-web to map the log collection onto the
// containers/X/log/ namespace.
- r.Header.Set("X-Webdav-Prefix", "/arvados/v1/containers/"+ctr.UUID+"/log")
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log")
+ if len(opts.Path) >= 28 && opts.Path[6:13] == "-dz642-" {
+ // "/arvados/v1/container_requests/{crUUID}/log/{cUUID}..."
+ // proxies to
+ // "/log for container {cUUID}..."
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+opts.Path[1:28])
+ r.Header.Set("X-Webdav-Source", "/log for container "+opts.Path[1:28]+"/")
+ }
},
}
if conn.cluster.TLS.Insecure {
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
index 4884eda46..dc8a8cea0 100644
--- a/lib/controller/localdb/container_gateway_test.go
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -39,6 +39,7 @@ var _ = check.Suite(&ContainerGatewaySuite{})
type ContainerGatewaySuite struct {
localdbSuite
+ reqUUID string
ctrUUID string
srv *httptest.Server
gw *crunchrun.Gateway
@@ -47,7 +48,29 @@ type ContainerGatewaySuite struct {
func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
s.localdbSuite.SetUpTest(c)
- s.ctrUUID = arvadostest.QueuedContainerUUID
+ cr, err := s.localdb.ContainerRequestCreate(s.userctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", time.Now().Format(time.RFC3339Nano)},
+ "container_count_max": 1,
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "output_path": "/out",
+ "priority": 1,
+ "state": arvados.ContainerRequestStateCommitted,
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ }}})
+ c.Assert(err, check.IsNil)
+ s.reqUUID = cr.UUID
+ s.ctrUUID = cr.ContainerUUID
h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
fmt.Fprint(h, s.ctrUUID)
@@ -75,15 +98,14 @@ func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
ArvadosClient: ac,
}
c.Assert(s.gw.Start(), check.IsNil)
+
rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
- // OK if this line fails (because state is already Running
- // from a previous test case) as long as the following line
- // succeeds:
- s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
"state": arvados.ContainerStateLocked}})
- _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ c.Assert(err, check.IsNil)
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
"state": arvados.ContainerStateRunning,
@@ -243,18 +265,23 @@ func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
_, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
- "log": coll.PortableDataHash,
- "gateway_address": "",
+ "state": arvados.ContainerStateComplete,
+ "exit_code": 0,
+ "log": coll.PortableDataHash,
}})
c.Assert(err, check.IsNil)
- // gateway_address="" above already ensures localdb
- // can't circumvent the keep-web proxy test by getting
- // content from the container gateway; this is just
- // extra insurance.
+ updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
+ c.Assert(err, check.IsNil)
+ c.Logf("container request log UUID is %s", updatedReq.LogUUID)
+ crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
+ c.Assert(err, check.IsNil)
+ c.Logf("collection log manifest:\n%s", crLog.ManifestText)
+ // Ensure localdb can't circumvent the keep-web proxy test by
+ // getting content from the container gateway.
s.gw.LogCollection = nil
}
-func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
forceProxyForTest = true
defer func() { forceProxyForTest = false }()
@@ -271,9 +298,9 @@ func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
}
- handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
- UUID: s.ctrUUID,
- WebDAVOptions: arvados.WebDAVOptions{Path: "/stderr.txt"},
+ handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
+ UUID: s.reqUUID,
+ WebDAVOptions: arvados.WebDAVOptions{Path: "/" + s.ctrUUID + "/stderr.txt"},
})
if broken {
c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
@@ -281,7 +308,7 @@ func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
}
c.Check(err, check.IsNil)
c.Assert(handler, check.NotNil)
- r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log/stderr.txt", nil)
+ r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
c.Assert(err, check.IsNil)
r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
rec := httptest.NewRecorder()
@@ -294,18 +321,18 @@ func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
}
}
-func (s *ContainerGatewaySuite) TestContainerLogViaGateway(c *check.C) {
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
s.setupLogCollection(c)
- s.testContainerLog(c)
+ s.testContainerRequestLog(c)
}
-func (s *ContainerGatewaySuite) TestContainerLogViaKeepWeb(c *check.C) {
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
s.setupLogCollection(c)
s.saveLogAndCloseGateway(c)
- s.testContainerLog(c)
+ s.testContainerRequestLog(c)
}
-func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
+func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
for _, trial := range []struct {
method string
path string
@@ -316,7 +343,7 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
}{
{
method: "GET",
- path: "/stderr.txt",
+ path: s.ctrUUID + "/stderr.txt",
expectStatus: http.StatusOK,
expectBodyRe: "hello world\n",
expectHeader: http.Header{
@@ -325,7 +352,7 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
},
{
method: "GET",
- path: "/stderr.txt",
+ path: s.ctrUUID + "/stderr.txt",
header: http.Header{
"Range": {"bytes=-6"},
},
@@ -338,7 +365,7 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
},
{
method: "OPTIONS",
- path: "/stderr.txt",
+ path: s.ctrUUID + "/stderr.txt",
expectStatus: http.StatusOK,
expectBodyRe: "",
expectHeader: http.Header{
@@ -348,25 +375,34 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
},
{
method: "PROPFIND",
- path: "",
+ path: s.ctrUUID + "/",
+ expectStatus: http.StatusMultiStatus,
+ expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
+ expectHeader: http.Header{
+ "Content-Type": {"text/xml; charset=utf-8"},
+ },
+ },
+ {
+ method: "PROPFIND",
+ path: s.ctrUUID,
expectStatus: http.StatusMultiStatus,
- expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*`,
+ expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
expectHeader: http.Header{
"Content-Type": {"text/xml; charset=utf-8"},
},
},
{
method: "PROPFIND",
- path: "/a/b/c/",
+ path: s.ctrUUID + "/a/b/c/",
expectStatus: http.StatusMultiStatus,
- expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*`,
+ expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
expectHeader: http.Header{
"Content-Type": {"text/xml; charset=utf-8"},
},
},
{
method: "GET",
- path: "/a/b/c/d.html",
+ path: s.ctrUUID + "/a/b/c/d.html",
expectStatus: http.StatusOK,
expectBodyRe: "<html></html>\n",
expectHeader: http.Header{
@@ -375,13 +411,13 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
},
} {
c.Logf("trial %#v", trial)
- handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
- UUID: s.ctrUUID,
- WebDAVOptions: arvados.WebDAVOptions{Path: trial.path},
+ handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
+ UUID: s.reqUUID,
+ WebDAVOptions: arvados.WebDAVOptions{Path: "/" + trial.path},
})
c.Assert(err, check.IsNil)
c.Assert(handler, check.NotNil)
- r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log"+trial.path, nil)
+ r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
c.Assert(err, check.IsNil)
for k := range trial.header {
r.Header.Set(k, trial.header.Get(k))
@@ -399,19 +435,19 @@ func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
}
}
-func (s *ContainerGatewaySuite) TestContainerLogViaCadaver(c *check.C) {
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
s.setupLogCollection(c)
- out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/containers/"+s.ctrUUID+"/log", "ls")
+ out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
- out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
s.saveLogAndCloseGateway(c)
- out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 2cbd9b88d..b9eb23d05 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -209,13 +209,6 @@ func (rtr *router) addRoutes() {
return rtr.backend.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
},
},
- {
- arvados.EndpointContainerLog,
- func() interface{} { return &arvados.ContainerLogOptions{} },
- func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.backend.ContainerLog(ctx, *opts.(*arvados.ContainerLogOptions))
- },
- },
{
arvados.EndpointContainerSSH,
func() interface{} { return &arvados.ContainerSSHOptions{} },
@@ -290,6 +283,13 @@ func (rtr *router) addRoutes() {
return rtr.backend.ContainerRequestDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
+ {
+ arvados.EndpointContainerRequestLog,
+ func() interface{} { return &arvados.ContainerLogOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.ContainerRequestLog(ctx, *opts.(*arvados.ContainerLogOptions))
+ },
+ },
{
arvados.EndpointGroupCreate,
func() interface{} { return &arvados.CreateOptions{} },
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index b194bd222..d4ee8794c 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -177,7 +177,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav GET root",
method: "GET",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log/",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "/",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -189,7 +189,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav GET root without trailing slash",
method: "GET",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -201,7 +201,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav OPTIONS root",
method: "OPTIONS",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log/",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "/",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -213,7 +213,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav OPTIONS root without trailing slash",
method: "OPTIONS",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID,
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -225,7 +225,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav PROPFIND root",
method: "PROPFIND",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log/",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "/",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -237,7 +237,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav PROPFIND root without trailing slash",
method: "PROPFIND",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -249,7 +249,7 @@ func (s *RouterSuite) TestOptions(c *check.C) {
{
comment: "container log webdav no_forward=true",
method: "GET",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/log/?no_forward=true",
+ path: "/arvados/v1/container_requests/" + arvadostest.CompletedContainerRequestUUID + "/log/" + arvadostest.CompletedContainerUUID + "/?no_forward=true",
shouldCall: "ContainerLog",
withOptions: arvados.ContainerLogOptions{
UUID: arvadostest.CompletedContainerUUID,
@@ -260,9 +260,9 @@ func (s *RouterSuite) TestOptions(c *check.C) {
Path: "/"}},
},
{
- comment: "/logX does not route to ContainerLog",
+ comment: "/logX does not route to ContainerRequestLog",
method: "GET",
- path: "/arvados/v1/containers/" + arvadostest.CompletedContainerUUID + "/logX",
+ path: "/arvados/v1/containers/" + arvadostest.CompletedContainerRequestUUID + "/logX",
shouldStatus: http.StatusNotFound,
shouldCall: "",
},
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 70a936a6f..ca9f9856f 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -339,19 +339,6 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
return resp, err
}
-func (conn *Conn) ContainerLog(ctx context.Context, options arvados.ContainerLogOptions) (resp http.Handler, err error) {
- proxy := &httputil.ReverseProxy{
- Transport: conn.httpClient.Transport,
- Director: func(r *http.Request) {
- u := conn.baseURL
- u.Path = r.URL.Path
- u.RawQuery = fmt.Sprintf("no_forward=%v", options.NoForward)
- r.URL = &u
- },
- }
- return proxy, nil
-}
-
// ContainerSSH returns a connection to the out-of-band SSH server for
// a running container. If the returned error is nil, the caller is
// responsible for closing sshconn.Conn.
@@ -484,6 +471,19 @@ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.De
return resp, err
}
+func (conn *Conn) ContainerRequestLog(ctx context.Context, options arvados.ContainerLogOptions) (resp http.Handler, err error) {
+ proxy := &httputil.ReverseProxy{
+ Transport: conn.httpClient.Transport,
+ Director: func(r *http.Request) {
+ u := conn.baseURL
+ u.Path = r.URL.Path
+ u.RawQuery = fmt.Sprintf("no_forward=%v", options.NoForward)
+ r.URL = &u
+ },
+ }
+ return proxy, nil
+}
+
func (conn *Conn) GroupCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Group, error) {
ep := arvados.EndpointGroupCreate
var resp arvados.Group
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index 7fd82a8bf..30f8957a2 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -81,14 +81,13 @@ type Gateway struct {
// controller process at the other end of the tunnel.
UpdateTunnelURL func(url string)
- // Source for serving WebDAV requests at
- // /arvados/v1/containers/{uuid}/log/
+ // Source for serving WebDAV requests with
+ // X-Webdav-Source: /log
LogCollection arvados.CollectionFileSystem
sshConfig ssh.ServerConfig
requestAuth string
respondAuth string
- logPath string
}
// Start starts an http server that allows authenticated clients to open an
@@ -163,8 +162,6 @@ func (gw *Gateway) Start() error {
h.Write([]byte(gw.requestAuth))
gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
- gw.logPath = "/arvados/v1/containers/" + gw.ContainerUUID + "/log"
-
srv := &httpserver.Server{
Server: http.Server{
Handler: gw,
@@ -277,6 +274,7 @@ var webdavMethod = map[string]bool{
}
func (gw *Gateway) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ w.Header().Set("Vary", "X-Arvados-Authorization, X-Arvados-Container-Gateway-Uuid, X-Webdav-Prefix, X-Webdav-Source")
reqUUID := req.Header.Get("X-Arvados-Container-Gateway-Uuid")
if reqUUID == "" {
// older controller versions only send UUID as query param
@@ -295,7 +293,7 @@ func (gw *Gateway) ServeHTTP(w http.ResponseWriter, req *http.Request) {
switch {
case req.Method == "POST" && req.Header.Get("Upgrade") == "ssh":
gw.handleSSH(w, req)
- case req.URL.Path == gw.logPath || strings.HasPrefix(req.URL.Path, gw.logPath):
+ case req.Header.Get("X-Webdav-Source") == "/log":
if !webdavMethod[req.Method] {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
@@ -307,12 +305,17 @@ func (gw *Gateway) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
func (gw *Gateway) handleLogsWebDAV(w http.ResponseWriter, r *http.Request) {
+ prefix := r.Header.Get("X-Webdav-Prefix")
+ if !strings.HasPrefix(r.URL.Path, prefix) {
+ http.Error(w, "X-Webdav-Prefix header is not a prefix of the requested path", http.StatusBadRequest)
+ return
+ }
if gw.LogCollection == nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
wh := webdav.Handler{
- Prefix: gw.logPath,
+ Prefix: prefix,
FileSystem: &webdavfs.FS{
FileSystem: gw.LogCollection,
Prefix: "",
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 861b8e6ce..fa4a4ac4a 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -49,7 +49,6 @@ var (
EndpointContainerDelete = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
EndpointContainerLock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
EndpointContainerUnlock = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
- EndpointContainerLog = APIEndpoint{"GET", "arvados/v1/containers/{uuid}/log{path:|/.*}", ""}
EndpointContainerSSH = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/ssh", ""}
EndpointContainerSSHCompat = APIEndpoint{"POST", "arvados/v1/connect/{uuid}/ssh", ""} // for compatibility with arvados <2.7
EndpointContainerGatewayTunnel = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/gateway_tunnel", ""}
@@ -59,6 +58,7 @@ var (
EndpointContainerRequestGet = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}", ""}
EndpointContainerRequestList = APIEndpoint{"GET", "arvados/v1/container_requests", ""}
EndpointContainerRequestDelete = APIEndpoint{"DELETE", "arvados/v1/container_requests/{uuid}", ""}
+ EndpointContainerRequestLog = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}/log{path:|/.*}", ""}
EndpointGroupCreate = APIEndpoint{"POST", "arvados/v1/groups", "group"}
EndpointGroupUpdate = APIEndpoint{"PATCH", "arvados/v1/groups/{uuid}", "group"}
EndpointGroupGet = APIEndpoint{"GET", "arvados/v1/groups/{uuid}", ""}
@@ -285,7 +285,6 @@ type API interface {
ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
ContainerLock(ctx context.Context, options GetOptions) (Container, error)
ContainerUnlock(ctx context.Context, options GetOptions) (Container, error)
- ContainerLog(ctx context.Context, options ContainerLogOptions) (http.Handler, error)
ContainerSSH(ctx context.Context, options ContainerSSHOptions) (ConnectionResponse, error)
ContainerGatewayTunnel(ctx context.Context, options ContainerGatewayTunnelOptions) (ConnectionResponse, error)
ContainerRequestCreate(ctx context.Context, options CreateOptions) (ContainerRequest, error)
@@ -293,6 +292,7 @@ type API interface {
ContainerRequestGet(ctx context.Context, options GetOptions) (ContainerRequest, error)
ContainerRequestList(ctx context.Context, options ListOptions) (ContainerRequestList, error)
ContainerRequestDelete(ctx context.Context, options DeleteOptions) (ContainerRequest, error)
+ ContainerRequestLog(ctx context.Context, options ContainerLogOptions) (http.Handler, error)
GroupCreate(ctx context.Context, options CreateOptions) (Group, error)
GroupUpdate(ctx context.Context, options UpdateOptions) (Group, error)
GroupGet(ctx context.Context, options GetOptions) (Group, error)
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index 483832de5..544d622f4 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -116,22 +116,6 @@ func (as *APIStub) ContainerUnlock(ctx context.Context, options arvados.GetOptio
as.appendCall(ctx, as.ContainerUnlock, options)
return arvados.Container{}, as.Error
}
-func (as *APIStub) ContainerLog(ctx context.Context, options arvados.ContainerLogOptions) (http.Handler, error) {
- as.appendCall(ctx, as.ContainerLog, options)
- // Return a handler that responds with the configured
- // error/success status.
- return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- if as.Error == nil {
- w.WriteHeader(http.StatusOK)
- } else if err := httpserver.HTTPStatusError(nil); errors.As(as.Error, &err) {
- w.WriteHeader(err.HTTPStatus())
- io.WriteString(w, err.Error())
- } else {
- w.WriteHeader(http.StatusInternalServerError)
- io.WriteString(w, err.Error())
- }
- }), nil
-}
func (as *APIStub) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ConnectionResponse, error) {
as.appendCall(ctx, as.ContainerSSH, options)
return arvados.ConnectionResponse{}, as.Error
@@ -160,6 +144,22 @@ func (as *APIStub) ContainerRequestDelete(ctx context.Context, options arvados.D
as.appendCall(ctx, as.ContainerRequestDelete, options)
return arvados.ContainerRequest{}, as.Error
}
+func (as *APIStub) ContainerRequestLog(ctx context.Context, options arvados.ContainerLogOptions) (http.Handler, error) {
+ as.appendCall(ctx, as.ContainerRequestLog, options)
+ // Return a handler that responds with the configured
+ // error/success status.
+ return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if as.Error == nil {
+ w.WriteHeader(http.StatusOK)
+ } else if err := httpserver.HTTPStatusError(nil); errors.As(as.Error, &err) {
+ w.WriteHeader(err.HTTPStatus())
+ io.WriteString(w, err.Error())
+ } else {
+ w.WriteHeader(http.StatusInternalServerError)
+ io.WriteString(w, err.Error())
+ }
+ }), nil
+}
func (as *APIStub) GroupCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Group, error) {
as.appendCall(ctx, as.GroupCreate, options)
return arvados.Group{}, as.Error
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3b22a13b0..27981c487 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -360,6 +360,10 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
fsprefix = "by_id/" + collectionID + "/"
}
+ if src := r.Header.Get("X-Webdav-Source"); strings.HasPrefix(src, "/") && !strings.Contains(src, "//") && !strings.Contains(src, "/../") {
+ fsprefix += src[1:]
+ }
+
if tokens == nil {
tokens = reqTokens
if h.Cluster.Users.AnonymousUserToken != "" {
@@ -593,7 +597,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
},
LockSystem: webdavfs.NoLockSystem,
Logger: func(r *http.Request, err error) {
- if err != nil {
+ if err != nil && !os.IsNotExist(err) {
ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler")
}
},
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 9228c3628..c9b48f99a 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -93,6 +93,120 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
}
+func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
+ for _, trial := range []struct {
+ method string
+ path string
+ prefix string
+ source string
+ notFound bool
+ seeOther bool
+ }{
+ {
+ method: "PROPFIND",
+ path: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "/dir1",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/",
+ prefix: "/prefix",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1",
+ seeOther: true,
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "",
+ source: "/dir1",
+ notFound: true,
+ },
+ } {
+ c.Logf("trial %+v", trial)
+ u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
+ req := &http.Request{
+ Method: trial.method,
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveTokenV2},
+ "X-Webdav-Prefix": {trial.prefix},
+ "X-Webdav-Source": {trial.source},
+ },
+ Body: ioutil.NopCloser(bytes.NewReader(nil)),
+ }
+
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ if trial.notFound {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else if trial.method == "PROPFIND" {
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
+ } else if trial.seeOther {
+ c.Check(resp.Code, check.Equals, http.StatusSeeOther)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ }
+ }
+}
+
func (s *UnitSuite) TestEmptyResponse(c *check.C) {
for _, trial := range []struct {
dataExists bool
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list