[arvados] updated: 2.1.0-2849-g2375b76b1
git repository hosting
git at public.arvados.org
Fri Aug 12 00:43:48 UTC 2022
Summary of changes:
.licenseignore | 1 +
SECURITY.md | 42 ++++
apps/workbench/Gemfile.lock | 2 +-
.../app/controllers/actions_controller.rb | 2 +-
.../app/controllers/application_controller.rb | 6 +-
apps/workbench/app/helpers/application_helper.rb | 2 +-
apps/workbench/app/models/arvados_api_client.rb | 4 +-
apps/workbench/test/test_helper.rb | 2 +-
cmd/arvados-client/container_gateway.go | 5 +-
cmd/arvados-client/container_gateway_test.go | 9 +
cmd/arvados-server/cmd.go | 2 +
doc/_config.yml | 1 +
doc/admin/config-urls.html.textile.liquid | 3 +-
doc/admin/upgrading.html.textile.liquid | 24 ++-
doc/architecture/hpc.html.textile.liquid | 29 +++
doc/install/arvbox.html.textile.liquid | 9 +-
go.mod | 1 +
go.sum | 2 +
lib/boot/nginx.go | 27 +++
lib/controller/federation/conn.go | 6 +-
lib/controller/localdb/conn.go | 4 +
lib/controller/localdb/container_gateway.go | 234 ++++++++++++++++-----
lib/controller/localdb/container_gateway_test.go | 167 ++++++++++++++-
lib/controller/proxy.go | 9 +-
lib/controller/router/request.go | 1 +
lib/controller/router/router.go | 18 ++
lib/controller/rpc/conn.go | 82 +++++---
lib/crunchrun/container_gateway.go | 221 ++++++++++++++++---
lib/crunchrun/crunchrun.go | 67 +++++-
lib/crunchrun/executor_test.go | 17 ++
lib/crunchrun/integration_test.go | 21 +-
lib/diagnostics/cmd.go | 58 ++++-
lib/diagnostics/hello-world.tar | Bin 0 -> 24064 bytes
lib/install/deps.go | 1 +
lib/install/init.go | 23 --
lib/lsf/dispatch.go | 19 +-
sdk/cli/bin/arv | 2 +-
sdk/cwl/arvados_cwl/executor.py | 16 +-
sdk/cwl/arvados_cwl/runner.py | 204 ++++++++++++------
sdk/cwl/setup.py | 7 +-
sdk/go/arvados/api.go | 16 +-
sdk/go/arvados/container_gateway.go | 35 +--
sdk/go/arvadostest/api.go | 8 +-
.../org/arvados/client/logic/keep/KeepClient.java | 4 +-
sdk/python/arvados/collection.py | 2 +-
sdk/python/tests/nginx.conf | 8 +
sdk/python/tests/run_test_server.py | 2 +
sdk/python/tests/test_collections.py | 14 ++
services/api/Gemfile.lock | 2 +-
.../controllers/arvados/v1/groups_controller.rb | 25 ++-
services/api/app/models/container.rb | 4 +-
services/api/lib/record_filters.rb | 2 +-
.../api/test/functional/arvados/v1/filters_test.rb | 52 +++++
services/api/test/unit/container_test.rb | 1 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 9 +-
services/crunch-dispatch-slurm/script.go | 10 +-
services/crunch-dispatch-slurm/script_test.go | 3 +-
services/keep-web/s3.go | 2 +-
services/keep-web/s3_test.go | 7 +
.../aws/pillars/nginx_webshell_configuration.sls | 2 +-
...l.params.example.single_host_multiple_hostnames | 2 +-
...ocal.params.example.single_host_single_hostname | 2 +-
tools/salt-install/provision.sh | 2 +-
63 files changed, 1274 insertions(+), 290 deletions(-)
create mode 100644 SECURITY.md
create mode 100644 doc/architecture/hpc.html.textile.liquid
create mode 100644 lib/diagnostics/hello-world.tar
via 2375b76b13b2ee5a071f5941f7a27c90e5a35aa9 (commit)
via 9c65df788a099fcd8ed7d68dda2c35e3503ee365 (commit)
via bb5ce73fd625c761ef68388116da5063d430c655 (commit)
via 42c20b25e1325124b88e3b9b285544dc41122b56 (commit)
via c8cbf2509601da0890bccc7f9ef5f5a8eaa307d0 (commit)
via 8d0b26e44e50df56d63f489cc62f4c04fbe613e7 (commit)
via 067b16c3cb19f17cca368b1373977c5610511806 (commit)
via 750366f2b8978d52babc2345184a7797b4601a98 (commit)
via 101c02ace8036f92d07e3d5e22736267381c0489 (commit)
via 7822d4d431284d0912ba40d288da81a1eac68a3e (commit)
via 1cbf8cd312dd019809b060d83999c677e94dbe7e (commit)
via 513804e1a2bf43329dc7d37ee9374f3e02ffe169 (commit)
via c0924347a69157d3058a39d238fb0e0bacefa3a2 (commit)
via c6b4cfe6758ff5bc4fbc2ca72c60d9c8485f7267 (commit)
via 3fb6110248db3f87fb21f852c8d6bfebbb2910a0 (commit)
via 61c68cf08258d3292257b67c6b50a223b17f4bfd (commit)
via 7de4c3a96c6ea992df549efa8446bd89ce5bd667 (commit)
via 80b2655dbb69a4ceacce0a7f58845b3ee7fb5853 (commit)
via f1e7c6e0018d276aae506b52ce18e7c31bf48479 (commit)
via 4e5838bd9e1a7baa5b3e53e97e308140e4b6105f (commit)
via b334b065b36357dd08099adad9835f4aa7075337 (commit)
via a8f70f5f978641afa273adcbf995423228f0c7c4 (commit)
via 2fdb9c8541e96756604439f604b82a68e747a35a (commit)
via b23d2434ab8162ea67be50fc3299a0e4450e13ee (commit)
via 275e7919b78fd9d19c8f6b62c8ba97052bba589c (commit)
via d791fa7adb14991c972b6166f39155ff314b7d1e (commit)
via 381d8aa1544bebfc39761636eaf4ca07427be783 (commit)
via 2b95ec7a9a6516a7dd2554e2dd8ecf9ddcfbceb7 (commit)
via b8749c42d9ae650f4e2dbeeda2a8cdae9266ff2a (commit)
via 11227eb2ee3877648c946bedd9a427fe35950822 (commit)
via 20aa08040bc12e6387455755ce3fadc238c19d76 (commit)
via 159004e1fab103ce295146b172dd76149e95e845 (commit)
via 62612ee23efe146829a7bb64817cdd23f41775bb (commit)
via c9b8b9b9c78a77dd30b828914c8bee9fa8dcbb90 (commit)
via 1923ceda3c8845526f1ddcdd6275513760e0cd84 (commit)
via 3e95703e7444c1d2077eba748740ebe3df94b1d5 (commit)
via 6cd62b6e286d4470ef9e2b2c70653d78a05f8cf2 (commit)
via a46fce5c3bfdf6b5a7fcc817309970156bcdc5b8 (commit)
via bce07be48c8c9d8ae62cb1615f5a7cde2ac72249 (commit)
via 711d8642c478c29ffd4c9bc6facbf3bd4d168763 (commit)
via c296ba650406f32de3315159e4ecd7411fc3dcb9 (commit)
via 2e03d03bc55b5a612c2bf04d878a72f2ee420d99 (commit)
via 94081a34c4972cd65a20cbeb4d1837149f057378 (commit)
via 46aadf6a64881e932adb4c54cb1a8c4b22e84d67 (commit)
via 32b402114038bc6761c04c370afad786dbbb3125 (commit)
via 61d3aed1cd0286c7928167a84267c225938a37ec (commit)
via 736c58d66deda35079ad8b64df02ee0ef57232f0 (commit)
via e995446d93c8109d7894010d57c4cc32ae04a459 (commit)
via 5613df6b553c02f95e84b4536397e9be8d95007a (commit)
via 6dfa03b4c6211d265e15a4982831edd59eadf3af (commit)
via 6501c300ebaa01552e17619cc1c89ee38f7398a7 (commit)
via 420e857f8e8ac75beca258fa72b9edac680500cd (commit)
via e186a3539c02602333bb4c98690ab3b7527decc4 (commit)
via d8eeb0f02b967153790e54284fd3213b648def20 (commit)
via 748ee07068ed64fa2e12901ce43f548bd4ff213a (commit)
via 3218db7a7af5c34032d28ba03bffa1684f52897e (commit)
via 831540fd5eedb6226996b5c72a86f2dba64cb196 (commit)
via 13736aa4f0feea65abce2aedc7c4ca0d18e01061 (commit)
via df17ea1c927da5ac2955e9cc83b6d7ab88085fdb (commit)
via ef833210fd7400727e01c97551ab0e645773d3fb (commit)
via 2261d1fd9e1b69d0a60f1f7fe9029317aeb4cf52 (commit)
via dc70bbf9ea15395476107a3b8dff96f754a40998 (commit)
via 091ae55fc1df3ec50490becd437d512e38b0f972 (commit)
via a42604972cccf8dd9c8341c260927a6c48c62b84 (commit)
via 817ee84de15cdc960990e86af8ce705073fcafba (commit)
via 31a270c68c24b7de994655ee788478a64b6bdfb7 (commit)
via bad3a575113abaf7a232f85dd15c4d1e1b60c0e2 (commit)
via 3e27af4fa076222a18d602091060b9d11ee2079c (commit)
via 49f99d7b9a8706c48baedba3659d7ca98b02dde3 (commit)
via 87f3da84318306184165dae50f75ac6721d89285 (commit)
via 5e20c073d84304c3e84770bb7d89035bf1fb9626 (commit)
via 3fae0f0626c5152a5aa6f39f0874f0190f2131db (commit)
via 9587429b4ee56fe9a1ca3555ecebd04e0dae929d (commit)
via c4bae86d39f237df8ac6a5505323f6a93011514a (commit)
via f1f74069850d8c5e987ef7d7fc246735ff94d58d (commit)
via ca6cbc015e137e5e24c6ac5268e9fc72a61db84d (commit)
via 9a4705cbb5410ddddd97d19bdd77821755ff640c (commit)
via bdc29d3129f6d75aa9ce0a24ffb849a272b06f08 (commit)
via ced6d55c36132aee7da3a5fe65f608c9dbf33362 (commit)
from c966970d64c21d7adaf1c3c8b737aa9e7c166f0e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2375b76b13b2ee5a071f5941f7a27c90e5a35aa9
Merge: 11227eb2e 9c65df788
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 11 20:43:28 2022 -0400
17344: Merge branch 'main'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit 11227eb2ee3877648c946bedd9a427fe35950822
Author: Tom Clegg <tom at curii.com>
Date: Tue Aug 2 10:12:51 2022 -0400
17344: arvados-server boot: set X-External-Client header.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index b391c4dc8..9f1091eac 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -5,6 +5,7 @@
package boot
import (
+ "bytes"
"context"
"fmt"
"io/ioutil"
@@ -17,6 +18,7 @@ import (
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
)
// Run an Nginx process that proxies the supervisor's configured
@@ -46,6 +48,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
vars := map[string]string{
"LISTENHOST": extListenHost,
"UPSTREAMHOST": super.ListenHost,
+ "INTERNALSUBNETS": internalSubnets(super.logger),
"SSLCERT": filepath.Join(super.tempdir, "server.crt"),
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
@@ -150,3 +153,27 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
}
return waitForConnect(ctx, testurl.Host)
}
+
+// Return 0 or more local subnets as "geo" fragments for Nginx config,
+// e.g., "1.2.3.0/24 0; 10.1.0.0/16 0;".
+func internalSubnets(logger logrus.FieldLogger) string {
+ iproutes, err := exec.Command("ip", "route").CombinedOutput()
+ if err != nil {
+ logger.Warnf("treating all clients as external because `ip route` failed: %s (%q)", err, iproutes)
+ return ""
+ }
+ subnets := ""
+ for _, line := range bytes.Split(iproutes, []byte("\n")) {
+ fields := strings.Fields(string(line))
+ if len(fields) > 2 && fields[1] == "dev" {
+ // lan example:
+ // 192.168.86.0/24 dev ens3 proto kernel scope link src 192.168.86.196
+ // gcp example (private subnet):
+ // 10.47.0.0/24 dev eth0 proto kernel scope link src 10.47.0.5
+ // gcp example (no private subnet):
+ // 10.128.0.1 dev ens4 scope link
+ subnets += fields[0] + " 0; "
+ }
+ }
+ return subnets
+}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 4ad3eda42..a1a75bbcc 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -15,6 +15,11 @@ http {
fastcgi_temp_path "{{TMPDIR}}";
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
+ geo $external_client {
+ default 1;
+ 127.0.0.0/8 0;
+ {{INTERNALSUBNETS}}
+ }
upstream controller {
server {{UPSTREAMHOST}}:{{CONTROLLERPORT}};
}
@@ -26,7 +31,10 @@ http {
client_max_body_size 0;
location / {
proxy_pass http://controller;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
proxy_set_header Host $http_host;
+ proxy_set_header X-External-Client $external_client;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_redirect off;
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index e32d385f7..7147c7aa8 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -660,6 +660,7 @@ def run_nginx():
nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
nginxconf['ERRORLOG'] = _logfilename('nginx_error')
nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx'
+ nginxconf['INTERNALSUBNETS'] = '169.254.0.0/16 0;'
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list