[arvados] updated: 2.1.0-2849-g2375b76b1

git repository hosting git at public.arvados.org
Fri Aug 12 00:43:48 UTC 2022


Summary of changes:
 .licenseignore                                     |   1 +
 SECURITY.md                                        |  42 ++++
 apps/workbench/Gemfile.lock                        |   2 +-
 .../app/controllers/actions_controller.rb          |   2 +-
 .../app/controllers/application_controller.rb      |   6 +-
 apps/workbench/app/helpers/application_helper.rb   |   2 +-
 apps/workbench/app/models/arvados_api_client.rb    |   4 +-
 apps/workbench/test/test_helper.rb                 |   2 +-
 cmd/arvados-client/container_gateway.go            |   5 +-
 cmd/arvados-client/container_gateway_test.go       |   9 +
 cmd/arvados-server/cmd.go                          |   2 +
 doc/_config.yml                                    |   1 +
 doc/admin/config-urls.html.textile.liquid          |   3 +-
 doc/admin/upgrading.html.textile.liquid            |  24 ++-
 doc/architecture/hpc.html.textile.liquid           |  29 +++
 doc/install/arvbox.html.textile.liquid             |   9 +-
 go.mod                                             |   1 +
 go.sum                                             |   2 +
 lib/boot/nginx.go                                  |  27 +++
 lib/controller/federation/conn.go                  |   6 +-
 lib/controller/localdb/conn.go                     |   4 +
 lib/controller/localdb/container_gateway.go        | 234 ++++++++++++++++-----
 lib/controller/localdb/container_gateway_test.go   | 167 ++++++++++++++-
 lib/controller/proxy.go                            |   9 +-
 lib/controller/router/request.go                   |   1 +
 lib/controller/router/router.go                    |  18 ++
 lib/controller/rpc/conn.go                         |  82 +++++---
 lib/crunchrun/container_gateway.go                 | 221 ++++++++++++++++---
 lib/crunchrun/crunchrun.go                         |  67 +++++-
 lib/crunchrun/executor_test.go                     |  17 ++
 lib/crunchrun/integration_test.go                  |  21 +-
 lib/diagnostics/cmd.go                             |  58 ++++-
 lib/diagnostics/hello-world.tar                    | Bin 0 -> 24064 bytes
 lib/install/deps.go                                |   1 +
 lib/install/init.go                                |  23 --
 lib/lsf/dispatch.go                                |  19 +-
 sdk/cli/bin/arv                                    |   2 +-
 sdk/cwl/arvados_cwl/executor.py                    |  16 +-
 sdk/cwl/arvados_cwl/runner.py                      | 204 ++++++++++++------
 sdk/cwl/setup.py                                   |   7 +-
 sdk/go/arvados/api.go                              |  16 +-
 sdk/go/arvados/container_gateway.go                |  35 +--
 sdk/go/arvadostest/api.go                          |   8 +-
 .../org/arvados/client/logic/keep/KeepClient.java  |   4 +-
 sdk/python/arvados/collection.py                   |   2 +-
 sdk/python/tests/nginx.conf                        |   8 +
 sdk/python/tests/run_test_server.py                |   2 +
 sdk/python/tests/test_collections.py               |  14 ++
 services/api/Gemfile.lock                          |   2 +-
 .../controllers/arvados/v1/groups_controller.rb    |  25 ++-
 services/api/app/models/container.rb               |   4 +-
 services/api/lib/record_filters.rb                 |   2 +-
 .../api/test/functional/arvados/v1/filters_test.rb |  52 +++++
 services/api/test/unit/container_test.rb           |   1 +
 .../crunch-dispatch-slurm/crunch-dispatch-slurm.go |   9 +-
 services/crunch-dispatch-slurm/script.go           |  10 +-
 services/crunch-dispatch-slurm/script_test.go      |   3 +-
 services/keep-web/s3.go                            |   2 +-
 services/keep-web/s3_test.go                       |   7 +
 .../aws/pillars/nginx_webshell_configuration.sls   |   2 +-
 ...l.params.example.single_host_multiple_hostnames |   2 +-
 ...ocal.params.example.single_host_single_hostname |   2 +-
 tools/salt-install/provision.sh                    |   2 +-
 63 files changed, 1274 insertions(+), 290 deletions(-)
 create mode 100644 SECURITY.md
 create mode 100644 doc/architecture/hpc.html.textile.liquid
 create mode 100644 lib/diagnostics/hello-world.tar

       via  2375b76b13b2ee5a071f5941f7a27c90e5a35aa9 (commit)
       via  9c65df788a099fcd8ed7d68dda2c35e3503ee365 (commit)
       via  bb5ce73fd625c761ef68388116da5063d430c655 (commit)
       via  42c20b25e1325124b88e3b9b285544dc41122b56 (commit)
       via  c8cbf2509601da0890bccc7f9ef5f5a8eaa307d0 (commit)
       via  8d0b26e44e50df56d63f489cc62f4c04fbe613e7 (commit)
       via  067b16c3cb19f17cca368b1373977c5610511806 (commit)
       via  750366f2b8978d52babc2345184a7797b4601a98 (commit)
       via  101c02ace8036f92d07e3d5e22736267381c0489 (commit)
       via  7822d4d431284d0912ba40d288da81a1eac68a3e (commit)
       via  1cbf8cd312dd019809b060d83999c677e94dbe7e (commit)
       via  513804e1a2bf43329dc7d37ee9374f3e02ffe169 (commit)
       via  c0924347a69157d3058a39d238fb0e0bacefa3a2 (commit)
       via  c6b4cfe6758ff5bc4fbc2ca72c60d9c8485f7267 (commit)
       via  3fb6110248db3f87fb21f852c8d6bfebbb2910a0 (commit)
       via  61c68cf08258d3292257b67c6b50a223b17f4bfd (commit)
       via  7de4c3a96c6ea992df549efa8446bd89ce5bd667 (commit)
       via  80b2655dbb69a4ceacce0a7f58845b3ee7fb5853 (commit)
       via  f1e7c6e0018d276aae506b52ce18e7c31bf48479 (commit)
       via  4e5838bd9e1a7baa5b3e53e97e308140e4b6105f (commit)
       via  b334b065b36357dd08099adad9835f4aa7075337 (commit)
       via  a8f70f5f978641afa273adcbf995423228f0c7c4 (commit)
       via  2fdb9c8541e96756604439f604b82a68e747a35a (commit)
       via  b23d2434ab8162ea67be50fc3299a0e4450e13ee (commit)
       via  275e7919b78fd9d19c8f6b62c8ba97052bba589c (commit)
       via  d791fa7adb14991c972b6166f39155ff314b7d1e (commit)
       via  381d8aa1544bebfc39761636eaf4ca07427be783 (commit)
       via  2b95ec7a9a6516a7dd2554e2dd8ecf9ddcfbceb7 (commit)
       via  b8749c42d9ae650f4e2dbeeda2a8cdae9266ff2a (commit)
       via  11227eb2ee3877648c946bedd9a427fe35950822 (commit)
       via  20aa08040bc12e6387455755ce3fadc238c19d76 (commit)
       via  159004e1fab103ce295146b172dd76149e95e845 (commit)
       via  62612ee23efe146829a7bb64817cdd23f41775bb (commit)
       via  c9b8b9b9c78a77dd30b828914c8bee9fa8dcbb90 (commit)
       via  1923ceda3c8845526f1ddcdd6275513760e0cd84 (commit)
       via  3e95703e7444c1d2077eba748740ebe3df94b1d5 (commit)
       via  6cd62b6e286d4470ef9e2b2c70653d78a05f8cf2 (commit)
       via  a46fce5c3bfdf6b5a7fcc817309970156bcdc5b8 (commit)
       via  bce07be48c8c9d8ae62cb1615f5a7cde2ac72249 (commit)
       via  711d8642c478c29ffd4c9bc6facbf3bd4d168763 (commit)
       via  c296ba650406f32de3315159e4ecd7411fc3dcb9 (commit)
       via  2e03d03bc55b5a612c2bf04d878a72f2ee420d99 (commit)
       via  94081a34c4972cd65a20cbeb4d1837149f057378 (commit)
       via  46aadf6a64881e932adb4c54cb1a8c4b22e84d67 (commit)
       via  32b402114038bc6761c04c370afad786dbbb3125 (commit)
       via  61d3aed1cd0286c7928167a84267c225938a37ec (commit)
       via  736c58d66deda35079ad8b64df02ee0ef57232f0 (commit)
       via  e995446d93c8109d7894010d57c4cc32ae04a459 (commit)
       via  5613df6b553c02f95e84b4536397e9be8d95007a (commit)
       via  6dfa03b4c6211d265e15a4982831edd59eadf3af (commit)
       via  6501c300ebaa01552e17619cc1c89ee38f7398a7 (commit)
       via  420e857f8e8ac75beca258fa72b9edac680500cd (commit)
       via  e186a3539c02602333bb4c98690ab3b7527decc4 (commit)
       via  d8eeb0f02b967153790e54284fd3213b648def20 (commit)
       via  748ee07068ed64fa2e12901ce43f548bd4ff213a (commit)
       via  3218db7a7af5c34032d28ba03bffa1684f52897e (commit)
       via  831540fd5eedb6226996b5c72a86f2dba64cb196 (commit)
       via  13736aa4f0feea65abce2aedc7c4ca0d18e01061 (commit)
       via  df17ea1c927da5ac2955e9cc83b6d7ab88085fdb (commit)
       via  ef833210fd7400727e01c97551ab0e645773d3fb (commit)
       via  2261d1fd9e1b69d0a60f1f7fe9029317aeb4cf52 (commit)
       via  dc70bbf9ea15395476107a3b8dff96f754a40998 (commit)
       via  091ae55fc1df3ec50490becd437d512e38b0f972 (commit)
       via  a42604972cccf8dd9c8341c260927a6c48c62b84 (commit)
       via  817ee84de15cdc960990e86af8ce705073fcafba (commit)
       via  31a270c68c24b7de994655ee788478a64b6bdfb7 (commit)
       via  bad3a575113abaf7a232f85dd15c4d1e1b60c0e2 (commit)
       via  3e27af4fa076222a18d602091060b9d11ee2079c (commit)
       via  49f99d7b9a8706c48baedba3659d7ca98b02dde3 (commit)
       via  87f3da84318306184165dae50f75ac6721d89285 (commit)
       via  5e20c073d84304c3e84770bb7d89035bf1fb9626 (commit)
       via  3fae0f0626c5152a5aa6f39f0874f0190f2131db (commit)
       via  9587429b4ee56fe9a1ca3555ecebd04e0dae929d (commit)
       via  c4bae86d39f237df8ac6a5505323f6a93011514a (commit)
       via  f1f74069850d8c5e987ef7d7fc246735ff94d58d (commit)
       via  ca6cbc015e137e5e24c6ac5268e9fc72a61db84d (commit)
       via  9a4705cbb5410ddddd97d19bdd77821755ff640c (commit)
       via  bdc29d3129f6d75aa9ce0a24ffb849a272b06f08 (commit)
       via  ced6d55c36132aee7da3a5fe65f608c9dbf33362 (commit)
      from  c966970d64c21d7adaf1c3c8b737aa9e7c166f0e (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 2375b76b13b2ee5a071f5941f7a27c90e5a35aa9
Merge: 11227eb2e 9c65df788
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 11 20:43:28 2022 -0400

    17344: Merge branch 'main'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>


commit 11227eb2ee3877648c946bedd9a427fe35950822
Author: Tom Clegg <tom at curii.com>
Date:   Tue Aug 2 10:12:51 2022 -0400

    17344: arvados-server boot: set X-External-Client header.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index b391c4dc8..9f1091eac 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -5,6 +5,7 @@
 package boot
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -17,6 +18,7 @@ import (
 	"strings"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"github.com/sirupsen/logrus"
 )
 
 // Run an Nginx process that proxies the supervisor's configured
@@ -46,6 +48,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 	vars := map[string]string{
 		"LISTENHOST":       extListenHost,
 		"UPSTREAMHOST":     super.ListenHost,
+		"INTERNALSUBNETS":  internalSubnets(super.logger),
 		"SSLCERT":          filepath.Join(super.tempdir, "server.crt"),
 		"SSLKEY":           filepath.Join(super.tempdir, "server.key"),
 		"ACCESSLOG":        filepath.Join(super.tempdir, "nginx_access.log"),
@@ -150,3 +153,27 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 	}
 	return waitForConnect(ctx, testurl.Host)
 }
+
+// Return 0 or more local subnets as "geo" fragments for Nginx config,
+// e.g., "1.2.3.0/24 0; 10.1.0.0/16 0;".
+func internalSubnets(logger logrus.FieldLogger) string {
+	iproutes, err := exec.Command("ip", "route").CombinedOutput()
+	if err != nil {
+		logger.Warnf("treating all clients as external because `ip route` failed: %s (%q)", err, iproutes)
+		return ""
+	}
+	subnets := ""
+	for _, line := range bytes.Split(iproutes, []byte("\n")) {
+		fields := strings.Fields(string(line))
+		if len(fields) > 2 && fields[1] == "dev" {
+			// lan example:
+			// 192.168.86.0/24 dev ens3 proto kernel scope link src 192.168.86.196
+			// gcp example (private subnet):
+			// 10.47.0.0/24 dev eth0 proto kernel scope link src 10.47.0.5
+			// gcp example (no private subnet):
+			// 10.128.0.1 dev ens4 scope link
+			subnets += fields[0] + " 0; "
+		}
+	}
+	return subnets
+}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 4ad3eda42..a1a75bbcc 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -15,6 +15,11 @@ http {
   fastcgi_temp_path "{{TMPDIR}}";
   uwsgi_temp_path "{{TMPDIR}}";
   scgi_temp_path "{{TMPDIR}}";
+  geo $external_client {
+    default 1;
+    127.0.0.0/8 0;
+    {{INTERNALSUBNETS}}
+  }
   upstream controller {
     server {{UPSTREAMHOST}}:{{CONTROLLERPORT}};
   }
@@ -26,7 +31,10 @@ http {
     client_max_body_size 0;
     location  / {
       proxy_pass http://controller;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
       proxy_set_header Host $http_host;
+      proxy_set_header X-External-Client $external_client;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto https;
       proxy_redirect off;
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index e32d385f7..7147c7aa8 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -660,6 +660,7 @@ def run_nginx():
     nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
     nginxconf['ERRORLOG'] = _logfilename('nginx_error')
     nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx'
+    nginxconf['INTERNALSUBNETS'] = '169.254.0.0/16 0;'
 
     conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
     conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list