[arvados] updated: 2.7.0-5657-g8090581c64
git repository hosting
git at public.arvados.org
Thu Dec 28 18:31:54 UTC 2023
Summary of changes:
lib/service/cmd.go | 22 +++++++++++++++++++++-
lib/service/cmd_test.go | 34 ++++++++++++++++++++++++++++------
2 files changed, 49 insertions(+), 7 deletions(-)
via 8090581c645f62fec8337b3a2b9f72448fc24431 (commit)
from 02d65726493a9eadbf3375c23dde8e128472ccbe (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8090581c645f62fec8337b3a2b9f72448fc24431
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 28 13:31:43 2023 -0500
21285: Send all SSH and tunnel endpoints to tunnel queue.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index e40b47acbb..82e95fe0b4 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -16,6 +16,7 @@ import (
_ "net/http/pprof"
"net/url"
"os"
+ "regexp"
"strings"
"time"
@@ -297,7 +298,7 @@ func (c *command) requestLimiter(handler http.Handler, cluster *arvados.Cluster,
Priority: c.requestPriority,
Registry: reg,
Queue: func(req *http.Request) *httpserver.RequestQueue {
- if strings.HasPrefix(req.URL.Path, "/arvados/v1/connect/") {
+ if req.Method == http.MethodPost && reTunnelPath.MatchString(req.URL.Path) {
return rqTunnel
} else {
return rqAPI
@@ -306,6 +307,25 @@ func (c *command) requestLimiter(handler http.Handler, cluster *arvados.Cluster,
}
}
+// reTunnelPath matches paths of API endpoints that go in the "tunnel"
+// queue.
+var reTunnelPath = regexp.MustCompile(func() string {
+ rePathVar := regexp.MustCompile(`{.*?}`)
+ out := ""
+ for _, endpoint := range []arvados.APIEndpoint{
+ arvados.EndpointContainerGatewayTunnel,
+ arvados.EndpointContainerGatewayTunnelCompat,
+ arvados.EndpointContainerSSH,
+ arvados.EndpointContainerSSHCompat,
+ } {
+ if out != "" {
+ out += "|"
+ }
+ out += `\Q/` + rePathVar.ReplaceAllString(endpoint.Path, `\E[^/]*\Q`) + `\E`
+ }
+ return "^(" + out + ")$"
+}())
+
func (c *command) requestPriority(req *http.Request, queued time.Time) int64 {
switch {
case req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/containers/") && strings.HasSuffix(req.URL.Path, "/lock"):
diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index 0266752f38..9ead90019e 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -200,6 +200,15 @@ func (*Suite) TestCommand(c *check.C) {
c.Check(stderr.String(), check.Matches, `(?ms).*"msg":"CheckHealth called".*`)
}
+func (s *Suite) TestTunnelPathRegexp(c *check.C) {
+ c.Check(reTunnelPath.MatchString(`/arvados/v1/connect/zzzzz-dz642-aaaaaaaaaaaaaaa/gateway_tunnel`), check.Equals, true)
+ c.Check(reTunnelPath.MatchString(`/arvados/v1/containers/zzzzz-dz642-aaaaaaaaaaaaaaa/gateway_tunnel`), check.Equals, true)
+ c.Check(reTunnelPath.MatchString(`/arvados/v1/connect/zzzzz-dz642-aaaaaaaaaaaaaaa/ssh`), check.Equals, true)
+ c.Check(reTunnelPath.MatchString(`/arvados/v1/containers/zzzzz-dz642-aaaaaaaaaaaaaaa/ssh`), check.Equals, true)
+ c.Check(reTunnelPath.MatchString(`/blah/arvados/v1/containers/zzzzz-dz642-aaaaaaaaaaaaaaa/ssh`), check.Equals, false)
+ c.Check(reTunnelPath.MatchString(`/arvados/v1/containers/zzzzz-dz642-aaaaaaaaaaaaaaa`), check.Equals, false)
+}
+
func (s *Suite) TestRequestLimitsAndDumpRequests_Keepweb(c *check.C) {
s.testRequestLimitAndDumpRequests(c, arvados.ServiceNameKeepweb, "MaxConcurrentRequests")
}
@@ -244,7 +253,7 @@ Clusters:
started := make(chan bool, max+1)
hold := make(chan bool)
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if strings.HasPrefix(r.URL.Path, "/arvados/v1/connect") {
+ if strings.Contains(r.URL.Path, "/ssh") || strings.Contains(r.URL.Path, "/gateway_tunnel") {
<-hold
} else {
started <- true
@@ -283,10 +292,11 @@ Clusters:
activeReqs.Add(1)
go func() {
defer activeReqs.Done()
- resp, err := client.Get("http://localhost:" + port + "/testpath")
+ target := "http://localhost:" + port + "/testpath"
+ resp, err := client.Get(target)
for err != nil && strings.Contains(err.Error(), "dial tcp") && deadline.After(time.Now()) {
time.Sleep(time.Second / 100)
- resp, err = client.Get("http://localhost:" + port + "/testpath")
+ resp, err = client.Get(target)
}
if c.Check(err, check.IsNil) {
if resp.StatusCode == http.StatusOK {
@@ -302,14 +312,22 @@ Clusters:
// API req limit
extraTunnelReqs := 20
var tunnelResp200, tunnelResp503 int64
+ var paths = []string{
+ "/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", "z1234-dz642-abcdeabcdeabcde", -1),
+ "/" + strings.Replace(arvados.EndpointContainerSSHCompat.Path, "{uuid}", "z1234-dz642-abcdeabcdeabcde", -1),
+ "/" + strings.Replace(arvados.EndpointContainerGatewayTunnel.Path, "{uuid}", "z1234-dz642-abcdeabcdeabcde", -1),
+ "/" + strings.Replace(arvados.EndpointContainerGatewayTunnelCompat.Path, "{uuid}", "z1234-dz642-abcdeabcdeabcde", -1),
+ }
for i := 0; i < maxTunnels+extraTunnelReqs; i++ {
+ i := i
activeReqs.Add(1)
go func() {
defer activeReqs.Done()
- resp, err := client.Get("http://localhost:" + port + "/arvados/v1/connect/...")
+ target := "http://localhost:" + port + paths[i%len(paths)]
+ resp, err := client.Post(target, "application/octet-stream", nil)
for err != nil && strings.Contains(err.Error(), "dial tcp") && deadline.After(time.Now()) {
time.Sleep(time.Second / 100)
- resp, err = client.Get("http://localhost:" + port + "/arvados/v1/connect/...")
+ resp, err = client.Post(target, "application/octet-stream", nil)
}
if c.Check(err, check.IsNil) {
if resp.StatusCode == http.StatusOK {
@@ -327,6 +345,10 @@ Clusters:
case <-started:
case <-time.After(time.Second):
c.Logf("%s", stderr.String())
+ c.Logf("apiResp200 %d", apiResp200)
+ c.Logf("apiResp503 %d", apiResp503)
+ c.Logf("tunnelResp200 %d", tunnelResp200)
+ c.Logf("tunnelResp503 %d", tunnelResp503)
c.Fatal("timed out")
}
}
@@ -345,7 +367,7 @@ Clusters:
c.Check(err, check.IsNil)
for i := 0; i < len(loaded); i++ {
- if strings.HasPrefix(loaded[i].URL, "/arvados/v1/connect/") {
+ if strings.Contains(loaded[i].URL, "/ssh") || strings.Contains(loaded[i].URL, "/gateway_tunnel") {
// Filter out a gateway tunnel req
// that doesn't count toward our API
// req limit
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list