[ARVADOS] updated: 1.1.4-599-g224ac505b
Git user
git at public.curoverse.com
Fri Jul 13 12:33:50 EDT 2018
Summary of changes:
sdk/python/tests/nginx.conf | 21 ++++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
discards 70138209b3dcb1d6d54f6cd2aa2e279c4441ecda (commit)
discards bc603ec9d2ec299c3bc05d005c54015bd6072b17 (commit)
discards 85f0a03adeacbe31b12f435509d2ed7931093bf7 (commit)
via 224ac505b1162837f2f84fe2735a959d71bd5ce9 (commit)
via f0d62ea064d32b980f723e37972788cbd693c2ff (commit)
via e7870334a9f2c44fcd8580d3e75074df216c647a (commit)
via f62cff30c9948815d3cf2b00294da01c146cb8f2 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (70138209b3dcb1d6d54f6cd2aa2e279c4441ecda)
\
N -- N -- N (224ac505b1162837f2f84fe2735a959d71bd5ce9)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 224ac505b1162837f2f84fe2735a959d71bd5ce9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Jul 13 10:51:43 2018 -0400
13497: Improve nginx logging in test suite.
Include request scheme://host:port, elapsed time, and upstream server
type.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 4fe0c7ae7..432326171 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -7,14 +7,17 @@ error_log "{{ERRORLOG}}" info; # Yes, must be specified here _and_ cmdl
events {
}
http {
- access_log "{{ACCESSLOG}}" combined;
+ log_format customlog
+ '[$time_local] $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
+ '"$http_referer" "$http_user_agent"';
+ access_log "{{ACCESSLOG}}" customlog;
client_body_temp_path "{{TMPDIR}}";
upstream arv-git-http {
server localhost:{{GITPORT}};
}
server {
listen *:{{GITSSLPORT}} ssl default_server;
- server_name _;
+ server_name arv-git-http;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -30,7 +33,7 @@ http {
}
server {
listen *:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name _;
+ server_name keepproxy;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -46,7 +49,7 @@ http {
}
server {
listen *:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name ~^(?<request_host>.*)$;
+ server_name keep-web;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -59,7 +62,7 @@ http {
}
server {
listen *:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name ~.*;
+ server_name keep-web-dl ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -75,7 +78,7 @@ http {
}
server {
listen *:{{WSSPORT}} ssl default_server;
- server_name ~^(?<request_host>.*)$;
+ server_name websocket;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -93,7 +96,7 @@ http {
}
server {
listen *:{{CONTROLLERSSLPORT}} ssl default_server;
- server_name _;
+ server_name controller;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
commit f0d62ea064d32b980f723e37972788cbd693c2ff
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jul 12 16:50:59 2018 -0400
13497: Avoid misdirecting POST to GET when request has double slash.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 30994130d..a1a69a88e 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -30,6 +30,20 @@ type Handler struct {
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
h.setupOnce.Do(h.setup)
+ if req.Method != "GET" && req.Method != "HEAD" {
+ // http.ServeMux returns 301 with a cleaned path if
+ // the incoming request has a double slash. Some
+ // clients (including the Go standard library) change
+ // the request method to GET when following a 301
+ // redirect if the original method was not HEAD
+ // (RFC7231 6.4.2 specifically allows this in the case
+ // of POST). Thus "POST //foo" gets misdirected to
+ // "GET /foo". To avoid this, eliminate double slashes
+ // before passing the request to ServeMux.
+ for strings.Contains(req.URL.Path, "//") {
+ req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
+ }
+ }
h.handlerStack.ServeHTTP(w, req)
}
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 9ee99903c..59e8de3bc 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -99,7 +99,7 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad
}
var updated arvados.Collection
defer c.pdhs.Remove(coll.UUID)
- err := client.RequestAndDecode(&updated, "PATCH", "/arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
+ err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
if err == nil {
c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
expire: time.Now().Add(time.Duration(c.TTL)),
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 3814a459d..0e2f17c35 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -74,7 +74,7 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
- err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
+ err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
readPath, writePath, pdhPath := pathFunc(newCollection)
commit e7870334a9f2c44fcd8580d3e75074df216c647a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jul 12 16:43:37 2018 -0400
13497: Use X-Forwarded-Proto as scheme for keep-web redirect URLs.
This avoids relying on the downstream proxy to rewrite redirects.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 36d2477d1..6e99f23d7 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -174,7 +174,7 @@ def find_available_port():
sock.close()
return port
-def _wait_until_port_listens(port, timeout=10):
+def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
If nothing listens on the port within the specified timeout (given
@@ -196,11 +196,13 @@ def _wait_until_port_listens(port, timeout=10):
except subprocess.CalledProcessError:
time.sleep(0.1)
continue
- return
- print(
- "WARNING: Nothing is listening on port {} (waited {} seconds).".
- format(port, timeout),
- file=sys.stderr)
+ return True
+ if warn:
+ print(
+ "WARNING: Nothing is listening on port {} (waited {} seconds).".
+ format(port, timeout),
+ file=sys.stderr)
+ return False
def _logfilename(label):
"""Set up a labelled log file, and return a path to write logs to.
@@ -376,6 +378,10 @@ def reset():
headers={'Authorization': 'OAuth2 {}'.format(token)})
os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
os.environ['ARVADOS_API_TOKEN'] = token
+ if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
+ os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
+ else:
+ os.environ['ARVADOS_API_HOST'] = existing_api_host
def stop(force=False):
"""Stop the API server, if one is running.
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 517ec1a2a..7d17be6e7 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -183,6 +183,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
remoteAddr = xff + "," + remoteAddr
}
+ if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" {
+ r.URL.Scheme = xfp
+ }
w := httpserver.WrapResponseWriter(wOrig)
defer func() {
@@ -773,6 +776,7 @@ func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, loc
u = newu
}
redir := (&url.URL{
+ Scheme: r.URL.Scheme,
Host: r.Host,
Path: u.Path,
RawQuery: redirQuery.Encode(),
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index f86f81bfa..206bf6f43 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -513,7 +513,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
if resp.Code != http.StatusSeeOther {
return resp
}
- c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
+ c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
u, _ = u.Parse(resp.Header().Get("Location"))
commit f62cff30c9948815d3cf2b00294da01c146cb8f2
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jul 12 14:09:59 2018 -0400
13497: Fix Rails redirect targets.
Rails relies on the X-Forwarded-Proto request header, so we need to
set that in the test suite's Nginx config, like we do in the install
guide.
Also tidy up needlessly divergent proxy directives in Nginx test
config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 92ded6301..30994130d 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -94,6 +94,9 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
xff = xffIn + "," + xff
}
hdrOut.Set("X-Forwarded-For", xff)
+ if hdrOut.Get("X-Forwarded-Proto") == "" {
+ hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
+ }
hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
ctx := reqIn.Context()
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index ce1929fdf..4fe0c7ae7 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -19,6 +19,10 @@ http {
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://arv-git-http;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
upstream keepproxy {
@@ -31,6 +35,10 @@ http {
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://keepproxy;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
upstream keep-web {
@@ -43,8 +51,10 @@ http {
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://keep-web;
- proxy_set_header Host $request_host:{{KEEPWEBPORT}};
+ proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
server {
@@ -56,7 +66,8 @@ http {
proxy_pass http://keep-web;
proxy_set_header Host download:{{KEEPWEBPORT}};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_redirect //download:{{KEEPWEBPORT}}/ https://$host:{{KEEPWEBDLSSLPORT}}/;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect https://download:{{KEEPWEBPORT}}/ https://$host:{{KEEPWEBDLSSLPORT}}/;
}
}
upstream ws {
@@ -71,8 +82,10 @@ http {
proxy_pass http://ws;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
- proxy_set_header Host $request_host:{{WSPORT}};
+ proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
upstream controller {
@@ -85,7 +98,10 @@ http {
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_pass http://controller;
+ proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
}
}
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list