[ARVADOS] updated: 1.1.4-374-gc9455b6d7
Git user
git at public.curoverse.com
Wed Jun 13 17:01:09 EDT 2018
Summary of changes:
apps/workbench/test/test_helper.rb | 30 +++----------------------
build/run-tests.sh | 9 ++++++--
lib/controller/handler.go | 18 ++++++++++++++-
lib/controller/handler_test.go | 18 ++++++++++++++-
sdk/python/tests/nginx.conf | 14 ++++++++++++
sdk/python/tests/run_test_server.py | 44 +++++++++++++++++++++++++++++++++++++
6 files changed, 102 insertions(+), 31 deletions(-)
via c9455b6d7cb2f20a7653f868d1ab8d6e7539c11c (commit)
via 15ecb85308b18e3a15fffb94cb2a0752c5ed7a70 (commit)
via a44dd92e008308938258fed8bc1eac2c8a5c98b8 (commit)
via 851cb6af920a43bb4b7fa0395cf785032537316d (commit)
via aa945483655dacf6ba88dc54a547a03b488a30fb (commit)
from d636833ef26fd6568acc49e32ee9d040bea67f92 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c9455b6d7cb2f20a7653f868d1ab8d6e7539c11c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 13 16:59:52 2018 -0400
13497: Set usable path for nginx request body buffering.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index d818c5f9c..bda67e630 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -8,6 +8,7 @@ events {
}
http {
access_log {{ACCESSLOG}} combined;
+ client_body_temp_path "{{TMPDIR}}";
upstream arv-git-http {
server localhost:{{GITPORT}};
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d8a21204a..f0fbfe742 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -646,6 +646,7 @@ def run_nginx():
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
+ nginxconf['TMPDIR'] = TEST_TMPDIR
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
commit 15ecb85308b18e3a15fffb94cb2a0752c5ed7a70
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 13 16:43:41 2018 -0400
13497: Remove alternate integration test glue in Workbench.
Expect run-tests.sh to have set everything up correctly instead.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index 60dadec61..2fd926ff1 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -177,38 +177,14 @@ class ApiServerForTests
end
def run_test_server
- env_script = nil
Dir.chdir PYTHON_TESTS_DIR do
- # These are no-ops if we're running within run-tests.sh (except
- # that we do get a useful env_script back from "start", even
- # though it doesn't need to start up a new server).
- env_script = check_output %w(python ./run_test_server.py start --auth admin)
- check_output %w(python ./run_test_server.py start_arv-git-httpd)
- check_output %w(python ./run_test_server.py start_keep-web)
- check_output %w(python ./run_test_server.py start_nginx)
- # This one isn't a no-op, even under run-tests.sh.
check_output %w(python ./run_test_server.py start_keep)
end
- test_env = {}
- env_script.each_line do |line|
- line = line.chomp
- if 0 == line.index('export ')
- toks = line.sub('export ', '').split '=', 2
- $stderr.puts "run_test_server.py: #{toks[0]}=#{toks[1]}"
- test_env[toks[0]] = toks[1]
- end
- end
- test_env
end
def stop_test_server
Dir.chdir PYTHON_TESTS_DIR do
check_output %w(python ./run_test_server.py stop_keep)
- # These are no-ops if we're running within run-tests.sh
- check_output %w(python ./run_test_server.py stop_nginx)
- check_output %w(python ./run_test_server.py stop_arv-git-httpd)
- check_output %w(python ./run_test_server.py stop_keep-web)
- check_output %w(python ./run_test_server.py stop)
end
@@server_is_running = false
end
@@ -223,9 +199,9 @@ class ApiServerForTests
stop_test_server
end
- test_env = run_test_server
- $application_config['arvados_login_base'] = "https://#{test_env['ARVADOS_API_HOST']}/login"
- $application_config['arvados_v1_base'] = "https://#{test_env['ARVADOS_API_HOST']}/arvados/v1"
+ run_test_server
+ $application_config['arvados_login_base'] = "https://#{ENV['ARVADOS_API_HOST']}/login"
+ $application_config['arvados_v1_base'] = "https://#{ENV['ARVADOS_API_HOST']}/arvados/v1"
$application_config['arvados_insecure_host'] = true
ActiveSupport::TestCase.reset_application_config
commit a44dd92e008308938258fed8bc1eac2c8a5c98b8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 13 15:51:30 2018 -0400
13497: Route API traffic through controller in test suites.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9bcc5ba05..52269fd6e 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -349,15 +349,19 @@ start_services() {
rm -f "$WORKSPACE/tmp/api.pid"
fi
cd "$WORKSPACE" \
- && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+ && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
+ && python sdk/python/tests/run_test_server.py start_controller \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
- && python sdk/python/tests/run_test_server.py start_nginx \
+ && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \
&& (env | egrep ^ARVADOS)
+ if [[ -n "$fail" ]]; then
+ return 1
+ fi
}
stop_services() {
@@ -371,6 +375,7 @@ stop_services() {
&& python sdk/python/tests/run_test_server.py stop_ws \
&& python sdk/python/tests/run_test_server.py stop_keep-web \
&& python sdk/python/tests/run_test_server.py stop_keep_proxy \
+ && python sdk/python/tests/run_test_server.py stop_controller \
&& python sdk/python/tests/run_test_server.py stop
}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index dcd4d26a3..70a337a6c 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -38,7 +38,7 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
SystemNodes: map[string]arvados.SystemNode{
"*": {
Controller: arvados.SystemServiceInstance{Listen: ":"},
- RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST"), TLS: true},
+ RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true},
},
},
}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 780968cb8..d818c5f9c 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -74,4 +74,17 @@ http {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
+ upstream controller {
+ server localhost:{{CONTROLLERPORT}};
+ }
+ server {
+ listen *:{{CONTROLLERSSLPORT}} ssl default_server;
+ server_name _;
+ ssl_certificate {{SSLCERT}};
+ ssl_certificate_key {{SSLKEY}};
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+ }
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 567b3b3bf..d8a21204a 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -377,6 +377,40 @@ def stop(force=False):
kill_server_pid(_pidfile('api'))
my_api_host = None
+def run_controller():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ stop_controller()
+ rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
+ port = find_available_port()
+ conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
+ with open(conf, 'w') as f:
+ f.write("""
+Clusters:
+ zzzzz:
+ SystemNodes:
+ "*":
+ "arvados-controller":
+ Listen: ":{}"
+ "arvados-api-server":
+ Listen: ":{}"
+ TLS: true
+ """.format(port, rails_api_port))
+ logf = open(_fifo2stderr('controller'), 'w')
+ controller = subprocess.Popen(
+ ["arvados-server", "controller", "-config", conf],
+ stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ with open(_pidfile('controller'), 'w') as f:
+ f.write(str(controller.pid))
+ _wait_until_port_listens(port)
+ _setport('controller', port)
+ return port
+
+def stop_controller():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('controller'))
+
def run_ws():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
@@ -598,6 +632,8 @@ def run_nginx():
return
stop_nginx()
nginxconf = {}
+ nginxconf['CONTROLLERPORT'] = _getport('controller')
+ nginxconf['CONTROLLERSSLPORT'] = find_available_port()
nginxconf['KEEPWEBPORT'] = _getport('keep-web')
nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
nginxconf['KEEPWEBSSLPORT'] = find_available_port()
@@ -628,6 +664,7 @@ def run_nginx():
'-g', 'pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
_setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
_setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
_setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
@@ -766,6 +803,7 @@ if __name__ == "__main__":
actions = [
'start', 'stop',
'start_ws', 'stop_ws',
+ 'start_controller', 'stop_controller',
'start_keep', 'stop_keep',
'start_keep_proxy', 'stop_keep_proxy',
'start_keep-web', 'stop_keep-web',
@@ -802,6 +840,10 @@ if __name__ == "__main__":
run_ws()
elif args.action == 'stop_ws':
stop_ws()
+ elif args.action == 'start_controller':
+ run_controller()
+ elif args.action == 'stop_controller':
+ stop_controller()
elif args.action == 'start_keep':
run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
elif args.action == 'stop_keep':
@@ -820,6 +862,7 @@ if __name__ == "__main__":
stop_keep_web()
elif args.action == 'start_nginx':
run_nginx()
+ print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
elif args.action == 'stop_nginx':
stop_nginx()
else:
commit 851cb6af920a43bb4b7fa0395cf785032537316d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 13 15:51:07 2018 -0400
13497: Don't propagate connection-oriented headers when proxying.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 013d293f2..ad765bafa 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -43,6 +43,19 @@ func (h *Handler) setup() {
h.handlerStack = mux
}
+// headers that shouldn't be forwarded when proxying. See
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
+var dropHeaders = map[string]bool{
+ "Connection": true,
+ "Keep-Alive": true,
+ "Proxy-Authenticate": true,
+ "Proxy-Authorization": true,
+ "TE": true,
+ "Trailer": true,
+ "Transfer-Encoding": true,
+ "Upgrade": true,
+}
+
func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
urlOut, err := findRailsAPI(h.Cluster, h.Node)
if err != nil {
@@ -61,7 +74,9 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
// headers like Via and X-Forwarded-For.
hdrOut := http.Header{}
for k, v := range reqIn.Header {
- hdrOut[k] = v
+ if !dropHeaders[k] {
+ hdrOut[k] = v
+ }
}
xff := reqIn.RemoteAddr
if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" {
commit aa945483655dacf6ba88dc54a547a03b488a30fb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 13 15:49:26 2018 -0400
13497: Send request body when proxying.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 6e4f0e3b4..013d293f2 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -81,6 +81,7 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
Method: reqIn.Method,
URL: urlOut,
Header: hdrOut,
+ Body: reqIn.Body,
}).WithContext(ctx)
resp, err := arvados.InsecureHTTPClient.Do(reqOut)
if err != nil {
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index a187ba443..dcd4d26a3 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -8,7 +8,9 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
+ "net/url"
"os"
+ "strings"
"testing"
"time"
@@ -94,6 +96,20 @@ func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
}
+func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
+ req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
+ "_method": {"GET"},
+ "api_token": {arvadostest.ActiveToken},
+ }.Encode()))
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var u arvados.User
+ err := json.Unmarshal(resp.Body.Bytes(), &u)
+ c.Check(err, check.IsNil)
+ c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
+}
+
func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
resp := httptest.NewRecorder()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list