[ARVADOS] updated: 3399e630e78d09fa553a7d0876e2cddb4e154472
Git user
git at public.curoverse.com
Tue Sep 20 22:39:49 EDT 2016
Summary of changes:
sdk/go/arvados/client.go | 4 +-
sdk/go/config/load.go | 23 +++++++
services/keep-web/anonymous.go | 35 -----------
services/keep-web/doc.go | 63 ++++++++++++-------
services/keep-web/handler.go | 51 ++++++++--------
services/keep-web/handler_test.go | 55 ++++++++---------
.../keep-web.service} | 4 +-
services/keep-web/main.go | 66 +++++++++++++++++++-
services/keep-web/server.go | 17 +-----
services/keep-web/server_test.go | 18 ++++--
services/keep-web/usage.go | 71 ++++++++++++++++++++++
11 files changed, 268 insertions(+), 139 deletions(-)
create mode 100644 sdk/go/config/load.go
delete mode 100644 services/keep-web/anonymous.go
copy services/{crunch-dispatch-slurm/crunch-dispatch-slurm.service => keep-web/keep-web.service} (61%)
create mode 100644 services/keep-web/usage.go
via 3399e630e78d09fa553a7d0876e2cddb4e154472 (commit)
via 30140496ddefee85305437ba4826c7e2fae0ecb2 (commit)
via 2b9be1ee00a28fa03c19dbc53d3c4fed45e65a33 (commit)
via 87fcb4c388573967b0dedf6a23557fcf9888d998 (commit)
from 6802d84dd40245f2c605364dcd6d42849c409324 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3399e630e78d09fa553a7d0876e2cddb4e154472
Merge: 6802d84 3014049
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Sep 20 22:39:22 2016 -0400
Merge branch '9957-keep-web-config' closes #9957
commit 30140496ddefee85305437ba4826c7e2fae0ecb2
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Sep 19 14:16:14 2016 -0400
9957: Clarify AuthToken is not used.
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
index e7f90f0..1cd4d56 100644
--- a/services/keep-web/usage.go
+++ b/services/keep-web/usage.go
@@ -36,7 +36,7 @@ Client.APIHost:
Client.AuthToken:
- Should be empty.
+ Unused. Normally empty, or omitted entirely.
Client.Insecure:
commit 2b9be1ee00a28fa03c19dbc53d3c4fed45e65a33
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Sep 19 13:55:08 2016 -0400
9957: Clarify anonymous token explanation.
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 4483c60..37f81b1 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -67,8 +67,9 @@
// Anonymous downloads
//
// The "AnonymousTokens" configuration entry is an array of tokens to
-// use when clients try to retrieve files without providing their own
-// Arvados API token.
+// use when processing anonymous requests, i.e., whenever a web client
+// does not supply its own Arvados API token via path, query string,
+// cookie, or request header.
//
// "AnonymousTokens":["xxxxxxxxxxxxxxxxxxxxxxx"]
//
commit 87fcb4c388573967b0dedf6a23557fcf9888d998
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 15 23:41:39 2016 -0400
9957: Refactor keep-web to load config from a file, with legacy support for command line flags.
Add systemd unit file.
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index f95152b..36f4eb5 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -23,7 +23,7 @@ import (
type Client struct {
// HTTP client used to make requests. If nil,
// DefaultSecureClient or InsecureHTTPClient will be used.
- Client *http.Client
+ Client *http.Client `json:"-"`
// Hostname (or host:port) of Arvados API server.
APIHost string
@@ -40,7 +40,7 @@ type Client struct {
// discovering keep services so this is just a convenience for
// callers who use a Client to initialize an
// arvadosclient.ArvadosClient.)
- KeepServiceURIs []string
+ KeepServiceURIs []string `json:",omitempty"`
}
// The default http.Client used by a Client with Insecure==true and
diff --git a/sdk/go/config/load.go b/sdk/go/config/load.go
new file mode 100644
index 0000000..143be8b
--- /dev/null
+++ b/sdk/go/config/load.go
@@ -0,0 +1,23 @@
+package config
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+)
+
+// LoadFile loads configuration from the file given by configPath and
+// decodes it into cfg.
+//
+// Currently, only JSON is supported. Support for YAML is anticipated.
+func LoadFile(cfg interface{}, configPath string) error {
+ buf, err := ioutil.ReadFile(configPath)
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(buf, cfg)
+ if err != nil {
+ return fmt.Errorf("Error decoding config %q: %v", configPath, err)
+ }
+ return nil
+}
diff --git a/services/keep-web/anonymous.go b/services/keep-web/anonymous.go
deleted file mode 100644
index 15a98c2..0000000
--- a/services/keep-web/anonymous.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "strconv"
-)
-
-var anonymousTokens tokenSet
-
-type tokenSet []string
-
-func (ts *tokenSet) Set(s string) error {
- v, err := strconv.ParseBool(s)
- if v && len(*ts) == 0 {
- *ts = append(*ts, os.Getenv("ARVADOS_API_TOKEN"))
- } else if !v {
- *ts = (*ts)[:0]
- }
- return err
-}
-
-func (ts *tokenSet) String() string {
- return fmt.Sprintf("%v", len(*ts) > 0)
-}
-
-func (ts *tokenSet) IsBoolFlag() bool {
- return true
-}
-
-func init() {
- flag.Var(&anonymousTokens, "allow-anonymous",
- "Serve public data to anonymous clients. Try the token supplied in the ARVADOS_API_TOKEN environment variable when none of the tokens provided in an HTTP request succeed in reading the desired collection.")
-}
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 9ca732f..4483c60 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -6,17 +6,35 @@
//
// See http://doc.arvados.org/install/install-keep-web.html.
//
-// Run "keep-web -help" to show all supported options.
+// Configuration
+//
+// The default configuration file location is
+// /etc/arvados/keep-web/config.json.
+//
+// Example configuration file
+//
+// {
+// "Client": {
+// "APIHost": "zzzzz.arvadosapi.com:443",
+// "AuthToken": "",
+// "Insecure": false
+// },
+// "Listen":":1234",
+// "AnonymousTokens":["xxxxxxxxxxxxxxxxxxxx"],
+// "AttachmentOnlyHost":"",
+// "TrustAllContent":false
+// }
//
// Starting the server
//
-// Serve HTTP requests at port 1234 on all interfaces:
+// Start a server using the default config file
+// /etc/arvados/keep-web/config.json:
//
-// keep-web -listen=:1234
+// keep-web
//
-// Serve HTTP requests at port 1234 on the interface with IP address 1.2.3.4:
+// Start a server using the config file /path/to/config.json:
//
-// keep-web -listen=1.2.3.4:1234
+// keep-web -config /path/to/config.json
//
// Proxy configuration
//
@@ -48,12 +66,11 @@
//
// Anonymous downloads
//
-// Use the -allow-anonymous flag with an ARVADOS_API_TOKEN environment
-// variable to specify a token to use when clients try to retrieve
-// files without providing their own Arvados API token.
+// The "AnonymousTokens" configuration entry is an array of tokens to
+// use when clients try to retrieve files without providing their own
+// Arvados API token.
//
-// export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-// keep-web [...] -allow-anonymous
+// "AnonymousTokens":["xxxxxxxxxxxxxxxxxxxxxxx"]
//
// See http://doc.arvados.org/install/install-keep-web.html for examples.
//
@@ -211,30 +228,31 @@
// only when the designated origin matches exactly the Host header
// provided by the client or downstream proxy.
//
-// keep-web -listen :9999 -attachment-only-host domain.example:9999
+// "AttachmentOnlyHost":"domain.example:9999"
//
// Trust All Content mode
//
-// In "trust all content" mode, Keep-web will accept credentials (API
+// In TrustAllContent mode, Keep-web will accept credentials (API
// tokens) and serve any collection X at
-// "https://collections.example.com/c=X/path/file.ext".
-// This is UNSAFE except in the special case where everyone who is
-// able write ANY data to Keep, and every JavaScript and HTML file
-// written to Keep, is also trusted to read ALL of the data in Keep.
+// "https://collections.example.com/c=X/path/file.ext". This is
+// UNSAFE except in the special case where everyone who is able write
+// ANY data to Keep, and every JavaScript and HTML file written to
+// Keep, is also trusted to read ALL of the data in Keep.
//
// In such cases you can enable trust-all-content mode.
//
-// keep-web -listen :9999 -trust-all-content
+// "TrustAllContent":true
//
-// When using trust-all-content mode, the only effect of the
-// -attachment-only-host option is to add a "Content-Disposition:
+// When TrustAllContent is enabled, the only effect of the
+// AttachmentOnlyHost flag is to add a "Content-Disposition:
// attachment" header.
//
-// keep-web -listen :9999 -attachment-only-host domain.example:9999 -trust-all-content
+// "AttachmentOnlyHost":"domain.example:9999",
+// "TrustAllContent":true
//
// Depending on your site configuration, you might also want to enable
-// "trust all content" setting on Workbench. Normally, Workbench
+// the "trust all content" setting in Workbench. Normally, Workbench
// avoids redirecting requests to keep-web if they depend on
-// -trust-all-content being set.
+// TrustAllContent being enabled.
//
package main
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 6f5f66a..11d0d96 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,7 +1,6 @@
package main
import (
- "flag"
"fmt"
"html"
"io"
@@ -12,6 +11,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -19,23 +19,14 @@ import (
"git.curoverse.com/arvados.git/sdk/go/keepclient"
)
-type handler struct{}
-
-var (
- clientPool = arvadosclient.MakeClientPool()
- trustAllContent = false
- attachmentOnlyHost = ""
-)
-
-func init() {
- flag.StringVar(&attachmentOnlyHost, "attachment-only-host", "",
- "Accept credentials, and add \"Content-Disposition: attachment\" response headers, for requests at this hostname:port. Prohibiting inline display makes it possible to serve untrusted and non-public content from a single origin, i.e., without wildcard DNS or SSL.")
- flag.BoolVar(&trustAllContent, "trust-all-content", false,
- "Serve non-public content from a single origin. Dangerous: read docs before using!")
+type handler struct {
+ Config *Config
+ clientPool *arvadosclient.ClientPool
+ setupOnce sync.Once
}
-// return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
-// otherwise return "".
+// parseCollectionIDFromDNSName returns a UUID or PDH if s begins with
+// a UUID or URL-encoded PDH; otherwise "".
func parseCollectionIDFromDNSName(s string) string {
// Strip domain.
if i := strings.IndexRune(s, '.'); i >= 0 {
@@ -58,8 +49,9 @@ func parseCollectionIDFromDNSName(s string) string {
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
-// return a UUID or PDH if s is a UUID or a PDH (even if it is a PDH
-// with "+" replaced by " " or "-"); otherwise return "".
+// parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
+// PDH (even if it is a PDH with "+" replaced by " " or "-");
+// otherwise "".
func parseCollectionIDFromURL(s string) string {
if arvadosclient.UUIDMatch(s) {
return s
@@ -70,7 +62,14 @@ func parseCollectionIDFromURL(s string) string {
return ""
}
+func (h *handler) setup() {
+ h.clientPool = arvadosclient.MakeClientPool()
+}
+
+// ServeHTTP implements http.Handler.
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+ h.setupOnce.Do(h.setup)
+
var statusCode = 0
var statusText string
@@ -109,12 +108,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
- arv := clientPool.Get()
+ arv := h.clientPool.Get()
if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
return
}
- defer clientPool.Put(arv)
+ defer h.clientPool.Put(arv)
pathParts := strings.Split(r.URL.Path[1:], "/")
@@ -124,9 +123,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
var attachment bool
- credentialsOK := trustAllContent
+ credentialsOK := h.Config.TrustAllContent
- if r.Host != "" && r.Host == attachmentOnlyHost {
+ if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
credentialsOK = true
attachment = true
} else if r.FormValue("disposition") == "attachment" {
@@ -151,7 +150,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else {
// /collections/ID/PATH...
targetID = pathParts[1]
- tokens = anonymousTokens
+ tokens = h.Config.AnonymousTokens
targetPath = pathParts[2:]
}
} else {
@@ -186,7 +185,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// It is not safe to copy the provided token
// into a cookie unless the current vhost
// (origin) serves only a single collection or
- // we are in trustAllContent mode.
+ // we are in TrustAllContent mode.
statusCode = http.StatusBadRequest
return
}
@@ -246,7 +245,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if credentialsOK {
reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
}
- tokens = append(reqTokens, anonymousTokens...)
+ tokens = append(reqTokens, h.Config.AnonymousTokens...)
}
if len(targetPath) > 0 && targetPath[0] == "_" {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index d04c5c2..b3e17e8 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -38,7 +38,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
URL: u,
RequestURI: u.RequestURI(),
}
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
c.Check(resp.Body.String(), check.Equals, "")
}
@@ -51,7 +51,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
type authorizer func(*http.Request, string) int
func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
- doVhostRequests(c, authzViaAuthzHeader)
+ s.doVhostRequests(c, authzViaAuthzHeader)
}
func authzViaAuthzHeader(r *http.Request, tok string) int {
r.Header.Add("Authorization", "OAuth2 "+tok)
@@ -59,7 +59,7 @@ func authzViaAuthzHeader(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
- doVhostRequests(c, authzViaCookieValue)
+ s.doVhostRequests(c, authzViaCookieValue)
}
func authzViaCookieValue(r *http.Request, tok string) int {
r.AddCookie(&http.Cookie{
@@ -70,7 +70,7 @@ func authzViaCookieValue(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
- doVhostRequests(c, authzViaPath)
+ s.doVhostRequests(c, authzViaPath)
}
func authzViaPath(r *http.Request, tok string) int {
r.URL.Path = "/t=" + tok + r.URL.Path
@@ -78,7 +78,7 @@ func authzViaPath(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
- doVhostRequests(c, authzViaQueryString)
+ s.doVhostRequests(c, authzViaQueryString)
}
func authzViaQueryString(r *http.Request, tok string) int {
r.URL.RawQuery = "api_token=" + tok
@@ -86,7 +86,7 @@ func authzViaQueryString(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
- doVhostRequests(c, authzViaPOST)
+ s.doVhostRequests(c, authzViaPOST)
}
func authzViaPOST(r *http.Request, tok string) int {
r.Method = "POST"
@@ -97,7 +97,7 @@ func authzViaPOST(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
- doVhostRequests(c, authzViaPOST)
+ s.doVhostRequests(c, authzViaPOST)
}
func authzViaXHRPOST(r *http.Request, tok string) int {
r.Method = "POST"
@@ -113,7 +113,7 @@ func authzViaXHRPOST(r *http.Request, tok string) int {
// Try some combinations of {url, token} using the given authorization
// mechanism, and verify the result is correct.
-func doVhostRequests(c *check.C, authz authorizer) {
+func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
for _, hostPath := range []string{
arvadostest.FooCollection + ".example.com/foo",
arvadostest.FooCollection + "--collections.example.com/foo",
@@ -123,11 +123,11 @@ func doVhostRequests(c *check.C, authz authorizer) {
arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
} {
c.Log("doRequests: ", hostPath)
- doVhostRequestsWithHostPath(c, authz, hostPath)
+ s.doVhostRequestsWithHostPath(c, authz, hostPath)
}
}
-func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
+func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
for _, tok := range []string{
arvadostest.ActiveToken,
arvadostest.ActiveToken[:15],
@@ -144,7 +144,7 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
Header: http.Header{},
}
failCode := authz(req, tok)
- req, resp := doReq(req)
+ req, resp := s.doReq(req)
code, body := resp.Code, resp.Body.String()
// If the initial request had a (non-empty) token
@@ -173,9 +173,9 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
}
}
-func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
+func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return req, resp
}
@@ -191,7 +191,7 @@ func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
for _, c := range cookies {
req.AddCookie(c)
}
- return doReq(req)
+ return s.doReq(req)
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
@@ -270,10 +270,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- defer func(orig bool) {
- trustAllContent = orig
- }(trustAllContent)
- trustAllContent = true
+ s.testServer.Config.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
@@ -285,10 +282,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- defer func(orig string) {
- attachmentOnlyHost = orig
- }(attachmentOnlyHost)
- attachmentOnlyHost = "example.com:1234"
+ s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
@@ -333,7 +327,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- anonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
@@ -345,7 +339,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+ s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
@@ -357,6 +351,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
}
func (s *IntegrationSuite) TestRange(c *check.C) {
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
u, _ := url.Parse("http://example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt")
req := &http.Request{
Method: "GET",
@@ -366,7 +361,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
Header: http.Header{"Range": {"bytes=0-4"}},
}
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusPartialContent)
c.Check(resp.Body.String(), check.Equals, "Hello")
c.Check(resp.Header().Get("Content-Length"), check.Equals, "5")
@@ -374,7 +369,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
req.Header.Set("Range", "bytes=0-")
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
// 200 and 206 are both correct:
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "Hello world\n")
@@ -389,7 +384,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
} {
req.Header.Set("Range", hdr)
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "Hello world\n")
c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
@@ -420,7 +415,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
}.Encode())),
}
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
@@ -443,7 +438,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
c.Check(resp.Body.String(), check.Equals, expectRespBody)
}()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return resp
}
@@ -463,7 +458,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
}
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Header().Get("Location"), check.Equals, "")
return resp
}
diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
new file mode 100644
index 0000000..da56212
--- /dev/null
+++ b/services/keep-web/keep-web.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Arvados Keep web gateway
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/keep-web
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 135f01b..2d563bd 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -4,8 +4,38 @@ import (
"flag"
"log"
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
+ "github.com/coreos/go-systemd/daemon"
+)
+
+var (
+ defaultConfigPath = "/etc/arvados/keep-web/config.json"
)
+// Config specifies server configuration.
+type Config struct {
+ Client arvados.Client
+
+ Listen string
+
+ AnonymousTokens []string
+ AttachmentOnlyHost string
+ TrustAllContent bool
+
+ // Hack to support old command line flag, which is a bool
+ // meaning "get actual token from environment".
+ deprecatedAllowAnonymous bool
+}
+
+// DefaultConfig returns the default configuration.
+func DefaultConfig() *Config {
+ return &Config{
+ Listen: ":80",
+ }
+}
+
func init() {
// MakeArvadosClient returns an error if this env var isn't
// available as a default token (even if we explicitly set a
@@ -18,14 +48,44 @@ func init() {
}
func main() {
+ cfg := DefaultConfig()
+
+ var configPath string
+ deprecated := " (DEPRECATED -- use config file instead)"
+ flag.StringVar(&configPath, "config", defaultConfigPath,
+ "`path` to json configuration file")
+ flag.StringVar(&cfg.Listen, "listen", "",
+ "address:port or :port to listen on"+deprecated)
+ flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false,
+ "Load an anonymous token from the ARVADOS_API_TOKEN environment variable"+deprecated)
+ flag.StringVar(&cfg.AttachmentOnlyHost, "attachment-only-host", "",
+ "Only serve attachments at the given `host:port`"+deprecated)
+ flag.BoolVar(&cfg.TrustAllContent, "trust-all-content", false,
+ "Serve non-public content from a single origin. Dangerous: read docs before using!"+deprecated)
+ flag.Usage = usage
flag.Parse()
- if os.Getenv("ARVADOS_API_HOST") == "" {
- log.Fatal("ARVADOS_API_HOST environment variable must be set.")
+
+ if err := config.LoadFile(cfg, configPath); err != nil {
+ if h := os.Getenv("ARVADOS_API_HOST"); h != "" && configPath == defaultConfigPath {
+ log.Printf("DEPRECATED: Using ARVADOS_API_HOST environment variable. Use config file instead.")
+ cfg.Client.APIHost = h
+ } else {
+ log.Fatal(err)
+ }
+ }
+ if cfg.deprecatedAllowAnonymous {
+ log.Printf("DEPRECATED: Using -allow-anonymous command line flag with ARVADOS_API_TOKEN environment variable. Use config file instead.")
+ cfg.AnonymousTokens = []string{os.Getenv("ARVADOS_API_TOKEN")}
}
- srv := &server{}
+
+ os.Setenv("ARVADOS_API_HOST", cfg.Client.APIHost)
+ srv := &server{Config: cfg}
if err := srv.Start(); err != nil {
log.Fatal(err)
}
+ if _, err := daemon.SdNotify("READY=1"); err != nil {
+ log.Printf("Error notifying init daemon: %v", err)
+ }
log.Println("Listening at", srv.Addr)
if err := srv.Wait(); err != nil {
log.Fatal(err)
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 1009008..babc68b 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -1,27 +1,16 @@
package main
import (
- "flag"
- "net/http"
-
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
-var address string
-
-func init() {
- flag.StringVar(&address, "listen", ":80",
- "Address to listen on: \"host:port\", or \":port\" to listen on all interfaces.")
-}
-
type server struct {
httpserver.Server
+ Config *Config
}
func (srv *server) Start() error {
- mux := http.NewServeMux()
- mux.Handle("/", &handler{})
- srv.Handler = mux
- srv.Addr = address
+ srv.Handler = &handler{Config: srv.Config}
+ srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 324588a..bddd6db 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -6,16 +6,20 @@ import (
"io"
"io/ioutil"
"net"
+ "os"
"os/exec"
"strings"
"testing"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
+var testAPIHost = os.Getenv("ARVADOS_API_HOST")
+
var _ = check.Suite(&IntegrationSuite{})
// IntegrationSuite tests need an API server and a keep-web server
@@ -137,7 +141,7 @@ type curlCase struct {
}
func (s *IntegrationSuite) Test200(c *check.C) {
- anonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
for _, spec := range []curlCase{
// My collection
{
@@ -307,10 +311,14 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- s.testServer = &server{}
- var err error
- address = "127.0.0.1:0"
- err = s.testServer.Start()
+ s.testServer = &server{Config: &Config{
+ Client: arvados.Client{
+ APIHost: testAPIHost,
+ Insecure: true,
+ },
+ Listen: "127.0.0.1:0",
+ }}
+ err := s.testServer.Start()
c.Assert(err, check.Equals, nil)
}
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
new file mode 100644
index 0000000..e7f90f0
--- /dev/null
+++ b/services/keep-web/usage.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+)
+
+func usage() {
+ c := DefaultConfig()
+ c.AnonymousTokens = []string{"xxxxxxxxxxxxxxxxxxxxxxx"}
+ c.Client.APIHost = "zzzzz.arvadosapi.com:443"
+ exampleConfigFile, err := json.MarshalIndent(c, " ", " ")
+ if err != nil {
+ panic(err)
+ }
+ fmt.Fprintf(os.Stderr, `
+
+Keep-web provides read-only HTTP access to files stored in Keep; see
+https://godoc.org/github.com/curoverse/arvados/services/keep-web and
+http://doc.arvados.org/install/install-keep-web.html
+
+Usage: keep-web -config path/to/config.json
+
+Options:
+`)
+ flag.PrintDefaults()
+ fmt.Fprintf(os.Stderr, `
+Example config file:
+ %s
+
+Client.APIHost:
+
+ Address (or address:port) of the Arvados API endpoint.
+
+Client.AuthToken:
+
+ Should be empty.
+
+Client.Insecure:
+
+ True if your Arvados API endpoint uses an unverifiable SSL/TLS
+ certificate.
+
+Listen:
+
+ Local port to listen on. Can be "address", "address:port", or
+ ":port", where "address" is a host IP address or name and "port"
+ is a port number or name.
+
+AnonymousTokens:
+
+ Array of tokens to try when a client does not provide a token.
+
+AttachmentOnlyHost:
+
+ Accept credentials, and add "Content-Disposition: attachment"
+ response headers, for requests at this hostname:port.
+
+ This prohibits inline display, which makes it possible to serve
+ untrusted and non-public content from a single origin, i.e.,
+ without wildcard DNS or SSL.
+
+TrustAllContent:
+
+ Serve non-public content from a single origin. Dangerous: read
+ docs before using!
+
+`, exampleConfigFile)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list