[ARVADOS] updated: caba74a47f9819c6c214729f09aa9925926ca7c3
Git user
git at public.curoverse.com
Fri Sep 16 16:55:29 EDT 2016
Summary of changes:
sdk/go/arvados/client.go | 4 +--
services/keep-web/main.go | 20 ++++++------
services/keep-web/usage.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 90 insertions(+), 13 deletions(-)
create mode 100644 services/keep-web/usage.go
discards 089aa29a1b16b833a30e8384900e6b6ccc55fb4f (commit)
discards 3f19bd3d240272fb0c9e7e207df926fc659335ac (commit)
via caba74a47f9819c6c214729f09aa9925926ca7c3 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (089aa29a1b16b833a30e8384900e6b6ccc55fb4f)
\
N -- N -- N (caba74a47f9819c6c214729f09aa9925926ca7c3)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit caba74a47f9819c6c214729f09aa9925926ca7c3
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 15 23:41:39 2016 -0400
9957: Refactor keep-web to load config from a file, with legacy support for command line flags.
Add systemd unit file.
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index f95152b..36f4eb5 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -23,7 +23,7 @@ import (
type Client struct {
// HTTP client used to make requests. If nil,
// DefaultSecureClient or InsecureHTTPClient will be used.
- Client *http.Client
+ Client *http.Client `json:"-"`
// Hostname (or host:port) of Arvados API server.
APIHost string
@@ -40,7 +40,7 @@ type Client struct {
// discovering keep services so this is just a convenience for
// callers who use a Client to initialize an
// arvadosclient.ArvadosClient.)
- KeepServiceURIs []string
+ KeepServiceURIs []string `json:",omitempty"`
}
// The default http.Client used by a Client with Insecure==true and
diff --git a/sdk/go/configfile/load.go b/sdk/go/configfile/load.go
new file mode 100644
index 0000000..c76d906
--- /dev/null
+++ b/sdk/go/configfile/load.go
@@ -0,0 +1,19 @@
+package configfile
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+)
+
+func LoadFile(cfg interface{}, configPath string) error {
+ buf, err := ioutil.ReadFile(configPath)
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(buf, cfg)
+ if err != nil {
+ return fmt.Errorf("Error decoding config %q: %v", configPath, err)
+ }
+ return nil
+}
diff --git a/services/keep-web/anonymous.go b/services/keep-web/anonymous.go
deleted file mode 100644
index 15a98c2..0000000
--- a/services/keep-web/anonymous.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "strconv"
-)
-
-var anonymousTokens tokenSet
-
-type tokenSet []string
-
-func (ts *tokenSet) Set(s string) error {
- v, err := strconv.ParseBool(s)
- if v && len(*ts) == 0 {
- *ts = append(*ts, os.Getenv("ARVADOS_API_TOKEN"))
- } else if !v {
- *ts = (*ts)[:0]
- }
- return err
-}
-
-func (ts *tokenSet) String() string {
- return fmt.Sprintf("%v", len(*ts) > 0)
-}
-
-func (ts *tokenSet) IsBoolFlag() bool {
- return true
-}
-
-func init() {
- flag.Var(&anonymousTokens, "allow-anonymous",
- "Serve public data to anonymous clients. Try the token supplied in the ARVADOS_API_TOKEN environment variable when none of the tokens provided in an HTTP request succeed in reading the desired collection.")
-}
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 9ca732f..59dd784 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -6,17 +6,34 @@
//
// See http://doc.arvados.org/install/install-keep-web.html.
//
-// Run "keep-web -help" to show all supported options.
+// Configuration
+//
+// The default configuration file location is
+// /etc/arvados/keep-web/config.json.
+//
+// Example configuration file
+//
+// {
+// "Client": {
+// "APIHost": "zzzzz.arvadosapi.com:443",
+// "AuthToken": "",
+// "Insecure": false
+// },
+// "AllowAnonymous":false,
+// "AttachmentOnlyHost":"",
+// "TrustAllContent":false
+// }
//
// Starting the server
//
-// Serve HTTP requests at port 1234 on all interfaces:
+// Start a server using the default config file
+// /etc/arvados/keep-web/config.json:
//
-// keep-web -listen=:1234
+// keep-web
//
-// Serve HTTP requests at port 1234 on the interface with IP address 1.2.3.4:
+// Start a server using the config file /path/to/config.json:
//
-// keep-web -listen=1.2.3.4:1234
+// keep-web -config /path/to/config.json
//
// Proxy configuration
//
@@ -48,12 +65,11 @@
//
// Anonymous downloads
//
-// Use the -allow-anonymous flag with an ARVADOS_API_TOKEN environment
-// variable to specify a token to use when clients try to retrieve
-// files without providing their own Arvados API token.
+// If "AnonymousToken" is provided, that token will be used when
+// clients try to retrieve files without providing their own Arvados
+// API token.
//
-// export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-// keep-web [...] -allow-anonymous
+// "AnonymousToken":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
//
// See http://doc.arvados.org/install/install-keep-web.html for examples.
//
@@ -211,30 +227,31 @@
// only when the designated origin matches exactly the Host header
// provided by the client or downstream proxy.
//
-// keep-web -listen :9999 -attachment-only-host domain.example:9999
+// "AttachmentOnlyHost":"domain.example:9999"
//
// Trust All Content mode
//
-// In "trust all content" mode, Keep-web will accept credentials (API
+// In TrustAllContent mode, Keep-web will accept credentials (API
// tokens) and serve any collection X at
-// "https://collections.example.com/c=X/path/file.ext".
-// This is UNSAFE except in the special case where everyone who is
-// able write ANY data to Keep, and every JavaScript and HTML file
-// written to Keep, is also trusted to read ALL of the data in Keep.
+// "https://collections.example.com/c=X/path/file.ext". This is
+// UNSAFE except in the special case where everyone who is able write
+// ANY data to Keep, and every JavaScript and HTML file written to
+// Keep, is also trusted to read ALL of the data in Keep.
//
// In such cases you can enable trust-all-content mode.
//
-// keep-web -listen :9999 -trust-all-content
+// "TrustAllContent":true
//
-// When using trust-all-content mode, the only effect of the
-// -attachment-only-host option is to add a "Content-Disposition:
+// When TrustAllContent is enabled, the only effect of the
+// AttachmentOnlyHost flag is to add a "Content-Disposition:
// attachment" header.
//
-// keep-web -listen :9999 -attachment-only-host domain.example:9999 -trust-all-content
+// "AttachmentOnlyHost":"domain.example:9999",
+// "TrustAllContent":true
//
// Depending on your site configuration, you might also want to enable
-// "trust all content" setting on Workbench. Normally, Workbench
+// the "trust all content" setting in Workbench. Normally, Workbench
// avoids redirecting requests to keep-web if they depend on
-// -trust-all-content being set.
+// TrustAllContent being enabled.
//
package main
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 6f5f66a..38df1d8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,7 +1,6 @@
package main
import (
- "flag"
"fmt"
"html"
"io"
@@ -12,6 +11,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -19,19 +19,10 @@ import (
"git.curoverse.com/arvados.git/sdk/go/keepclient"
)
-type handler struct{}
-
-var (
- clientPool = arvadosclient.MakeClientPool()
- trustAllContent = false
- attachmentOnlyHost = ""
-)
-
-func init() {
- flag.StringVar(&attachmentOnlyHost, "attachment-only-host", "",
- "Accept credentials, and add \"Content-Disposition: attachment\" response headers, for requests at this hostname:port. Prohibiting inline display makes it possible to serve untrusted and non-public content from a single origin, i.e., without wildcard DNS or SSL.")
- flag.BoolVar(&trustAllContent, "trust-all-content", false,
- "Serve non-public content from a single origin. Dangerous: read docs before using!")
+type handler struct {
+ Config *Config
+ clientPool *arvadosclient.ClientPool
+ setupOnce sync.Once
}
// return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
@@ -70,7 +61,13 @@ func parseCollectionIDFromURL(s string) string {
return ""
}
+func (h *handler) setup() {
+ h.clientPool = arvadosclient.MakeClientPool()
+}
+
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+ h.setupOnce.Do(h.setup)
+
var statusCode = 0
var statusText string
@@ -109,12 +106,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
- arv := clientPool.Get()
+ arv := h.clientPool.Get()
if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
return
}
- defer clientPool.Put(arv)
+ defer h.clientPool.Put(arv)
pathParts := strings.Split(r.URL.Path[1:], "/")
@@ -124,9 +121,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
var attachment bool
- credentialsOK := trustAllContent
+ credentialsOK := h.Config.TrustAllContent
- if r.Host != "" && r.Host == attachmentOnlyHost {
+ if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
credentialsOK = true
attachment = true
} else if r.FormValue("disposition") == "attachment" {
@@ -151,7 +148,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else {
// /collections/ID/PATH...
targetID = pathParts[1]
- tokens = anonymousTokens
+ tokens = h.Config.AnonymousTokens
targetPath = pathParts[2:]
}
} else {
@@ -186,7 +183,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// It is not safe to copy the provided token
// into a cookie unless the current vhost
// (origin) serves only a single collection or
- // we are in trustAllContent mode.
+ // we are in TrustAllContent mode.
statusCode = http.StatusBadRequest
return
}
@@ -246,7 +243,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if credentialsOK {
reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
}
- tokens = append(reqTokens, anonymousTokens...)
+ tokens = append(reqTokens, h.Config.AnonymousTokens...)
}
if len(targetPath) > 0 && targetPath[0] == "_" {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index d04c5c2..b3e17e8 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -38,7 +38,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
URL: u,
RequestURI: u.RequestURI(),
}
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
c.Check(resp.Body.String(), check.Equals, "")
}
@@ -51,7 +51,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
type authorizer func(*http.Request, string) int
func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
- doVhostRequests(c, authzViaAuthzHeader)
+ s.doVhostRequests(c, authzViaAuthzHeader)
}
func authzViaAuthzHeader(r *http.Request, tok string) int {
r.Header.Add("Authorization", "OAuth2 "+tok)
@@ -59,7 +59,7 @@ func authzViaAuthzHeader(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
- doVhostRequests(c, authzViaCookieValue)
+ s.doVhostRequests(c, authzViaCookieValue)
}
func authzViaCookieValue(r *http.Request, tok string) int {
r.AddCookie(&http.Cookie{
@@ -70,7 +70,7 @@ func authzViaCookieValue(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
- doVhostRequests(c, authzViaPath)
+ s.doVhostRequests(c, authzViaPath)
}
func authzViaPath(r *http.Request, tok string) int {
r.URL.Path = "/t=" + tok + r.URL.Path
@@ -78,7 +78,7 @@ func authzViaPath(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
- doVhostRequests(c, authzViaQueryString)
+ s.doVhostRequests(c, authzViaQueryString)
}
func authzViaQueryString(r *http.Request, tok string) int {
r.URL.RawQuery = "api_token=" + tok
@@ -86,7 +86,7 @@ func authzViaQueryString(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
- doVhostRequests(c, authzViaPOST)
+ s.doVhostRequests(c, authzViaPOST)
}
func authzViaPOST(r *http.Request, tok string) int {
r.Method = "POST"
@@ -97,7 +97,7 @@ func authzViaPOST(r *http.Request, tok string) int {
}
func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
- doVhostRequests(c, authzViaPOST)
+ s.doVhostRequests(c, authzViaPOST)
}
func authzViaXHRPOST(r *http.Request, tok string) int {
r.Method = "POST"
@@ -113,7 +113,7 @@ func authzViaXHRPOST(r *http.Request, tok string) int {
// Try some combinations of {url, token} using the given authorization
// mechanism, and verify the result is correct.
-func doVhostRequests(c *check.C, authz authorizer) {
+func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
for _, hostPath := range []string{
arvadostest.FooCollection + ".example.com/foo",
arvadostest.FooCollection + "--collections.example.com/foo",
@@ -123,11 +123,11 @@ func doVhostRequests(c *check.C, authz authorizer) {
arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
} {
c.Log("doRequests: ", hostPath)
- doVhostRequestsWithHostPath(c, authz, hostPath)
+ s.doVhostRequestsWithHostPath(c, authz, hostPath)
}
}
-func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
+func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
for _, tok := range []string{
arvadostest.ActiveToken,
arvadostest.ActiveToken[:15],
@@ -144,7 +144,7 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
Header: http.Header{},
}
failCode := authz(req, tok)
- req, resp := doReq(req)
+ req, resp := s.doReq(req)
code, body := resp.Code, resp.Body.String()
// If the initial request had a (non-empty) token
@@ -173,9 +173,9 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
}
}
-func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
+func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return req, resp
}
@@ -191,7 +191,7 @@ func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
for _, c := range cookies {
req.AddCookie(c)
}
- return doReq(req)
+ return s.doReq(req)
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
@@ -270,10 +270,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- defer func(orig bool) {
- trustAllContent = orig
- }(trustAllContent)
- trustAllContent = true
+ s.testServer.Config.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
@@ -285,10 +282,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- defer func(orig string) {
- attachmentOnlyHost = orig
- }(attachmentOnlyHost)
- attachmentOnlyHost = "example.com:1234"
+ s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
@@ -333,7 +327,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- anonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
@@ -345,7 +339,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+ s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
@@ -357,6 +351,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
}
func (s *IntegrationSuite) TestRange(c *check.C) {
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
u, _ := url.Parse("http://example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt")
req := &http.Request{
Method: "GET",
@@ -366,7 +361,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
Header: http.Header{"Range": {"bytes=0-4"}},
}
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusPartialContent)
c.Check(resp.Body.String(), check.Equals, "Hello")
c.Check(resp.Header().Get("Content-Length"), check.Equals, "5")
@@ -374,7 +369,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
req.Header.Set("Range", "bytes=0-")
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
// 200 and 206 are both correct:
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "Hello world\n")
@@ -389,7 +384,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
} {
req.Header.Set("Range", hdr)
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "Hello world\n")
c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
@@ -420,7 +415,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
}.Encode())),
}
resp := httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
@@ -443,7 +438,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
c.Check(resp.Body.String(), check.Equals, expectRespBody)
}()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return resp
}
@@ -463,7 +458,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
}
resp = httptest.NewRecorder()
- (&handler{}).ServeHTTP(resp, req)
+ s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Header().Get("Location"), check.Equals, "")
return resp
}
diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
new file mode 100644
index 0000000..da56212
--- /dev/null
+++ b/services/keep-web/keep-web.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Arvados Keep web gateway
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/keep-web
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 135f01b..73f1e25 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -4,8 +4,35 @@ import (
"flag"
"log"
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/configfile"
+ "github.com/coreos/go-systemd/daemon"
+)
+
+var (
+ defaultConfigPath = "/etc/arvados/keep-web/config.json"
)
+type Config struct {
+ Client arvados.Client
+
+ Listen string
+
+ AnonymousTokens []string
+ AttachmentOnlyHost string
+ TrustAllContent bool
+
+ deprecatedAllowAnonymous bool
+}
+
+// DefaultConfig returns the default configuration.
+func DefaultConfig() *Config {
+ return &Config{
+ Listen: ":80",
+ }
+}
+
func init() {
// MakeArvadosClient returns an error if this env var isn't
// available as a default token (even if we explicitly set a
@@ -18,14 +45,44 @@ func init() {
}
func main() {
+ cfg := DefaultConfig()
+
+ var configPath string
+ deprecated := " (DEPRECATED -- use config file instead)"
+ flag.StringVar(&configPath, "config", defaultConfigPath,
+ "`path` to json configuration file")
+ flag.StringVar(&cfg.Listen, "listen", "",
+ "address:port or :port to listen on"+deprecated)
+ flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false,
+ "Load an anonymous token from the ARVADOS_API_TOKEN environment variable"+deprecated)
+ flag.StringVar(&cfg.AttachmentOnlyHost, "attachment-only-host", "",
+ "Only serve attachments at the given `host:port`"+deprecated)
+ flag.BoolVar(&cfg.TrustAllContent, "trust-all-content", false,
+ "Serve non-public content from a single origin. Dangerous: read docs before using!"+deprecated)
+ flag.Usage = usage
flag.Parse()
- if os.Getenv("ARVADOS_API_HOST") == "" {
- log.Fatal("ARVADOS_API_HOST environment variable must be set.")
+
+ if err := configfile.LoadFile(cfg, configPath); err != nil {
+ if h := os.Getenv("ARVADOS_API_HOST"); h != "" && configPath == defaultConfigPath {
+ log.Printf("DEPRECATED: Using ARVADOS_API_HOST environment variable. Use config file instead.")
+ cfg.Client.APIHost = h
+ } else {
+ log.Fatal(err)
+ }
+ }
+ if cfg.deprecatedAllowAnonymous {
+ log.Printf("DEPRECATED: Using -allow-anonymous command line flag with ARVADOS_API_TOKEN environment variable. Use config file instead.")
+ cfg.AnonymousTokens = []string{os.Getenv("ARVADOS_API_TOKEN")}
}
- srv := &server{}
+
+ os.Setenv("ARVADOS_API_HOST", cfg.Client.APIHost)
+ srv := &server{Config: cfg}
if err := srv.Start(); err != nil {
log.Fatal(err)
}
+ if _, err := daemon.SdNotify("READY=1"); err != nil {
+ log.Printf("Error notifying init daemon: %v", err)
+ }
log.Println("Listening at", srv.Addr)
if err := srv.Wait(); err != nil {
log.Fatal(err)
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 1009008..babc68b 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -1,27 +1,16 @@
package main
import (
- "flag"
- "net/http"
-
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
-var address string
-
-func init() {
- flag.StringVar(&address, "listen", ":80",
- "Address to listen on: \"host:port\", or \":port\" to listen on all interfaces.")
-}
-
type server struct {
httpserver.Server
+ Config *Config
}
func (srv *server) Start() error {
- mux := http.NewServeMux()
- mux.Handle("/", &handler{})
- srv.Handler = mux
- srv.Addr = address
+ srv.Handler = &handler{Config: srv.Config}
+ srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 324588a..bddd6db 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -6,16 +6,20 @@ import (
"io"
"io/ioutil"
"net"
+ "os"
"os/exec"
"strings"
"testing"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
+var testAPIHost = os.Getenv("ARVADOS_API_HOST")
+
var _ = check.Suite(&IntegrationSuite{})
// IntegrationSuite tests need an API server and a keep-web server
@@ -137,7 +141,7 @@ type curlCase struct {
}
func (s *IntegrationSuite) Test200(c *check.C) {
- anonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
for _, spec := range []curlCase{
// My collection
{
@@ -307,10 +311,14 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- s.testServer = &server{}
- var err error
- address = "127.0.0.1:0"
- err = s.testServer.Start()
+ s.testServer = &server{Config: &Config{
+ Client: arvados.Client{
+ APIHost: testAPIHost,
+ Insecure: true,
+ },
+ Listen: "127.0.0.1:0",
+ }}
+ err := s.testServer.Start()
c.Assert(err, check.Equals, nil)
}
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
new file mode 100644
index 0000000..05ebb39
--- /dev/null
+++ b/services/keep-web/usage.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+)
+
+var exampleConfigFile = []byte(`
+ {
+ "Client": {
+ "APIHost": "zzzzz.arvadosapi.com:443",
+ "Insecure": false
+ },
+ }`)
+
+func usage() {
+ c := DefaultConfig()
+ c.AnonymousTokens = []string{"xxxxxxxxxxxxxxxxxxxxxxx"}
+ c.Client.APIHost = "zzzzz.arvadosapi.com:443"
+ exampleConfigFile, err := json.MarshalIndent(c, " ", " ")
+ if err != nil {
+ panic(err)
+ }
+ fmt.Fprintf(os.Stderr, `
+
+Keep-web provides read-only HTTP access to files stored in Keep; see
+https://godoc.org/github.com/curoverse/arvados/services/keep-web and
+http://doc.arvados.org/install/install-keep-web.html
+
+Usage: keep-web -config path/to/config.json
+
+Options:
+`)
+ flag.PrintDefaults()
+ fmt.Fprintf(os.Stderr, `
+Example config file:
+ %s
+
+Client.APIHost:
+
+ Address (or address:port) of the Arvados API endpoint.
+
+Client.AuthToken:
+
+ Should be empty.
+
+Client.Insecure:
+
+ True if your Arvados API endpoint uses an unverifiable SSL/TLS
+ certificate.
+
+Listen:
+
+ Local port to listen on. Can be "address", "address:port", or
+ ":port", where "address" is a host IP address or name and "port"
+ is a port number or name.
+
+AnonymousTokens:
+
+ Array of tokens to try when a client does not provide a token.
+
+AttachmentOnlyHost:
+
+ Accept credentials, and add "Content-Disposition: attachment"
+ response headers, for requests at this hostname:port.
+
+ This prohibits inline display, which makes it possible to serve
+ untrusted and non-public content from a single origin, i.e.,
+ without wildcard DNS or SSL.
+
+TrustAllContent:
+
+ Serve non-public content from a single origin. Dangerous: read
+ docs before using!
+
+`, exampleConfigFile)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list