[ARVADOS] updated: caba74a47f9819c6c214729f09aa9925926ca7c3

Git user git at public.curoverse.com
Fri Sep 16 16:55:29 EDT 2016


Summary of changes:
 sdk/go/arvados/client.go   |  4 +--
 services/keep-web/main.go  | 20 ++++++------
 services/keep-web/usage.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+), 13 deletions(-)
 create mode 100644 services/keep-web/usage.go

  discards  089aa29a1b16b833a30e8384900e6b6ccc55fb4f (commit)
  discards  3f19bd3d240272fb0c9e7e207df926fc659335ac (commit)
       via  caba74a47f9819c6c214729f09aa9925926ca7c3 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (089aa29a1b16b833a30e8384900e6b6ccc55fb4f)
            \
             N -- N -- N (caba74a47f9819c6c214729f09aa9925926ca7c3)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit caba74a47f9819c6c214729f09aa9925926ca7c3
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 15 23:41:39 2016 -0400

    9957: Refactor keep-web to load config from a file, with legacy support for command line flags.
    
    Add systemd unit file.

diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index f95152b..36f4eb5 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -23,7 +23,7 @@ import (
 type Client struct {
 	// HTTP client used to make requests. If nil,
 	// DefaultSecureClient or InsecureHTTPClient will be used.
-	Client *http.Client
+	Client *http.Client `json:"-"`
 
 	// Hostname (or host:port) of Arvados API server.
 	APIHost string
@@ -40,7 +40,7 @@ type Client struct {
 	// discovering keep services so this is just a convenience for
 	// callers who use a Client to initialize an
 	// arvadosclient.ArvadosClient.)
-	KeepServiceURIs []string
+	KeepServiceURIs []string `json:",omitempty"`
 }
 
 // The default http.Client used by a Client with Insecure==true and
diff --git a/sdk/go/configfile/load.go b/sdk/go/configfile/load.go
new file mode 100644
index 0000000..c76d906
--- /dev/null
+++ b/sdk/go/configfile/load.go
@@ -0,0 +1,19 @@
+package configfile
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+)
+
+func LoadFile(cfg interface{}, configPath string) error {
+	buf, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(buf, cfg)
+	if err != nil {
+		return fmt.Errorf("Error decoding config %q: %v", configPath, err)
+	}
+	return nil
+}
diff --git a/services/keep-web/anonymous.go b/services/keep-web/anonymous.go
deleted file mode 100644
index 15a98c2..0000000
--- a/services/keep-web/anonymous.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"strconv"
-)
-
-var anonymousTokens tokenSet
-
-type tokenSet []string
-
-func (ts *tokenSet) Set(s string) error {
-	v, err := strconv.ParseBool(s)
-	if v && len(*ts) == 0 {
-		*ts = append(*ts, os.Getenv("ARVADOS_API_TOKEN"))
-	} else if !v {
-		*ts = (*ts)[:0]
-	}
-	return err
-}
-
-func (ts *tokenSet) String() string {
-	return fmt.Sprintf("%v", len(*ts) > 0)
-}
-
-func (ts *tokenSet) IsBoolFlag() bool {
-	return true
-}
-
-func init() {
-	flag.Var(&anonymousTokens, "allow-anonymous",
-		"Serve public data to anonymous clients. Try the token supplied in the ARVADOS_API_TOKEN environment variable when none of the tokens provided in an HTTP request succeed in reading the desired collection.")
-}
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 9ca732f..59dd784 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -6,17 +6,34 @@
 //
 // See http://doc.arvados.org/install/install-keep-web.html.
 //
-// Run "keep-web -help" to show all supported options.
+// Configuration
+//
+// The default configuration file location is
+// /etc/arvados/keep-web/config.json.
+//
+// Example configuration file
+//
+//   {
+//     "Client": {
+//       "APIHost": "zzzzz.arvadosapi.com:443",
+//       "AuthToken": "",
+//       "Insecure": false
+//     },
+//     "AllowAnonymous":false,
+//     "AttachmentOnlyHost":"",
+//     "TrustAllContent":false
+//   }
 //
 // Starting the server
 //
-// Serve HTTP requests at port 1234 on all interfaces:
+// Start a server using the default config file
+// /etc/arvados/keep-web/config.json:
 //
-//   keep-web -listen=:1234
+//   keep-web
 //
-// Serve HTTP requests at port 1234 on the interface with IP address 1.2.3.4:
+// Start a server using the config file /path/to/config.json:
 //
-//   keep-web -listen=1.2.3.4:1234
+//   keep-web -config /path/to/config.json
 //
 // Proxy configuration
 //
@@ -48,12 +65,11 @@
 //
 // Anonymous downloads
 //
-// Use the -allow-anonymous flag with an ARVADOS_API_TOKEN environment
-// variable to specify a token to use when clients try to retrieve
-// files without providing their own Arvados API token.
+// If "AnonymousToken" is provided, that token will be used when
+// clients try to retrieve files without providing their own Arvados
+// API token.
 //
-//   export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-//   keep-web [...] -allow-anonymous
+//   "AnonymousToken":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
 //
 // See http://doc.arvados.org/install/install-keep-web.html for examples.
 //
@@ -211,30 +227,31 @@
 // only when the designated origin matches exactly the Host header
 // provided by the client or downstream proxy.
 //
-//   keep-web -listen :9999 -attachment-only-host domain.example:9999
+//   "AttachmentOnlyHost":"domain.example:9999"
 //
 // Trust All Content mode
 //
-// In "trust all content" mode, Keep-web will accept credentials (API
+// In TrustAllContent mode, Keep-web will accept credentials (API
 // tokens) and serve any collection X at
-// "https://collections.example.com/c=X/path/file.ext".
-// This is UNSAFE except in the special case where everyone who is
-// able write ANY data to Keep, and every JavaScript and HTML file
-// written to Keep, is also trusted to read ALL of the data in Keep.
+// "https://collections.example.com/c=X/path/file.ext".  This is
+// UNSAFE except in the special case where everyone who is able write
+// ANY data to Keep, and every JavaScript and HTML file written to
+// Keep, is also trusted to read ALL of the data in Keep.
 //
 // In such cases you can enable trust-all-content mode.
 //
-//   keep-web -listen :9999 -trust-all-content
+//   "TrustAllContent":true
 //
-// When using trust-all-content mode, the only effect of the
-// -attachment-only-host option is to add a "Content-Disposition:
+// When TrustAllContent is enabled, the only effect of the
+// AttachmentOnlyHost flag is to add a "Content-Disposition:
 // attachment" header.
 //
-//   keep-web -listen :9999 -attachment-only-host domain.example:9999 -trust-all-content
+//   "AttachmentOnlyHost":"domain.example:9999",
+//   "TrustAllContent":true
 //
 // Depending on your site configuration, you might also want to enable
-// "trust all content" setting on Workbench. Normally, Workbench
+// the "trust all content" setting in Workbench. Normally, Workbench
 // avoids redirecting requests to keep-web if they depend on
-// -trust-all-content being set.
+// TrustAllContent being enabled.
 //
 package main
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 6f5f66a..38df1d8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"flag"
 	"fmt"
 	"html"
 	"io"
@@ -12,6 +11,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -19,19 +19,10 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 )
 
-type handler struct{}
-
-var (
-	clientPool         = arvadosclient.MakeClientPool()
-	trustAllContent    = false
-	attachmentOnlyHost = ""
-)
-
-func init() {
-	flag.StringVar(&attachmentOnlyHost, "attachment-only-host", "",
-		"Accept credentials, and add \"Content-Disposition: attachment\" response headers, for requests at this hostname:port. Prohibiting inline display makes it possible to serve untrusted and non-public content from a single origin, i.e., without wildcard DNS or SSL.")
-	flag.BoolVar(&trustAllContent, "trust-all-content", false,
-		"Serve non-public content from a single origin. Dangerous: read docs before using!")
+type handler struct {
+	Config     *Config
+	clientPool *arvadosclient.ClientPool
+	setupOnce  sync.Once
 }
 
 // return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
@@ -70,7 +61,13 @@ func parseCollectionIDFromURL(s string) string {
 	return ""
 }
 
+func (h *handler) setup() {
+	h.clientPool = arvadosclient.MakeClientPool()
+}
+
 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+	h.setupOnce.Do(h.setup)
+
 	var statusCode = 0
 	var statusText string
 
@@ -109,12 +106,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 	}
 
-	arv := clientPool.Get()
+	arv := h.clientPool.Get()
 	if arv == nil {
-		statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+		statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
 		return
 	}
-	defer clientPool.Put(arv)
+	defer h.clientPool.Put(arv)
 
 	pathParts := strings.Split(r.URL.Path[1:], "/")
 
@@ -124,9 +121,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	var reqTokens []string
 	var pathToken bool
 	var attachment bool
-	credentialsOK := trustAllContent
+	credentialsOK := h.Config.TrustAllContent
 
-	if r.Host != "" && r.Host == attachmentOnlyHost {
+	if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
 		credentialsOK = true
 		attachment = true
 	} else if r.FormValue("disposition") == "attachment" {
@@ -151,7 +148,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 		} else {
 			// /collections/ID/PATH...
 			targetID = pathParts[1]
-			tokens = anonymousTokens
+			tokens = h.Config.AnonymousTokens
 			targetPath = pathParts[2:]
 		}
 	} else {
@@ -186,7 +183,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 			// It is not safe to copy the provided token
 			// into a cookie unless the current vhost
 			// (origin) serves only a single collection or
-			// we are in trustAllContent mode.
+			// we are in TrustAllContent mode.
 			statusCode = http.StatusBadRequest
 			return
 		}
@@ -246,7 +243,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 		if credentialsOK {
 			reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
 		}
-		tokens = append(reqTokens, anonymousTokens...)
+		tokens = append(reqTokens, h.Config.AnonymousTokens...)
 	}
 
 	if len(targetPath) > 0 && targetPath[0] == "_" {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index d04c5c2..b3e17e8 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -38,7 +38,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
 			URL:        u,
 			RequestURI: u.RequestURI(),
 		}
-		(&handler{}).ServeHTTP(resp, req)
+		s.testServer.Handler.ServeHTTP(resp, req)
 		c.Check(resp.Code, check.Equals, http.StatusNotFound)
 		c.Check(resp.Body.String(), check.Equals, "")
 	}
@@ -51,7 +51,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
 type authorizer func(*http.Request, string) int
 
 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
-	doVhostRequests(c, authzViaAuthzHeader)
+	s.doVhostRequests(c, authzViaAuthzHeader)
 }
 func authzViaAuthzHeader(r *http.Request, tok string) int {
 	r.Header.Add("Authorization", "OAuth2 "+tok)
@@ -59,7 +59,7 @@ func authzViaAuthzHeader(r *http.Request, tok string) int {
 }
 
 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
-	doVhostRequests(c, authzViaCookieValue)
+	s.doVhostRequests(c, authzViaCookieValue)
 }
 func authzViaCookieValue(r *http.Request, tok string) int {
 	r.AddCookie(&http.Cookie{
@@ -70,7 +70,7 @@ func authzViaCookieValue(r *http.Request, tok string) int {
 }
 
 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
-	doVhostRequests(c, authzViaPath)
+	s.doVhostRequests(c, authzViaPath)
 }
 func authzViaPath(r *http.Request, tok string) int {
 	r.URL.Path = "/t=" + tok + r.URL.Path
@@ -78,7 +78,7 @@ func authzViaPath(r *http.Request, tok string) int {
 }
 
 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
-	doVhostRequests(c, authzViaQueryString)
+	s.doVhostRequests(c, authzViaQueryString)
 }
 func authzViaQueryString(r *http.Request, tok string) int {
 	r.URL.RawQuery = "api_token=" + tok
@@ -86,7 +86,7 @@ func authzViaQueryString(r *http.Request, tok string) int {
 }
 
 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
-	doVhostRequests(c, authzViaPOST)
+	s.doVhostRequests(c, authzViaPOST)
 }
 func authzViaPOST(r *http.Request, tok string) int {
 	r.Method = "POST"
@@ -97,7 +97,7 @@ func authzViaPOST(r *http.Request, tok string) int {
 }
 
 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
-	doVhostRequests(c, authzViaPOST)
+	s.doVhostRequests(c, authzViaPOST)
 }
 func authzViaXHRPOST(r *http.Request, tok string) int {
 	r.Method = "POST"
@@ -113,7 +113,7 @@ func authzViaXHRPOST(r *http.Request, tok string) int {
 
 // Try some combinations of {url, token} using the given authorization
 // mechanism, and verify the result is correct.
-func doVhostRequests(c *check.C, authz authorizer) {
+func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 	for _, hostPath := range []string{
 		arvadostest.FooCollection + ".example.com/foo",
 		arvadostest.FooCollection + "--collections.example.com/foo",
@@ -123,11 +123,11 @@ func doVhostRequests(c *check.C, authz authorizer) {
 		arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 	} {
 		c.Log("doRequests: ", hostPath)
-		doVhostRequestsWithHostPath(c, authz, hostPath)
+		s.doVhostRequestsWithHostPath(c, authz, hostPath)
 	}
 }
 
-func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
+func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 	for _, tok := range []string{
 		arvadostest.ActiveToken,
 		arvadostest.ActiveToken[:15],
@@ -144,7 +144,7 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
 			Header:     http.Header{},
 		}
 		failCode := authz(req, tok)
-		req, resp := doReq(req)
+		req, resp := s.doReq(req)
 		code, body := resp.Code, resp.Body.String()
 
 		// If the initial request had a (non-empty) token
@@ -173,9 +173,9 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
 	}
 }
 
-func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
+func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 	resp := httptest.NewRecorder()
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	if resp.Code != http.StatusSeeOther {
 		return req, resp
 	}
@@ -191,7 +191,7 @@ func doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 	for _, c := range cookies {
 		req.AddCookie(c)
 	}
-	return doReq(req)
+	return s.doReq(req)
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
@@ -270,10 +270,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
-	defer func(orig bool) {
-		trustAllContent = orig
-	}(trustAllContent)
-	trustAllContent = true
+	s.testServer.Config.TrustAllContent = true
 	s.testVhostRedirectTokenToCookie(c, "GET",
 		"example.com/c="+arvadostest.FooCollection+"/foo",
 		"?api_token="+arvadostest.ActiveToken,
@@ -285,10 +282,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
-	defer func(orig string) {
-		attachmentOnlyHost = orig
-	}(attachmentOnlyHost)
-	attachmentOnlyHost = "example.com:1234"
+	s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
 
 	s.testVhostRedirectTokenToCookie(c, "GET",
 		"example.com/c="+arvadostest.FooCollection+"/foo",
@@ -333,7 +327,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
 }
 
 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
-	anonymousTokens = []string{arvadostest.AnonymousToken}
+	s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 	s.testVhostRedirectTokenToCookie(c, "GET",
 		"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 		"",
@@ -345,7 +339,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
-	anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+	s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 	s.testVhostRedirectTokenToCookie(c, "GET",
 		"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 		"",
@@ -357,6 +351,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestRange(c *check.C) {
+	s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 	u, _ := url.Parse("http://example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt")
 	req := &http.Request{
 		Method:     "GET",
@@ -366,7 +361,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
 		Header:     http.Header{"Range": {"bytes=0-4"}},
 	}
 	resp := httptest.NewRecorder()
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	c.Check(resp.Code, check.Equals, http.StatusPartialContent)
 	c.Check(resp.Body.String(), check.Equals, "Hello")
 	c.Check(resp.Header().Get("Content-Length"), check.Equals, "5")
@@ -374,7 +369,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
 
 	req.Header.Set("Range", "bytes=0-")
 	resp = httptest.NewRecorder()
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	// 200 and 206 are both correct:
 	c.Check(resp.Code, check.Equals, http.StatusOK)
 	c.Check(resp.Body.String(), check.Equals, "Hello world\n")
@@ -389,7 +384,7 @@ func (s *IntegrationSuite) TestRange(c *check.C) {
 	} {
 		req.Header.Set("Range", hdr)
 		resp = httptest.NewRecorder()
-		(&handler{}).ServeHTTP(resp, req)
+		s.testServer.Handler.ServeHTTP(resp, req)
 		c.Check(resp.Code, check.Equals, http.StatusOK)
 		c.Check(resp.Body.String(), check.Equals, "Hello world\n")
 		c.Check(resp.Header().Get("Content-Length"), check.Equals, "12")
@@ -420,7 +415,7 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 		}.Encode())),
 	}
 	resp := httptest.NewRecorder()
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	c.Check(resp.Code, check.Equals, http.StatusOK)
 	c.Check(resp.Body.String(), check.Equals, "foo")
 	c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
@@ -443,7 +438,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
 		c.Check(resp.Body.String(), check.Equals, expectRespBody)
 	}()
 
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	if resp.Code != http.StatusSeeOther {
 		return resp
 	}
@@ -463,7 +458,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
 	}
 
 	resp = httptest.NewRecorder()
-	(&handler{}).ServeHTTP(resp, req)
+	s.testServer.Handler.ServeHTTP(resp, req)
 	c.Check(resp.Header().Get("Location"), check.Equals, "")
 	return resp
 }
diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
new file mode 100644
index 0000000..da56212
--- /dev/null
+++ b/services/keep-web/keep-web.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Arvados Keep web gateway
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/keep-web
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 135f01b..73f1e25 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -4,8 +4,35 @@ import (
 	"flag"
 	"log"
 	"os"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/configfile"
+	"github.com/coreos/go-systemd/daemon"
+)
+
+var (
+	defaultConfigPath = "/etc/arvados/keep-web/config.json"
 )
 
+type Config struct {
+	Client arvados.Client
+
+	Listen string
+
+	AnonymousTokens    []string
+	AttachmentOnlyHost string
+	TrustAllContent    bool
+
+	deprecatedAllowAnonymous bool
+}
+
+// DefaultConfig returns the default configuration.
+func DefaultConfig() *Config {
+	return &Config{
+		Listen: ":80",
+	}
+}
+
 func init() {
 	// MakeArvadosClient returns an error if this env var isn't
 	// available as a default token (even if we explicitly set a
@@ -18,14 +45,44 @@ func init() {
 }
 
 func main() {
+	cfg := DefaultConfig()
+
+	var configPath string
+	deprecated := " (DEPRECATED -- use config file instead)"
+	flag.StringVar(&configPath, "config", defaultConfigPath,
+		"`path` to json configuration file")
+	flag.StringVar(&cfg.Listen, "listen", "",
+		"address:port or :port to listen on"+deprecated)
+	flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false,
+		"Load an anonymous token from the ARVADOS_API_TOKEN environment variable"+deprecated)
+	flag.StringVar(&cfg.AttachmentOnlyHost, "attachment-only-host", "",
+		"Only serve attachments at the given `host:port`"+deprecated)
+	flag.BoolVar(&cfg.TrustAllContent, "trust-all-content", false,
+		"Serve non-public content from a single origin. Dangerous: read docs before using!"+deprecated)
+	flag.Usage = usage
 	flag.Parse()
-	if os.Getenv("ARVADOS_API_HOST") == "" {
-		log.Fatal("ARVADOS_API_HOST environment variable must be set.")
+
+	if err := configfile.LoadFile(cfg, configPath); err != nil {
+		if h := os.Getenv("ARVADOS_API_HOST"); h != "" && configPath == defaultConfigPath {
+			log.Printf("DEPRECATED: Using ARVADOS_API_HOST environment variable. Use config file instead.")
+			cfg.Client.APIHost = h
+		} else {
+			log.Fatal(err)
+		}
+	}
+	if cfg.deprecatedAllowAnonymous {
+		log.Printf("DEPRECATED: Using -allow-anonymous command line flag with ARVADOS_API_TOKEN environment variable. Use config file instead.")
+		cfg.AnonymousTokens = []string{os.Getenv("ARVADOS_API_TOKEN")}
 	}
-	srv := &server{}
+
+	os.Setenv("ARVADOS_API_HOST", cfg.Client.APIHost)
+	srv := &server{Config: cfg}
 	if err := srv.Start(); err != nil {
 		log.Fatal(err)
 	}
+	if _, err := daemon.SdNotify("READY=1"); err != nil {
+		log.Printf("Error notifying init daemon: %v", err)
+	}
 	log.Println("Listening at", srv.Addr)
 	if err := srv.Wait(); err != nil {
 		log.Fatal(err)
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 1009008..babc68b 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -1,27 +1,16 @@
 package main
 
 import (
-	"flag"
-	"net/http"
-
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
-var address string
-
-func init() {
-	flag.StringVar(&address, "listen", ":80",
-		"Address to listen on: \"host:port\", or \":port\" to listen on all interfaces.")
-}
-
 type server struct {
 	httpserver.Server
+	Config *Config
 }
 
 func (srv *server) Start() error {
-	mux := http.NewServeMux()
-	mux.Handle("/", &handler{})
-	srv.Handler = mux
-	srv.Addr = address
+	srv.Handler = &handler{Config: srv.Config}
+	srv.Addr = srv.Config.Listen
 	return srv.Server.Start()
 }
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 324588a..bddd6db 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -6,16 +6,20 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
+	"os"
 	"os/exec"
 	"strings"
 	"testing"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	check "gopkg.in/check.v1"
 )
 
+var testAPIHost = os.Getenv("ARVADOS_API_HOST")
+
 var _ = check.Suite(&IntegrationSuite{})
 
 // IntegrationSuite tests need an API server and a keep-web server
@@ -137,7 +141,7 @@ type curlCase struct {
 }
 
 func (s *IntegrationSuite) Test200(c *check.C) {
-	anonymousTokens = []string{arvadostest.AnonymousToken}
+	s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 	for _, spec := range []curlCase{
 		// My collection
 		{
@@ -307,10 +311,14 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 
 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 	arvadostest.ResetEnv()
-	s.testServer = &server{}
-	var err error
-	address = "127.0.0.1:0"
-	err = s.testServer.Start()
+	s.testServer = &server{Config: &Config{
+		Client: arvados.Client{
+			APIHost:  testAPIHost,
+			Insecure: true,
+		},
+		Listen: "127.0.0.1:0",
+	}}
+	err := s.testServer.Start()
 	c.Assert(err, check.Equals, nil)
 }
 
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
new file mode 100644
index 0000000..05ebb39
--- /dev/null
+++ b/services/keep-web/usage.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+)
+
+var exampleConfigFile = []byte(`
+    {
+	"Client": {
+	    "APIHost": "zzzzz.arvadosapi.com:443",
+	    "Insecure": false
+	},
+    }`)
+
+func usage() {
+	c := DefaultConfig()
+	c.AnonymousTokens = []string{"xxxxxxxxxxxxxxxxxxxxxxx"}
+	c.Client.APIHost = "zzzzz.arvadosapi.com:443"
+	exampleConfigFile, err := json.MarshalIndent(c, "    ", "  ")
+	if err != nil {
+		panic(err)
+	}
+	fmt.Fprintf(os.Stderr, `
+
+Keep-web provides read-only HTTP access to files stored in Keep; see
+https://godoc.org/github.com/curoverse/arvados/services/keep-web and
+http://doc.arvados.org/install/install-keep-web.html
+
+Usage: keep-web -config path/to/config.json
+
+Options:
+`)
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+Example config file:
+    %s
+
+Client.APIHost:
+
+    Address (or address:port) of the Arvados API endpoint.
+
+Client.AuthToken:
+
+    Should be empty.
+
+Client.Insecure:
+
+    True if your Arvados API endpoint uses an unverifiable SSL/TLS
+    certificate.
+
+Listen:
+
+    Local port to listen on. Can be "address", "address:port", or
+    ":port", where "address" is a host IP address or name and "port"
+    is a port number or name.
+
+AnonymousTokens:
+
+    Array of tokens to try when a client does not provide a token.
+
+AttachmentOnlyHost:
+
+    Accept credentials, and add "Content-Disposition: attachment"
+    response headers, for requests at this hostname:port.
+
+    This prohibits inline display, which makes it possible to serve
+    untrusted and non-public content from a single origin, i.e.,
+    without wildcard DNS or SSL.
+
+TrustAllContent:
+
+    Serve non-public content from a single origin. Dangerous: read
+    docs before using!
+
+`, exampleConfigFile)
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list