[ARVADOS] updated: e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 29 16:31:12 EDT 2015
Summary of changes:
doc/install/install-keep-web.html.textile.liquid | 22 ++--
doc/install/install-keepproxy.html.textile.liquid | 5 +-
sdk/go/arvadostest/fixtures.go | 19 ++--
sdk/go/auth/auth.go | 12 +-
sdk/go/keepclient/collectionreader.go | 131 ++++++++++++++--------
sdk/go/keepclient/collectionreader_test.go | 16 +--
sdk/go/keepclient/keepclient.go | 2 +-
sdk/go/keepclient/support.go | 6 +-
services/api/test/fixtures/collections.yml | 8 +-
services/crunchstat/crunchstat_test.go | 2 +-
services/datamanager/keep/keep.go | 2 +-
services/keep-web/.gitignore | 2 +-
services/keep-web/doc.go | 100 ++++++++++-------
services/keep-web/handler.go | 41 +++----
services/keep-web/handler_test.go | 102 ++++++++++++++---
services/keep-web/server_test.go | 53 +++++----
services/keepproxy/keepproxy.go | 8 +-
services/keepproxy/keepproxy_test.go | 4 +-
18 files changed, 343 insertions(+), 192 deletions(-)
via e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98 (commit)
via f0fdf6e7bdfd252b22c6347c292ddd511bfa31eb (commit)
via a58956e93618463e01733acaa9e8d6ce0afbc5ab (commit)
via 6cef86f2d748be96695293b48afac61e2ae640df (commit)
via 585bd329a7aea0cad03b99431e14fc6ca919a76f (commit)
via 0f668000c8cacec50804cb0019dbe7d7dc1d2b36 (commit)
via a8caa6e1b359893e99226a8b21f356eee6edd529 (commit)
via a62773375ef74bbdd6a77f5dfa2023e122e717db (commit)
via 405c8505ccf36f7a1cfec68f305fae2edc439fa0 (commit)
via c33fa89b10dad03c8dce3459358ff7fe4825aa99 (commit)
from 5f822d9281e5afea05b1c87e59740687c0ee6692 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 29 14:47:16 2015 -0400
5824: Add some clarifying comments and golint/vet/fmt fixes.
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index 0d05b8a..68ecc6e 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -8,6 +8,13 @@ import (
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
+// ReadCloserWithLen extends io.ReadCloser with a Len() method that
+// returns the total number of bytes available to read.
+type ReadCloserWithLen interface {
+ io.ReadCloser
+ Len() uint64
+}
+
const (
// After reading a data block from Keep, cfReader slices it up
// and sends the slices to a buffered channel to be consumed
@@ -24,71 +31,76 @@ const (
// parameter when retrieving the collection record).
var ErrNoManifest = errors.New("Collection has no manifest")
-// CollectionFileReader returns an io.Reader that reads file content
-// from a collection. The filename must be given relative to the root
-// of the collection, without a leading "./".
-func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (*cfReader, error) {
+// CollectionFileReader returns a ReadCloserWithLen that reads file
+// content from a collection. The filename must be given relative to
+// the root of the collection, without a leading "./".
+func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (ReadCloserWithLen, error) {
mText, ok := collection["manifest_text"].(string)
if !ok {
return nil, ErrNoManifest
}
m := manifest.Manifest{Text: mText}
rdrChan := make(chan *cfReader)
- go func() {
- // q is a queue of FileSegments that we have received but
- // haven't yet been able to send to toGet.
- var q []*manifest.FileSegment
- var r *cfReader
- for seg := range m.FileSegmentIterByName(filename) {
- if r == nil {
- // We've just discovered that the
- // requested filename does appear in
- // the manifest, so we can return a
- // real reader (not nil) from
- // CollectionFileReader().
- r = newCFReader(kc)
- rdrChan <- r
- }
- q = append(q, seg)
- r.totalSize += uint64(seg.Len)
- // Send toGet as many segments as we can until
- // it blocks.
- Q:
- for len(q) > 0 {
- select {
- case r.toGet <- q[0]:
- q = q[1:]
- default:
- break Q
- }
- }
- }
+ go kc.queueSegmentsToGet(m, filename, rdrChan)
+ r, ok := <-rdrChan
+ if !ok {
+ return nil, os.ErrNotExist
+ }
+ return r, nil
+}
+
+// Send segments for the specified file to r.toGet. Send a *cfReader
+// to rdrChan if the specified file is found (even if it's empty).
+// Then, close rdrChan.
+func (kc *KeepClient) queueSegmentsToGet(m manifest.Manifest, filename string, rdrChan chan *cfReader) {
+ defer close(rdrChan)
+
+ // q is a queue of FileSegments that we have received but
+ // haven't yet been able to send to toGet.
+ var q []*manifest.FileSegment
+ var r *cfReader
+ for seg := range m.FileSegmentIterByName(filename) {
if r == nil {
- // File not found
- rdrChan <- nil
- return
+ // We've just discovered that the requested
+ // filename does appear in the manifest, so we
+ // can return a real reader (not nil) from
+ // CollectionFileReader().
+ r = newCFReader(kc)
+ rdrChan <- r
}
- close(r.countDone)
- for _, seg := range q {
- r.toGet <- seg
+ q = append(q, seg)
+ r.totalSize += uint64(seg.Len)
+ // Send toGet as many segments as we can until it
+ // blocks.
+ Q:
+ for len(q) > 0 {
+ select {
+ case r.toGet <- q[0]:
+ q = q[1:]
+ default:
+ break Q
+ }
}
- close(r.toGet)
- }()
- // Before returning a reader, wait until we know whether the
- // file exists here:
- r := <-rdrChan
+ }
if r == nil {
- return nil, os.ErrNotExist
+ // File not found.
+ return
}
- return r, nil
+ close(r.countDone)
+ for _, seg := range q {
+ r.toGet <- seg
+ }
+ close(r.toGet)
}
type cfReader struct {
keepClient *KeepClient
+
// doGet() reads FileSegments from toGet, gets the data from
// Keep, and sends byte slices to toRead to be consumed by
// Read().
toGet chan *manifest.FileSegment
+
// toRead is a buffered channel, sized to fit one full Keep
// block. This lets us verify checksums without having a
// store-and-forward delay between blocks: by the time the
@@ -96,17 +108,22 @@ type cfReader struct {
// starting to fetch block N+1. A larger buffer would be
// useful for a caller whose read speed varies a lot.
toRead chan []byte
+
// bytes ready to send next time someone calls Read()
buf []byte
+
// Total size of the file being read. Not safe to read this
// until countDone is closed.
totalSize uint64
countDone chan struct{}
+
// First error encountered.
err error
+
// errNotNil is closed IFF err contains a non-nil error.
// Receiving from it will block until an error occurs.
errNotNil chan struct{}
+
// rdrClosed is closed IFF the reader's Close() method has
// been called. Any goroutines associated with the reader will
// stop and free up resources when they notice this channel is
@@ -116,31 +133,49 @@ type cfReader struct {
func (r *cfReader) Read(outbuf []byte) (int, error) {
if r.Error() != nil {
+ // Short circuit: the caller might as well find out
+ // now that we hit an error, even if there's buffered
+ // data we could return.
return 0, r.Error()
}
- for r.buf == nil || len(r.buf) == 0 {
+ for len(r.buf) == 0 {
+ // Private buffer was emptied out by the last Read()
+ // (or this is the first Read() and r.buf is nil).
+ // Read from r.toRead until we get a non-empty slice
+ // or hit an error.
var ok bool
r.buf, ok = <-r.toRead
if r.Error() != nil {
+ // Error encountered while waiting for bytes
return 0, r.Error()
} else if !ok {
+ // No more bytes to read, no error encountered
return 0, io.EOF
}
}
+ // Copy as much as possible from our private buffer to the
+ // caller's buffer
n := len(r.buf)
if len(r.buf) > len(outbuf) {
n = len(outbuf)
}
copy(outbuf[:n], r.buf[:n])
+
+ // Next call to Read() will continue where we left off
r.buf = r.buf[n:]
+
return n, nil
}
+// Close releases resources. It returns a non-nil error if an error
+// was encountered by the reader.
func (r *cfReader) Close() error {
close(r.rdrClosed)
return r.Error()
}
+// Error returns an error if one has been encountered, otherwise
+// nil. It is safe to call from any goroutine.
func (r *cfReader) Error() error {
select {
case <-r.errNotNil:
@@ -150,6 +185,8 @@ func (r *cfReader) Error() error {
}
}
+// Len returns the total number of bytes in the file being read. If
+// necessary, it waits for manifest parsing to finish.
func (r *cfReader) Len() uint64 {
// Wait for all segments to be counted
<-r.countDone
diff --git a/sdk/go/keepclient/collectionreader_test.go b/sdk/go/keepclient/collectionreader_test.go
index 94e41e2..9fb0d86 100644
--- a/sdk/go/keepclient/collectionreader_test.go
+++ b/sdk/go/keepclient/collectionreader_test.go
@@ -36,7 +36,7 @@ func (s *CollectionReaderUnit) SetUpTest(c *check.C) {
s.handler = SuccessHandler{
disk: make(map[string][]byte),
lock: make(chan struct{}, 1),
- ops: new(int),
+ ops: new(int),
}
localRoots := make(map[string]string)
for i, k := range RunSomeFakeKeepServers(s.handler, 4) {
@@ -47,8 +47,8 @@ func (s *CollectionReaderUnit) SetUpTest(c *check.C) {
type SuccessHandler struct {
disk map[string][]byte
- lock chan struct{} // channel with buffer==1: full when an operation is in progress.
- ops *int // number of operations completed
+ lock chan struct{} // channel with buffer==1: full when an operation is in progress.
+ ops *int // number of operations completed
}
func (h SuccessHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
@@ -65,7 +65,7 @@ func (h SuccessHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if h.ops != nil {
(*h.ops)++
}
- <- h.lock
+ <-h.lock
resp.Write([]byte(pdh))
case "GET":
pdh := req.URL.Path[1:]
@@ -74,7 +74,7 @@ func (h SuccessHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if h.ops != nil {
(*h.ops)++
}
- <- h.lock
+ <-h.lock
if !ok {
resp.WriteHeader(http.StatusNotFound)
} else {
@@ -192,7 +192,7 @@ func (s *CollectionReaderUnit) TestCollectionReaderCloseEarly(c *check.C) {
}()
err = rdr.Close()
c.Assert(err, check.IsNil)
- c.Assert(rdr.Error(), check.IsNil)
+ c.Assert(rdr.(*cfReader).Error(), check.IsNil)
// Release the stub server's lock. The first GET operation will proceed.
<-s.handler.lock
@@ -202,7 +202,7 @@ func (s *CollectionReaderUnit) TestCollectionReaderCloseEarly(c *check.C) {
<-firstReadDone
// doGet() should close toRead before sending any more bufs to it.
- if what, ok := <-rdr.toRead; ok {
+ if what, ok := <-rdr.(*cfReader).toRead; ok {
c.Errorf("Got %q, expected toRead to be closed", string(what))
}
@@ -217,7 +217,7 @@ func (s *CollectionReaderUnit) TestCollectionReaderDataError(c *check.C) {
c.Check(err, check.IsNil)
for i := 0; i < 2; i++ {
_, err = io.ReadFull(rdr, buf)
- c.Check(err, check.Not(check.IsNil))
+ c.Check(err, check.NotNil)
c.Check(err, check.Not(check.Equals), io.EOF)
}
}
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 67c304d..e74ae2c 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -169,7 +169,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
retryList = append(retryList, host)
} else if resp.StatusCode != http.StatusOK {
var respbody []byte
- respbody, _ = ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+ respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
url, resp.StatusCode, bytes.TrimSpace(respbody)))
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 0791d3c..f555461 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -169,7 +169,7 @@ type uploadStatus struct {
response string
}
-func (this KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
+func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
upload_status chan<- uploadStatus, expectedLength int64, requestId string) {
var req *http.Request
@@ -214,7 +214,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
defer resp.Body.Close()
defer io.Copy(ioutil.Discard, resp.Body)
- respbody, err2 := ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+ respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, err2.Error(), response)
@@ -228,7 +228,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
}
}
-func (this KeepClient) putReplicas(
+func (this *KeepClient) putReplicas(
hash string,
tr *streamer.AsyncStream,
expectedLength int64) (locator string, replicas int, err error) {
diff --git a/services/crunchstat/crunchstat_test.go b/services/crunchstat/crunchstat_test.go
index 13f4dc6..69f31af 100644
--- a/services/crunchstat/crunchstat_test.go
+++ b/services/crunchstat/crunchstat_test.go
@@ -101,7 +101,7 @@ func TestCopyPipeToChildLogLongLines(t *testing.T) {
}
if after, err := rcv.ReadBytes('\n'); err != nil || string(after) != "after\n" {
- t.Fatal("\"after\n\" not received (got \"%s\", %s)", after, err)
+ t.Fatalf("\"after\n\" not received (got \"%s\", %s)", after, err)
}
select {
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 86c2b08..3a9c21a 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -23,7 +23,7 @@ import (
// ServerAddress struct
type ServerAddress struct {
- SSL bool `json:service_ssl_flag`
+ SSL bool `json:"service_ssl_flag"`
Host string `json:"service_host"`
Port int `json:"service_port"`
UUID string `json:"uuid"`
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 9df78be..2704263 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -304,7 +304,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", t)
}
}
- w.Header().Set("Content-Length", fmt.Sprintf("%d", rdr.Len()))
+ if rdr, ok := rdr.(keepclient.ReadCloserWithLen); ok {
+ w.Header().Set("Content-Length", fmt.Sprintf("%d", rdr.Len()))
+ }
if attachment {
w.Header().Set("Content-Disposition", "attachment")
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index f877ee1..392de94 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -294,7 +294,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
anonymousTokens = []string{arvadostest.AnonymousToken}
s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
"",
"",
@@ -306,7 +306,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
"",
"",
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 8e734f7..f7677ec 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -201,7 +201,7 @@ func GetRemoteAddress(req *http.Request) string {
return req.RemoteAddr
}
-func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
+func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
var auth string
if auth = req.Header.Get("Authorization"); auth == "" {
return false, ""
@@ -331,7 +331,7 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
var pass bool
var tok string
- if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+ if pass, tok = CheckAuthorizationHeader(&kc, this.ApiTokenCache, req); !pass {
status, err = http.StatusForbidden, BadAuthorizationHeader
return
}
@@ -432,7 +432,7 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
var pass bool
var tok string
- if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+ if pass, tok = CheckAuthorizationHeader(&kc, this.ApiTokenCache, req); !pass {
err = BadAuthorizationHeader
status = http.StatusForbidden
return
@@ -515,7 +515,7 @@ func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
kc := *handler.KeepClient
- ok, token := CheckAuthorizationHeader(kc, handler.ApiTokenCache, req)
+ ok, token := CheckAuthorizationHeader(&kc, handler.ApiTokenCache, req)
if !ok {
status, err = http.StatusForbidden, BadAuthorizationHeader
return
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 6fe8fe7..917f012 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -103,7 +103,7 @@ func setupProxyService() {
}
}
-func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient {
+func runProxy(c *C, args []string, port int, bogusClientToken bool) *keepclient.KeepClient {
if bogusClientToken {
os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
}
@@ -138,7 +138,7 @@ func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.K
go main()
}
- return kc
+ return &kc
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
commit f0fdf6e7bdfd252b22c6347c292ddd511bfa31eb
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 29 13:40:18 2015 -0400
5824: Add comments and fix variable names, cf. golint.
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 3a6dfb2..d0270a6 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -1,5 +1,6 @@
package arvadostest
+// IDs of API server's test fixtures
const (
SpectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
ActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
@@ -10,11 +11,14 @@ const (
FooBarDirCollection = "zzzzz-4zz18-foonbarfilesdir"
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
- PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 73feffa4b7f6bb68e44cf984c85f6e88+3+Z+K at xyzzy acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero at 0 0:1:f 1:0:zero at 1 1:4:ooba 4:0:zero at 4 5:1:r 5:4:rbaz 9:0:zero at 9\n" +
- "./overlapReverse acbd18db4cc2f85cedef654fccc4a4d8+3 acbd18db4cc2f85cedef654fccc4a4d8+3 5:1:o 4:2:oo 2:4:ofoo\n" +
- "./segmented acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:1:frob 5:1:frob 1:1:frob 1:2:oof 0:1:oof 5:0:frob 3:1:frob\n" +
- `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:baz` + "\n" +
- `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:b\141z\040w\141z` + "\n" +
- "./foo acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero 0:3:foo\n" +
- ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo/zero 0:3:foo/foo\n"
)
+
+// A valid manifest designed to test various edge cases and parsing
+// requirements
+const PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 73feffa4b7f6bb68e44cf984c85f6e88+3+Z+K at xyzzy acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero at 0 0:1:f 1:0:zero at 1 1:4:ooba 4:0:zero at 4 5:1:r 5:4:rbaz 9:0:zero at 9\n" +
+ "./overlapReverse acbd18db4cc2f85cedef654fccc4a4d8+3 acbd18db4cc2f85cedef654fccc4a4d8+3 5:1:o 4:2:oo 2:4:ofoo\n" +
+ "./segmented acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:1:frob 5:1:frob 1:1:frob 1:2:oof 0:1:oof 5:0:frob 3:1:frob\n" +
+ `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:baz` + "\n" +
+ `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:b\141z\040w\141z` + "\n" +
+ "./foo acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero 0:3:foo\n" +
+ ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo/zero 0:3:foo/foo\n"
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index ddbc9b1..9df78be 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -36,7 +36,7 @@ func init() {
// return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
// otherwise return "".
-func parseCollectionIdFromDNSName(s string) string {
+func parseCollectionIDFromDNSName(s string) string {
// Strip domain.
if i := strings.IndexRune(s, '.'); i >= 0 {
s = s[:i]
@@ -60,7 +60,7 @@ var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
// return a UUID or PDH if s is a UUID or a PDH (even if it is a PDH
// with "+" replaced by " " or "-"); otherwise return "".
-func parseCollectionIdFromURL(s string) string {
+func parseCollectionIDFromURL(s string) string {
if arvadosclient.UUIDMatch(s) {
return s
}
@@ -109,7 +109,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
- var targetId string
+ var targetID string
var targetPath []string
var tokens []string
var reqTokens []string
@@ -124,24 +124,24 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
attachment = true
}
- if targetId = parseCollectionIdFromDNSName(r.Host); targetId != "" {
+ if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
targetPath = pathParts
} else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID/PATH...
- targetId = parseCollectionIdFromURL(pathParts[0][2:])
+ targetID = parseCollectionIDFromURL(pathParts[0][2:])
targetPath = pathParts[1:]
} else if len(pathParts) >= 3 && pathParts[0] == "collections" {
if len(pathParts) >= 5 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetId = pathParts[2]
+ targetID = pathParts[2]
tokens = []string{pathParts[3]}
targetPath = pathParts[4:]
pathToken = true
} else {
// /collections/ID/PATH...
- targetId = pathParts[1]
+ targetID = pathParts[1]
tokens = anonymousTokens
targetPath = pathParts[2:]
}
@@ -228,7 +228,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
collection := make(map[string]interface{})
found := false
for _, arv.ApiToken = range tokens {
- err := arv.Get("collections", targetId, nil, &collection)
+ err := arv.Get("collections", targetID, nil, &collection)
if err == nil {
// Success
found = true
commit a58956e93618463e01733acaa9e8d6ce0afbc5ab
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 29 12:06:09 2015 -0400
5824: Add tests.
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 5e38f25..f877ee1 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -172,9 +172,48 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
arvadostest.FooCollection+".example.com/foo",
"?api_token="+arvadostest.ActiveToken,
- "text/plain",
+ "",
"",
http.StatusOK,
+ "foo",
+ )
+}
+
+func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
+ "",
+ "",
+ "",
+ http.StatusOK,
+ "foo",
+ )
+}
+
+// Bad token in URL is 404 Not Found because it doesn't make sense to
+// retry the same URL with different authorization.
+func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
+ "",
+ "",
+ "",
+ http.StatusNotFound,
+ "",
+ )
+}
+
+// Bad token in a cookie (even if it got there via our own
+// query-string-to-cookie redirect) is, in principle, retryable at the
+// same URL so it's 401 Unauthorized.
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken",
+ "",
+ "",
+ http.StatusUnauthorized,
+ "",
)
}
@@ -182,9 +221,10 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "text/plain",
+ "",
"",
http.StatusBadRequest,
+ "",
)
}
@@ -196,9 +236,10 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "text/plain",
+ "",
"",
http.StatusOK,
+ "foo",
)
}
@@ -211,17 +252,19 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "text/plain",
+ "",
"",
http.StatusBadRequest,
+ "",
)
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"example.com:1234/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
- "text/plain",
+ "",
"",
http.StatusOK,
+ "foo",
)
c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
}
@@ -233,6 +276,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
http.StatusOK,
+ "foo",
)
}
@@ -243,23 +287,52 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
+ "",
+ )
+}
+
+func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
+ anonymousTokens = []string{arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ "",
+ "",
+ "",
+ http.StatusOK,
+ "Hello world\n",
+ )
+}
+
+func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
+ anonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ "",
+ "",
+ "",
+ http.StatusNotFound,
+ "",
)
}
-func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, body string, expectStatus int) *httptest.ResponseRecorder {
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
u, _ := url.Parse(`http://` + hostPath + queryString)
req := &http.Request{
Method: method,
Host: u.Host,
URL: u,
Header: http.Header{"Content-Type": {contentType}},
- Body: ioutil.NopCloser(strings.NewReader(body)),
+ Body: ioutil.NopCloser(strings.NewReader(reqBody)),
}
resp := httptest.NewRecorder()
+ defer func() {
+ c.Check(resp.Code, check.Equals, expectStatus)
+ c.Check(resp.Body.String(), check.Equals, expectRespBody)
+ }()
+
(&handler{}).ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
- c.Assert(resp.Code, check.Equals, expectStatus)
return resp
}
c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`".*`)
@@ -279,9 +352,5 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
resp = httptest.NewRecorder()
(&handler{}).ServeHTTP(resp, req)
c.Check(resp.Header().Get("Location"), check.Equals, "")
- c.Check(resp.Code, check.Equals, expectStatus)
- if expectStatus == http.StatusOK {
- c.Check(resp.Body.String(), check.Equals, "foo")
- }
return resp
}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 0a38384..8e3a21a 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -50,14 +50,18 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
// really works against the server.
func (s *IntegrationSuite) Test404(c *check.C) {
for _, uri := range []string{
- // Routing errors
+ // Routing errors (always 404 regardless of what's stored in Keep)
"/",
"/foo",
"/download",
"/collections",
"/collections/",
+ // Implicit/generated index is not implemented yet;
+ // until then, return 404.
"/collections/" + arvadostest.FooCollection,
"/collections/" + arvadostest.FooCollection + "/",
+ "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
+ "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
// Non-existent file in collection
"/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
"/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
@@ -120,7 +124,6 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
}
type curlCase struct {
- id string
auth string
host string
path string
@@ -138,6 +141,12 @@ func (s *IntegrationSuite) Test200(c *check.C) {
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
+ auth: arvadostest.ActiveToken,
+ host: arvadostest.FooCollection + ".collections.example.com",
+ path: "/foo",
+ dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
+ },
+ {
host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
path: "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
@@ -170,7 +179,7 @@ func (s *IntegrationSuite) Test200(c *check.C) {
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
- // Anonymously accessible user agreement
+ // Anonymously accessible data
{
path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
commit 6cef86f2d748be96695293b48afac61e2ae640df
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 29 11:09:11 2015 -0400
5824: Add test for file in subdir.
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 3040e0a..3a6dfb2 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -7,6 +7,7 @@ const (
FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
NonexistentCollection = "zzzzz-4zz18-totallynotexist"
HelloWorldCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+ FooBarDirCollection = "zzzzz-4zz18-foonbarfilesdir"
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 73feffa4b7f6bb68e44cf984c85f6e88+3+Z+K at xyzzy acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero at 0 0:1:f 1:0:zero at 1 1:4:ooba 4:0:zero at 4 5:1:r 5:4:rbaz 9:0:zero at 9\n" +
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index d7f6f92..4ba5a3b 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -299,7 +299,7 @@ collection_with_files_in_subdir:
modified_by_user_uuid: zzzzz-tpzed-user1withloadab
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
- manifest_text: ". 85877ca2d7e05498dd3d109baf2df106+95+A3a4e26a366ee7e4ed3e476ccf05354761be2e4ae at 545a9920 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file1_in_subdir4.txt 32:32:file2_in_subdir4.txt"
+ manifest_text: ". 85877ca2d7e05498dd3d109baf2df106+95 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir4.txt 32:32:file2_in_subdir4.txt"
graph_test_collection1:
uuid: zzzzz-4zz18-bv31uwvy3neko22
@@ -481,7 +481,7 @@ collection_with_repeated_filenames_and_contents_in_two_dirs_1:
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
name: collection_with_repeated_filenames_and_contents_in_two_dirs_1
- manifest_text: "./dir1 92b53930db60fe94be2a73fc771ba921+34+Af966b611a1e6a7df18e0f20ac742a255c27744b7 at 550a3f11 0:12:alice 12:12:alice.txt 24:10:bob.txt\n./dir2 56ac2557b1ded11ccab7293dc47d1e88+44+A1780092551dadcb9c74190a793a779cea84d632d at 550a3f11 0:27:alice.txt\n"
+ manifest_text: "./dir1 92b53930db60fe94be2a73fc771ba921+34 0:12:alice 12:12:alice.txt 24:10:bob.txt\n./dir2 56ac2557b1ded11ccab7293dc47d1e88+44 0:27:alice.txt\n"
collection_with_repeated_filenames_and_contents_in_two_dirs_2:
uuid: zzzzz-4zz18-duplicatenames2
@@ -493,7 +493,7 @@ collection_with_repeated_filenames_and_contents_in_two_dirs_2:
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
name: collection_with_repeated_filenames_and_contents_in_two_dirs_2
- manifest_text: "./dir1 92b53930db60fe94be2a73fc771ba921+34+Af966b611a1e6a7df18e0f20ac742a255c27744b7 at 550a3f11 0:12:alice 12:12:alice.txt 24:10:carol.txt\n./dir2 56ac2557b1ded11ccab7293dc47d1e88+44+A1780092551dadcb9c74190a793a779cea84d632d at 550a3f11 0:27:alice.txt\n"
+ manifest_text: "./dir1 92b53930db60fe94be2a73fc771ba921+34 0:12:alice 12:12:alice.txt 24:10:carol.txt\n./dir2 56ac2557b1ded11ccab7293dc47d1e88+44 0:27:alice.txt\n"
foo_and_bar_files_in_dir:
uuid: zzzzz-4zz18-foonbarfilesdir
@@ -505,7 +505,7 @@ foo_and_bar_files_in_dir:
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
name: foo_file_in_dir
- manifest_text: "./dir1 a84b928ebdbae3f658518c711beaec02+28+A0cff02249e70e8cd6e55dba49fef4afa3f5bfdfb at 550acd28 0:3:bar 12:16:foo\n"
+ manifest_text: "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n"
multi_level_to_combine:
uuid: zzzzz-4zz18-pyw8yp9g3ujh45f
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 09c731c..5e38f25 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -103,6 +103,7 @@ func doVhostRequests(c *check.C, authz authorizer) {
arvadostest.FooCollection + "--collections.example.com/_/foo",
arvadostest.FooPdh + ".example.com/foo",
strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
+ arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
} {
c.Log("doRequests: ", hostPath)
doVhostRequestsWithHostPath(c, authz, hostPath)
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 80d95c0..0a38384 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -129,13 +129,6 @@ type curlCase struct {
func (s *IntegrationSuite) Test200(c *check.C) {
anonymousTokens = []string{arvadostest.AnonymousToken}
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.Equals, nil)
- arv.ApiToken = arvadostest.ActiveToken
- kc, err := keepclient.MakeKeepClient(&arv)
- c.Assert(err, check.Equals, nil)
- kc.PutB([]byte("Hello world\n"))
- kc.PutB([]byte("foo"))
for _, spec := range []curlCase{
// My collection
{
@@ -281,6 +274,15 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
arvadostest.StartKeep()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, check.Equals, nil)
+ kc.PutB([]byte("Hello world\n"))
+ kc.PutB([]byte("foo"))
+ kc.PutB([]byte("foobar"))
}
func (s *IntegrationSuite) TearDownSuite(c *check.C) {
commit 585bd329a7aea0cad03b99431e14fc6ca919a76f
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 29 11:08:59 2015 -0400
5824: Clarify docs.
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index bd0e2ca..7e0a00f 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -49,9 +49,10 @@
// Download URLs
//
// The following "same origin" URL patterns are supported for public
-// collections (i.e., collections which can be served by keep-web
-// without making use of any credentials supplied by the client). See
-// "Same-origin URLs" below.
+// collections and collections shared anonymously via secret links
+// (i.e., collections which can be served by keep-web without making
+// use of any implicit credentials like cookies). See "Same-origin
+// URLs" below.
//
// http://collections.example.com/c=uuid_or_pdh/path/file.txt
// http://collections.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
@@ -63,16 +64,18 @@
// http://uuid_or_pdh--collections.example.com/t=TOKEN/path/file.txt
//
// In the "multiple origin" form, the string "--" can be replaced with
-// "." with identical results (assuming the upstream proxy is
+// "." with identical results (assuming the downstream proxy is
// configured accordingly). These two are equivalent:
//
// http://uuid_or_pdh--collections.example.com/path/file.txt
// http://uuid_or_pdh.collections.example.com/path/file.txt
//
-// The first form ("uuid_or_pdh--collections.example.com") minimizes
-// the cost and effort of deploying a wildcard TLS certificate for
-// *.collections.example.com. The second form is likely to be easier
-// to configure, and more efficient to run, on an upstream proxy.
+// The first form (with "--" instead of ".") avoids the cost and
+// effort of deploying a wildcard TLS certificate for
+// *.collections.example.com at sites that already have a wildcard
+// certificate for *.example.com. The second form is likely to be
+// easier to configure, and more efficient to run, on a downstream
+// proxy.
//
// In all of the above forms, the "collections.example.com" part can
// be anything at all: keep-web itself ignores everything after the
@@ -173,8 +176,8 @@
// In such cases -- for example, a site which is not reachable from
// the internet, where some data is world-readable from Arvados's
// perspective but is intended to be available only to users within
-// the local network -- the upstream proxy should configured to return
-// 401 for all paths beginning with "/c=".
+// the local network -- the downstream proxy should configured to
+// return 401 for all paths beginning with "/c=".
//
// Same-origin URLs
//
@@ -195,7 +198,7 @@
// will be accepted and all responses will have a
// "Content-Disposition: attachment" header. This behavior is invoked
// only when the designated origin matches exactly the Host header
-// provided by the client or upstream proxy.
+// provided by the client or downstream proxy.
//
// keep-web -address :9999 -attachment-only-host domain.example:9999
//
commit 0f668000c8cacec50804cb0019dbe7d7dc1d2b36
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Oct 28 12:17:54 2015 -0400
5824: Rename conventional dl.* to collections.*
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index c73b58a..08e7af2 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -1,15 +1,15 @@
---
layout: default
navsection: installguide
-title: Install the download server
+title: Install the keep-web server
...
The keep-web server provides read-only HTTP access to files stored in Keep. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
-By convention, we use the following hostname for the download service:
+By convention, we use the following hostname for the keep-web service:
<notextile>
-<pre><code>dl.<span class="userinput">uuid_prefix</span>.your.domain
+<pre><code>collections.<span class="userinput">uuid_prefix</span>.your.domain
</code></pre>
</notextile>
@@ -79,7 +79,7 @@ upstream keep-web {
server {
listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name dl.<span class="userinput">uuid_prefix</span>.your.domain *.dl.<span class="userinput">uuid_prefix</span>.your.domain ~.*--dl.<span class="userinput">uuid_prefix</span>.your.domain;
+ server_name collections.<span class="userinput">uuid_prefix</span>.your.domain *.collections.<span class="userinput">uuid_prefix</span>.your.domain ~.*--collections.<span class="userinput">uuid_prefix</span>.your.domain;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
@@ -99,17 +99,17 @@ server {
h3. Configure DNS
Configure your DNS servers so the following names resolve to your Nginx proxy's public IP address.
-* @*--dl.uuid_prefix.your.domain@, if your DNS server allows this without interfering with other DNS names; or
-* @*.dl.uuid_prefix.your.domain@, if you have a wildcard SSL certificate valid for these names; or
-* @dl.uuid_prefix.your.domain@, if neither of the above options is feasible. In this case, only unauthenticated requests will be served, i.e., public data and collection sharing links.
+* @*--collections.uuid_prefix.your.domain@, if your DNS server allows this without interfering with other DNS names; or
+* @*.collections.uuid_prefix.your.domain@, if you have a wildcard SSL certificate valid for these names; or
+* @collections.uuid_prefix.your.domain@, if neither of the above options is feasible. In this case, only unauthenticated requests will be served, i.e., public data and collection sharing links.
h3. Tell Workbench about the keep-web service
Add *one* of the following entries to your Workbench configuration file (@/etc/arvados/workbench/application.yml@), depending on your DNS setup:
<notextile>
-<pre><code>keep_web_url: https://%{uuid_or_pdh}--dl.<span class="userinput">uuid_prefix</span>.your.domain
-keep_web_url: https://%{uuid_or_pdh}.dl.<span class="userinput">uuid_prefix</span>.your.domain
-keep_web_url: https://dl.<span class="userinput">uuid_prefix</span>.your.domain
+<pre><code>keep_web_url: https://%{uuid_or_pdh}--collections.<span class="userinput">uuid_prefix</span>.your.domain
+keep_web_url: https://%{uuid_or_pdh}.collections.<span class="userinput">uuid_prefix</span>.your.domain
+keep_web_url: https://collections.<span class="userinput">uuid_prefix</span>.your.domain
</code></pre>
</notextile>
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 937e498..bd0e2ca 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -31,7 +31,7 @@
// }
// server {
// listen *:443 ssl;
-// server_name dl.example.com *.dl.example.com ~.*--dl.example.com;
+// server_name collections.example.com *.collections.example.com ~.*--collections.example.com;
// ssl_certificate /root/wildcard.example.com.crt;
// ssl_certificate_key /root/wildcard.example.com.key;
// location / {
@@ -53,31 +53,31 @@
// without making use of any credentials supplied by the client). See
// "Same-origin URLs" below.
//
-// http://dl.example.com/c=uuid_or_pdh/path/file.txt
-// http://dl.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
+// http://collections.example.com/c=uuid_or_pdh/path/file.txt
+// http://collections.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
//
// The following "multiple origin" URL patterns are supported for all
// collections:
//
-// http://uuid_or_pdh--dl.example.com/path/file.txt
-// http://uuid_or_pdh--dl.example.com/t=TOKEN/path/file.txt
+// http://uuid_or_pdh--collections.example.com/path/file.txt
+// http://uuid_or_pdh--collections.example.com/t=TOKEN/path/file.txt
//
// In the "multiple origin" form, the string "--" can be replaced with
// "." with identical results (assuming the upstream proxy is
// configured accordingly). These two are equivalent:
//
-// http://uuid_or_pdh--dl.example.com/path/file.txt
-// http://uuid_or_pdh.dl.example.com/path/file.txt
+// http://uuid_or_pdh--collections.example.com/path/file.txt
+// http://uuid_or_pdh.collections.example.com/path/file.txt
//
-// The first form ("uuid_or_pdh--dl.example.com") minimizes the cost
-// and effort of deploying a wildcard TLS certificate for
-// *.dl.example.com. The second form is likely to be easier to
-// configure, and more efficient to run, on an upstream proxy.
+// The first form ("uuid_or_pdh--collections.example.com") minimizes
+// the cost and effort of deploying a wildcard TLS certificate for
+// *.collections.example.com. The second form is likely to be easier
+// to configure, and more efficient to run, on an upstream proxy.
//
-// In all of the above forms, the "dl.example.com" part can be
-// anything at all: keep-web itself ignores everything after the first
-// "." or "--". (Of course, in order for clients to connect at all,
-// DNS and any relevant proxies must be configured accordingly.)
+// In all of the above forms, the "collections.example.com" part can
+// be anything at all: keep-web itself ignores everything after the
+// first "." or "--". (Of course, in order for clients to connect at
+// all, DNS and any relevant proxies must be configured accordingly.)
//
// In all of the above forms, the "uuid_or_pdh" part can be either a
// collection UUID or a portable data hash with the "+" character
@@ -96,9 +96,9 @@
// 1f4b0bc7583c2a7f9102c395f4ffc5e3+45, the following URLs are
// interchangeable:
//
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/foo/bar.txt
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/_/foo/bar.txt
-// http://zzzzz-4zz18-znfnqtbbv4spc3w--dl.example.com/_/foo/bar.txt
+// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
+// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
+// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
//
@@ -106,13 +106,13 @@
// convenient to maintain support for existing Workbench download
// links:
//
-// http://dl.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt
+// http://collections.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt
//
// A regular Workbench "download" link is also accepted, but
// credentials passed via cookie, header, etc. are ignored. Only
// public data can be served this way:
//
-// http://dl.example.com/collections/uuid_or_pdh/foo/bar.txt
+// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
//
// Authorization mechanisms
//
@@ -142,6 +142,14 @@
// the token stripped from the query string and added to a cookie
// instead.
//
+// Indexes
+//
+// Currently, keep-web does not generate HTML index listings, nor does
+// it serve a default file like "index.html" when a directory is
+// requested. These features are likely to be added in future
+// versions. Until then, keep-web responds with 404 if a directory
+// name (or any path ending with "/") is requested.
+//
// Compatibility
//
// Client-provided authorization tokens are ignored if the client does
@@ -175,8 +183,8 @@
// current viewer's credentials to download additional data from
// collection Y -- data which is accessible to the current viewer, but
// not to the author of collection X -- from the same origin
-// (``https://dl.example.com/'') and upload it to some other site
-// chosen by the author of collection X.
+// (``https://collections.example.com/'') and upload it to some other
+// site chosen by the author of collection X.
//
// Attachment-Only host
//
@@ -195,10 +203,10 @@
//
// In "trust all content" mode, Keep-web will accept credentials (API
// tokens) and serve any collection X at
-// "https://dl.example.com/collections/X/path/file.ext". This is
-// UNSAFE except in the special case where everyone who is able write
-// ANY data to Keep, and every JavaScript and HTML file written to
-// Keep, is also trusted to read ALL of the data in Keep.
+// "https://collections.example.com/collections/X/path/file.ext".
+// This is UNSAFE except in the special case where everyone who is
+// able write ANY data to Keep, and every JavaScript and HTML file
+// written to Keep, is also trusted to read ALL of the data in Keep.
//
// In such cases you can enable trust-all-content mode.
//
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5f546ce..ddbc9b1 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -41,9 +41,9 @@ func parseCollectionIdFromDNSName(s string) string {
if i := strings.IndexRune(s, '.'); i >= 0 {
s = s[:i]
}
- // Names like {uuid}--dl.example.com serve the same purpose as
- // {uuid}.dl.example.com but can reduce cost/effort of using
- // [additional] wildcard certificates.
+ // Names like {uuid}--collections.example.com serve the same
+ // purpose as {uuid}.collections.example.com but can reduce
+ // cost/effort of using [additional] wildcard certificates.
if i := strings.Index(s, "--"); i >= 0 {
s = s[:i]
}
@@ -125,7 +125,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if targetId = parseCollectionIdFromDNSName(r.Host); targetId != "" {
- // http://ID.dl.example/PATH...
+ // http://ID.collections.example/PATH...
credentialsOK = true
targetPath = pathParts
} else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
@@ -216,10 +216,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if len(targetPath) > 0 && targetPath[0] == "_" {
// If a collection has a directory called "t=foo" or
- // "_", it can be served at //dl.example/_/t=foo/ or
- // //dl.example/_/_/ respectively: //dl.example/t=foo/
- // won't work because t=foo will be interpreted as a
- // token "foo".
+ // "_", it can be served at
+ // //collections.example/_/t=foo/ or
+ // //collections.example/_/_/ respectively:
+ // //collections.example/t=foo/ won't work because
+ // t=foo will be interpreted as a token "foo".
targetPath = targetPath[1:]
}
@@ -273,7 +274,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// someone trying (anonymously) to download public
// data that has been deleted. Allow a referrer to
// provide this context somehow?
- w.Header().Add("WWW-Authenticate", "Basic realm=\"dl\"")
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
statusCode = http.StatusUnauthorized
return
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 5b2a825..09c731c 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -99,10 +99,10 @@ func authzViaPOST(r *http.Request, tok string) int {
func doVhostRequests(c *check.C, authz authorizer) {
for _, hostPath := range []string{
arvadostest.FooCollection + ".example.com/foo",
- arvadostest.FooCollection + "--dl.example.com/foo",
- arvadostest.FooCollection + "--dl.example.com/_/foo",
+ arvadostest.FooCollection + "--collections.example.com/foo",
+ arvadostest.FooCollection + "--collections.example.com/_/foo",
arvadostest.FooPdh + ".example.com/foo",
- strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--dl.example.com/foo",
+ strings.Replace(arvadostest.FooPdh, "+", "-", -1) + "--collections.example.com/foo",
} {
c.Log("doRequests: ", hostPath)
doVhostRequestsWithHostPath(c, authz, hostPath)
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 740d243..80d95c0 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -28,17 +28,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
"",
"bogustoken",
} {
- hdr, body, _ := s.runCurl(c, token, "dl.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
+ hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
if token != "" {
- hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
+ hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
}
- hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/bad-route")
+ hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
}
@@ -67,7 +67,7 @@ func (s *IntegrationSuite) Test404(c *check.C) {
"/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
"/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
} {
- hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "dl.example.com", uri)
+ hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
c.Check(body, check.Equals, "")
}
@@ -112,7 +112,7 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
c.Assert(err, check.Equals, nil)
uuid := coll["uuid"].(string)
- hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".dl.example.com", "/testdata.bin")
+ hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
@@ -140,12 +140,12 @@ func (s *IntegrationSuite) Test200(c *check.C) {
// My collection
{
auth: arvadostest.ActiveToken,
- host: arvadostest.FooCollection + "--dl.example.com",
+ host: arvadostest.FooCollection + "--collections.example.com",
path: "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".dl.example.com",
+ host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
path: "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
@@ -183,12 +183,12 @@ func (s *IntegrationSuite) Test200(c *check.C) {
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
+ host: arvadostest.HelloWorldCollection + ".collections.example.com",
path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
+ host: arvadostest.HelloWorldCollection + ".collections.example.com",
path: "/_/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
@@ -208,7 +208,7 @@ func (s *IntegrationSuite) Test200(c *check.C) {
},
{
auth: arvadostest.SpectatorToken,
- host: arvadostest.HelloWorldCollection + "--dl.example.com",
+ host: arvadostest.HelloWorldCollection + "--collections.example.com",
path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
@@ -220,7 +220,7 @@ func (s *IntegrationSuite) Test200(c *check.C) {
} {
host := spec.host
if host == "" {
- host = "dl.example.com"
+ host = "collections.example.com"
}
hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
commit a8caa6e1b359893e99226a8b21f356eee6edd529
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Oct 28 11:37:47 2015 -0400
5824: Rename cookie to arvados_api_token.
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 71f4c48..ca4eb94 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -68,7 +68,7 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
}
func (a *Credentials) loadTokenFromCookie(r *http.Request) {
- cookie, err := r.Cookie("api_token")
+ cookie, err := r.Cookie("arvados_api_token")
if err != nil || len(cookie.Value) == 0 {
return
}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 9751cd1..5f546ce 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -178,7 +178,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// resulting page.
http.SetCookie(w, &http.Cookie{
- Name: "api_token",
+ Name: "arvados_api_token",
Value: auth.EncodeTokenCookie([]byte(t)),
Path: "/",
Expires: time.Now().AddDate(10, 0, 0),
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 9b5ab2a..5b2a825 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -61,7 +61,7 @@ func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
}
func authzViaCookieValue(r *http.Request, tok string) int {
r.AddCookie(&http.Cookie{
- Name: "api_token",
+ Name: "arvados_api_token",
Value: auth.EncodeTokenCookie([]byte(tok)),
})
return http.StatusUnauthorized
commit a62773375ef74bbdd6a77f5dfa2023e122e717db
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Oct 28 10:52:34 2015 -0400
5824: Rename keepdl to keep-web.
diff --git a/services/keep-web/.gitignore b/services/keep-web/.gitignore
index 173e306..53997c2 100644
--- a/services/keep-web/.gitignore
+++ b/services/keep-web/.gitignore
@@ -1 +1 @@
-keepdl
+keep-web
commit 405c8505ccf36f7a1cfec68f305fae2edc439fa0
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Oct 27 17:15:16 2015 -0400
5824: Clarify docs.
diff --git a/doc/install/install-keepproxy.html.textile.liquid b/doc/install/install-keepproxy.html.textile.liquid
index 3b658f8..e6e2b10 100644
--- a/doc/install/install-keepproxy.html.textile.liquid
+++ b/doc/install/install-keepproxy.html.textile.liquid
@@ -4,7 +4,10 @@ navsection: installguide
title: Install Keepproxy server
...
-The Keepproxy server is a gateway into your Keep storage. Unlike the Keepstore servers, which are only accessible on the local LAN, Keepproxy is suitable for lower-bandwidth clients located elsewhere on the internet: a client sends a single copy of a data block, and Keepproxy sends copies to the appropriate Keepstore servers. Keepproxy also accepts requests from clients that do not compute data hashes before uploading data: notably, the browser-based upload feature in Workbench requires Keepproxy.
+The Keepproxy server is a gateway into your Keep storage. Unlike the Keepstore servers, which are only accessible on the local LAN, Keepproxy is suitable for clients located elsewhere on the internet. Specifically, in contrast to Keepstore:
+* A client writing through Keepproxy generates less network traffic: the client sends a single copy of a data block, and Keepproxy sends copies to the appropriate Keepstore servers.
+* A client can write through Keepproxy without precomputing content hashes. Notably, the browser-based upload feature in Workbench requires Keepproxy.
+* Keepproxy checks API token validity before processing requests. (Clients that can connect directly to Keepstore can use it as scratch space even without a valid API token.)
By convention, we use the following hostname for the Keepproxy server:
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 41cfb99..71f4c48 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -67,11 +67,6 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// secret is known)
}
-// TODO: LoadTokensFromHttpRequestBody(). We can't assume in
-// LoadTokensFromHttpRequest() that [or how] we should read and parse
-// the request body. This has to be requested explicitly by the
-// application.
-
func (a *Credentials) loadTokenFromCookie(r *http.Request) {
cookie, err := r.Cookie("api_token")
if err != nil || len(cookie.Value) == 0 {
@@ -83,3 +78,8 @@ func (a *Credentials) loadTokenFromCookie(r *http.Request) {
}
a.Tokens = append(a.Tokens, string(token))
}
+
+// TODO: LoadTokensFromHttpRequestBody(). We can't assume in
+// LoadTokensFromHttpRequest() that [or how] we should read and parse
+// the request body. This has to be requested explicitly by the
+// application.
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index cc47ebe..937e498 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -6,6 +6,8 @@
//
// See http://doc.arvados.org/install/install-keep-web.html.
//
+// Run "keep-web -help" to show all supported options.
+//
// Starting the server
//
// Serve HTTP requests at port 1234 on all interfaces:
@@ -67,18 +69,21 @@
// http://uuid_or_pdh--dl.example.com/path/file.txt
// http://uuid_or_pdh.dl.example.com/path/file.txt
//
-// The first form minimizes the cost and effort of deploying a
-// wildcard TLS certificate for *.dl.example.com. The second form is
-// likely to be easier to configure, and more efficient to run, on an
-// upstream proxy.
+// The first form ("uuid_or_pdh--dl.example.com") minimizes the cost
+// and effort of deploying a wildcard TLS certificate for
+// *.dl.example.com. The second form is likely to be easier to
+// configure, and more efficient to run, on an upstream proxy.
//
// In all of the above forms, the "dl.example.com" part can be
-// anything at all: keep-web ignores everything after the first "." or
-// "--".
+// anything at all: keep-web itself ignores everything after the first
+// "." or "--". (Of course, in order for clients to connect at all,
+// DNS and any relevant proxies must be configured accordingly.)
//
// In all of the above forms, the "uuid_or_pdh" part can be either a
// collection UUID or a portable data hash with the "+" character
-// replaced by "-".
+// optionally replaced by "-". (Replacing "+" with "-" is mandatory
+// when "uuid_or_pdh" appears in the domain name only because "+" is
+// not a valid character in a domain name.)
//
// In all of the above forms, a top level directory called "_" is
// skipped. In cases where the "path/file.txt" part might start with
@@ -91,23 +96,23 @@
// 1f4b0bc7583c2a7f9102c395f4ffc5e3+45, the following URLs are
// interchangeable:
//
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/foo
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/_/foo
-// http://zzzzz-4zz18-znfnqtbbv4spc3w--dl.example.com/_/foo
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo
+// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/foo/bar.txt
+// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/_/foo/bar.txt
+// http://zzzzz-4zz18-znfnqtbbv4spc3w--dl.example.com/_/foo/bar.txt
+// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
+// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
//
// An additional form is supported specifically to make it more
// convenient to maintain support for existing Workbench download
// links:
//
-// http://dl.example.com/collections/download/uuid_or_pdh/TOKEN/path/file.txt
+// http://dl.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt
//
// A regular Workbench "download" link is also accepted, but
// credentials passed via cookie, header, etc. are ignored. Only
// public data can be served this way:
//
-// http://dl.example.com/collections/uuid_or_pdh/path/file.txt
+// http://dl.example.com/collections/uuid_or_pdh/foo/bar.txt
//
// Authorization mechanisms
//
@@ -121,13 +126,13 @@
//
// A token can be provided in an URL-encoded query string:
//
-// GET /foo.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+// GET /foo/bar.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
//
// A suitably encoded token can be provided in a POST body if the
// request has a content type of application/x-www-form-urlencoded or
// multipart/form-data:
//
-// POST /foo.txt
+// POST /foo/bar.txt
// Content-Type: application/x-www-form-urlencoded
// [...]
// api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
@@ -184,7 +189,7 @@
// only when the designated origin matches exactly the Host header
// provided by the client or upstream proxy.
//
-// keep-web -attachment-only-host domain.example:9999
+// keep-web -address :9999 -attachment-only-host domain.example:9999
//
// Trust All Content mode
//
@@ -197,6 +202,12 @@
//
// In such cases you can enable trust-all-content mode.
//
-// keep-web -trust-all-content [...]
+// keep-web -address :9999 -trust-all-content
+//
+// When using trust-all-content mode, the only effect of the
+// -attachment-only-host option is to add a "Content-Disposition:
+// attachment" header.
+//
+// keep-web -address :9999 -attachment-only-host domain.example:9999 -trust-all-content
//
package main
commit c33fa89b10dad03c8dce3459358ff7fe4825aa99
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Oct 27 10:33:08 2015 -0400
5824: Fix wrong title.
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index 4f2a129..c73b58a 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -103,7 +103,7 @@ Configure your DNS servers so the following names resolve to your Nginx proxy's
* @*.dl.uuid_prefix.your.domain@, if you have a wildcard SSL certificate valid for these names; or
* @dl.uuid_prefix.your.domain@, if neither of the above options is feasible. In this case, only unauthenticated requests will be served, i.e., public data and collection sharing links.
-h3. Tell the API server about the keep-web service
+h3. Tell Workbench about the keep-web service
Add *one* of the following entries to your Workbench configuration file (@/etc/arvados/workbench/application.yml@), depending on your DNS setup:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list