[ARVADOS] updated: 2dc0e004e32b05b69472085600d13af5ef1db86c
Git user
git at public.curoverse.com
Mon Jun 12 16:59:33 EDT 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 26 ++--
sdk/go/arvados/collection_fs_test.go | 118 ++++++++++++++++++
services/keep-web/handler.go | 236 +++++++++++++++++++++++++++--------
services/keep-web/handler_test.go | 86 ++++++++++---
4 files changed, 388 insertions(+), 78 deletions(-)
create mode 100644 sdk/go/arvados/collection_fs_test.go
via 2dc0e004e32b05b69472085600d13af5ef1db86c (commit)
via e5d5b6d3e963057f6f72e51b89546f64a168e96a (commit)
via 66b416e62f84ea1413c888b359b2cde04f59ee99 (commit)
via ae650952dcf66ce9026855a7198f21027ad06056 (commit)
from 1cc0242418e2682ecb9520be292bf08ac3e12c76 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2dc0e004e32b05b69472085600d13af5ef1db86c
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 12 16:59:09 2017 -0400
8784: wget args, styling
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index a37cfb7..be6049a 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -340,7 +340,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
h.seeOtherWithCookie(w, r, basename+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, &coll, fs, openPath)
+ h.serveDirectory(w, r, &coll, fs, openPath, stripParts)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if int64(w.WroteBodyBytes()) != stat.Size() {
@@ -352,7 +352,27 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
var dirListingTemplate = `<!DOCTYPE HTML>
-<HTML><HEAD><TITLE>{{ .Collection.Name }}</TITLE></HEAD>
+<HTML><HEAD>
+ <META name="robots" content="NOINDEX">
+ <TITLE>{{ .Collection.Name }}</TITLE>
+ <STYLE type="text/css">
+ body {
+ margin: 1.5em;
+ }
+ pre {
+ background-color: #D9EDF7;
+ border-radius: .25em;
+ padding: .75em;
+ overflow: auto;
+ }
+ .footer p {
+ font-size: 82%;
+ }
+ ul li {
+ font-family: monospace;
+ }
+ </STYLE>
+</HEAD>
<BODY>
<H1>{{ .Collection.Name }}</H1>
@@ -360,7 +380,7 @@ var dirListingTemplate = `<!DOCTYPE HTML>
Arvados. You can download individual files listed below. To download
the entire collection with wget, try:</P>
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs=3 {{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
<H2>File Listing</H2>
@@ -368,9 +388,10 @@ the entire collection with wget, try:</P>
{{range .Files}} <LI><A href="{{.}}">{{.}}</A></LI>{{end}}
</UL>
+<HR noshade>
<DIV class="footer">
- <H2>About Arvados</H2>
<P>
+ About Arvados:
Arvados is a free and open source software bioinformatics platform.
To learn more, visit arvados.org.
Arvados is not responsible for the files listed on this page.
@@ -380,7 +401,7 @@ the entire collection with wget, try:</P>
</BODY>
`
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collection *arvados.Collection, fs http.FileSystem, base string) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collection *arvados.Collection, fs http.FileSystem, base string, stripParts int) {
var files []string
var walk func(string) error
if !strings.HasSuffix(base, "/") {
@@ -426,6 +447,7 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
"Collection": collection,
"Files": files,
"Request": r,
+ "StripParts": stripParts,
})
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 508c9cb..d394659 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -1,6 +1,7 @@
package main
import (
+ "fmt"
"html"
"io/ioutil"
"net/http"
@@ -486,49 +487,58 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
for _, trial := range []struct {
- uri string
- header http.Header
- expect []string
+ uri string
+ header http.Header
+ expect []string
+ cutDirs int
}{
{
- uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
- header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 0,
},
{
- uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 0,
},
{
- uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
- header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
+ uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 2,
},
{
- uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
- header: nil,
- expect: []string{"dir1/foo", "dir1/bar"},
+ uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 4,
},
{
- uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
- header: nil,
- expect: []string{"dir1/foo", "dir1/bar"},
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 2,
},
{
- uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
},
{
- uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 2,
},
{
- uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
- header: authHeader,
- expect: []string{"foo", "bar"},
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 0,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
@@ -571,6 +581,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
for _, e := range trial.expect {
c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
}
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
}
}
}
commit e5d5b6d3e963057f6f72e51b89546f64a168e96a
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 12 16:42:17 2017 -0400
8784: WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3361d91..a37cfb7 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -6,7 +6,6 @@ import (
"html"
"html/template"
"io"
- "log"
"net/http"
"net/url"
"os"
@@ -211,52 +210,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// token in an HttpOnly cookie, and redirect to the
// same URL with the query param redacted and method =
// GET.
-
- if !credentialsOK {
- // It is not safe to copy the provided token
- // into a cookie unless the current vhost
- // (origin) serves only a single collection or
- // we are in TrustAllContent mode.
- statusCode = http.StatusBadRequest
- return
- }
-
- // The HttpOnly flag is necessary to prevent
- // JavaScript code (included in, or loaded by, a page
- // in the collection being served) from employing the
- // user's token beyond reading other files in the same
- // domain, i.e., same collection.
- //
- // The 303 redirect is necessary in the case of a GET
- // request to avoid exposing the token in the Location
- // bar, and in the case of a POST request to avoid
- // raising warnings when the user refreshes the
- // resulting page.
-
- http.SetCookie(w, &http.Cookie{
- Name: "arvados_api_token",
- Value: auth.EncodeTokenCookie([]byte(formToken)),
- Path: "/",
- HttpOnly: true,
- })
-
- // Propagate query parameters (except api_token) from
- // the original request.
- redirQuery := r.URL.Query()
- redirQuery.Del("api_token")
-
- redir := (&url.URL{
- Host: r.Host,
- Path: r.URL.Path,
- RawQuery: redirQuery.Encode(),
- }).String()
-
- w.Header().Add("Location", redir)
- statusCode, statusText = http.StatusSeeOther, redir
- w.WriteHeader(statusCode)
- io.WriteString(w, `<A href="`)
- io.WriteString(w, html.EscapeString(redir))
- io.WriteString(w, `">Continue</A>`)
+ h.seeOtherWithCookie(w, r, "", credentialsOK)
return
}
@@ -377,7 +331,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}, kc)
openPath := "/" + strings.Join(targetPath, "/")
- log.Printf("targetPath %q openPath %q", targetPath, openPath)
if f, err := fs.Open(openPath); os.IsNotExist(err) {
statusCode = http.StatusNotFound
} else if err != nil {
@@ -385,26 +338,60 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if stat, err := f.Stat(); err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
} else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
- http.Redirect(w, r, basename+"/", http.StatusFound)
+ h.seeOtherWithCookie(w, r, basename+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, fs, openPath)
+ h.serveDirectory(w, r, &coll, fs, openPath)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
+ if int64(w.WroteBodyBytes()) != stat.Size() {
+ n, err := f.Read(make([]byte, 1024))
+ statusCode, statusText = http.StatusInternalServerError, fmt.Sprintf("f.Size()==%d but only wrote %d bytes; read(1024) returns %d, %s", stat.Size(), w.WroteBodyBytes(), n, err)
+
+ }
}
}
-var dirListingTemplate = `<UL>
-{{range .}}
-<LI><A href="{{.}}">{{.}}</A></LI>
-{{end}}
+var dirListingTemplate = `<!DOCTYPE HTML>
+<HTML><HEAD><TITLE>{{ .Collection.Name }}</TITLE></HEAD>
+<BODY>
+<H1>{{ .Collection.Name }}</H1>
+
+<P>This collection of data files is being shared with you through
+Arvados. You can download individual files listed below. To download
+the entire collection with wget, try:</P>
+
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs=3 {{ .Request.URL }}</PRE>
+
+<H2>File Listing</H2>
+
+<UL>
+{{range .Files}} <LI><A href="{{.}}">{{.}}</A></LI>{{end}}
</UL>
+
+<DIV class="footer">
+ <H2>About Arvados</H2>
+ <P>
+ Arvados is a free and open source software bioinformatics platform.
+ To learn more, visit arvados.org.
+ Arvados is not responsible for the files listed on this page.
+ </P>
+</DIV>
+
+</BODY>
`
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, fs http.FileSystem, base string) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collection *arvados.Collection, fs http.FileSystem, base string) {
var files []string
var walk func(string) error
+ if !strings.HasSuffix(base, "/") {
+ base = base + "/"
+ }
walk = func(path string) error {
- d, err := fs.Open(base + path)
+ dirname := base + path
+ if dirname != "/" {
+ dirname = strings.TrimSuffix(dirname, "/")
+ }
+ d, err := fs.Open(dirname)
if err != nil {
return err
}
@@ -414,12 +401,12 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, fs http
}
for _, ent := range ents {
if ent.IsDir() {
- err = walk(path + "/" + ent.Name())
+ err = walk(path + ent.Name() + "/")
if err != nil {
return err
}
} else {
- files = append(files, path+"/"+ent.Name())
+ files = append(files, path+ent.Name())
}
}
return nil
@@ -434,7 +421,12 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, fs http
return
}
sort.Strings(files)
- tmpl.Execute(w, files)
+ w.WriteHeader(http.StatusOK)
+ tmpl.Execute(w, map[string]interface{}{
+ "Collection": collection,
+ "Files": files,
+ "Request": r,
+ })
}
func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
@@ -454,3 +446,61 @@ func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename
w.Header().Set("Content-Disposition", disposition)
}
}
+
+func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
+ if !credentialsOK {
+ // It is not safe to copy the provided token
+ // into a cookie unless the current vhost
+ // (origin) serves only a single collection or
+ // we are in TrustAllContent mode.
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ if formToken := r.FormValue("api_token"); formToken != "" {
+ // The HttpOnly flag is necessary to prevent
+ // JavaScript code (included in, or loaded by, a page
+ // in the collection being served) from employing the
+ // user's token beyond reading other files in the same
+ // domain, i.e., same collection.
+ //
+ // The 303 redirect is necessary in the case of a GET
+ // request to avoid exposing the token in the Location
+ // bar, and in the case of a POST request to avoid
+ // raising warnings when the user refreshes the
+ // resulting page.
+
+ http.SetCookie(w, &http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(formToken)),
+ Path: "/",
+ HttpOnly: true,
+ })
+ }
+
+ // Propagate query parameters (except api_token) from
+ // the original request.
+ redirQuery := r.URL.Query()
+ redirQuery.Del("api_token")
+
+ u := r.URL
+ if location != "" {
+ newu, err := u.Parse(location)
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ u = newu
+ }
+ redir := (&url.URL{
+ Host: r.Host,
+ Path: u.Path,
+ RawQuery: redirQuery.Encode(),
+ }).String()
+
+ w.Header().Add("Location", redir)
+ w.WriteHeader(http.StatusSeeOther)
+ io.WriteString(w, `<A href="`)
+ io.WriteString(w, html.EscapeString(redir))
+ io.WriteString(w, `">Continue</A>`)
+}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 21c6d87..508c9cb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -481,28 +481,58 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
}
func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+ authHeader := http.Header{
+ "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
+ }
for _, trial := range []struct {
uri string
+ header http.Header
expect []string
}{
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
+ header: authHeader,
expect: []string{"dir1/foo", "dir1/bar"},
},
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
+ header: authHeader,
expect: []string{"foo", "bar"},
},
{
+ uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ },
+ {
+ uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ },
+ {
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
+ header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
},
{
- uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ },
+ {
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ },
+ {
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
expect: []string{"foo", "bar"},
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
+ header: authHeader,
expect: nil,
},
} {
@@ -514,18 +544,32 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: http.Header{
- "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
- },
+ Header: trial.header,
}
s.testServer.Handler.ServeHTTP(resp, req)
- c.Logf("resp.Body.String() == %q", resp.Body.String())
+ var cookies []*http.Cookie
+ for resp.Code == http.StatusSeeOther {
+ u, _ := req.URL.Parse(resp.Header().Get("Location"))
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{},
+ }
+ cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
+ for _, c := range cookies {
+ req.AddCookie(c)
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ }
if trial.expect == nil {
c.Check(resp.Code, check.Equals, http.StatusNotFound)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK)
for _, e := range trial.expect {
- c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
}
}
}
commit 66b416e62f84ea1413c888b359b2cde04f59ee99
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 12 10:44:55 2017 -0400
8784: WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index fb5c316..01d5b65 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -22,12 +22,6 @@ type keepClient interface {
ManifestFileReader(manifest.Manifest, string) (File, error)
}
-type collectionFS struct {
- collection *Collection
- client *Client
- kc keepClient
-}
-
type collectionFile struct {
File
collection *Collection
@@ -99,6 +93,7 @@ func (cd *collectionDir) Seek(int64, int) (int64, error) {
return 0, nil
}
+// collectionDirent implements os.FileInfo.
type collectionDirent struct {
collection *Collection
name string
@@ -107,12 +102,12 @@ type collectionDirent struct {
size int64
}
-// Name implements os.FileInfo
+// Name implements os.FileInfo.
func (e collectionDirent) Name() string {
return e.name
}
-// ModTime implements os.FileInfo
+// ModTime implements os.FileInfo.
func (e collectionDirent) ModTime() time.Time {
if e.collection.ModifiedAt == nil {
return time.Now()
@@ -120,7 +115,7 @@ func (e collectionDirent) ModTime() time.Time {
return *e.collection.ModifiedAt
}
-// Mode implements os.FileInfo
+// Mode implements os.FileInfo.
func (e collectionDirent) Mode() os.FileMode {
if e.isDir {
return 0555
@@ -129,20 +124,29 @@ func (e collectionDirent) Mode() os.FileMode {
}
}
-// IsDir implements os.FileInfo
+// IsDir implements os.FileInfo.
func (e collectionDirent) IsDir() bool {
return e.isDir
}
+// Size implements os.FileInfo.
func (e collectionDirent) Size() int64 {
return e.size
}
-// Sys implements os.FileInfo
+// Sys implements os.FileInfo.
func (e collectionDirent) Sys() interface{} {
return nil
}
+// collectionFS implements http.FileSystem.
+type collectionFS struct {
+ collection *Collection
+ client *Client
+ kc keepClient
+}
+
+// FileSystem returns an http.FileSystem for the collection.
func (c *Collection) FileSystem(client *Client, kc keepClient) http.FileSystem {
return &collectionFS{
collection: c,
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 64ee699..3361d91 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -4,9 +4,13 @@ import (
"encoding/json"
"fmt"
"html"
+ "html/template"
"io"
+ "log"
"net/http"
"net/url"
+ "os"
+ "sort"
"strconv"
"strings"
"sync"
@@ -367,14 +371,70 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if err != nil {
panic(err)
}
- if strings.HasSuffix(r.URL.Path, "/") {
+ fs := coll.FileSystem(&arvados.Client{
+ APIHost: arv.ApiServer,
+ AuthToken: arv.ApiToken,
+ Insecure: arv.ApiInsecure,
+ }, kc)
+ openPath := "/" + strings.Join(targetPath, "/")
+ log.Printf("targetPath %q openPath %q", targetPath, openPath)
+ if f, err := fs.Open(openPath); os.IsNotExist(err) {
+ statusCode = http.StatusNotFound
+ } else if err != nil {
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ } else if stat, err := f.Stat(); err != nil {
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ } else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
+ http.Redirect(w, r, basename+"/", http.StatusFound)
+ } else if stat.IsDir() {
+ h.serveDirectory(w, r, fs, openPath)
+ } else {
+ http.ServeContent(w, r, basename, stat.ModTime(), f)
+ }
+}
+
+var dirListingTemplate = `<UL>
+{{range .}}
+<LI><A href="{{.}}">{{.}}</A></LI>
+{{end}}
+</UL>
+`
+
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, fs http.FileSystem, base string) {
+ var files []string
+ var walk func(string) error
+ walk = func(path string) error {
+ d, err := fs.Open(base + path)
+ if err != nil {
+ return err
+ }
+ ents, err := d.Readdir(-1)
+ if err != nil {
+ return err
+ }
+ for _, ent := range ents {
+ if ent.IsDir() {
+ err = walk(path + "/" + ent.Name())
+ if err != nil {
+ return err
+ }
+ } else {
+ files = append(files, path+"/"+ent.Name())
+ }
+ }
+ return nil
}
- var stripPrefix string
- if stripParts > 0 {
- stripPrefix = "/" + strings.Join(pathParts[:stripParts], "/")
+ if err := walk(""); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ tmpl, err := template.New("dir").Parse(dirListingTemplate)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
}
- fs := coll.FileSystem(&arvados.Client{APIHost: arv.ApiServer, AuthToken: arv.ApiToken}, kc)
- http.StripPrefix(stripPrefix, http.FileServer(fs)).ServeHTTP(w, r)
+ sort.Strings(files)
+ tmpl.Execute(w, files)
}
func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 02d7b5f..21c6d87 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -487,22 +487,22 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
}{
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
- expect: []string{"dir1"},
+ expect: []string{"dir1/foo", "dir1/bar"},
},
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
expect: []string{"foo", "bar"},
},
{
- uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/",
- expect: []string{"dir1"},
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
+ expect: []string{"dir1/foo", "dir1/bar"},
},
{
- uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
expect: []string{"foo", "bar"},
},
{
- uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/theperthcountyconspiracydoesnotexist/",
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
expect: nil,
},
} {
@@ -519,6 +519,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
},
}
s.testServer.Handler.ServeHTTP(resp, req)
+ c.Logf("resp.Body.String() == %q", resp.Body.String())
if trial.expect == nil {
c.Check(resp.Code, check.Equals, http.StatusNotFound)
} else {
commit ae650952dcf66ce9026855a7198f21027ad06056
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 9 17:23:33 2017 -0400
8784: dir listings, WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
new file mode 100644
index 0000000..8cfd21e
--- /dev/null
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -0,0 +1,118 @@
+package arvados
+
+import (
+ "io"
+ "net/http"
+ "os"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CollectionFSSuite{})
+
+type CollectionFSSuite struct {
+ client *Client
+ coll Collection
+ fs http.FileSystem
+}
+
+func (s *CollectionFSSuite) SetUpTest(c *check.C) {
+ s.client = NewClientFromEnv()
+ err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
+ c.Assert(err, check.IsNil)
+ s.fs = s.coll.FileSystem(s.client, nil)
+}
+
+func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
+ f, err := s.fs.Open("/dir1")
+ c.Assert(err, check.IsNil)
+
+ st, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(st.Size(), check.Equals, int64(2))
+ c.Check(st.IsDir(), check.Equals, true)
+
+ fis, err := f.Readdir(0)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 2)
+ if len(fis) > 0 {
+ c.Check(fis[0].Size(), check.Equals, int64(3))
+ }
+}
+
+func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
+ f, err := s.fs.Open("./dir1")
+ c.Assert(err, check.IsNil)
+ for i := 0; i < 2; i++ {
+ fis, err := f.Readdir(1)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 1)
+ if len(fis) > 0 {
+ c.Check(fis[0].Size(), check.Equals, int64(3))
+ }
+ }
+ fis, err := f.Readdir(1)
+ c.Check(len(fis), check.Equals, 0)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.Equals, io.EOF)
+
+ f, err = s.fs.Open("dir1")
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(1)
+ c.Check(len(fis), check.Equals, 1)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(2)
+ c.Check(len(fis), check.Equals, 1)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(2)
+ c.Check(len(fis), check.Equals, 0)
+ c.Assert(err, check.Equals, io.EOF)
+}
+
+func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
+ for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
+ f, err := s.fs.Open(path)
+ c.Assert(err, check.IsNil)
+
+ st, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(st.Size(), check.Equals, int64(1))
+ c.Check(st.IsDir(), check.Equals, true)
+ }
+ for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
+ c.Logf("%q", path)
+ f, err := s.fs.Open(path)
+ c.Assert(err, check.IsNil)
+
+ st, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(st.Size(), check.Equals, int64(2))
+ c.Check(st.IsDir(), check.Equals, true)
+ }
+}
+
+func (s *CollectionFSSuite) TestNotExist(c *check.C) {
+ for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
+ f, err := s.fs.Open(path)
+ c.Assert(f, check.IsNil)
+ c.Assert(err, check.NotNil)
+ c.Assert(os.IsNotExist(err), check.Equals, true)
+ }
+}
+
+func (s *CollectionFSSuite) TestOpenFile(c *check.C) {
+ c.Skip("cannot test files with nil keepclient")
+
+ f, err := s.fs.Open("/foo.txt")
+ c.Assert(err, check.IsNil)
+ st, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(st.Size(), check.Equals, int64(3))
+}
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list