[ARVADOS] updated: 7506ffaf8bb3e7aa2c07be4b2812dbe8bd1d54f4
Git user
git at public.curoverse.com
Thu Jun 8 17:06:10 EDT 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 42 ++++++++++++++++++-----------
sdk/go/arvadostest/fixtures.go | 3 +++
sdk/go/keepclient/collectionreader.go | 19 +++++--------
services/keep-web/handler.go | 39 +++++++++++++--------------
services/keep-web/handler_test.go | 50 +++++++++++++++++++++++++++++++++++
5 files changed, 103 insertions(+), 50 deletions(-)
via 7506ffaf8bb3e7aa2c07be4b2812dbe8bd1d54f4 (commit)
via 0e9d72a55ed5219ce9e792fd5f41e47526e9f381 (commit)
from 0788c2724ccf317932c5368f60ba709ab289b202 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7506ffaf8bb3e7aa2c07be4b2812dbe8bd1d54f4
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jun 8 17:05:54 2017 -0400
8784: dir listings, WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 405360c..9724e99 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -11,7 +11,7 @@ import (
)
type keepClient interface {
- ManifestFileReader(manifest.Manifest, string) (http.File, error)
+ ManifestFileReader(manifest.Manifest, string) (Reader, error)
}
type collectionFS struct {
@@ -20,7 +20,10 @@ type collectionFS struct {
kc keepClient
}
-type reader interface {
+// A Reader implements, io.Reader, io.Seeker, and io.Closer, and has a
+// Size() method that returns the total number of bytes available to
+// read.
+type Reader interface {
io.Reader
io.Seeker
io.Closer
@@ -28,7 +31,7 @@ type reader interface {
}
type collectionFile struct {
- reader
+ Reader
collection *Collection
name string
}
@@ -56,7 +59,7 @@ func (cf *collectionFile) ModTime() time.Time {
if cf.collection.ModifiedAt == nil {
return time.Now()
}
- return cf.modtime
+ return *cf.collection.ModifiedAt
}
// IsDir implements os.FileInfo
@@ -84,8 +87,8 @@ func (c *collectionFS) Open(name string) (http.File, error) {
return nil, err
}
return &collectionFile{
- reader: reader,
- collection: c,
+ Reader: reader,
+ collection: c.collection,
name: path.Base(name),
- }
+ }, nil
}
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 299d186..7e21da4 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -18,6 +18,9 @@ const (
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
+ FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
+ FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
+
Dispatch1Token = "kwi8oowusvbutahacwk2geulqewy5oaqmpalczfna4b6bb0hfw"
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index 0e825cd..90e3068 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -6,19 +6,11 @@ import (
"io"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
-// A Reader implements, io.Reader, io.Seeker, and io.Closer, and has a
-// Len() method that returns the total number of bytes available to
-// read.
-type Reader interface {
- io.Reader
- io.Seeker
- io.Closer
- Len() uint64
- Size() int64
-}
+type Reader arvados.Reader
const (
// After reading a data block from Keep, cfReader slices it up
@@ -48,7 +40,7 @@ func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, fi
return kc.ManifestFileReader(m, filename)
}
-func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (Reader, error) {
+func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.Reader, error) {
f := &file{
kc: kc,
}
@@ -165,11 +157,6 @@ func (f *file) Seek(offset int64, whence int) (int64, error) {
return f.offset, nil
}
-// Len returns the file size in bytes.
-func (f *file) Len() uint64 {
- return uint64(f.size)
-}
-
// Size returns the file size in bytes.
func (f *file) Size() int64 {
return f.size
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index a740908..e79533e 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -5,15 +5,14 @@ import (
"fmt"
"html"
"io"
+ "log"
"net/http"
"net/url"
- "os"
- "path"
"strconv"
"strings"
"sync"
- "time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -164,12 +163,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
+ } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID/PATH...
targetID = parseCollectionIDFromURL(pathParts[0][2:])
targetPath = pathParts[1:]
- } else if len(pathParts) >= 3 && pathParts[0] == "collections" {
- if len(pathParts) >= 5 && pathParts[1] == "download" {
+ } else if len(pathParts) >= 2 && pathParts[0] == "collections" {
+ if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
targetID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
@@ -183,6 +182,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+ log.Printf("rHost %q targetID %q", r.Host, targetID)
if targetID == "" {
statusCode = http.StatusNotFound
return
@@ -349,45 +349,28 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
}
- filename := strings.Join(targetPath, "/")
- if strings.HasSuffix(filename, "/") {
- j, err := json.Marshal(collection)
- if err != nil {
- panic(err)
- }
- var coll arvados.Collection
- err = json.Unmarshal(j, &coll)
- if err != nil {
- panic(err)
- }
- http.FileServer(coll.FileSystem(arvados.Client{APIHost: arv.ApiHost}, kc)).ServeHTTP(w, r)
- return
- }
-
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- rdr, err := kc.CollectionFileReader(collection, filename)
- if os.IsNotExist(err) {
- statusCode = http.StatusNotFound
- return
- } else if err != nil {
- statusCode, statusText = http.StatusBadGateway, err.Error()
- return
- }
- defer rdr.Close()
- basename := path.Base(filename)
+ basename := targetPath[len(targetPath)-1]
applyContentDispositionHdr(w, r, basename, attachment)
- modstr, _ := collection["modified_at"].(string)
- modtime, err := time.Parse(time.RFC3339Nano, modstr)
+ j, err := json.Marshal(collection)
if err != nil {
- modtime = time.Now()
+ panic(err)
+ }
+ var coll arvados.Collection
+ err = json.Unmarshal(j, &coll)
+ if err != nil {
+ panic(err)
+ }
+ if strings.HasSuffix(r.URL.Path, "/") {
}
- http.ServeContent(w, r, basename, modtime, rdr)
+ fs := coll.FileSystem(&arvados.Client{APIHost: arv.ApiServer, AuthToken: arv.ApiToken}, kc)
+ http.FileServer(fs).ServeHTTP(w, r)
}
func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index df0346b..02d7b5f 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -479,3 +479,53 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
c.Check(resp.Header().Get("Location"), check.Equals, "")
return resp
}
+
+func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
+ for _, trial := range []struct {
+ uri string
+ expect []string
+ }{
+ {
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
+ expect: []string{"dir1"},
+ },
+ {
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
+ expect: []string{"foo", "bar"},
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/",
+ expect: []string{"dir1"},
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
+ expect: []string{"foo", "bar"},
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/theperthcountyconspiracydoesnotexist/",
+ expect: nil,
+ },
+ } {
+ c.Logf("%q => %q", trial.uri, trial.expect)
+ resp := httptest.NewRecorder()
+ u := mustParseURL("//" + trial.uri)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
+ },
+ }
+ s.testServer.Handler.ServeHTTP(resp, req)
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ for _, e := range trial.expect {
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*`+e+`.*`)
+ }
+ }
+ }
+}
commit 0e9d72a55ed5219ce9e792fd5f41e47526e9f381
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jun 8 09:35:57 2017 -0400
8784: dir listings, WIP
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 867323b..405360c 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -1,22 +1,34 @@
package arvados
import (
+ "io"
"net/http"
"os"
"path"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "git.curoverse.com/arvados.git/sdk/go/manifest"
)
+type keepClient interface {
+ ManifestFileReader(manifest.Manifest, string) (http.File, error)
+}
+
type collectionFS struct {
collection *Collection
client *Client
+ kc keepClient
+}
+
+type reader interface {
+ io.Reader
+ io.Seeker
+ io.Closer
+ Size() int64
}
type collectionFile struct {
- keepclient.CollectionReader
+ reader
collection *Collection
name string
}
@@ -34,11 +46,6 @@ func (cf *collectionFile) Name() string {
return cf.name
}
-// Size implements os.FileInfo
-func (cf *collectionFile) Size() int64 {
- return int64(cf.CollectionReader.Len())
-}
-
// Mode implements os.FileInfo
func (cf *collectionFile) Mode() os.FileMode {
return 0444
@@ -62,23 +69,23 @@ func (cf *collectionFile) Sys() interface{} {
return nil
}
-func (c *Collection) FileSystem(client *Client) http.FileSystem {
+func (c *Collection) FileSystem(client *Client, kc keepClient) http.FileSystem {
return &collectionFS{
collection: c,
client: client,
+ kc: kc,
}
}
func (c *collectionFS) Open(name string) (http.File, error) {
- kc := keepclient.New(arvadosclient.New(c.client))
m := manifest.Manifest{Text: c.collection.ManifestText}
- reader, err := kc.ManifestFileReader(m, name)
+ reader, err := c.kc.ManifestFileReader(m, name)
if err != nil {
return nil, err
}
return &collectionFile{
- CollectionReader: reader,
- collection: c,
- name: path.Base(name),
+ reader: reader,
+ collection: c,
+ name: path.Base(name),
}
}
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index 344a70c..0e825cd 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -17,6 +17,7 @@ type Reader interface {
io.Seeker
io.Closer
Len() uint64
+ Size() int64
}
const (
@@ -169,6 +170,11 @@ func (f *file) Len() uint64 {
return uint64(f.size)
}
+// Size returns the file size in bytes.
+func (f *file) Size() int64 {
+ return f.size
+}
+
func (f *file) load(m manifest.Manifest, path string) error {
f.segments = nil
f.size = 0
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 42c37b8..a740908 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -350,6 +350,20 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
filename := strings.Join(targetPath, "/")
+ if strings.HasSuffix(filename, "/") {
+ j, err := json.Marshal(collection)
+ if err != nil {
+ panic(err)
+ }
+ var coll arvados.Collection
+ err = json.Unmarshal(j, &coll)
+ if err != nil {
+ panic(err)
+ }
+ http.FileServer(coll.FileSystem(arvados.Client{APIHost: arv.ApiHost}, kc)).ServeHTTP(w, r)
+ return
+ }
+
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list