[ARVADOS] updated: 1cc0242418e2682ecb9520be292bf08ac3e12c76
Git user
git at public.curoverse.com
Fri Jun 9 17:23:12 EDT 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 201 ++++++++++++++++++++++++++++------
sdk/go/keepclient/collectionreader.go | 6 +-
services/crunch-run/crunchrun.go | 4 +-
services/keep-web/handler.go | 22 ++--
services/keep-web/server_test.go | 4 +-
5 files changed, 189 insertions(+), 48 deletions(-)
via 1cc0242418e2682ecb9520be292bf08ac3e12c76 (commit)
via 367ec4406e2481d715ace439b7965683f2465c77 (commit)
from 7506ffaf8bb3e7aa2c07be4b2812dbe8bd1d54f4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1cc0242418e2682ecb9520be292bf08ac3e12c76
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 9 17:22:50 2017 -0400
8784: dir listings, WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 9724e99..fb5c316 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -5,13 +5,21 @@ import (
"net/http"
"os"
"path"
+ "strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
+type File interface {
+ io.Reader
+ io.Closer
+ io.Seeker
+ Size() int64
+}
+
type keepClient interface {
- ManifestFileReader(manifest.Manifest, string) (Reader, error)
+ ManifestFileReader(manifest.Manifest, string) (File, error)
}
type collectionFS struct {
@@ -20,55 +28,118 @@ type collectionFS struct {
kc keepClient
}
-// A Reader implements, io.Reader, io.Seeker, and io.Closer, and has a
-// Size() method that returns the total number of bytes available to
-// read.
-type Reader interface {
- io.Reader
- io.Seeker
- io.Closer
- Size() int64
-}
-
type collectionFile struct {
- Reader
+ File
collection *Collection
name string
+ size int64
+}
+
+func (cf *collectionFile) Size() int64 {
+ return cf.size
}
func (cf *collectionFile) Readdir(count int) ([]os.FileInfo, error) {
- return []os.FileInfo{}, nil
+ return nil, io.EOF
}
func (cf *collectionFile) Stat() (os.FileInfo, error) {
- return cf, nil
+ return collectionDirent{
+ collection: cf.collection,
+ name: cf.name,
+ size: cf.size,
+ isDir: false,
+ }, nil
}
-// Name implements os.FileInfo
-func (cf *collectionFile) Name() string {
- return cf.name
+type collectionDir struct {
+ collection *Collection
+ stream string
+ dirents []os.FileInfo
}
-// Mode implements os.FileInfo
-func (cf *collectionFile) Mode() os.FileMode {
- return 0444
+// Readdir implements os.File.
+func (cd *collectionDir) Readdir(count int) ([]os.FileInfo, error) {
+ ret := cd.dirents
+ if count <= 0 {
+ cd.dirents = nil
+ return ret, nil
+ } else if len(ret) == 0 {
+ return nil, io.EOF
+ }
+ if count > len(ret) {
+ count = len(ret)
+ }
+ cd.dirents = cd.dirents[count:]
+ return ret[:count], nil
+}
+
+// Stat implements os.File.
+func (cd *collectionDir) Stat() (os.FileInfo, error) {
+ return collectionDirent{
+ collection: cd.collection,
+ name: path.Base(cd.stream),
+ isDir: true,
+ size: int64(len(cd.dirents)),
+ }, nil
+}
+
+// Close implements os.File.
+func (cd *collectionDir) Close() error {
+ return nil
+}
+
+// Read implements os.File.
+func (cd *collectionDir) Read([]byte) (int, error) {
+ return 0, nil
+}
+
+// Seek implements os.File.
+func (cd *collectionDir) Seek(int64, int) (int64, error) {
+ return 0, nil
+}
+
+type collectionDirent struct {
+ collection *Collection
+ name string
+ isDir bool
+ mode os.FileMode
+ size int64
+}
+
+// Name implements os.FileInfo
+func (e collectionDirent) Name() string {
+ return e.name
}
// ModTime implements os.FileInfo
-func (cf *collectionFile) ModTime() time.Time {
- if cf.collection.ModifiedAt == nil {
+func (e collectionDirent) ModTime() time.Time {
+ if e.collection.ModifiedAt == nil {
return time.Now()
}
- return *cf.collection.ModifiedAt
+ return *e.collection.ModifiedAt
+}
+
+// Mode implements os.FileInfo
+func (e collectionDirent) Mode() os.FileMode {
+ if e.isDir {
+ return 0555
+ } else {
+ return 0444
+ }
}
// IsDir implements os.FileInfo
-func (cf *collectionFile) IsDir() bool {
- return false
+func (e collectionDirent) IsDir() bool {
+ return e.isDir
+}
+
+func (e collectionDirent) Size() int64 {
+ return e.size
}
// Sys implements os.FileInfo
-func (cf *collectionFile) Sys() interface{} {
+func (e collectionDirent) Sys() interface{} {
return nil
}
@@ -81,14 +152,80 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) http.FileSystem {
}
func (c *collectionFS) Open(name string) (http.File, error) {
+ // Ensure name looks the way it does in a manifest.
+ name = path.Clean("/" + name)
+ if name == "/" || name == "./" {
+ name = "."
+ } else if strings.HasPrefix(name, "/") {
+ name = "." + name
+ }
+
m := manifest.Manifest{Text: c.collection.ManifestText}
- reader, err := c.kc.ManifestFileReader(m, name)
- if err != nil {
- return nil, err
+
+ filesizes := c.fileSizes()
+
+ // Return a file if it exists.
+ if size, ok := filesizes[name]; ok {
+ reader, err := c.kc.ManifestFileReader(m, name)
+ if err != nil {
+ return nil, err
+ }
+ return &collectionFile{
+ File: reader,
+ collection: c.collection,
+ name: path.Base(name),
+ size: size,
+ }, nil
+ }
+
+ // Return a directory if it's the root dir or there are file
+ // entries below it.
+ children := map[string]collectionDirent{}
+ for fnm, size := range filesizes {
+ if fnm == name {
+ }
+ if !strings.HasPrefix(fnm, name+"/") {
+ continue
+ }
+ isDir := false
+ ent := fnm[len(name)+1:]
+ if i := strings.Index(ent, "/"); i >= 0 {
+ ent = ent[:i]
+ isDir = true
+ }
+ e := children[ent]
+ e.collection = c.collection
+ e.isDir = isDir
+ e.name = ent
+ e.size = size
+ children[ent] = e
}
- return &collectionFile{
- Reader: reader,
+ if len(children) == 0 && name != "." {
+ return nil, os.ErrNotExist
+ }
+ dirents := make([]os.FileInfo, 0, len(children))
+ for _, ent := range children {
+ dirents = append(dirents, ent)
+ }
+ return &collectionDir{
collection: c.collection,
- name: path.Base(name),
+ stream: name,
+ dirents: dirents,
}, nil
}
+
+// fileSizes returns a map of files that can be opened. Each key
+// starts with "./".
+func (c *collectionFS) fileSizes() map[string]int64 {
+ var sizes map[string]int64
+ m := manifest.Manifest{Text: c.collection.ManifestText}
+ for ms := range m.StreamIter() {
+ for _, fss := range ms.FileStreamSegments {
+ if sizes == nil {
+ sizes = map[string]int64{}
+ }
+ sizes[ms.StreamName+"/"+fss.Name] += int64(fss.SegLen)
+ }
+ }
+ return sizes
+}
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index 90e3068..527318e 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -10,8 +10,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
-type Reader arvados.Reader
-
const (
// After reading a data block from Keep, cfReader slices it up
// and sends the slices to a buffered channel to be consumed
@@ -31,7 +29,7 @@ var ErrNoManifest = errors.New("Collection has no manifest")
// CollectionFileReader returns a Reader that reads content from a single file
// in the collection. The filename must be relative to the root of the
// collection. A leading prefix of "/" or "./" in the filename is ignored.
-func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (Reader, error) {
+func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (arvados.File, error) {
mText, ok := collection["manifest_text"].(string)
if !ok {
return nil, ErrNoManifest
@@ -40,7 +38,7 @@ func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, fi
return kc.ManifestFileReader(m, filename)
}
-func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.Reader, error) {
+func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
f := &file{
kc: kc,
}
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index aea93df..4a91401 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -49,7 +49,7 @@ var ErrCancelled = errors.New("Cancelled")
// IKeepClient is the minimal Keep API methods used by crunch-run.
type IKeepClient interface {
PutHB(hash string, buf []byte) (string, int, error)
- ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error)
+ ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
}
// NewLogWriter is a factory function to create a new log writer.
@@ -676,7 +676,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
runner.CrunchLog.Print("Attaching container streams")
// If stdin mount is provided, attach it to the docker container
- var stdinRdr keepclient.Reader
+ var stdinRdr arvados.File
var stdinJson []byte
if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok {
if stdinMnt.Kind == "collection" {
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 35d31dd..64ee699 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -5,7 +5,6 @@ import (
"fmt"
"html"
"io"
- "log"
"net/http"
"net/url"
"strconv"
@@ -181,7 +180,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
- log.Printf("rHost %q targetID %q", r.Host, targetID)
if targetID == "" {
statusCode = http.StatusNotFound
return
commit 367ec4406e2481d715ace439b7965683f2465c77
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jun 8 21:57:32 2017 -0400
8784: dir listings, WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index e79533e..35d31dd 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -141,8 +141,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
+ var stripParts int
var targetID string
- var targetPath []string
var tokens []string
var reqTokens []string
var pathToken bool
@@ -159,26 +159,25 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
- targetPath = pathParts
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID/PATH...
targetID = parseCollectionIDFromURL(pathParts[0][2:])
- targetPath = pathParts[1:]
+ stripParts = 1
} else if len(pathParts) >= 2 && pathParts[0] == "collections" {
if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
targetID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
- targetPath = pathParts[4:]
+ stripParts = 4
pathToken = true
} else {
// /collections/ID/PATH...
targetID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
- targetPath = pathParts[2:]
+ stripParts = 2
}
}
@@ -259,7 +258,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
}
- if tokens == nil && strings.HasPrefix(targetPath[0], "t=") {
+ targetPath := pathParts[stripParts:]
+ if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
// http://ID.example/t=TOKEN/PATH...
// /c=ID/t=TOKEN/PATH...
//
@@ -269,6 +269,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokens = []string{targetPath[0][2:]}
pathToken = true
targetPath = targetPath[1:]
+ stripParts++
}
if tokens == nil {
@@ -286,6 +287,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// //collections.example/t=foo/ won't work because
// t=foo will be interpreted as a token "foo".
targetPath = targetPath[1:]
+ stripParts++
}
forceReload := false
@@ -369,8 +371,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if strings.HasSuffix(r.URL.Path, "/") {
}
+ var stripPrefix string
+ if stripParts > 0 {
+ stripPrefix = "/" + strings.Join(pathParts[:stripParts], "/")
+ }
fs := coll.FileSystem(&arvados.Client{APIHost: arv.ApiServer, AuthToken: arv.ApiToken}, kc)
- http.FileServer(fs).ServeHTTP(w, r)
+ http.StripPrefix(stripPrefix, http.FileServer(fs)).ServeHTTP(w, r)
}
func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 52fe459..500561d 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -77,7 +77,9 @@ func (s *IntegrationSuite) Test404(c *check.C) {
} {
hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
- c.Check(body, check.Equals, "")
+ if len(body) > 0 {
+ c.Check(body, check.Equals, "404 page not found\n")
+ }
}
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list