[ARVADOS] updated: 1.1.0-20-g991d7d7
Git user
git at public.curoverse.com
Sat Oct 14 00:38:13 EDT 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 65 ++++++++++++++++++++++++++-------
services/keep-web/handler.go | 2 +-
services/keep-web/webdav.go | 80 ++++++++++++++++++++++++++++++++++-------
3 files changed, 121 insertions(+), 26 deletions(-)
via 991d7d7967cef46bf5aebf95f946e3072aa1e933 (commit)
from 3d5ee53dbf5ae2806cd6540afecc6328267da62f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 991d7d7967cef46bf5aebf95f946e3072aa1e933
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Oct 14 00:29:38 2017 -0400
12216: Avoid opening all files when generating webdav dir lists.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 8ecbe9c..1acf274 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -146,7 +146,13 @@ func (e collectionDirent) Sys() interface{} {
return nil
}
-// collectionFS implements http.FileSystem.
+// A CollectionFileSystem is an http.Filesystem with an added Stat() method.
+type CollectionFileSystem interface {
+ http.FileSystem
+ Stat(name string) (os.FileInfo, error)
+}
+
+// collectionFS implements CollectionFileSystem.
type collectionFS struct {
collection *Collection
client *Client
@@ -155,8 +161,8 @@ type collectionFS struct {
sizesOnce sync.Once
}
-// FileSystem returns an http.FileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) http.FileSystem {
+// FileSystem returns a CollectionFileSystem for the collection.
+func (c *Collection) FileSystem(client *Client, kc keepClient) CollectionFileSystem {
return &collectionFS{
collection: c,
client: client,
@@ -164,21 +170,44 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) http.FileSystem {
}
}
+func (c *collectionFS) Stat(name string) (os.FileInfo, error) {
+ name = canonicalName(name)
+ if name == "." {
+ return collectionDirent{
+ collection: c.collection,
+ name: "/",
+ isDir: true,
+ }, nil
+ }
+ if size, ok := c.fileSizes()[name]; ok {
+ return collectionDirent{
+ collection: c.collection,
+ name: path.Base(name),
+ size: size,
+ isDir: false,
+ }, nil
+ }
+ for fnm := range c.fileSizes() {
+ if !strings.HasPrefix(fnm, name+"/") {
+ continue
+ }
+ return collectionDirent{
+ collection: c.collection,
+ name: path.Base(name),
+ isDir: true,
+ }, nil
+ }
+ return nil, os.ErrNotExist
+}
+
func (c *collectionFS) Open(name string) (http.File, error) {
// Ensure name looks the way it does in a manifest.
- name = path.Clean("/" + name)
- if name == "/" || name == "./" {
- name = "."
- } else if strings.HasPrefix(name, "/") {
- name = "." + name
- }
+ name = canonicalName(name)
m := manifest.Manifest{Text: c.collection.ManifestText}
- filesizes := c.fileSizes()
-
// Return a file if it exists.
- if size, ok := filesizes[name]; ok {
+ if size, ok := c.fileSizes()[name]; ok {
reader, err := c.kc.ManifestFileReader(m, name)
if err != nil {
return nil, err
@@ -194,7 +223,7 @@ func (c *collectionFS) Open(name string) (http.File, error) {
// Return a directory if it's the root dir or there are file
// entries below it.
children := map[string]collectionDirent{}
- for fnm, size := range filesizes {
+ for fnm, size := range c.fileSizes() {
if !strings.HasPrefix(fnm, name+"/") {
continue
}
@@ -239,3 +268,13 @@ func (c *collectionFS) fileSizes() map[string]int64 {
})
return c.sizes
}
+
+func canonicalName(name string) string {
+ name = path.Clean("/" + name)
+ if name == "/" || name == "./" {
+ name = "."
+ } else if strings.HasPrefix(name, "/") {
+ name = "." + name
+ }
+ return name
+}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 28dbb66..ab04568 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -357,7 +357,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if webdavMethod[r.Method] {
h := webdav.Handler{
Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
- FileSystem: &webdavFS{httpfs: fs},
+ FileSystem: &webdavFS{collfs: fs},
LockSystem: h.webdavLS,
Logger: func(_ *http.Request, err error) {
if os.IsNotExist(err) {
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 0a7b782..57f3f53 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -11,9 +11,12 @@ import (
prand "math/rand"
"net/http"
"os"
+ "sync"
"sync/atomic"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+
"golang.org/x/net/context"
"golang.org/x/net/webdav"
)
@@ -24,10 +27,10 @@ var (
errReadOnly = errors.New("read-only filesystem")
)
-// webdavFS implements a read-only webdav.FileSystem by wrapping
-// http.Filesystem.
+// webdavFS implements a read-only webdav.FileSystem by wrapping an
+// arvados.CollectionFilesystem.
type webdavFS struct {
- httpfs http.FileSystem
+ collfs arvados.CollectionFileSystem
}
var _ webdav.FileSystem = &webdavFS{}
@@ -37,11 +40,11 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
}
func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
- f, err := fs.httpfs.Open(name)
+ fi, err := fs.collfs.Stat(name)
if err != nil {
return nil, err
}
- return &webdavFile{File: f}, nil
+ return &webdavFile{collfs: fs.collfs, fileInfo: fi, name: name}, nil
}
func (fs *webdavFS) RemoveAll(ctx context.Context, name string) error {
@@ -53,23 +56,76 @@ func (fs *webdavFS) Rename(ctx context.Context, oldName, newName string) error {
}
func (fs *webdavFS) Stat(ctx context.Context, name string) (os.FileInfo, error) {
- if f, err := fs.httpfs.Open(name); err != nil {
- return nil, err
- } else {
- return f.Stat()
- }
+ return fs.collfs.Stat(name)
}
// webdavFile implements a read-only webdav.File by wrapping
-// http.File. Writes fail.
+// http.File.
+//
+// The http.File is opened from an arvados.CollectionFileSystem, but
+// not until Seek, Read, or Readdir is called. This deferred-open
+// strategy makes webdav's OpenFile-Stat-Close cycle fast even though
+// the collfs's Open method is slow. This is relevant because webdav
+// does OpenFile-Stat-Close on each file when preparing directory
+// listings.
+//
+// Writes to a webdavFile always fail.
type webdavFile struct {
- http.File
+ // fields populated by (*webdavFS).OpenFile()
+ collfs http.FileSystem
+ fileInfo os.FileInfo
+ name string
+
+ // internal fields
+ file http.File
+ loadOnce sync.Once
+ err error
+}
+
+func (f *webdavFile) load() {
+ f.file, f.err = f.collfs.Open(f.name)
}
func (f *webdavFile) Write([]byte) (int, error) {
return 0, errReadOnly
}
+func (f *webdavFile) Seek(offset int64, whence int) (int64, error) {
+ f.loadOnce.Do(f.load)
+ if f.err != nil {
+ return 0, f.err
+ }
+ return f.file.Seek(offset, whence)
+}
+
+func (f *webdavFile) Read(buf []byte) (int, error) {
+ f.loadOnce.Do(f.load)
+ if f.err != nil {
+ return 0, f.err
+ }
+ return f.file.Read(buf)
+}
+
+func (f *webdavFile) Close() error {
+ if f.file == nil {
+ // We never called load(), or load() failed
+ return f.err
+ }
+ return f.file.Close()
+}
+
+func (f *webdavFile) Readdir(n int) ([]os.FileInfo, error) {
+ f.loadOnce.Do(f.load)
+ if f.err != nil {
+ return nil, f.err
+ }
+ return f.file.Readdir(n)
+}
+
+func (f *webdavFile) Stat() (os.FileInfo, error) {
+ return f.fileInfo, nil
+}
+
// noLockSystem implements webdav.LockSystem by returning success for
// every possible locking operation, even though it has no side
// effects such as actually locking anything. This works for a
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list