[ARVADOS] updated: 1.1.1-255-g0a27883
Git user
git at public.curoverse.com
Wed Dec 20 14:18:18 EST 2017
Summary of changes:
lib/mount/fs.go | 41 +++++++++-
sdk/go/arvados/fs_base.go | 134 ++++++++++++++++++++++---------
sdk/go/arvados/fs_collection.go | 169 ++++++++++++++++++----------------------
sdk/go/arvados/fs_filehandle.go | 5 ++
sdk/go/arvados/fs_site.go | 26 +++++--
5 files changed, 235 insertions(+), 140 deletions(-)
via 0a278831cc02f83d17dc57541713f540488335d2 (commit)
from 600421bc73da06efe5ccfc54af02d35bea1bfb02 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0a278831cc02f83d17dc57541713f540488335d2
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 19 09:30:56 2017 -0500
go-fuse: Save on sync(). WIP
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 8e488f5..13e2a06 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -2,7 +2,9 @@ package mount
import (
"io"
+ "log"
"os"
+ "runtime/debug"
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -55,10 +57,12 @@ func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
}
func (fs *keepFS) Init() {
+ defer fs.debugPanics()
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
}
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS, invalidFH
}
@@ -72,6 +76,7 @@ func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint
}
func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
return -fuse.EROFS, invalidFH
}
@@ -88,6 +93,7 @@ func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
}
func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -120,6 +126,7 @@ func (fs *keepFS) errCode(err error) int {
}
func (fs *keepFS) Mkdir(path string, mode uint32) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -132,6 +139,7 @@ func (fs *keepFS) Mkdir(path string, mode uint32) int {
}
func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ defer fs.debugPanics()
f, err := fs.root.OpenFile(path, 0, 0)
if err != nil {
return fs.errCode(err), invalidFH
@@ -145,10 +153,12 @@ func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
}
func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
return fs.Release(path, fh)
}
func (fs *keepFS) Release(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
fs.Lock()
defer fs.Unlock()
defer delete(fs.open, fh)
@@ -162,6 +172,7 @@ func (fs *keepFS) Release(path string, fh uint64) (errc int) {
}
func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -169,6 +180,7 @@ func (fs *keepFS) Rename(oldname, newname string) (errc int) {
}
func (fs *keepFS) Unlink(path string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -176,6 +188,7 @@ func (fs *keepFS) Unlink(path string) (errc int) {
}
func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -196,6 +209,7 @@ func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
}
func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ defer fs.debugPanics()
var fi os.FileInfo
var err error
if f := fs.lookupFH(fh); f != nil {
@@ -218,7 +232,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
if fi, err := fs.root.Stat(path); err != nil {
return fs.errCode(err)
- } else if (os.FileMode(mode)^fi.Mode())&os.ModePerm != 0 {
+ } else if mode & ^uint32(fuse.S_IFREG|fuse.S_IFDIR|0777) != 0 || (fi.Mode()&os.ModeDir != 0) != (mode&fuse.S_IFDIR != 0) {
return -fuse.ENOSYS
} else {
return 0
@@ -226,6 +240,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ defer fs.debugPanics()
var m uint32
if fi.IsDir() {
m = m | fuse.S_IFDIR
@@ -252,6 +267,7 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -270,6 +286,7 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -288,6 +305,7 @@ func (fs *keepFS) Readdir(path string,
fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
ofst int64,
fh uint64) (errc int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -305,3 +323,24 @@ func (fs *keepFS) Readdir(path string,
}
return 0
}
+
+func (fs *keepFS) Fsync(path string, datasync bool, fh uint64) int {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ return fs.errCode(f.Sync())
+}
+
+func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
+ return fs.Fsync(path, datasync, fh)
+}
+
+func (fs *keepFS) debugPanics() {
+ if err := recover(); err != nil {
+ log.Printf("(%T) %v", err, err)
+ debug.PrintStack()
+ panic(err)
+ }
+}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 3be3f5e..8d987d4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path"
@@ -40,6 +41,7 @@ type File interface {
Readdir(int) ([]os.FileInfo, error)
Stat() (os.FileInfo, error)
Truncate(int64) error
+ Sync() error
}
// A FileSystem is an http.Filesystem plus Stat() and support for
@@ -47,8 +49,10 @@ type File interface {
// goroutines.
type FileSystem interface {
http.FileSystem
+ fsBackend
- inode
+ newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+ newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -75,10 +79,12 @@ type FileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+ Sync() error
}
type inode interface {
Parent() inode
+ FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
@@ -186,6 +192,7 @@ func (*nullnode) Child(name string, replace func(inode) inode) inode {
}
type treenode struct {
+ fs FileSystem
parent inode
inodes map[string]inode
fileinfo fileinfo
@@ -193,6 +200,10 @@ type treenode struct {
nullnode
}
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -239,7 +250,8 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
type fileSystem struct {
- inode
+ root inode
+ fsBackend
}
// OpenFile is analogous to os.OpenFile().
@@ -248,12 +260,11 @@ func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, e
}
func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
+ parent := rlookup(fs.root, dirname)
if parent == nil {
return nil, os.ErrNotExist
}
@@ -294,20 +305,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -322,10 +323,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
} else if flag&os.O_TRUNC != 0 {
if !writable {
return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
+ } else if n.IsDir() {
return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
}
}
return &filehandle{
@@ -346,7 +347,7 @@ func (fs *fileSystem) Create(name string) (File, error) {
func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
+ n := rlookup(fs.root, dirname)
if n == nil {
return os.ErrNotExist
}
@@ -355,12 +356,8 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
if n.Child(name, nil) != nil {
return os.ErrExist
}
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
+ child, err = fs.newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -372,7 +369,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
+ node := rlookup(fs.root, name)
if node == nil {
err = os.ErrNotExist
} else {
@@ -414,7 +411,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
for _, f := range []*filehandle{olddirf, newdirf} {
node := f.inode
needLock = append(needLock, node)
- for node.Parent() != node {
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
needLock = append(needLock, node)
}
@@ -428,10 +425,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
err = nil
olddirf.inode.Child(oldname, func(oldinode inode) inode {
if oldinode == nil {
@@ -450,20 +443,18 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
oldinode.Lock()
defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
+ n.fileinfo.name = newname
case *filenode:
- n.parent = newdn
+ n.parent = newdirf.inode
n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
+ //TODO: olddirf.setModTime(time.Now())
+ //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
@@ -488,7 +479,7 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs, dirname)
+ dir := rlookup(fs.root, dirname)
if dir == nil {
return os.ErrNotExist
}
@@ -507,3 +498,70 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
})
return err
}
+
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ treenode: treenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
+ }
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ }
+ return
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e451189..fc00335 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"os"
"path"
"regexp"
@@ -20,11 +21,28 @@ import (
var maxBlockSize = 1 << 26
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
+
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -38,28 +56,32 @@ type CollectionFileSystem interface {
}
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
} else {
modTime = *c.ModifiedAt
}
+ fs := &collectionFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ uuid: c.UUID,
+ }
dn := &dirnode{
- client: client,
- kc: kc,
treenode: treenode{
+ fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
},
- parent: nil,
inodes: make(map[string]inode),
},
}
dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ fs.fileSystem.root = dn
if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -68,9 +90,31 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
type collectionFileSystem struct {
fileSystem
+ uuid string
}
-func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
+}
+
+func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
if name == ".arvados#collection" {
return &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -86,13 +130,13 @@ func (fs collectionFileSystem) Child(name string, replace func(inode) inode) ino
return data, err
}}
}
- return fs.fileSystem.Child(name, replace)
+ return fs.fileSystem.root.Child(name, replace)
}
-func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
- fs.fileSystem.inode.Lock()
- defer fs.fileSystem.inode.Unlock()
- return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -170,8 +214,9 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
// filenode implements inode.
type filenode struct {
+ parent inode
+ fs FileSystem
fileinfo fileinfo
- parent *dirnode
segments []segment
// number of times `segments` has changed in a
// way that might invalidate a filenodePtr
@@ -193,6 +238,10 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -438,7 +487,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -447,7 +496,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.parent.(fsBackend),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -458,8 +507,6 @@ func (fn *filenode) pruneMemSegments() {
type dirnode struct {
treenode
- client *Client
- kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -480,7 +527,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
@@ -488,7 +535,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
@@ -709,7 +756,7 @@ func (dn *dirnode) loadManifest(txt string) error {
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
@@ -738,7 +785,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
- node := dn
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -758,16 +805,13 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
continue
}
node.Child(name, func(child inode) inode {
- switch child.(type) {
- case nil:
- node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if child == nil {
+ node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
- case *dirnode:
- node = child.(*dirnode)
- case *filenode:
+ } else if !child.IsDir() {
err = ErrFileExists
- default:
- err = ErrInvalidOperation
+ } else {
+ node = child
}
return child
})
@@ -778,8 +822,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
- return fn
+ child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ fn = child.(*filenode)
+ return child
case *filenode:
fn = child
return child
@@ -794,68 +839,6 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return
}
-// rlookup (recursive lookup) returns the inode for the file/directory
-// with the given name (which may contain "/" separators). If no such
-// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
- node = start
- for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
- if node.IsDir() {
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- }
- node = func() inode {
- node.RLock()
- defer node.RUnlock()
- return node.Child(name, nil)
- }()
- }
- return
-}
-
-// Caller must have lock, and must have already ensured
-// Children(name,nil) is nil.
-func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- client: dn.client,
- kc: dn.kc,
- treenode: treenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
@@ -920,7 +903,7 @@ func (me *memSegment) ReadAt(p []byte, off int64) (n int, err error) {
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index 56963b6..d586531 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -97,3 +97,8 @@ func (f *filehandle) Stat() (os.FileInfo, error) {
func (f *filehandle) Close() error {
return nil
}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 66856b7..5326131 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -18,7 +18,11 @@ import (
// collections, these changes are not persistent or visible to other
// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ fs := &fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ }
root := &treenode{
+ fs: fs,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -28,8 +32,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
root.parent = root
root.Child("by_id", func(inode) inode {
- return &vdirnode{
+ var vn inode
+ vn = &vdirnode{
treenode: treenode{
+ fs: fs,
parent: root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
@@ -39,25 +45,29 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: func(name string) inode {
- return newEntByID(c, kc, name)
+ return newEntByID(vn, name)
},
}
+ return vn
})
- return &fileSystem{inode: root}
+ fs.root = root
+ return fs
}
-func newEntByID(c *Client, kc keepClient, id string) inode {
+func newEntByID(parent inode, id string) inode {
var coll Collection
- err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(c, kc)
- fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ fs, err := coll.FileSystem(parent.FS(), parent.FS())
if err != nil {
return nil
}
- return fs
+ root := fs.(*collectionFileSystem).root.(*dirnode)
+ root.fileinfo.name = id
+ root.parent = parent
+ return root
}
type vdirnode struct {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list