[ARVADOS] updated: 1.1.1-258-ge0825fa
Git user
git at public.curoverse.com
Wed Dec 20 23:05:28 EST 2017
Summary of changes:
lib/mount/fs.go | 11 ++++
sdk/go/arvados/fs_base.go | 80 +++++++++----------------
sdk/go/arvados/fs_collection.go | 129 +++++++++++++++++++++++-----------------
sdk/go/arvados/fs_site.go | 54 +++++++++++------
4 files changed, 148 insertions(+), 126 deletions(-)
discards 0a278831cc02f83d17dc57541713f540488335d2 (commit)
via e0825fa4883606d61a01e0291d669c3e40b56d3e (commit)
via 789b260e4200eb9342548134e6289e53f430e701 (commit)
via 1653a7fa7722c9698118af6c36a610249100e7ab (commit)
via a81ef188f57d9d75b207b8f982f4b90a31dd9f8f (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (0a278831cc02f83d17dc57541713f540488335d2)
\
N -- N -- N (e0825fa4883606d61a01e0291d669c3e40b56d3e)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e0825fa4883606d61a01e0291d669c3e40b56d3e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:31 2017 -0500
go-fuse: Tidy up, add comments.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 13e2a06..0d90689 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -20,6 +20,7 @@ type sharedFile struct {
sync.Mutex
}
+// keepFS implements cgofuse's FileSystemInterface.
type keepFS struct {
fuse.FileSystemBase
Client *arvados.Client
@@ -38,6 +39,8 @@ var (
invalidFH = ^uint64(0)
)
+// newFH wraps f in a sharedFile, adds it to fs's lookup table using a
+// new handle number, and returns the handle number.
func (fs *keepFS) newFH(f arvados.File) uint64 {
fs.Lock()
defer fs.Unlock()
@@ -157,6 +160,11 @@ func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
return fs.Release(path, fh)
}
+func (fs *keepFS) Rmdir(path string) int {
+ defer fs.debugPanics()
+ return fs.errCode(fs.root.Remove(path))
+}
+
func (fs *keepFS) Release(path string, fh uint64) (errc int) {
defer fs.debugPanics()
fs.Lock()
@@ -337,6 +345,9 @@ func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
return fs.Fsync(path, datasync, fh)
}
+// debugPanics (when deferred by keepFS handlers) prints an error and
+// stack trace on stderr when a handler crashes. (Without this,
+// cgofuse recovers from panics silently and returns EIO.)
func (fs *keepFS) debugPanics() {
if err := recover(); err != nil {
log.Printf("(%T) %v", err, err)
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index a64ce8c..af0068f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -21,28 +21,6 @@ import (
var maxBlockSize = 1 << 26
-type fsBackend interface {
- keepClient
- apiClient
-}
-
-// Ideally *Client would do everything; meanwhile keepBackend
-// implements fsBackend by merging the two kinds of arvados client.
-type keepBackend struct {
- keepClient
- apiClient
-}
-
-type keepClient interface {
- ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
-}
-
-type apiClient interface {
- RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -55,6 +33,11 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
+type collectionFileSystem struct {
+ fileSystem
+ uuid string
+}
+
// FileSystem returns a CollectionFileSystem for the collection.
func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
@@ -88,11 +71,6 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
return fs, nil
}
-type collectionFileSystem struct {
- fileSystem
- uuid string
-}
-
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
@@ -142,27 +120,6 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
- if dn == dn.fs.rootnode() && name == ".arvados#collection" {
- gn := &getternode{Getter: func() ([]byte, error) {
- var coll Collection
- var err error
- coll.ManifestText, err = dn.fs.MarshalManifest(".")
- if err != nil {
- return nil, err
- }
- data, err := json.Marshal(&coll)
- if err == nil {
- data = append(data, 10)
- }
- return data, err
- }}
- gn.SetParent(dn)
- return gn
- }
- return dn.treenode.Child(name, replace)
-}
-
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
@@ -550,6 +507,27 @@ func (dn *dirnode) FS() FileSystem {
return dn.fs
}
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ gn.SetParent(dn)
+ return gn
+ }
+ return dn.treenode.Child(name, replace)
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index fd842f0..974dad8 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -56,11 +56,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- return nil, ErrInvalidOperation
-}
-
-func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
commit 789b260e4200eb9342548134e6289e53f430e701
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:09 2017 -0500
go-fuse: Combine newDirnode and newFIlenode to just newNode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fe185a4..ebf2ad6 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -52,8 +52,9 @@ type FileSystem interface {
fsBackend
rootnode() inode
- newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
- newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+
+ // create a new node with nil parent.
+ newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -319,11 +320,7 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- if perm.IsDir() {
- n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
- } else {
- n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
- }
+ n, err = parent.FS().newNode(name, perm|0755, time.Now())
return n
})
if err != nil {
@@ -371,7 +368,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = n.FS().newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
return
})
if err != nil {
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 36a92af..a64ce8c 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -93,37 +93,32 @@ type collectionFileSystem struct {
uuid string
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
}
- return &dirnode{
- fs: fs,
- treenode: treenode{
+ if perm.IsDir() {
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+ } else {
+ return &filenode{
+ fs: fs,
fileinfo: fileinfo{
name: name,
- mode: perm | os.ModeDir,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
+ }, nil
}
- return &filenode{
- fs: fs,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
}
func (fs *collectionFileSystem) Sync() error {
@@ -852,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
}
node.Child(name, func(child inode) inode {
if child == nil {
- node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ node, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
} else if !child.IsDir() {
err = ErrFileExists
@@ -868,7 +863,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
fn = child.(*filenode)
return child
case *filenode:
commit 1653a7fa7722c9698118af6c36a610249100e7ab
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 17:11:07 2017 -0500
go-fuse: Clear up some type assertions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 8d987d4..fe185a4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -51,6 +51,7 @@ type FileSystem interface {
http.FileSystem
fsBackend
+ rootnode() inode
newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
@@ -83,6 +84,7 @@ type FileSystem interface {
}
type inode interface {
+ SetParent(inode)
Parent() inode
FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
@@ -204,6 +206,12 @@ func (n *treenode) FS() FileSystem {
return n.fs
}
+func (n *treenode) SetParent(p inode) {
+ n.RLock()
+ defer n.RUnlock()
+ n.parent = p
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -218,11 +226,13 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
// TODO: special treatment for "", ".", ".."
child = n.inodes[name]
if replace != nil {
- child = replace(child)
- if child == nil {
+ newchild := replace(child)
+ if newchild == nil {
delete(n.inodes, name)
- } else {
- n.inodes[name] = child
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ newchild.SetParent(n)
+ child = newchild
}
}
return
@@ -254,6 +264,10 @@ type fileSystem struct {
fsBackend
}
+func (fs *fileSystem) rootnode() inode {
+ return fs.root
+}
+
// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.openFile(name, flag, perm)
@@ -306,9 +320,9 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
var err error
n = parent.Child(name, func(inode) inode {
if perm.IsDir() {
- n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -357,7 +371,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = fs.newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -404,12 +418,12 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// When acquiring locks on multiple nodes, all common
// ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
+ // deadlock. This is assured by locking the path from
+ // filesystem root to newdir, then locking the path from
+ // filesystem root to olddir, skipping any already-locked
+ // nodes.
needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
+ for _, node := range []inode{olddirf.inode, newdirf.inode} {
needLock = append(needLock, node)
for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
@@ -499,41 +513,6 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
return err
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- treenode: treenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
func (fs *fileSystem) Sync() error {
log.Printf("TODO: sync fileSystem")
return ErrInvalidOperation
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fc00335..36a92af 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -64,14 +64,14 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
modTime = *c.ModifiedAt
}
fs := &collectionFileSystem{
+ uuid: c.UUID,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: client, keepClient: kc},
},
- uuid: c.UUID,
}
- dn := &dirnode{
+ root := &dirnode{
+ fs: fs,
treenode: treenode{
- fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
@@ -80,11 +80,11 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
inodes: make(map[string]inode),
},
}
- dn.parent = dn
- fs.fileSystem.root = dn
- if err := dn.loadManifest(c.ManifestText); err != nil {
+ root.SetParent(root)
+ if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ fs.root = root
return fs, nil
}
@@ -93,6 +93,39 @@ type collectionFileSystem struct {
uuid string
}
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: fs,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
func (fs *collectionFileSystem) Sync() error {
log.Printf("cfs.Sync()")
if fs.uuid == "" {
@@ -114,12 +147,12 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
- if name == ".arvados#collection" {
- return &getternode{Getter: func() ([]byte, error) {
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
var err error
- coll.ManifestText, err = fs.MarshalManifest(".")
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
if err != nil {
return nil, err
}
@@ -129,8 +162,10 @@ func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) in
}
return data, err
}}
+ gn.SetParent(dn)
+ return gn
}
- return fs.fileSystem.root.Child(name, replace)
+ return dn.treenode.Child(name, replace)
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
@@ -232,6 +267,12 @@ func (fn *filenode) appendSegment(e segment) {
fn.fileinfo.size += int64(e.Len())
}
+func (fn *filenode) SetParent(p inode) {
+ fn.RLock()
+ defer fn.RUnlock()
+ fn.parent = p
+}
+
func (fn *filenode) Parent() inode {
fn.RLock()
defer fn.RUnlock()
@@ -496,7 +537,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.(fsBackend),
+ kc: fn.parent.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -506,9 +547,14 @@ func (fn *filenode) pruneMemSegments() {
}
type dirnode struct {
+ fs *collectionFileSystem
treenode
}
+func (dn *dirnode) FS() FileSystem {
+ return dn.fs
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
@@ -801,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
// can't be sure parent will be a *dirnode
return nil, ErrInvalidArgument
}
- node = node.Parent().(*dirnode)
+ node = node.Parent()
continue
}
node.Child(name, func(child inode) inode {
@@ -832,7 +878,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
err = ErrIsDirectory
return child
default:
- err = ErrInvalidOperation
+ err = ErrInvalidArgument
return child
}
})
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 5326131..fd842f0 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,17 +9,21 @@ import (
"time"
)
+type siteFileSystem struct {
+ fileSystem
+}
+
// SiteFileSystem returns a FileSystem that maps collections and other
// Arvados objects onto a filesystem layout.
//
// This is experimental: the filesystem layout is not stable, and
// there are significant known bugs and shortcomings. For example,
-// although the FileSystem allows files to be added and modified in
-// collections, these changes are not persistent or visible to other
-// Arvados clients.
+// writes are not persisted until Sync() is called.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
- fs := &fileSystem{
- fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ fs := &siteFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ },
}
root := &treenode{
fs: fs,
@@ -34,7 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root.Child("by_id", func(inode) inode {
var vn inode
vn = &vdirnode{
- treenode: treenode{
+ inode: &treenode{
fs: fs,
parent: root,
inodes: make(map[string]inode),
@@ -44,9 +48,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
mode: 0755 | os.ModeDir,
},
},
- create: func(name string) inode {
- return newEntByID(vn, name)
- },
+ create: fs.mountCollection,
}
return vn
})
@@ -54,33 +56,51 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func newEntByID(parent inode, id string) inode {
+func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
- err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(parent.FS(), parent.FS())
+ cfs, err := coll.FileSystem(fs, fs)
if err != nil {
return nil
}
- root := fs.(*collectionFileSystem).root.(*dirnode)
- root.fileinfo.name = id
- root.parent = parent
+ root := cfs.rootnode()
+ root.SetParent(parent)
+ root.(*dirnode).fileinfo.name = id
return root
}
+// vdirnode wraps an inode by ignoring any requests to add/replace
+// children, and calling a create() func when a non-existing child is
+// looked up.
+//
+// create() can return either a new node, which will be added to the
+// treenode, or nil for ENOENT.
type vdirnode struct {
- treenode
- create func(string) inode
+ inode
+ create func(parent inode, name string) inode
}
func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
- return vn.treenode.Child(name, func(existing inode) inode {
+ return vn.inode.Child(name, func(existing inode) inode {
if existing != nil {
return existing
} else {
- return vn.create(name)
+ n := vn.create(vn, name)
+ if n != nil {
+ n.SetParent(vn)
+ }
+ return n
}
})
}
commit a81ef188f57d9d75b207b8f982f4b90a31dd9f8f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 19 09:30:56 2017 -0500
go-fuse: Save on sync().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 8e488f5..13e2a06 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -2,7 +2,9 @@ package mount
import (
"io"
+ "log"
"os"
+ "runtime/debug"
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -55,10 +57,12 @@ func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
}
func (fs *keepFS) Init() {
+ defer fs.debugPanics()
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
}
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS, invalidFH
}
@@ -72,6 +76,7 @@ func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint
}
func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
return -fuse.EROFS, invalidFH
}
@@ -88,6 +93,7 @@ func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
}
func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -120,6 +126,7 @@ func (fs *keepFS) errCode(err error) int {
}
func (fs *keepFS) Mkdir(path string, mode uint32) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -132,6 +139,7 @@ func (fs *keepFS) Mkdir(path string, mode uint32) int {
}
func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ defer fs.debugPanics()
f, err := fs.root.OpenFile(path, 0, 0)
if err != nil {
return fs.errCode(err), invalidFH
@@ -145,10 +153,12 @@ func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
}
func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
return fs.Release(path, fh)
}
func (fs *keepFS) Release(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
fs.Lock()
defer fs.Unlock()
defer delete(fs.open, fh)
@@ -162,6 +172,7 @@ func (fs *keepFS) Release(path string, fh uint64) (errc int) {
}
func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -169,6 +180,7 @@ func (fs *keepFS) Rename(oldname, newname string) (errc int) {
}
func (fs *keepFS) Unlink(path string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -176,6 +188,7 @@ func (fs *keepFS) Unlink(path string) (errc int) {
}
func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -196,6 +209,7 @@ func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
}
func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ defer fs.debugPanics()
var fi os.FileInfo
var err error
if f := fs.lookupFH(fh); f != nil {
@@ -218,7 +232,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
if fi, err := fs.root.Stat(path); err != nil {
return fs.errCode(err)
- } else if (os.FileMode(mode)^fi.Mode())&os.ModePerm != 0 {
+ } else if mode & ^uint32(fuse.S_IFREG|fuse.S_IFDIR|0777) != 0 || (fi.Mode()&os.ModeDir != 0) != (mode&fuse.S_IFDIR != 0) {
return -fuse.ENOSYS
} else {
return 0
@@ -226,6 +240,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ defer fs.debugPanics()
var m uint32
if fi.IsDir() {
m = m | fuse.S_IFDIR
@@ -252,6 +267,7 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -270,6 +286,7 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -288,6 +305,7 @@ func (fs *keepFS) Readdir(path string,
fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
ofst int64,
fh uint64) (errc int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -305,3 +323,24 @@ func (fs *keepFS) Readdir(path string,
}
return 0
}
+
+func (fs *keepFS) Fsync(path string, datasync bool, fh uint64) int {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ return fs.errCode(f.Sync())
+}
+
+func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
+ return fs.Fsync(path, datasync, fh)
+}
+
+func (fs *keepFS) debugPanics() {
+ if err := recover(); err != nil {
+ log.Printf("(%T) %v", err, err)
+ debug.PrintStack()
+ panic(err)
+ }
+}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 3be3f5e..8d987d4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path"
@@ -40,6 +41,7 @@ type File interface {
Readdir(int) ([]os.FileInfo, error)
Stat() (os.FileInfo, error)
Truncate(int64) error
+ Sync() error
}
// A FileSystem is an http.Filesystem plus Stat() and support for
@@ -47,8 +49,10 @@ type File interface {
// goroutines.
type FileSystem interface {
http.FileSystem
+ fsBackend
- inode
+ newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+ newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -75,10 +79,12 @@ type FileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+ Sync() error
}
type inode interface {
Parent() inode
+ FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
@@ -186,6 +192,7 @@ func (*nullnode) Child(name string, replace func(inode) inode) inode {
}
type treenode struct {
+ fs FileSystem
parent inode
inodes map[string]inode
fileinfo fileinfo
@@ -193,6 +200,10 @@ type treenode struct {
nullnode
}
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -239,7 +250,8 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
type fileSystem struct {
- inode
+ root inode
+ fsBackend
}
// OpenFile is analogous to os.OpenFile().
@@ -248,12 +260,11 @@ func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, e
}
func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
+ parent := rlookup(fs.root, dirname)
if parent == nil {
return nil, os.ErrNotExist
}
@@ -294,20 +305,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -322,10 +323,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
} else if flag&os.O_TRUNC != 0 {
if !writable {
return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
+ } else if n.IsDir() {
return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
}
}
return &filehandle{
@@ -346,7 +347,7 @@ func (fs *fileSystem) Create(name string) (File, error) {
func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
+ n := rlookup(fs.root, dirname)
if n == nil {
return os.ErrNotExist
}
@@ -355,12 +356,8 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
if n.Child(name, nil) != nil {
return os.ErrExist
}
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
+ child, err = fs.newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -372,7 +369,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
+ node := rlookup(fs.root, name)
if node == nil {
err = os.ErrNotExist
} else {
@@ -414,7 +411,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
for _, f := range []*filehandle{olddirf, newdirf} {
node := f.inode
needLock = append(needLock, node)
- for node.Parent() != node {
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
needLock = append(needLock, node)
}
@@ -428,10 +425,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
err = nil
olddirf.inode.Child(oldname, func(oldinode inode) inode {
if oldinode == nil {
@@ -450,20 +443,18 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
oldinode.Lock()
defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
+ n.fileinfo.name = newname
case *filenode:
- n.parent = newdn
+ n.parent = newdirf.inode
n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
+ //TODO: olddirf.setModTime(time.Now())
+ //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
@@ -488,7 +479,7 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs, dirname)
+ dir := rlookup(fs.root, dirname)
if dir == nil {
return os.ErrNotExist
}
@@ -507,3 +498,70 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
})
return err
}
+
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ treenode: treenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
+ }
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ }
+ return
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e451189..fc00335 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"os"
"path"
"regexp"
@@ -20,11 +21,28 @@ import (
var maxBlockSize = 1 << 26
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
+
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -38,28 +56,32 @@ type CollectionFileSystem interface {
}
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
} else {
modTime = *c.ModifiedAt
}
+ fs := &collectionFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ uuid: c.UUID,
+ }
dn := &dirnode{
- client: client,
- kc: kc,
treenode: treenode{
+ fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
},
- parent: nil,
inodes: make(map[string]inode),
},
}
dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ fs.fileSystem.root = dn
if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -68,9 +90,31 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
type collectionFileSystem struct {
fileSystem
+ uuid string
}
-func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
+}
+
+func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
if name == ".arvados#collection" {
return &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -86,13 +130,13 @@ func (fs collectionFileSystem) Child(name string, replace func(inode) inode) ino
return data, err
}}
}
- return fs.fileSystem.Child(name, replace)
+ return fs.fileSystem.root.Child(name, replace)
}
-func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
- fs.fileSystem.inode.Lock()
- defer fs.fileSystem.inode.Unlock()
- return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -170,8 +214,9 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
// filenode implements inode.
type filenode struct {
+ parent inode
+ fs FileSystem
fileinfo fileinfo
- parent *dirnode
segments []segment
// number of times `segments` has changed in a
// way that might invalidate a filenodePtr
@@ -193,6 +238,10 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -438,7 +487,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -447,7 +496,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.parent.(fsBackend),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -458,8 +507,6 @@ func (fn *filenode) pruneMemSegments() {
type dirnode struct {
treenode
- client *Client
- kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -480,7 +527,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
@@ -488,7 +535,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
@@ -709,7 +756,7 @@ func (dn *dirnode) loadManifest(txt string) error {
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
@@ -738,7 +785,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
- node := dn
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -758,16 +805,13 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
continue
}
node.Child(name, func(child inode) inode {
- switch child.(type) {
- case nil:
- node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if child == nil {
+ node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
- case *dirnode:
- node = child.(*dirnode)
- case *filenode:
+ } else if !child.IsDir() {
err = ErrFileExists
- default:
- err = ErrInvalidOperation
+ } else {
+ node = child
}
return child
})
@@ -778,8 +822,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
- return fn
+ child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ fn = child.(*filenode)
+ return child
case *filenode:
fn = child
return child
@@ -794,68 +839,6 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return
}
-// rlookup (recursive lookup) returns the inode for the file/directory
-// with the given name (which may contain "/" separators). If no such
-// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
- node = start
- for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
- if node.IsDir() {
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- }
- node = func() inode {
- node.RLock()
- defer node.RUnlock()
- return node.Child(name, nil)
- }()
- }
- return
-}
-
-// Caller must have lock, and must have already ensured
-// Children(name,nil) is nil.
-func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- client: dn.client,
- kc: dn.kc,
- treenode: treenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
@@ -920,7 +903,7 @@ func (me *memSegment) ReadAt(p []byte, off int64) (n int, err error) {
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index 56963b6..d586531 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -97,3 +97,8 @@ func (f *filehandle) Stat() (os.FileInfo, error) {
func (f *filehandle) Close() error {
return nil
}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 66856b7..5326131 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -18,7 +18,11 @@ import (
// collections, these changes are not persistent or visible to other
// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ fs := &fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ }
root := &treenode{
+ fs: fs,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -28,8 +32,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
root.parent = root
root.Child("by_id", func(inode) inode {
- return &vdirnode{
+ var vn inode
+ vn = &vdirnode{
treenode: treenode{
+ fs: fs,
parent: root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
@@ -39,25 +45,29 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: func(name string) inode {
- return newEntByID(c, kc, name)
+ return newEntByID(vn, name)
},
}
+ return vn
})
- return &fileSystem{inode: root}
+ fs.root = root
+ return fs
}
-func newEntByID(c *Client, kc keepClient, id string) inode {
+func newEntByID(parent inode, id string) inode {
var coll Collection
- err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(c, kc)
- fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ fs, err := coll.FileSystem(parent.FS(), parent.FS())
if err != nil {
return nil
}
- return fs
+ root := fs.(*collectionFileSystem).root.(*dirnode)
+ root.fileinfo.name = id
+ root.parent = parent
+ return root
}
type vdirnode struct {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list