[ARVADOS] created: 1.1.1-245-g7054b8a
Git user
git at public.curoverse.com
Sat Dec 16 03:17:17 EST 2017
at 7054b8ac176da77107f4195ddce2b46aacd2af81 (commit)
commit 7054b8ac176da77107f4195ddce2b46aacd2af81
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 09:22:03 2017 -0500
go-fuse: Mount collections on demand in mnt/by_id/.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/base_fs.go
new file mode 100644
index 0000000..4de1bf8
--- /dev/null
+++ b/sdk/go/arvados/base_fs.go
@@ -0,0 +1,97 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+)
+
+type nullnode struct{}
+
+func (*nullnode) Mkdir(string, os.FileMode) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) Read([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Write([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Truncate(int64) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) FileInfo() os.FileInfo {
+ return fileinfo{}
+}
+
+func (*nullnode) IsDir() bool {
+ return false
+}
+
+func (*nullnode) Readdir() []os.FileInfo {
+ return nil
+}
+
+func (*nullnode) Child(name string, replace func(inode) inode) inode {
+ return nil
+}
+
+type treenode struct {
+ parent inode
+ inodes map[string]inode
+ fileinfo fileinfo
+ sync.RWMutex
+ nullnode
+}
+
+func (n *treenode) Parent() inode {
+ n.RLock()
+ defer n.RUnlock()
+ return n.parent
+}
+
+func (n *treenode) IsDir() bool {
+ return true
+}
+
+func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
+ // TODO: special treatment for "", ".", ".."
+ child = n.inodes[name]
+ if replace != nil {
+ child = replace(child)
+ if child == nil {
+ delete(n.inodes, name)
+ } else {
+ n.inodes[name] = child
+ }
+ }
+ return
+}
+
+func (n *treenode) Size() int64 {
+ return n.FileInfo().Size()
+}
+
+func (n *treenode) FileInfo() os.FileInfo {
+ n.Lock()
+ defer n.Unlock()
+ n.fileinfo.size = int64(len(n.inodes))
+ return n.fileinfo
+}
+
+func (n *treenode) Readdir() (fi []os.FileInfo) {
+ n.RLock()
+ defer n.RUnlock()
+ fi = make([]os.FileInfo, 0, len(n.inodes))
+ for _, inode := range n.inodes {
+ fi = append(fi, inode.FileInfo())
+ }
+ return
+}
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 7bbbaa4..dcfd098 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -89,12 +89,14 @@ func (fi fileinfo) Sys() interface{} {
return nil
}
-// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files. All methods are safe to call
-// from multiple goroutines.
-type CollectionFileSystem interface {
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
http.FileSystem
+ inode
+
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -120,6 +122,12 @@ type CollectionFileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+}
+
+// A CollectionFileSystem is a FileSystem that can be serialized as a
+// manifest and stored as a collection.
+type CollectionFileSystem interface {
+ FileSystem
// Flush all file data to Keep and return a snapshot of the
// filesystem suitable for saving as (Collection)ManifestText.
@@ -129,39 +137,288 @@ type CollectionFileSystem interface {
}
type fileSystem struct {
- dirnode
+ inode
+}
+
+type collectionFileSystem struct {
+ fileSystem
}
+func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.inode.Lock()
+ defer fs.fileSystem.inode.Unlock()
+ return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+}
+
+// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.dirnode.OpenFile(name, flag, perm)
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
}
func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.dirnode.OpenFile(name, os.O_RDONLY, 0)
+ return fs.OpenFile(name, os.O_RDONLY, 0)
}
func (fs *fileSystem) Create(name string) (File, error) {
- return fs.dirnode.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := fs.dirnode.lookupPath(name)
+ node := rlookup(fs.inode, name)
if node == nil {
err = os.ErrNotExist
} else {
- fi = node.Stat()
+ fi = node.FileInfo()
}
return
}
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ case *filenode:
+ n.parent = newdirf.inode.(*dirnode)
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
+
type inode interface {
Parent() inode
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
+ IsDir() bool
Readdir() []os.FileInfo
Size() int64
- Stat() os.FileInfo
+ FileInfo() os.FileInfo
+ // Caller must have lock (or rlock if func is nil)
+ Child(string, func(inode) inode) inode
sync.Locker
RLock()
RUnlock()
@@ -177,6 +434,7 @@ type filenode struct {
repacked int64
memsize int64 // bytes in memSegments
sync.RWMutex
+ nullnode
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -264,10 +522,6 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
-func (fn *filenode) Readdir() []os.FileInfo {
- return nil
-}
-
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -302,7 +556,7 @@ func (fn *filenode) Size() int64 {
return fn.fileinfo.Size()
}
-func (fn *filenode) Stat() os.FileInfo {
+func (fn *filenode) FileInfo() os.FileInfo {
fn.RLock()
defer fn.RUnlock()
return fn.fileinfo
@@ -539,19 +793,22 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
} else {
modTime = *c.ModifiedAt
}
- fs := &fileSystem{dirnode: dirnode{
+ dn := &dirnode{
client: client,
kc: kc,
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
},
- parent: nil,
- inodes: make(map[string]inode),
- }}
- fs.dirnode.parent = &fs.dirnode
- if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
return fs, nil
@@ -622,7 +879,7 @@ func (f *filehandle) Write(p []byte) (n int, err error) {
}
func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.Stat().IsDir() {
+ if !f.inode.IsDir() {
return nil, ErrInvalidOperation
}
if count <= 0 {
@@ -643,7 +900,7 @@ func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
}
func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.Stat(), nil
+ return f.inode.FileInfo(), nil
}
func (f *filehandle) Close() error {
@@ -651,12 +908,9 @@ func (f *filehandle) Close() error {
}
type dirnode struct {
- fileinfo fileinfo
- parent *dirnode
- client *Client
- kc keepClient
- inodes map[string]inode
- sync.RWMutex
+ treenode
+ client *Client
+ kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -735,12 +989,6 @@ func (dn *dirnode) sync() error {
return flush(pending)
}
-func (dn *dirnode) MarshalManifest(prefix string) (string, error) {
- dn.Lock()
- defer dn.Unlock()
- return dn.marshalManifest(prefix)
-}
-
// caller must have read lock.
func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var streamLen int64
@@ -941,6 +1189,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
+ node := dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -950,332 +1199,111 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
for _, name := range names[:len(names)-1] {
switch name {
case "", ".":
+ continue
case "..":
- dn = dn.parent
- default:
- switch node := dn.inodes[name].(type) {
+ if node == dn {
+ // can't be sure parent will be a *dirnode
+ return nil, ErrInvalidArgument
+ }
+ node = node.Parent().(*dirnode)
+ continue
+ }
+ node.Child(name, func(child inode) inode {
+ switch child.(type) {
case nil:
- dn = dn.newDirnode(name, 0755, dn.fileinfo.modTime)
+ node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child = node
case *dirnode:
- dn = node
+ node = child.(*dirnode)
case *filenode:
err = ErrFileExists
- return
+ default:
+ err = ErrInvalidOperation
}
+ return child
+ })
+ if err != nil {
+ return
}
}
- switch node := dn.inodes[basename].(type) {
- case nil:
- fn = dn.newFilenode(basename, 0755, dn.fileinfo.modTime)
- case *filenode:
- fn = node
- case *dirnode:
- err = ErrIsDirectory
- }
- return
-}
-
-func (dn *dirnode) mkdir(name string) (*filehandle, error) {
- return dn.OpenFile(name, os.O_CREATE|os.O_EXCL, os.ModeDir|0755)
-}
-
-func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
- f, err := dn.mkdir(name)
- if err == nil {
- err = f.Close()
- }
- return err
-}
-
-func (dn *dirnode) Remove(name string) error {
- return dn.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (dn *dirnode) RemoveAll(name string) error {
- err := dn.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (dn *dirnode) remove(name string, recursive bool) error {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return os.ErrNotExist
- }
- dn.Lock()
- defer dn.Unlock()
- switch node := dn.inodes[name].(type) {
- case nil:
- return os.ErrNotExist
- case *dirnode:
- node.RLock()
- defer node.RUnlock()
- if !recursive && len(node.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ node.Child(basename, func(child inode) inode {
+ switch child := child.(type) {
+ case nil:
+ fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ return fn
+ case *filenode:
+ fn = child
+ return child
+ case *dirnode:
+ err = ErrIsDirectory
+ return child
+ default:
+ err = ErrInvalidOperation
+ return child
}
- }
- delete(dn.inodes, name)
- return nil
+ })
+ return
}
-func (dn *dirnode) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := dn.OpenFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := dn.OpenFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
}
- }
-
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- oldinode, ok := olddn.inodes[oldname]
- if !ok {
- return os.ErrNotExist
- }
- if existing, ok := newdn.inodes[newname]; ok {
- // overwriting an existing file or dir
- if dn, ok := existing.(*dirnode); ok {
- if !oldinode.Stat().IsDir() {
- return ErrIsDirectory
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
}
- dn.RLock()
- defer dn.RUnlock()
- if len(dn.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ if name == ".." {
+ node = node.Parent()
+ continue
}
}
- } else {
- if newdn.inodes == nil {
- newdn.inodes = make(map[string]inode)
- }
- newdn.fileinfo.size++
- }
- newdn.inodes[newname] = oldinode
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdn
- case *filenode:
- n.parent = newdn
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- delete(olddn.inodes, oldname)
- olddn.fileinfo.size--
- return nil
-}
-
-func (dn *dirnode) Parent() inode {
- dn.RLock()
- defer dn.RUnlock()
- return dn.parent
-}
-
-func (dn *dirnode) Readdir() (fi []os.FileInfo) {
- dn.RLock()
- defer dn.RUnlock()
- fi = make([]os.FileInfo, 0, len(dn.inodes))
- for _, inode := range dn.inodes {
- fi = append(fi, inode.Stat())
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
}
return
}
-func (dn *dirnode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Write(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Size() int64 {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo.Size()
-}
-
-func (dn *dirnode) Stat() os.FileInfo {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo
-}
-
-func (dn *dirnode) Truncate(int64) error {
- return ErrInvalidOperation
-}
-
-// lookupPath returns the inode for the file/directory with the given
-// name (which may contain "/" separators), along with its parent
-// node. If no such file/directory exists, the returned node is nil.
-func (dn *dirnode) lookupPath(path string) (node inode) {
- node = dn
- for _, name := range strings.Split(path, "/") {
- dn, ok := node.(*dirnode)
- if !ok {
- return nil
- }
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- dn.RLock()
- node = dn.inodes[name]
- dn.RUnlock()
+// Caller must have lock, and must have already ensured
+// Children(name,nil) is nil.
+func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
}
- return
-}
-
-func (dn *dirnode) newDirnode(name string, perm os.FileMode, modTime time.Time) *dirnode {
- child := &dirnode{
- parent: dn,
+ return &dirnode{
client: dn.client,
kc: dn.kc,
- fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | perm,
- modTime: modTime,
+ treenode: treenode{
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
+ }, nil
}
-func (dn *dirnode) newFilenode(name string, perm os.FileMode, modTime time.Time) *filenode {
- child := &filenode{
- parent: dn,
+func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ parent: parent,
fileinfo: fileinfo{
name: name,
- mode: perm,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
-}
-
-// OpenFile is analogous to os.OpenFile().
-func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: dn}, nil
- case "..":
- return &filehandle{inode: dn.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- dn.Lock()
- defer dn.Unlock()
- } else {
- dn.RLock()
- defer dn.RUnlock()
- }
- n, ok := dn.inodes[name]
- if !ok {
- if !createMode {
- return nil, os.ErrNotExist
- }
- if perm.IsDir() {
- n = dn.newDirnode(name, 0755, time.Now())
- } else {
- n = dn.newFilenode(name, 0755, time.Now())
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
}, nil
}
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 57ba325..836335f 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -465,6 +465,10 @@ func (s *CollectionFSSuite) TestMkdir(c *check.C) {
}
func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
maxBlockSize = 8
defer func() { maxBlockSize = 2 << 26 }()
@@ -990,6 +994,10 @@ var _ = check.Suite(&CollectionFSUnitSuite{})
// expect ~2 seconds to load a manifest with 256K files
func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
const (
dirCount = 512
fileCount = 512
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/site_fs.go
new file mode 100644
index 0000000..853bd32
--- /dev/null
+++ b/sdk/go/arvados/site_fs.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "time"
+)
+
+func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ root := &treenode{
+ fileinfo: fileinfo{
+ name: "/",
+ mode: os.ModeDir | 0755,
+ modTime: time.Now(),
+ },
+ inodes: make(map[string]inode),
+ }
+ root.parent = root
+ root.Child("by_id", func(inode) inode {
+ return &vdirnode{
+ treenode: treenode{
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: "by_id",
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ create: func(name string) inode {
+ return newEntByID(c, kc, name)
+ },
+ }
+ })
+ return &fileSystem{inode: root}
+}
+
+func newEntByID(c *Client, kc keepClient, id string) inode {
+ var coll Collection
+ err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ if err != nil {
+ return nil
+ }
+ fs, err := coll.FileSystem(c, kc)
+ fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ if err != nil {
+ return nil
+ }
+ return fs
+}
+
+type vdirnode struct {
+ treenode
+ create func(string) inode
+}
+
+func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
+ return vn.treenode.Child(name, func(existing inode) inode {
+ if existing != nil {
+ return existing
+ } else {
+ return vn.create(name)
+ }
+ })
+}
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/site_fs_test.go
new file mode 100644
index 0000000..a8c369f
--- /dev/null
+++ b/sdk/go/arvados/site_fs_test.go
@@ -0,0 +1,58 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&SiteFSSuite{})
+
+type SiteFSSuite struct {
+ client *Client
+ fs FileSystem
+ kc keepClient
+}
+
+func (s *SiteFSSuite) SetUpTest(c *check.C) {
+ s.client = NewClientFromEnv()
+ s.kc = &keepClientStub{
+ blocks: map[string][]byte{
+ "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
+ }}
+ s.fs = s.client.SiteFileSystem(s.kc)
+}
+
+func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
+ _, ok := s.fs.(http.FileSystem)
+ c.Check(ok, check.Equals, true)
+}
+
+func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Equals, 0)
+}
+
+func (s *SiteFSSuite) TestByUUID(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 0)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.FooCollection)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(-1)
+ var names []string
+ for _, fi := range fis {
+ names = append(names, fi.Name())
+ }
+ c.Check(names, check.DeepEquals, []string{"foo"})
+}
diff --git a/services/mount/fs.go b/services/mount/fs.go
index a5f4082..4789976 100644
--- a/services/mount/fs.go
+++ b/services/mount/fs.go
@@ -6,12 +6,17 @@ import (
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
"github.com/billziss-gh/cgofuse/fuse"
)
type keepFS struct {
fuse.FileSystemBase
- root arvados.CollectionFileSystem
+ Collection arvados.Collection
+ Client *arvados.Client
+ KeepClient *keepclient.KeepClient
+
+ root arvados.FileSystem
open map[uint64]arvados.File
lastFH uint64
sync.Mutex
@@ -33,10 +38,16 @@ func (fs *keepFS) newFH(f arvados.File) uint64 {
return fh
}
+func (fs *keepFS) Init() {
+ fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
+}
+
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
- if err != nil {
- return -fuse.EPERM, invalidFH
+ if err == os.ErrExist {
+ return -fuse.EEXIST, invalidFH
+ } else if err != nil {
+ return -fuse.EINVAL, invalidFH
}
return 0, fs.newFH(f)
}
@@ -102,7 +113,28 @@ func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
m = m | uint32(fi.Mode()&os.ModePerm)
stat.Mode = m
+ stat.Nlink = 1
stat.Size = fi.Size()
+ t := fuse.NewTimespec(fi.ModTime())
+ stat.Mtim = t
+ stat.Ctim = t
+ stat.Atim = t
+ stat.Birthtim = t
+ stat.Blksize = 1024
+ stat.Blocks = (stat.Size + stat.Blksize - 1) / stat.Blksize
+}
+
+func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return 0
+ }
+ _, err := f.Seek(ofst, io.SeekStart)
+ if err != nil {
+ return 0
+ }
+ n, _ = f.Write(buf)
+ return
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
@@ -114,7 +146,7 @@ func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
if err != nil {
return 0
}
- n, err = f.Read(buf)
+ n, _ = f.Read(buf)
return
}
diff --git a/services/mount/main.go b/services/mount/main.go
index 6f31739..c0ef380 100644
--- a/services/mount/main.go
+++ b/services/mount/main.go
@@ -1,8 +1,8 @@
package main
import (
- "flag"
"log"
+ "os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -11,14 +11,7 @@ import (
)
func main() {
- var coll arvados.Collection
- flag.StringVar(&coll.UUID, "id", "", "collection `uuid` or pdh")
- flag.Parse()
client := arvados.NewClientFromEnv()
- err := client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
- if err != nil {
- log.Fatal(err)
- }
ac, err := arvadosclient.New(client)
if err != nil {
log.Fatal(err)
@@ -27,10 +20,9 @@ func main() {
if err != nil {
log.Fatal(err)
}
- fs, err := coll.FileSystem(client, kc)
- if err != nil {
- log.Fatal(err)
- }
- host := fuse.NewFileSystemHost(&keepFS{root: fs})
- host.Mount("", flag.Args())
+ host := fuse.NewFileSystemHost(&keepFS{
+ Client: client,
+ KeepClient: kc,
+ })
+ host.Mount("", os.Args[1:])
}
commit 49e07ee61634db9839fa3c7409bc6994b579dd74
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 01:14:43 2017 -0500
go-fuse: Mount a single collection readonly.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/mount/.gitignore b/services/mount/.gitignore
new file mode 100644
index 0000000..8c9846d
--- /dev/null
+++ b/services/mount/.gitignore
@@ -0,0 +1 @@
+mount-*
diff --git a/services/mount/Makefile b/services/mount/Makefile
new file mode 100644
index 0000000..3730673
--- /dev/null
+++ b/services/mount/Makefile
@@ -0,0 +1,5 @@
+all:
+ go get .
+ docker build --tag=cgofuse $(GOPATH)/src/github.com/billziss-gh/cgofuse
+ go get github.com/karalabe/xgo
+ xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
diff --git a/services/mount/fs.go b/services/mount/fs.go
new file mode 100644
index 0000000..a5f4082
--- /dev/null
+++ b/services/mount/fs.go
@@ -0,0 +1,148 @@
+package main
+
+import (
+ "io"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+type keepFS struct {
+ fuse.FileSystemBase
+ root arvados.CollectionFileSystem
+ open map[uint64]arvados.File
+ lastFH uint64
+ sync.Mutex
+}
+
+var (
+ invalidFH = ^uint64(0)
+)
+
+func (fs *keepFS) newFH(f arvados.File) uint64 {
+ fs.Lock()
+ defer fs.Unlock()
+ if fs.open == nil {
+ fs.open = make(map[uint64]arvados.File)
+ }
+ fs.lastFH++
+ fh := fs.lastFH
+ fs.open[fh] = f
+ return fh
+}
+
+func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
+ if err != nil {
+ return -fuse.EPERM, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, flags, 0)
+ if err != nil {
+ return -fuse.ENOENT, invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return -fuse.EIO, invalidFH
+ } else if fi.IsDir() {
+ f.Close()
+ return -fuse.EISDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, 0, 0)
+ if err != nil {
+ return -fuse.ENOENT, invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return -fuse.EIO, invalidFH
+ } else if !fi.IsDir() {
+ f.Close()
+ return -fuse.ENOTDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Releasedir(path string, fh uint64) int {
+ return fs.Release(path, fh)
+}
+
+func (fs *keepFS) Release(path string, fh uint64) int {
+ fs.Lock()
+ defer fs.Unlock()
+ defer delete(fs.open, fh)
+ if f := fs.open[fh]; f != nil {
+ err := f.Close()
+ if err != nil {
+ return -fuse.EIO
+ }
+ }
+ return 0
+}
+
+func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ fi, err := fs.root.Stat(path)
+ if err != nil {
+ return -fuse.ENOENT
+ }
+ fs.fillStat(stat, fi)
+ return 0
+}
+
+func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ var m uint32
+ if fi.IsDir() {
+ m = m | fuse.S_IFDIR
+ } else {
+ m = m | fuse.S_IFREG
+ }
+ m = m | uint32(fi.Mode()&os.ModePerm)
+ stat.Mode = m
+ stat.Size = fi.Size()
+}
+
+func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return 0
+ }
+ _, err := f.Seek(ofst, io.SeekStart)
+ if err != nil {
+ return 0
+ }
+ n, err = f.Read(buf)
+ return
+}
+
+func (fs *keepFS) Readdir(path string,
+ fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
+ ofst int64,
+ fh uint64) (errc int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ fill(".", nil, 0)
+ fill("..", nil, 0)
+ var stat fuse.Stat_t
+ fis, err := f.Readdir(-1)
+ if err != nil {
+ return -fuse.ENOSYS // ???
+ }
+ for _, fi := range fis {
+ fs.fillStat(&stat, fi)
+ //fill(fi.Name(), &stat, 0)
+ fill(fi.Name(), nil, 0)
+ }
+ return 0
+}
+
+func (fs *keepFS) lookupFH(fh uint64) arvados.File {
+ fs.Lock()
+ defer fs.Unlock()
+ return fs.open[fh]
+}
diff --git a/services/mount/fs_test.go b/services/mount/fs_test.go
new file mode 100644
index 0000000..fadda1a
--- /dev/null
+++ b/services/mount/fs_test.go
@@ -0,0 +1,7 @@
+package main
+
+import (
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+var _ fuse.FileSystem = &keepFS{}
diff --git a/services/mount/main.go b/services/mount/main.go
new file mode 100644
index 0000000..6f31739
--- /dev/null
+++ b/services/mount/main.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+ "flag"
+ "log"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+func main() {
+ var coll arvados.Collection
+ flag.StringVar(&coll.UUID, "id", "", "collection `uuid` or pdh")
+ flag.Parse()
+ client := arvados.NewClientFromEnv()
+ err := client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ log.Fatal(err)
+ }
+ kc, err := keepclient.MakeKeepClient(ac)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fs, err := coll.FileSystem(client, kc)
+ if err != nil {
+ log.Fatal(err)
+ }
+ host := fuse.NewFileSystemHost(&keepFS{root: fs})
+ host.Mount("", flag.Args())
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list