[ARVADOS] updated: 1.1.1-254-g600421b
Git user
git at public.curoverse.com
Mon Dec 18 03:03:38 EST 2017
Summary of changes:
lib/mount/fs.go | 3 +-
sdk/go/arvados/base_fs.go | 97 ----
sdk/go/arvados/fs_base.go | 509 ++++++++++++++++++
.../arvados/{collection_fs.go => fs_collection.go} | 577 +++------------------
...collection_fs_test.go => fs_collection_test.go} | 0
sdk/go/arvados/fs_filehandle.go | 99 ++++
sdk/go/arvados/fs_getternode.go | 66 +++
sdk/go/arvados/{site_fs.go => fs_site.go} | 8 +
.../arvados/{site_fs_test.go => fs_site_test.go} | 0
9 files changed, 743 insertions(+), 616 deletions(-)
delete mode 100644 sdk/go/arvados/base_fs.go
create mode 100644 sdk/go/arvados/fs_base.go
rename sdk/go/arvados/{collection_fs.go => fs_collection.go} (68%)
rename sdk/go/arvados/{collection_fs_test.go => fs_collection_test.go} (100%)
create mode 100644 sdk/go/arvados/fs_filehandle.go
create mode 100644 sdk/go/arvados/fs_getternode.go
rename sdk/go/arvados/{site_fs.go => fs_site.go} (77%)
rename sdk/go/arvados/{site_fs_test.go => fs_site_test.go} (100%)
discards c4995c2790925a9abe9c04f0d5efc30b3eef86e6 (commit)
via 600421bc73da06efe5ccfc54af02d35bea1bfb02 (commit)
via 6a6d167bdaca6428d9c61406545bc7db1b30e62c (commit)
via e35954da4b37d8cf954aebca8f55db7749676936 (commit)
via bb42507e20e4d7ca0ed01ac95ad6d2b2fc4f6d33 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (c4995c2790925a9abe9c04f0d5efc30b3eef86e6)
\
N -- N -- N (600421bc73da06efe5ccfc54af02d35bea1bfb02)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 600421bc73da06efe5ccfc54af02d35bea1bfb02
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:54:36 2017 -0500
go-fuse: Move code from fs_collection to fs_base and fs_filehandle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 4de1bf8..3be3f5e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -5,10 +5,152 @@
package arvados
import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
"os"
+ "path"
+ "strings"
"sync"
+ "time"
)
+var (
+ ErrReadOnlyFile = errors.New("read-only file")
+ ErrNegativeOffset = errors.New("cannot seek to negative offset")
+ ErrFileExists = errors.New("file exists")
+ ErrInvalidOperation = errors.New("invalid operation")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
+ ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrPermission = os.ErrPermission
+)
+
+// A File is an *os.File-like interface for reading and writing files
+// in a FileSystem.
+type File interface {
+ io.Reader
+ io.Writer
+ io.Closer
+ io.Seeker
+ Size() int64
+ Readdir(int) ([]os.FileInfo, error)
+ Stat() (os.FileInfo, error)
+ Truncate(int64) error
+}
+
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
+ http.FileSystem
+
+ inode
+
+ // analogous to os.Stat()
+ Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
+ Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
+ OpenFile(name string, flag int, perm os.FileMode) (File, error)
+
+ Mkdir(name string, perm os.FileMode) error
+ Remove(name string) error
+ RemoveAll(name string) error
+ Rename(oldname, newname string) error
+}
+
+type inode interface {
+ Parent() inode
+ Read([]byte, filenodePtr) (int, filenodePtr, error)
+ Write([]byte, filenodePtr) (int, filenodePtr, error)
+ Truncate(int64) error
+ IsDir() bool
+ Readdir() []os.FileInfo
+ Size() int64
+ FileInfo() os.FileInfo
+
+ // Child() performs lookups and updates of named child nodes.
+ //
+ // If replace is non-nil, Child calls replace(x) where x is
+ // the current child inode with the given name. If possible,
+ // the child inode is replaced with the one returned by
+ // replace().
+ //
+ // Nil represents "no child". replace(nil) signifies that no
+ // child with this name exists yet. If replace() returns nil,
+ // the existing child should be deleted if possible.
+ //
+ // An implementation of Child() is permitted to ignore
+ // replace() or its return value. For example, a regular file
+ // inode does not have children, so Child() always returns
+ // nil.
+ //
+ // Child() returns the child, if any, with the given name: if
+ // a child was added or changed, the new child is returned.
+ //
+ // Caller must have lock (or rlock if replace is nil).
+ Child(name string, replace func(inode) inode) inode
+
+ sync.Locker
+ RLock()
+ RUnlock()
+}
+
+type fileinfo struct {
+ name string
+ mode os.FileMode
+ size int64
+ modTime time.Time
+}
+
+// Name implements os.FileInfo.
+func (fi fileinfo) Name() string {
+ return fi.name
+}
+
+// ModTime implements os.FileInfo.
+func (fi fileinfo) ModTime() time.Time {
+ return fi.modTime
+}
+
+// Mode implements os.FileInfo.
+func (fi fileinfo) Mode() os.FileMode {
+ return fi.mode
+}
+
+// IsDir implements os.FileInfo.
+func (fi fileinfo) IsDir() bool {
+ return fi.mode&os.ModeDir != 0
+}
+
+// Size implements os.FileInfo.
+func (fi fileinfo) Size() int64 {
+ return fi.size
+}
+
+// Sys implements os.FileInfo.
+func (fi fileinfo) Sys() interface{} {
+ return nil
+}
+
type nullnode struct{}
func (*nullnode) Mkdir(string, os.FileMode) error {
@@ -95,3 +237,273 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
return
}
+
+type fileSystem struct {
+ inode
+}
+
+// OpenFile is analogous to os.OpenFile().
+func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
+}
+
+func (fs *fileSystem) Open(name string) (http.File, error) {
+ return fs.OpenFile(name, os.O_RDONLY, 0)
+}
+
+func (fs *fileSystem) Create(name string) (File, error) {
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
+}
+
+func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
+ node := rlookup(fs.inode, name)
+ if node == nil {
+ err = os.ErrNotExist
+ } else {
+ fi = node.FileInfo()
+ }
+ return
+}
+
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ oldinode.Lock()
+ defer oldinode.Unlock()
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ n.treenode.fileinfo.name = newname
+ case *filenode:
+ n.parent = newdn
+ n.fileinfo.name = newname
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ olddn.treenode.fileinfo.modTime = time.Now()
+ newdn.treenode.fileinfo.modTime = time.Now()
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 72ef49f..e451189 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -6,10 +6,8 @@ package arvados
import (
"encoding/json"
- "errors"
"fmt"
"io"
- "net/http"
"os"
"path"
"regexp"
@@ -20,111 +18,13 @@ import (
"time"
)
-var (
- ErrReadOnlyFile = errors.New("read-only file")
- ErrNegativeOffset = errors.New("cannot seek to negative offset")
- ErrFileExists = errors.New("file exists")
- ErrInvalidOperation = errors.New("invalid operation")
- ErrInvalidArgument = errors.New("invalid argument")
- ErrDirectoryNotEmpty = errors.New("directory not empty")
- ErrWriteOnlyMode = errors.New("file is O_WRONLY")
- ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
- ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
- ErrPermission = os.ErrPermission
-
- maxBlockSize = 1 << 26
-)
-
-// A File is an *os.File-like interface for reading and writing files
-// in a CollectionFileSystem.
-type File interface {
- io.Reader
- io.Writer
- io.Closer
- io.Seeker
- Size() int64
- Readdir(int) ([]os.FileInfo, error)
- Stat() (os.FileInfo, error)
- Truncate(int64) error
-}
+var maxBlockSize = 1 << 26
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
-type fileinfo struct {
- name string
- mode os.FileMode
- size int64
- modTime time.Time
-}
-
-// Name implements os.FileInfo.
-func (fi fileinfo) Name() string {
- return fi.name
-}
-
-// ModTime implements os.FileInfo.
-func (fi fileinfo) ModTime() time.Time {
- return fi.modTime
-}
-
-// Mode implements os.FileInfo.
-func (fi fileinfo) Mode() os.FileMode {
- return fi.mode
-}
-
-// IsDir implements os.FileInfo.
-func (fi fileinfo) IsDir() bool {
- return fi.mode&os.ModeDir != 0
-}
-
-// Size implements os.FileInfo.
-func (fi fileinfo) Size() int64 {
- return fi.size
-}
-
-// Sys implements os.FileInfo.
-func (fi fileinfo) Sys() interface{} {
- return nil
-}
-
-// A FileSystem is an http.Filesystem plus Stat() and support for
-// opening writable files. All methods are safe to call from multiple
-// goroutines.
-type FileSystem interface {
- http.FileSystem
-
- inode
-
- // analogous to os.Stat()
- Stat(name string) (os.FileInfo, error)
-
- // analogous to os.Create(): create/truncate a file and open it O_RDWR.
- Create(name string) (File, error)
-
- // Like os.OpenFile(): create or open a file or directory.
- //
- // If flag&os.O_EXCL==0, it opens an existing file or
- // directory if one exists. If flag&os.O_CREATE!=0, it creates
- // a new empty file or directory if one does not already
- // exist.
- //
- // When creating a new item, perm&os.ModeDir determines
- // whether it is a file or a directory.
- //
- // A file can be opened multiple times and used concurrently
- // from multiple goroutines. However, each File object should
- // be used by only one goroutine at a time.
- OpenFile(name string, flag int, perm os.FileMode) (File, error)
-
- Mkdir(name string, perm os.FileMode) error
- Remove(name string) error
- RemoveAll(name string) error
- Rename(oldname, newname string) error
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -137,8 +37,33 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
-type fileSystem struct {
- inode
+// FileSystem returns a CollectionFileSystem for the collection.
+func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+ var modTime time.Time
+ if c.ModifiedAt == nil {
+ modTime = time.Now()
+ } else {
+ modTime = *c.ModifiedAt
+ }
+ dn := &dirnode{
+ client: client,
+ kc: kc,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
+ },
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
+ return nil, err
+ }
+ return fs, nil
}
type collectionFileSystem struct {
@@ -170,301 +95,6 @@ func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
}
-// OpenFile is analogous to os.OpenFile().
-func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.openFile(name, flag, perm)
-}
-
-func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
- if parent == nil {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable && parent.IsDir() {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: parent}, nil
- case "..":
- return &filehandle{inode: parent.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- parent.Lock()
- defer parent.Unlock()
- } else {
- parent.RLock()
- defer parent.RUnlock()
- }
- n := parent.Child(name, nil)
- if n == nil {
- if !createMode {
- return nil, os.ErrNotExist
- }
- var err error
- n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
- if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
- } else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
- }
- return n
- })
- if err != nil {
- return nil, err
- } else if n == nil {
- // parent rejected new child
- return nil, ErrInvalidOperation
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
- }, nil
-}
-
-func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.OpenFile(name, os.O_RDONLY, 0)
-}
-
-func (fs *fileSystem) Create(name string) (File, error) {
- return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
-}
-
-func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
- dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
- if n == nil {
- return os.ErrNotExist
- }
- n.Lock()
- defer n.Unlock()
- if n.Child(name, nil) != nil {
- return os.ErrExist
- }
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
- child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
- return
- })
- if err != nil {
- return err
- } else if child == nil {
- return ErrInvalidArgument
- }
- return nil
-}
-
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
- if node == nil {
- err = os.ErrNotExist
- } else {
- fi = node.FileInfo()
- }
- return
-}
-
-func (fs *fileSystem) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
-
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
- }
- }
-
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
- err = nil
- olddirf.inode.Child(oldname, func(oldinode inode) inode {
- if oldinode == nil {
- err = os.ErrNotExist
- return nil
- }
- newdirf.inode.Child(newname, func(existing inode) inode {
- if existing != nil && existing.IsDir() {
- err = ErrIsDirectory
- return existing
- }
- return oldinode
- })
- if err != nil {
- return oldinode
- }
- oldinode.Lock()
- defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
- case *filenode:
- n.parent = newdn
- n.fileinfo.name = newname
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
- return nil
- })
- return err
-}
-
-func (fs *fileSystem) Remove(name string) error {
- return fs.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (fs *fileSystem) RemoveAll(name string) error {
- err := fs.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (fs *fileSystem) remove(name string, recursive bool) (err error) {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dir := rlookup(fs, dirname)
- if dir == nil {
- return os.ErrNotExist
- }
- dir.Lock()
- defer dir.Unlock()
- dir.Child(name, func(node inode) inode {
- if node == nil {
- err = os.ErrNotExist
- return nil
- }
- if !recursive && node.IsDir() && node.Size() > 0 {
- err = ErrDirectoryNotEmpty
- return node
- }
- return nil
- })
- return err
-}
-
-type inode interface {
- Parent() inode
- Read([]byte, filenodePtr) (int, filenodePtr, error)
- Write([]byte, filenodePtr) (int, filenodePtr, error)
- Truncate(int64) error
- IsDir() bool
- Readdir() []os.FileInfo
- Size() int64
- FileInfo() os.FileInfo
- // Caller must have lock (or rlock if func is nil)
- Child(string, func(inode) inode) inode
- sync.Locker
- RLock()
- RUnlock()
-}
-
-// filenode implements inode.
-type filenode struct {
- fileinfo fileinfo
- parent *dirnode
- segments []segment
- // number of times `segments` has changed in a
- // way that might invalidate a filenodePtr
- repacked int64
- memsize int64 // bytes in memSegments
- sync.RWMutex
- nullnode
-}
-
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
@@ -538,6 +168,19 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
return
}
+// filenode implements inode.
+type filenode struct {
+ fileinfo fileinfo
+ parent *dirnode
+ segments []segment
+ // number of times `segments` has changed in a
+ // way that might invalidate a filenodePtr
+ repacked int64
+ memsize int64 // bytes in memSegments
+ sync.RWMutex
+ nullnode
+}
+
// caller must have lock
func (fn *filenode) appendSegment(e segment) {
fn.segments = append(fn.segments, e)
@@ -813,128 +456,6 @@ func (fn *filenode) pruneMemSegments() {
}
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
- var modTime time.Time
- if c.ModifiedAt == nil {
- modTime = time.Now()
- } else {
- modTime = *c.ModifiedAt
- }
- dn := &dirnode{
- client: client,
- kc: kc,
- treenode: treenode{
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
- },
- parent: nil,
- inodes: make(map[string]inode),
- },
- }
- dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
- if err := dn.loadManifest(c.ManifestText); err != nil {
- return nil, err
- }
- return fs, nil
-}
-
-type filehandle struct {
- inode
- ptr filenodePtr
- append bool
- readable bool
- writable bool
- unreaddirs []os.FileInfo
-}
-
-func (f *filehandle) Read(p []byte) (n int, err error) {
- if !f.readable {
- return 0, ErrWriteOnlyMode
- }
- f.inode.RLock()
- defer f.inode.RUnlock()
- n, f.ptr, err = f.inode.Read(p, f.ptr)
- return
-}
-
-func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
- size := f.inode.Size()
- ptr := f.ptr
- switch whence {
- case io.SeekStart:
- ptr.off = off
- case io.SeekCurrent:
- ptr.off += off
- case io.SeekEnd:
- ptr.off = size + off
- }
- if ptr.off < 0 {
- return f.ptr.off, ErrNegativeOffset
- }
- if ptr.off != f.ptr.off {
- f.ptr = ptr
- // force filenode to recompute f.ptr fields on next
- // use
- f.ptr.repacked = -1
- }
- return f.ptr.off, nil
-}
-
-func (f *filehandle) Truncate(size int64) error {
- return f.inode.Truncate(size)
-}
-
-func (f *filehandle) Write(p []byte) (n int, err error) {
- if !f.writable {
- return 0, ErrReadOnlyFile
- }
- f.inode.Lock()
- defer f.inode.Unlock()
- if fn, ok := f.inode.(*filenode); ok && f.append {
- f.ptr = filenodePtr{
- off: fn.fileinfo.size,
- segmentIdx: len(fn.segments),
- segmentOff: 0,
- repacked: fn.repacked,
- }
- }
- n, f.ptr, err = f.inode.Write(p, f.ptr)
- return
-}
-
-func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.IsDir() {
- return nil, ErrInvalidOperation
- }
- if count <= 0 {
- return f.inode.Readdir(), nil
- }
- if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
- }
- if len(f.unreaddirs) == 0 {
- return nil, io.EOF
- }
- if count > len(f.unreaddirs) {
- count = len(f.unreaddirs)
- }
- ret := f.unreaddirs[:count]
- f.unreaddirs = f.unreaddirs[count:]
- return ret, nil
-}
-
-func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.FileInfo(), nil
-}
-
-func (f *filehandle) Close() error {
- return nil
-}
-
type dirnode struct {
treenode
client *Client
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
new file mode 100644
index 0000000..56963b6
--- /dev/null
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -0,0 +1,99 @@
+package arvados
+
+import (
+ "io"
+ "os"
+)
+
+type filehandle struct {
+ inode
+ ptr filenodePtr
+ append bool
+ readable bool
+ writable bool
+ unreaddirs []os.FileInfo
+}
+
+func (f *filehandle) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
+ n, f.ptr, err = f.inode.Read(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
+ size := f.inode.Size()
+ ptr := f.ptr
+ switch whence {
+ case io.SeekStart:
+ ptr.off = off
+ case io.SeekCurrent:
+ ptr.off += off
+ case io.SeekEnd:
+ ptr.off = size + off
+ }
+ if ptr.off < 0 {
+ return f.ptr.off, ErrNegativeOffset
+ }
+ if ptr.off != f.ptr.off {
+ f.ptr = ptr
+ // force filenode to recompute f.ptr fields on next
+ // use
+ f.ptr.repacked = -1
+ }
+ return f.ptr.off, nil
+}
+
+func (f *filehandle) Truncate(size int64) error {
+ return f.inode.Truncate(size)
+}
+
+func (f *filehandle) Write(p []byte) (n int, err error) {
+ if !f.writable {
+ return 0, ErrReadOnlyFile
+ }
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ segmentIdx: len(fn.segments),
+ segmentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
+ n, f.ptr, err = f.inode.Write(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
+ if !f.inode.IsDir() {
+ return nil, ErrInvalidOperation
+ }
+ if count <= 0 {
+ return f.inode.Readdir(), nil
+ }
+ if f.unreaddirs == nil {
+ f.unreaddirs = f.inode.Readdir()
+ }
+ if len(f.unreaddirs) == 0 {
+ return nil, io.EOF
+ }
+ if count > len(f.unreaddirs) {
+ count = len(f.unreaddirs)
+ }
+ ret := f.unreaddirs[:count]
+ f.unreaddirs = f.unreaddirs[count:]
+ return ret, nil
+}
+
+func (f *filehandle) Stat() (os.FileInfo, error) {
+ return f.inode.FileInfo(), nil
+}
+
+func (f *filehandle) Close() error {
+ return nil
+}
commit 6a6d167bdaca6428d9c61406545bc7db1b30e62c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:18:01 2017 -0500
go-fuse: Rearrange source files.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/fs_base.go
similarity index 100%
rename from sdk/go/arvados/base_fs.go
rename to sdk/go/arvados/fs_base.go
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/fs_collection.go
similarity index 100%
rename from sdk/go/arvados/collection_fs.go
rename to sdk/go/arvados/fs_collection.go
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/fs_collection_test.go
similarity index 100%
rename from sdk/go/arvados/collection_fs_test.go
rename to sdk/go/arvados/fs_collection_test.go
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/fs_site.go
similarity index 77%
rename from sdk/go/arvados/site_fs.go
rename to sdk/go/arvados/fs_site.go
index 853bd32..66856b7 100644
--- a/sdk/go/arvados/site_fs.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,6 +9,14 @@ import (
"time"
)
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// although the FileSystem allows files to be added and modified in
+// collections, these changes are not persistent or visible to other
+// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root := &treenode{
fileinfo: fileinfo{
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/fs_site_test.go
similarity index 100%
rename from sdk/go/arvados/site_fs_test.go
rename to sdk/go/arvados/fs_site_test.go
commit e35954da4b37d8cf954aebca8f55db7749676936
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 00:49:36 2017 -0500
go-fuse: .arvados#collection special file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 4647055..72ef49f 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -5,6 +5,7 @@
package arvados
import (
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -144,6 +145,25 @@ type collectionFileSystem struct {
fileSystem
}
+func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+ if name == ".arvados#collection" {
+ return &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ }
+ return fs.fileSystem.Child(name, replace)
+}
+
func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.inode.Lock()
defer fs.fileSystem.inode.Unlock()
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs_getternode.go
new file mode 100644
index 0000000..c9ffb38
--- /dev/null
+++ b/sdk/go/arvados/fs_getternode.go
@@ -0,0 +1,66 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "os"
+ "time"
+)
+
+// A getternode is a read-only character device that returns whatever
+// data is returned by the supplied function.
+type getternode struct {
+ Getter func() ([]byte, error)
+
+ treenode
+ data *bytes.Reader
+}
+
+func (*getternode) IsDir() bool {
+ return false
+}
+
+func (*getternode) Child(string, func(inode) inode) inode {
+ return nil
+}
+
+func (gn *getternode) get() error {
+ if gn.data != nil {
+ return nil
+ }
+ data, err := gn.Getter()
+ if err != nil {
+ return err
+ }
+ gn.data = bytes.NewReader(data)
+ return nil
+}
+
+func (gn *getternode) Size() int64 {
+ return gn.FileInfo().Size()
+}
+
+func (gn *getternode) FileInfo() os.FileInfo {
+ gn.Lock()
+ defer gn.Unlock()
+ var size int64
+ if gn.get() == nil {
+ size = gn.data.Size()
+ }
+ return fileinfo{
+ modTime: time.Now(),
+ mode: 0444,
+ size: size,
+ }
+}
+
+func (gn *getternode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
+ if err := gn.get(); err != nil {
+ return 0, ptr, err
+ }
+ n, err := gn.data.ReadAt(p, ptr.off)
+ return n, filenodePtr{off: ptr.off + int64(n)}, err
+}
commit bb42507e20e4d7ca0ed01ac95ad6d2b2fc4f6d33
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 20:14:54 2017 -0500
go-fuse: Fix seek/read/write race.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 331c3cb..8e488f5 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -10,6 +10,14 @@ import (
"github.com/curoverse/cgofuse/fuse"
)
+// sharedFile wraps arvados.File with a sync.Mutex, so fuse can safely
+// use a single filehandle concurrently on behalf of multiple
+// threads/processes.
+type sharedFile struct {
+ arvados.File
+ sync.Mutex
+}
+
type keepFS struct {
fuse.FileSystemBase
Client *arvados.Client
@@ -19,7 +27,7 @@ type keepFS struct {
Gid int
root arvados.FileSystem
- open map[uint64]arvados.File
+ open map[uint64]*sharedFile
lastFH uint64
sync.Mutex
}
@@ -32,14 +40,20 @@ func (fs *keepFS) newFH(f arvados.File) uint64 {
fs.Lock()
defer fs.Unlock()
if fs.open == nil {
- fs.open = make(map[uint64]arvados.File)
+ fs.open = make(map[uint64]*sharedFile)
}
fs.lastFH++
fh := fs.lastFH
- fs.open[fh] = f
+ fs.open[fh] = &sharedFile{File: f}
return fh
}
+func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
+ fs.Lock()
+ defer fs.Unlock()
+ return fs.open[fh]
+}
+
func (fs *keepFS) Init() {
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
}
@@ -165,14 +179,19 @@ func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
if fs.ReadOnly {
return -fuse.EROFS
}
- f := fs.lookupFH(fh)
- if f == nil {
- var err error
- if f, err = fs.root.OpenFile(path, os.O_RDWR, 0); err != nil {
- return fs.errCode(err)
- }
- defer f.Close()
+
+ // Sometimes fh is a valid filehandle and we don't need to
+ // waste a name lookup.
+ if f := fs.lookupFH(fh); f != nil {
+ return fs.errCode(f.Truncate(size))
}
+
+ // Other times, fh is invalid and we need to lookup path.
+ f, err := fs.root.OpenFile(path, os.O_RDWR, 0)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ defer f.Close()
return fs.errCode(f.Truncate(size))
}
@@ -180,8 +199,10 @@ func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int)
var fi os.FileInfo
var err error
if f := fs.lookupFH(fh); f != nil {
+ // Valid filehandle -- ignore path.
fi, err = f.Stat()
} else {
+ // Invalid filehandle -- lookup path.
fi, err = fs.root.Stat(path)
}
if err != nil {
@@ -233,9 +254,14 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
if fs.ReadOnly {
return -fuse.EROFS
- } else if f := fs.lookupFH(fh); f == nil {
+ }
+ f := fs.lookupFH(fh)
+ if f == nil {
return -fuse.EBADF
- } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
} else {
n, _ = f.Write(buf)
@@ -244,9 +270,13 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
- if f := fs.lookupFH(fh); f == nil {
+ f := fs.lookupFH(fh)
+ if f == nil {
return -fuse.EBADF
- } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
} else {
n, _ = f.Read(buf)
@@ -275,9 +305,3 @@ func (fs *keepFS) Readdir(path string,
}
return 0
}
-
-func (fs *keepFS) lookupFH(fh uint64) arvados.File {
- fs.Lock()
- defer fs.Unlock()
- return fs.open[fh]
-}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list