[ARVADOS] created: 1.1.2-32-g9c6b450
Git user
git at public.curoverse.com
Wed Jan 3 01:31:14 EST 2018
at 9c6b4501a9c1c686ecac72fed5026686171d9d29 (commit)
commit 9c6b4501a9c1c686ecac72fed5026686171d9d29
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 3 01:20:47 2018 -0500
12308: Port "rename a to a/b/c" fix from 11377b3b.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index a3df168..419be7f 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -444,6 +444,11 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = os.ErrNotExist
return nil
}
+ if locked[oldinode] {
+ // oldinode cannot become a descendant of itself.
+ err = ErrInvalidArgument
+ return oldinode
+ }
accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e7e6f19..27ea904 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -539,7 +539,12 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
gn.SetParent(dn, name)
return gn
}
- return dn.treenode.Child(name, replace)
+ oldchild := dn.treenode.Child(name, nil)
+ child := dn.treenode.Child(name, replace)
+ if child != nil && child != oldchild {
+ child.SetParent(dn, name)
+ }
+ return child
}
// sync flushes in-memory data (for all files in the tree rooted at
commit d9ec8063857543da420907319d9a4f1b990f3dcb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:23:32 2017 -0500
12308: Update parent dir modtime when adding/removing/renaming.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 9978c43..a3df168 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -232,8 +232,9 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
if newchild == nil {
delete(n.inodes, name)
} else if newchild != child {
- n.inodes[name] = newchild
newchild.SetParent(n, name)
+ n.inodes[name] = newchild
+ n.fileinfo.modTime = time.Now()
child = newchild
}
}
@@ -458,8 +459,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// Leave oldinode in olddir.
return oldinode
}
- //TODO: olddirf.setModTime(time.Now())
- //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 0121d2d..e7e6f19 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -67,10 +67,23 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ backdateTree(root, modTime)
fs.root = root
return fs, nil
}
+func backdateTree(n inode, modTime time.Time) {
+ switch n := n.(type) {
+ case *filenode:
+ n.fileinfo.modTime = modTime
+ case *dirnode:
+ n.fileinfo.modTime = modTime
+ for _, n := range n.inodes {
+ backdateTree(n, modTime)
+ }
+ }
+}
+
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 8a54b44..d3ca510 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -94,6 +94,7 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
n := vn.create(vn, name)
if n != nil {
n.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
}
return n
}
commit 67ad6de632f9390d593dc289eca9e138bac563b1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:07:22 2017 -0500
12308: Sync name when changing parents.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index ebf2ad6..9978c43 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -85,7 +85,7 @@ type FileSystem interface {
}
type inode interface {
- SetParent(inode)
+ SetParent(parent inode, name string)
Parent() inode
FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
@@ -207,10 +207,11 @@ func (n *treenode) FS() FileSystem {
return n.fs
}
-func (n *treenode) SetParent(p inode) {
- n.RLock()
- defer n.RUnlock()
+func (n *treenode) SetParent(p inode, name string) {
+ n.Lock()
+ defer n.Unlock()
n.parent = p
+ n.fileinfo.name = name
}
func (n *treenode) Parent() inode {
@@ -232,7 +233,7 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
delete(n.inodes, name)
} else if newchild != child {
n.inodes[name] = newchild
- newchild.SetParent(n)
+ newchild.SetParent(n, name)
child = newchild
}
}
@@ -442,28 +443,21 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = os.ErrNotExist
return nil
}
- newdirf.inode.Child(newname, func(existing inode) inode {
+ accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
return existing
}
return oldinode
})
- if err != nil {
+ if accepted != oldinode {
+ if err == nil {
+ // newdirf didn't accept oldinode.
+ err = ErrInvalidArgument
+ }
+ // Leave oldinode in olddir.
return oldinode
}
- oldinode.Lock()
- defer oldinode.Unlock()
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdirf.inode
- n.fileinfo.name = newname
- case *filenode:
- n.parent = newdirf.inode
- n.fileinfo.name = newname
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
//TODO: olddirf.setModTime(time.Now())
//TODO: newdirf.setModTime(time.Now())
return nil
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 96977cb..0121d2d 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -63,7 +63,7 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
inodes: make(map[string]inode),
},
}
- root.SetParent(root)
+ root.SetParent(root, ".")
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -219,10 +219,11 @@ func (fn *filenode) appendSegment(e segment) {
fn.fileinfo.size += int64(e.Len())
}
-func (fn *filenode) SetParent(p inode) {
- fn.RLock()
- defer fn.RUnlock()
+func (fn *filenode) SetParent(p inode, name string) {
+ fn.Lock()
+ defer fn.Unlock()
fn.parent = p
+ fn.fileinfo.name = name
}
func (fn *filenode) Parent() inode {
@@ -522,7 +523,7 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
}
return data, err
}}
- gn.SetParent(dn)
+ gn.SetParent(dn, name)
return gn
}
return dn.treenode.Child(name, replace)
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 974dad8..8a54b44 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -71,8 +71,7 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return nil
}
root := cfs.rootnode()
- root.SetParent(parent)
- root.(*dirnode).fileinfo.name = id
+ root.SetParent(parent, id)
return root
}
@@ -94,7 +93,7 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
} else {
n := vn.create(vn, name)
if n != nil {
- n.SetParent(vn)
+ n.SetParent(vn, name)
}
return n
}
commit ac7d4334a2be25126e277529b0644a97ec8927cb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:01:44 2017 -0500
12308: Fix collectionfs crash.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index af0068f..96977cb 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -480,7 +480,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
+ locator, _, err := fn.FS().PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -489,7 +489,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.FS(),
+ kc: fn.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
commit db551f2d600ded8ee33b02fd3bcc3e0306ec3cff
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 23:16:34 2017 -0500
12308: Add missing copyright headers.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/cmd/arvados-client/Makefile b/cmd/arvados-client/Makefile
index d02fb78..33fbc40 100644
--- a/cmd/arvados-client/Makefile
+++ b/cmd/arvados-client/Makefile
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
all:
go get .
docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" "$(GOPATH)"/src/github.com/curoverse/cgofuse
diff --git a/lib/mount/command.go b/lib/mount/command.go
index ed105e4..498d1c2 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package mount
import (
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 0d90689..a6e2540 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package mount
import (
diff --git a/lib/mount/fs_test.go b/lib/mount/fs_test.go
index a178d6a..bcace0f 100644
--- a/lib/mount/fs_test.go
+++ b/lib/mount/fs_test.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package mount
import (
commit a3994591d74a61efab6cdb76af191a37a8e32399
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:31 2017 -0500
12308: Tidy up, add comments.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 13e2a06..0d90689 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -20,6 +20,7 @@ type sharedFile struct {
sync.Mutex
}
+// keepFS implements cgofuse's FileSystemInterface.
type keepFS struct {
fuse.FileSystemBase
Client *arvados.Client
@@ -38,6 +39,8 @@ var (
invalidFH = ^uint64(0)
)
+// newFH wraps f in a sharedFile, adds it to fs's lookup table using a
+// new handle number, and returns the handle number.
func (fs *keepFS) newFH(f arvados.File) uint64 {
fs.Lock()
defer fs.Unlock()
@@ -157,6 +160,11 @@ func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
return fs.Release(path, fh)
}
+func (fs *keepFS) Rmdir(path string) int {
+ defer fs.debugPanics()
+ return fs.errCode(fs.root.Remove(path))
+}
+
func (fs *keepFS) Release(path string, fh uint64) (errc int) {
defer fs.debugPanics()
fs.Lock()
@@ -337,6 +345,9 @@ func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
return fs.Fsync(path, datasync, fh)
}
+// debugPanics (when deferred by keepFS handlers) prints an error and
+// stack trace on stderr when a handler crashes. (Without this,
+// cgofuse recovers from panics silently and returns EIO.)
func (fs *keepFS) debugPanics() {
if err := recover(); err != nil {
log.Printf("(%T) %v", err, err)
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs_backend.go
new file mode 100644
index 0000000..301f0b4
--- /dev/null
+++ b/sdk/go/arvados/fs_backend.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "io"
+
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
+type keepClient interface {
+ ReadAt(locator string, p []byte, off int) (int, error)
+ PutB(p []byte) (string, int, error)
+}
+
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index a64ce8c..af0068f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -21,28 +21,6 @@ import (
var maxBlockSize = 1 << 26
-type fsBackend interface {
- keepClient
- apiClient
-}
-
-// Ideally *Client would do everything; meanwhile keepBackend
-// implements fsBackend by merging the two kinds of arvados client.
-type keepBackend struct {
- keepClient
- apiClient
-}
-
-type keepClient interface {
- ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
-}
-
-type apiClient interface {
- RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -55,6 +33,11 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
+type collectionFileSystem struct {
+ fileSystem
+ uuid string
+}
+
// FileSystem returns a CollectionFileSystem for the collection.
func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
@@ -88,11 +71,6 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
return fs, nil
}
-type collectionFileSystem struct {
- fileSystem
- uuid string
-}
-
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
@@ -142,27 +120,6 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
- if dn == dn.fs.rootnode() && name == ".arvados#collection" {
- gn := &getternode{Getter: func() ([]byte, error) {
- var coll Collection
- var err error
- coll.ManifestText, err = dn.fs.MarshalManifest(".")
- if err != nil {
- return nil, err
- }
- data, err := json.Marshal(&coll)
- if err == nil {
- data = append(data, 10)
- }
- return data, err
- }}
- gn.SetParent(dn)
- return gn
- }
- return dn.treenode.Child(name, replace)
-}
-
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
@@ -550,6 +507,27 @@ func (dn *dirnode) FS() FileSystem {
return dn.fs
}
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ gn.SetParent(dn)
+ return gn
+ }
+ return dn.treenode.Child(name, replace)
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index fd842f0..974dad8 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -56,11 +56,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- return nil, ErrInvalidOperation
-}
-
-func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
commit 961c50a5a808bafab4a975af5e5bf8a3e04bb448
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:09 2017 -0500
12308: Combine newDirnode and newFIlenode to just newNode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fe185a4..ebf2ad6 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -52,8 +52,9 @@ type FileSystem interface {
fsBackend
rootnode() inode
- newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
- newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+
+ // create a new node with nil parent.
+ newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -319,11 +320,7 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- if perm.IsDir() {
- n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
- } else {
- n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
- }
+ n, err = parent.FS().newNode(name, perm|0755, time.Now())
return n
})
if err != nil {
@@ -371,7 +368,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = n.FS().newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
return
})
if err != nil {
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 36a92af..a64ce8c 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -93,37 +93,32 @@ type collectionFileSystem struct {
uuid string
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
}
- return &dirnode{
- fs: fs,
- treenode: treenode{
+ if perm.IsDir() {
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+ } else {
+ return &filenode{
+ fs: fs,
fileinfo: fileinfo{
name: name,
- mode: perm | os.ModeDir,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
+ }, nil
}
- return &filenode{
- fs: fs,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
}
func (fs *collectionFileSystem) Sync() error {
@@ -852,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
}
node.Child(name, func(child inode) inode {
if child == nil {
- node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ node, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
} else if !child.IsDir() {
err = ErrFileExists
@@ -868,7 +863,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
fn = child.(*filenode)
return child
case *filenode:
commit 8ba6b8bf39075278c1e5f823e4defcc1131973a7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 17:11:07 2017 -0500
12308: Clear up some type assertions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 8d987d4..fe185a4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -51,6 +51,7 @@ type FileSystem interface {
http.FileSystem
fsBackend
+ rootnode() inode
newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
@@ -83,6 +84,7 @@ type FileSystem interface {
}
type inode interface {
+ SetParent(inode)
Parent() inode
FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
@@ -204,6 +206,12 @@ func (n *treenode) FS() FileSystem {
return n.fs
}
+func (n *treenode) SetParent(p inode) {
+ n.RLock()
+ defer n.RUnlock()
+ n.parent = p
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -218,11 +226,13 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
// TODO: special treatment for "", ".", ".."
child = n.inodes[name]
if replace != nil {
- child = replace(child)
- if child == nil {
+ newchild := replace(child)
+ if newchild == nil {
delete(n.inodes, name)
- } else {
- n.inodes[name] = child
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ newchild.SetParent(n)
+ child = newchild
}
}
return
@@ -254,6 +264,10 @@ type fileSystem struct {
fsBackend
}
+func (fs *fileSystem) rootnode() inode {
+ return fs.root
+}
+
// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.openFile(name, flag, perm)
@@ -306,9 +320,9 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
var err error
n = parent.Child(name, func(inode) inode {
if perm.IsDir() {
- n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -357,7 +371,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = fs.newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -404,12 +418,12 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// When acquiring locks on multiple nodes, all common
// ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
+ // deadlock. This is assured by locking the path from
+ // filesystem root to newdir, then locking the path from
+ // filesystem root to olddir, skipping any already-locked
+ // nodes.
needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
+ for _, node := range []inode{olddirf.inode, newdirf.inode} {
needLock = append(needLock, node)
for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
@@ -499,41 +513,6 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
return err
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- treenode: treenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
func (fs *fileSystem) Sync() error {
log.Printf("TODO: sync fileSystem")
return ErrInvalidOperation
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fc00335..36a92af 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -64,14 +64,14 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
modTime = *c.ModifiedAt
}
fs := &collectionFileSystem{
+ uuid: c.UUID,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: client, keepClient: kc},
},
- uuid: c.UUID,
}
- dn := &dirnode{
+ root := &dirnode{
+ fs: fs,
treenode: treenode{
- fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
@@ -80,11 +80,11 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
inodes: make(map[string]inode),
},
}
- dn.parent = dn
- fs.fileSystem.root = dn
- if err := dn.loadManifest(c.ManifestText); err != nil {
+ root.SetParent(root)
+ if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ fs.root = root
return fs, nil
}
@@ -93,6 +93,39 @@ type collectionFileSystem struct {
uuid string
}
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: fs,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
func (fs *collectionFileSystem) Sync() error {
log.Printf("cfs.Sync()")
if fs.uuid == "" {
@@ -114,12 +147,12 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
- if name == ".arvados#collection" {
- return &getternode{Getter: func() ([]byte, error) {
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
var err error
- coll.ManifestText, err = fs.MarshalManifest(".")
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
if err != nil {
return nil, err
}
@@ -129,8 +162,10 @@ func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) in
}
return data, err
}}
+ gn.SetParent(dn)
+ return gn
}
- return fs.fileSystem.root.Child(name, replace)
+ return dn.treenode.Child(name, replace)
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
@@ -232,6 +267,12 @@ func (fn *filenode) appendSegment(e segment) {
fn.fileinfo.size += int64(e.Len())
}
+func (fn *filenode) SetParent(p inode) {
+ fn.RLock()
+ defer fn.RUnlock()
+ fn.parent = p
+}
+
func (fn *filenode) Parent() inode {
fn.RLock()
defer fn.RUnlock()
@@ -496,7 +537,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.(fsBackend),
+ kc: fn.parent.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -506,9 +547,14 @@ func (fn *filenode) pruneMemSegments() {
}
type dirnode struct {
+ fs *collectionFileSystem
treenode
}
+func (dn *dirnode) FS() FileSystem {
+ return dn.fs
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
@@ -801,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
// can't be sure parent will be a *dirnode
return nil, ErrInvalidArgument
}
- node = node.Parent().(*dirnode)
+ node = node.Parent()
continue
}
node.Child(name, func(child inode) inode {
@@ -832,7 +878,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
err = ErrIsDirectory
return child
default:
- err = ErrInvalidOperation
+ err = ErrInvalidArgument
return child
}
})
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 5326131..fd842f0 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,17 +9,21 @@ import (
"time"
)
+type siteFileSystem struct {
+ fileSystem
+}
+
// SiteFileSystem returns a FileSystem that maps collections and other
// Arvados objects onto a filesystem layout.
//
// This is experimental: the filesystem layout is not stable, and
// there are significant known bugs and shortcomings. For example,
-// although the FileSystem allows files to be added and modified in
-// collections, these changes are not persistent or visible to other
-// Arvados clients.
+// writes are not persisted until Sync() is called.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
- fs := &fileSystem{
- fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ fs := &siteFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ },
}
root := &treenode{
fs: fs,
@@ -34,7 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root.Child("by_id", func(inode) inode {
var vn inode
vn = &vdirnode{
- treenode: treenode{
+ inode: &treenode{
fs: fs,
parent: root,
inodes: make(map[string]inode),
@@ -44,9 +48,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
mode: 0755 | os.ModeDir,
},
},
- create: func(name string) inode {
- return newEntByID(vn, name)
- },
+ create: fs.mountCollection,
}
return vn
})
@@ -54,33 +56,51 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func newEntByID(parent inode, id string) inode {
+func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
- err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(parent.FS(), parent.FS())
+ cfs, err := coll.FileSystem(fs, fs)
if err != nil {
return nil
}
- root := fs.(*collectionFileSystem).root.(*dirnode)
- root.fileinfo.name = id
- root.parent = parent
+ root := cfs.rootnode()
+ root.SetParent(parent)
+ root.(*dirnode).fileinfo.name = id
return root
}
+// vdirnode wraps an inode by ignoring any requests to add/replace
+// children, and calling a create() func when a non-existing child is
+// looked up.
+//
+// create() can return either a new node, which will be added to the
+// treenode, or nil for ENOENT.
type vdirnode struct {
- treenode
- create func(string) inode
+ inode
+ create func(parent inode, name string) inode
}
func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
- return vn.treenode.Child(name, func(existing inode) inode {
+ return vn.inode.Child(name, func(existing inode) inode {
if existing != nil {
return existing
} else {
- return vn.create(name)
+ n := vn.create(vn, name)
+ if n != nil {
+ n.SetParent(vn)
+ }
+ return n
}
})
}
commit eca8dff2d88265aac1d6f366b8c8c9759648a5b3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 19 09:30:56 2017 -0500
12308: Save on sync().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 8e488f5..13e2a06 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -2,7 +2,9 @@ package mount
import (
"io"
+ "log"
"os"
+ "runtime/debug"
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -55,10 +57,12 @@ func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
}
func (fs *keepFS) Init() {
+ defer fs.debugPanics()
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
}
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS, invalidFH
}
@@ -72,6 +76,7 @@ func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint
}
func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ defer fs.debugPanics()
if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
return -fuse.EROFS, invalidFH
}
@@ -88,6 +93,7 @@ func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
}
func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -120,6 +126,7 @@ func (fs *keepFS) errCode(err error) int {
}
func (fs *keepFS) Mkdir(path string, mode uint32) int {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -132,6 +139,7 @@ func (fs *keepFS) Mkdir(path string, mode uint32) int {
}
func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ defer fs.debugPanics()
f, err := fs.root.OpenFile(path, 0, 0)
if err != nil {
return fs.errCode(err), invalidFH
@@ -145,10 +153,12 @@ func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
}
func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
return fs.Release(path, fh)
}
func (fs *keepFS) Release(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
fs.Lock()
defer fs.Unlock()
defer delete(fs.open, fh)
@@ -162,6 +172,7 @@ func (fs *keepFS) Release(path string, fh uint64) (errc int) {
}
func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -169,6 +180,7 @@ func (fs *keepFS) Rename(oldname, newname string) (errc int) {
}
func (fs *keepFS) Unlink(path string) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -176,6 +188,7 @@ func (fs *keepFS) Unlink(path string) (errc int) {
}
func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -196,6 +209,7 @@ func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
}
func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ defer fs.debugPanics()
var fi os.FileInfo
var err error
if f := fs.lookupFH(fh); f != nil {
@@ -218,7 +232,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
if fi, err := fs.root.Stat(path); err != nil {
return fs.errCode(err)
- } else if (os.FileMode(mode)^fi.Mode())&os.ModePerm != 0 {
+ } else if mode & ^uint32(fuse.S_IFREG|fuse.S_IFDIR|0777) != 0 || (fi.Mode()&os.ModeDir != 0) != (mode&fuse.S_IFDIR != 0) {
return -fuse.ENOSYS
} else {
return 0
@@ -226,6 +240,7 @@ func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
}
func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ defer fs.debugPanics()
var m uint32
if fi.IsDir() {
m = m | fuse.S_IFDIR
@@ -252,6 +267,7 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
if fs.ReadOnly {
return -fuse.EROFS
}
@@ -270,6 +286,7 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -288,6 +305,7 @@ func (fs *keepFS) Readdir(path string,
fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
ofst int64,
fh uint64) (errc int) {
+ defer fs.debugPanics()
f := fs.lookupFH(fh)
if f == nil {
return -fuse.EBADF
@@ -305,3 +323,24 @@ func (fs *keepFS) Readdir(path string,
}
return 0
}
+
+func (fs *keepFS) Fsync(path string, datasync bool, fh uint64) int {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ return fs.errCode(f.Sync())
+}
+
+func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
+ return fs.Fsync(path, datasync, fh)
+}
+
+func (fs *keepFS) debugPanics() {
+ if err := recover(); err != nil {
+ log.Printf("(%T) %v", err, err)
+ debug.PrintStack()
+ panic(err)
+ }
+}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 3be3f5e..8d987d4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path"
@@ -40,6 +41,7 @@ type File interface {
Readdir(int) ([]os.FileInfo, error)
Stat() (os.FileInfo, error)
Truncate(int64) error
+ Sync() error
}
// A FileSystem is an http.Filesystem plus Stat() and support for
@@ -47,8 +49,10 @@ type File interface {
// goroutines.
type FileSystem interface {
http.FileSystem
+ fsBackend
- inode
+ newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+ newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -75,10 +79,12 @@ type FileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+ Sync() error
}
type inode interface {
Parent() inode
+ FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
@@ -186,6 +192,7 @@ func (*nullnode) Child(name string, replace func(inode) inode) inode {
}
type treenode struct {
+ fs FileSystem
parent inode
inodes map[string]inode
fileinfo fileinfo
@@ -193,6 +200,10 @@ type treenode struct {
nullnode
}
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -239,7 +250,8 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
type fileSystem struct {
- inode
+ root inode
+ fsBackend
}
// OpenFile is analogous to os.OpenFile().
@@ -248,12 +260,11 @@ func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, e
}
func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
+ parent := rlookup(fs.root, dirname)
if parent == nil {
return nil, os.ErrNotExist
}
@@ -294,20 +305,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -322,10 +323,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
} else if flag&os.O_TRUNC != 0 {
if !writable {
return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
+ } else if n.IsDir() {
return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
}
}
return &filehandle{
@@ -346,7 +347,7 @@ func (fs *fileSystem) Create(name string) (File, error) {
func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
+ n := rlookup(fs.root, dirname)
if n == nil {
return os.ErrNotExist
}
@@ -355,12 +356,8 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
if n.Child(name, nil) != nil {
return os.ErrExist
}
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
+ child, err = fs.newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -372,7 +369,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
+ node := rlookup(fs.root, name)
if node == nil {
err = os.ErrNotExist
} else {
@@ -414,7 +411,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
for _, f := range []*filehandle{olddirf, newdirf} {
node := f.inode
needLock = append(needLock, node)
- for node.Parent() != node {
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
needLock = append(needLock, node)
}
@@ -428,10 +425,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
err = nil
olddirf.inode.Child(oldname, func(oldinode inode) inode {
if oldinode == nil {
@@ -450,20 +443,18 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
oldinode.Lock()
defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
+ n.fileinfo.name = newname
case *filenode:
- n.parent = newdn
+ n.parent = newdirf.inode
n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
+ //TODO: olddirf.setModTime(time.Now())
+ //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
@@ -488,7 +479,7 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs, dirname)
+ dir := rlookup(fs.root, dirname)
if dir == nil {
return os.ErrNotExist
}
@@ -507,3 +498,70 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
})
return err
}
+
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ treenode: treenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
+ }
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ }
+ return
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e451189..fc00335 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"os"
"path"
"regexp"
@@ -20,11 +21,28 @@ import (
var maxBlockSize = 1 << 26
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
+
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -38,28 +56,32 @@ type CollectionFileSystem interface {
}
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
} else {
modTime = *c.ModifiedAt
}
+ fs := &collectionFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ uuid: c.UUID,
+ }
dn := &dirnode{
- client: client,
- kc: kc,
treenode: treenode{
+ fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
},
- parent: nil,
inodes: make(map[string]inode),
},
}
dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ fs.fileSystem.root = dn
if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -68,9 +90,31 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
type collectionFileSystem struct {
fileSystem
+ uuid string
}
-func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
+}
+
+func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
if name == ".arvados#collection" {
return &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -86,13 +130,13 @@ func (fs collectionFileSystem) Child(name string, replace func(inode) inode) ino
return data, err
}}
}
- return fs.fileSystem.Child(name, replace)
+ return fs.fileSystem.root.Child(name, replace)
}
-func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
- fs.fileSystem.inode.Lock()
- defer fs.fileSystem.inode.Unlock()
- return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -170,8 +214,9 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
// filenode implements inode.
type filenode struct {
+ parent inode
+ fs FileSystem
fileinfo fileinfo
- parent *dirnode
segments []segment
// number of times `segments` has changed in a
// way that might invalidate a filenodePtr
@@ -193,6 +238,10 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -438,7 +487,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -447,7 +496,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.parent.(fsBackend),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -458,8 +507,6 @@ func (fn *filenode) pruneMemSegments() {
type dirnode struct {
treenode
- client *Client
- kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -480,7 +527,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
@@ -488,7 +535,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
@@ -709,7 +756,7 @@ func (dn *dirnode) loadManifest(txt string) error {
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
@@ -738,7 +785,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
- node := dn
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -758,16 +805,13 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
continue
}
node.Child(name, func(child inode) inode {
- switch child.(type) {
- case nil:
- node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if child == nil {
+ node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
- case *dirnode:
- node = child.(*dirnode)
- case *filenode:
+ } else if !child.IsDir() {
err = ErrFileExists
- default:
- err = ErrInvalidOperation
+ } else {
+ node = child
}
return child
})
@@ -778,8 +822,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
- return fn
+ child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ fn = child.(*filenode)
+ return child
case *filenode:
fn = child
return child
@@ -794,68 +839,6 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return
}
-// rlookup (recursive lookup) returns the inode for the file/directory
-// with the given name (which may contain "/" separators). If no such
-// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
- node = start
- for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
- if node.IsDir() {
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- }
- node = func() inode {
- node.RLock()
- defer node.RUnlock()
- return node.Child(name, nil)
- }()
- }
- return
-}
-
-// Caller must have lock, and must have already ensured
-// Children(name,nil) is nil.
-func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- client: dn.client,
- kc: dn.kc,
- treenode: treenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
@@ -920,7 +903,7 @@ func (me *memSegment) ReadAt(p []byte, off int64) (n int, err error) {
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index 56963b6..d586531 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -97,3 +97,8 @@ func (f *filehandle) Stat() (os.FileInfo, error) {
func (f *filehandle) Close() error {
return nil
}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 66856b7..5326131 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -18,7 +18,11 @@ import (
// collections, these changes are not persistent or visible to other
// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ fs := &fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ }
root := &treenode{
+ fs: fs,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -28,8 +32,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
root.parent = root
root.Child("by_id", func(inode) inode {
- return &vdirnode{
+ var vn inode
+ vn = &vdirnode{
treenode: treenode{
+ fs: fs,
parent: root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
@@ -39,25 +45,29 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: func(name string) inode {
- return newEntByID(c, kc, name)
+ return newEntByID(vn, name)
},
}
+ return vn
})
- return &fileSystem{inode: root}
+ fs.root = root
+ return fs
}
-func newEntByID(c *Client, kc keepClient, id string) inode {
+func newEntByID(parent inode, id string) inode {
var coll Collection
- err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(c, kc)
- fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ fs, err := coll.FileSystem(parent.FS(), parent.FS())
if err != nil {
return nil
}
- return fs
+ root := fs.(*collectionFileSystem).root.(*dirnode)
+ root.fileinfo.name = id
+ root.parent = parent
+ return root
}
type vdirnode struct {
commit aa0ad923e691341b0dbcf16e157a1796efb56ea9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:54:36 2017 -0500
12308: Move code from fs_collection to fs_base and fs_filehandle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 4de1bf8..3be3f5e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -5,10 +5,152 @@
package arvados
import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
"os"
+ "path"
+ "strings"
"sync"
+ "time"
)
+var (
+ ErrReadOnlyFile = errors.New("read-only file")
+ ErrNegativeOffset = errors.New("cannot seek to negative offset")
+ ErrFileExists = errors.New("file exists")
+ ErrInvalidOperation = errors.New("invalid operation")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
+ ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrPermission = os.ErrPermission
+)
+
+// A File is an *os.File-like interface for reading and writing files
+// in a FileSystem.
+type File interface {
+ io.Reader
+ io.Writer
+ io.Closer
+ io.Seeker
+ Size() int64
+ Readdir(int) ([]os.FileInfo, error)
+ Stat() (os.FileInfo, error)
+ Truncate(int64) error
+}
+
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
+ http.FileSystem
+
+ inode
+
+ // analogous to os.Stat()
+ Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
+ Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
+ OpenFile(name string, flag int, perm os.FileMode) (File, error)
+
+ Mkdir(name string, perm os.FileMode) error
+ Remove(name string) error
+ RemoveAll(name string) error
+ Rename(oldname, newname string) error
+}
+
+type inode interface {
+ Parent() inode
+ Read([]byte, filenodePtr) (int, filenodePtr, error)
+ Write([]byte, filenodePtr) (int, filenodePtr, error)
+ Truncate(int64) error
+ IsDir() bool
+ Readdir() []os.FileInfo
+ Size() int64
+ FileInfo() os.FileInfo
+
+ // Child() performs lookups and updates of named child nodes.
+ //
+ // If replace is non-nil, Child calls replace(x) where x is
+ // the current child inode with the given name. If possible,
+ // the child inode is replaced with the one returned by
+ // replace().
+ //
+ // Nil represents "no child". replace(nil) signifies that no
+ // child with this name exists yet. If replace() returns nil,
+ // the existing child should be deleted if possible.
+ //
+ // An implementation of Child() is permitted to ignore
+ // replace() or its return value. For example, a regular file
+ // inode does not have children, so Child() always returns
+ // nil.
+ //
+ // Child() returns the child, if any, with the given name: if
+ // a child was added or changed, the new child is returned.
+ //
+ // Caller must have lock (or rlock if replace is nil).
+ Child(name string, replace func(inode) inode) inode
+
+ sync.Locker
+ RLock()
+ RUnlock()
+}
+
+type fileinfo struct {
+ name string
+ mode os.FileMode
+ size int64
+ modTime time.Time
+}
+
+// Name implements os.FileInfo.
+func (fi fileinfo) Name() string {
+ return fi.name
+}
+
+// ModTime implements os.FileInfo.
+func (fi fileinfo) ModTime() time.Time {
+ return fi.modTime
+}
+
+// Mode implements os.FileInfo.
+func (fi fileinfo) Mode() os.FileMode {
+ return fi.mode
+}
+
+// IsDir implements os.FileInfo.
+func (fi fileinfo) IsDir() bool {
+ return fi.mode&os.ModeDir != 0
+}
+
+// Size implements os.FileInfo.
+func (fi fileinfo) Size() int64 {
+ return fi.size
+}
+
+// Sys implements os.FileInfo.
+func (fi fileinfo) Sys() interface{} {
+ return nil
+}
+
type nullnode struct{}
func (*nullnode) Mkdir(string, os.FileMode) error {
@@ -95,3 +237,273 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
return
}
+
+type fileSystem struct {
+ inode
+}
+
+// OpenFile is analogous to os.OpenFile().
+func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
+}
+
+func (fs *fileSystem) Open(name string) (http.File, error) {
+ return fs.OpenFile(name, os.O_RDONLY, 0)
+}
+
+func (fs *fileSystem) Create(name string) (File, error) {
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
+}
+
+func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
+ node := rlookup(fs.inode, name)
+ if node == nil {
+ err = os.ErrNotExist
+ } else {
+ fi = node.FileInfo()
+ }
+ return
+}
+
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ oldinode.Lock()
+ defer oldinode.Unlock()
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ n.treenode.fileinfo.name = newname
+ case *filenode:
+ n.parent = newdn
+ n.fileinfo.name = newname
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ olddn.treenode.fileinfo.modTime = time.Now()
+ newdn.treenode.fileinfo.modTime = time.Now()
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 72ef49f..e451189 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -6,10 +6,8 @@ package arvados
import (
"encoding/json"
- "errors"
"fmt"
"io"
- "net/http"
"os"
"path"
"regexp"
@@ -20,111 +18,13 @@ import (
"time"
)
-var (
- ErrReadOnlyFile = errors.New("read-only file")
- ErrNegativeOffset = errors.New("cannot seek to negative offset")
- ErrFileExists = errors.New("file exists")
- ErrInvalidOperation = errors.New("invalid operation")
- ErrInvalidArgument = errors.New("invalid argument")
- ErrDirectoryNotEmpty = errors.New("directory not empty")
- ErrWriteOnlyMode = errors.New("file is O_WRONLY")
- ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
- ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
- ErrPermission = os.ErrPermission
-
- maxBlockSize = 1 << 26
-)
-
-// A File is an *os.File-like interface for reading and writing files
-// in a CollectionFileSystem.
-type File interface {
- io.Reader
- io.Writer
- io.Closer
- io.Seeker
- Size() int64
- Readdir(int) ([]os.FileInfo, error)
- Stat() (os.FileInfo, error)
- Truncate(int64) error
-}
+var maxBlockSize = 1 << 26
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
-type fileinfo struct {
- name string
- mode os.FileMode
- size int64
- modTime time.Time
-}
-
-// Name implements os.FileInfo.
-func (fi fileinfo) Name() string {
- return fi.name
-}
-
-// ModTime implements os.FileInfo.
-func (fi fileinfo) ModTime() time.Time {
- return fi.modTime
-}
-
-// Mode implements os.FileInfo.
-func (fi fileinfo) Mode() os.FileMode {
- return fi.mode
-}
-
-// IsDir implements os.FileInfo.
-func (fi fileinfo) IsDir() bool {
- return fi.mode&os.ModeDir != 0
-}
-
-// Size implements os.FileInfo.
-func (fi fileinfo) Size() int64 {
- return fi.size
-}
-
-// Sys implements os.FileInfo.
-func (fi fileinfo) Sys() interface{} {
- return nil
-}
-
-// A FileSystem is an http.Filesystem plus Stat() and support for
-// opening writable files. All methods are safe to call from multiple
-// goroutines.
-type FileSystem interface {
- http.FileSystem
-
- inode
-
- // analogous to os.Stat()
- Stat(name string) (os.FileInfo, error)
-
- // analogous to os.Create(): create/truncate a file and open it O_RDWR.
- Create(name string) (File, error)
-
- // Like os.OpenFile(): create or open a file or directory.
- //
- // If flag&os.O_EXCL==0, it opens an existing file or
- // directory if one exists. If flag&os.O_CREATE!=0, it creates
- // a new empty file or directory if one does not already
- // exist.
- //
- // When creating a new item, perm&os.ModeDir determines
- // whether it is a file or a directory.
- //
- // A file can be opened multiple times and used concurrently
- // from multiple goroutines. However, each File object should
- // be used by only one goroutine at a time.
- OpenFile(name string, flag int, perm os.FileMode) (File, error)
-
- Mkdir(name string, perm os.FileMode) error
- Remove(name string) error
- RemoveAll(name string) error
- Rename(oldname, newname string) error
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -137,8 +37,33 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
-type fileSystem struct {
- inode
+// FileSystem returns a CollectionFileSystem for the collection.
+func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+ var modTime time.Time
+ if c.ModifiedAt == nil {
+ modTime = time.Now()
+ } else {
+ modTime = *c.ModifiedAt
+ }
+ dn := &dirnode{
+ client: client,
+ kc: kc,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
+ },
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
+ return nil, err
+ }
+ return fs, nil
}
type collectionFileSystem struct {
@@ -170,301 +95,6 @@ func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
}
-// OpenFile is analogous to os.OpenFile().
-func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.openFile(name, flag, perm)
-}
-
-func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
- if parent == nil {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable && parent.IsDir() {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: parent}, nil
- case "..":
- return &filehandle{inode: parent.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- parent.Lock()
- defer parent.Unlock()
- } else {
- parent.RLock()
- defer parent.RUnlock()
- }
- n := parent.Child(name, nil)
- if n == nil {
- if !createMode {
- return nil, os.ErrNotExist
- }
- var err error
- n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
- if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
- } else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
- }
- return n
- })
- if err != nil {
- return nil, err
- } else if n == nil {
- // parent rejected new child
- return nil, ErrInvalidOperation
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
- }, nil
-}
-
-func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.OpenFile(name, os.O_RDONLY, 0)
-}
-
-func (fs *fileSystem) Create(name string) (File, error) {
- return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
-}
-
-func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
- dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
- if n == nil {
- return os.ErrNotExist
- }
- n.Lock()
- defer n.Unlock()
- if n.Child(name, nil) != nil {
- return os.ErrExist
- }
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
- child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
- return
- })
- if err != nil {
- return err
- } else if child == nil {
- return ErrInvalidArgument
- }
- return nil
-}
-
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
- if node == nil {
- err = os.ErrNotExist
- } else {
- fi = node.FileInfo()
- }
- return
-}
-
-func (fs *fileSystem) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
-
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
- }
- }
-
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
- err = nil
- olddirf.inode.Child(oldname, func(oldinode inode) inode {
- if oldinode == nil {
- err = os.ErrNotExist
- return nil
- }
- newdirf.inode.Child(newname, func(existing inode) inode {
- if existing != nil && existing.IsDir() {
- err = ErrIsDirectory
- return existing
- }
- return oldinode
- })
- if err != nil {
- return oldinode
- }
- oldinode.Lock()
- defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
- case *filenode:
- n.parent = newdn
- n.fileinfo.name = newname
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
- return nil
- })
- return err
-}
-
-func (fs *fileSystem) Remove(name string) error {
- return fs.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (fs *fileSystem) RemoveAll(name string) error {
- err := fs.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (fs *fileSystem) remove(name string, recursive bool) (err error) {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dir := rlookup(fs, dirname)
- if dir == nil {
- return os.ErrNotExist
- }
- dir.Lock()
- defer dir.Unlock()
- dir.Child(name, func(node inode) inode {
- if node == nil {
- err = os.ErrNotExist
- return nil
- }
- if !recursive && node.IsDir() && node.Size() > 0 {
- err = ErrDirectoryNotEmpty
- return node
- }
- return nil
- })
- return err
-}
-
-type inode interface {
- Parent() inode
- Read([]byte, filenodePtr) (int, filenodePtr, error)
- Write([]byte, filenodePtr) (int, filenodePtr, error)
- Truncate(int64) error
- IsDir() bool
- Readdir() []os.FileInfo
- Size() int64
- FileInfo() os.FileInfo
- // Caller must have lock (or rlock if func is nil)
- Child(string, func(inode) inode) inode
- sync.Locker
- RLock()
- RUnlock()
-}
-
-// filenode implements inode.
-type filenode struct {
- fileinfo fileinfo
- parent *dirnode
- segments []segment
- // number of times `segments` has changed in a
- // way that might invalidate a filenodePtr
- repacked int64
- memsize int64 // bytes in memSegments
- sync.RWMutex
- nullnode
-}
-
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
@@ -538,6 +168,19 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
return
}
+// filenode implements inode.
+type filenode struct {
+ fileinfo fileinfo
+ parent *dirnode
+ segments []segment
+ // number of times `segments` has changed in a
+ // way that might invalidate a filenodePtr
+ repacked int64
+ memsize int64 // bytes in memSegments
+ sync.RWMutex
+ nullnode
+}
+
// caller must have lock
func (fn *filenode) appendSegment(e segment) {
fn.segments = append(fn.segments, e)
@@ -813,128 +456,6 @@ func (fn *filenode) pruneMemSegments() {
}
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
- var modTime time.Time
- if c.ModifiedAt == nil {
- modTime = time.Now()
- } else {
- modTime = *c.ModifiedAt
- }
- dn := &dirnode{
- client: client,
- kc: kc,
- treenode: treenode{
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
- },
- parent: nil,
- inodes: make(map[string]inode),
- },
- }
- dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
- if err := dn.loadManifest(c.ManifestText); err != nil {
- return nil, err
- }
- return fs, nil
-}
-
-type filehandle struct {
- inode
- ptr filenodePtr
- append bool
- readable bool
- writable bool
- unreaddirs []os.FileInfo
-}
-
-func (f *filehandle) Read(p []byte) (n int, err error) {
- if !f.readable {
- return 0, ErrWriteOnlyMode
- }
- f.inode.RLock()
- defer f.inode.RUnlock()
- n, f.ptr, err = f.inode.Read(p, f.ptr)
- return
-}
-
-func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
- size := f.inode.Size()
- ptr := f.ptr
- switch whence {
- case io.SeekStart:
- ptr.off = off
- case io.SeekCurrent:
- ptr.off += off
- case io.SeekEnd:
- ptr.off = size + off
- }
- if ptr.off < 0 {
- return f.ptr.off, ErrNegativeOffset
- }
- if ptr.off != f.ptr.off {
- f.ptr = ptr
- // force filenode to recompute f.ptr fields on next
- // use
- f.ptr.repacked = -1
- }
- return f.ptr.off, nil
-}
-
-func (f *filehandle) Truncate(size int64) error {
- return f.inode.Truncate(size)
-}
-
-func (f *filehandle) Write(p []byte) (n int, err error) {
- if !f.writable {
- return 0, ErrReadOnlyFile
- }
- f.inode.Lock()
- defer f.inode.Unlock()
- if fn, ok := f.inode.(*filenode); ok && f.append {
- f.ptr = filenodePtr{
- off: fn.fileinfo.size,
- segmentIdx: len(fn.segments),
- segmentOff: 0,
- repacked: fn.repacked,
- }
- }
- n, f.ptr, err = f.inode.Write(p, f.ptr)
- return
-}
-
-func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.IsDir() {
- return nil, ErrInvalidOperation
- }
- if count <= 0 {
- return f.inode.Readdir(), nil
- }
- if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
- }
- if len(f.unreaddirs) == 0 {
- return nil, io.EOF
- }
- if count > len(f.unreaddirs) {
- count = len(f.unreaddirs)
- }
- ret := f.unreaddirs[:count]
- f.unreaddirs = f.unreaddirs[count:]
- return ret, nil
-}
-
-func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.FileInfo(), nil
-}
-
-func (f *filehandle) Close() error {
- return nil
-}
-
type dirnode struct {
treenode
client *Client
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
new file mode 100644
index 0000000..56963b6
--- /dev/null
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -0,0 +1,99 @@
+package arvados
+
+import (
+ "io"
+ "os"
+)
+
+type filehandle struct {
+ inode
+ ptr filenodePtr
+ append bool
+ readable bool
+ writable bool
+ unreaddirs []os.FileInfo
+}
+
+func (f *filehandle) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
+ n, f.ptr, err = f.inode.Read(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
+ size := f.inode.Size()
+ ptr := f.ptr
+ switch whence {
+ case io.SeekStart:
+ ptr.off = off
+ case io.SeekCurrent:
+ ptr.off += off
+ case io.SeekEnd:
+ ptr.off = size + off
+ }
+ if ptr.off < 0 {
+ return f.ptr.off, ErrNegativeOffset
+ }
+ if ptr.off != f.ptr.off {
+ f.ptr = ptr
+ // force filenode to recompute f.ptr fields on next
+ // use
+ f.ptr.repacked = -1
+ }
+ return f.ptr.off, nil
+}
+
+func (f *filehandle) Truncate(size int64) error {
+ return f.inode.Truncate(size)
+}
+
+func (f *filehandle) Write(p []byte) (n int, err error) {
+ if !f.writable {
+ return 0, ErrReadOnlyFile
+ }
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ segmentIdx: len(fn.segments),
+ segmentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
+ n, f.ptr, err = f.inode.Write(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
+ if !f.inode.IsDir() {
+ return nil, ErrInvalidOperation
+ }
+ if count <= 0 {
+ return f.inode.Readdir(), nil
+ }
+ if f.unreaddirs == nil {
+ f.unreaddirs = f.inode.Readdir()
+ }
+ if len(f.unreaddirs) == 0 {
+ return nil, io.EOF
+ }
+ if count > len(f.unreaddirs) {
+ count = len(f.unreaddirs)
+ }
+ ret := f.unreaddirs[:count]
+ f.unreaddirs = f.unreaddirs[count:]
+ return ret, nil
+}
+
+func (f *filehandle) Stat() (os.FileInfo, error) {
+ return f.inode.FileInfo(), nil
+}
+
+func (f *filehandle) Close() error {
+ return nil
+}
commit 28f6431aa5cdd2fa09dfd226e741c73c4c040e2f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:18:01 2017 -0500
12308: Rearrange source files.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/fs_base.go
similarity index 100%
rename from sdk/go/arvados/base_fs.go
rename to sdk/go/arvados/fs_base.go
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/fs_collection.go
similarity index 100%
rename from sdk/go/arvados/collection_fs.go
rename to sdk/go/arvados/fs_collection.go
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/fs_collection_test.go
similarity index 100%
rename from sdk/go/arvados/collection_fs_test.go
rename to sdk/go/arvados/fs_collection_test.go
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/fs_site.go
similarity index 77%
rename from sdk/go/arvados/site_fs.go
rename to sdk/go/arvados/fs_site.go
index 853bd32..66856b7 100644
--- a/sdk/go/arvados/site_fs.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,6 +9,14 @@ import (
"time"
)
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// although the FileSystem allows files to be added and modified in
+// collections, these changes are not persistent or visible to other
+// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root := &treenode{
fileinfo: fileinfo{
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/fs_site_test.go
similarity index 100%
rename from sdk/go/arvados/site_fs_test.go
rename to sdk/go/arvados/fs_site_test.go
commit 721ad80e9f349cea32415d6d8b2643b3478429fb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 00:49:36 2017 -0500
12308: .arvados#collection special file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 4647055..72ef49f 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -5,6 +5,7 @@
package arvados
import (
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -144,6 +145,25 @@ type collectionFileSystem struct {
fileSystem
}
+func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+ if name == ".arvados#collection" {
+ return &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ }
+ return fs.fileSystem.Child(name, replace)
+}
+
func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.inode.Lock()
defer fs.fileSystem.inode.Unlock()
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs_getternode.go
new file mode 100644
index 0000000..c9ffb38
--- /dev/null
+++ b/sdk/go/arvados/fs_getternode.go
@@ -0,0 +1,66 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "os"
+ "time"
+)
+
+// A getternode is a read-only character device that returns whatever
+// data is returned by the supplied function.
+type getternode struct {
+ Getter func() ([]byte, error)
+
+ treenode
+ data *bytes.Reader
+}
+
+func (*getternode) IsDir() bool {
+ return false
+}
+
+func (*getternode) Child(string, func(inode) inode) inode {
+ return nil
+}
+
+func (gn *getternode) get() error {
+ if gn.data != nil {
+ return nil
+ }
+ data, err := gn.Getter()
+ if err != nil {
+ return err
+ }
+ gn.data = bytes.NewReader(data)
+ return nil
+}
+
+func (gn *getternode) Size() int64 {
+ return gn.FileInfo().Size()
+}
+
+func (gn *getternode) FileInfo() os.FileInfo {
+ gn.Lock()
+ defer gn.Unlock()
+ var size int64
+ if gn.get() == nil {
+ size = gn.data.Size()
+ }
+ return fileinfo{
+ modTime: time.Now(),
+ mode: 0444,
+ size: size,
+ }
+}
+
+func (gn *getternode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
+ if err := gn.get(); err != nil {
+ return 0, ptr, err
+ }
+ n, err := gn.data.ReadAt(p, ptr.off)
+ return n, filenodePtr{off: ptr.off + int64(n)}, err
+}
commit cd48b440cbd6aa3f46a16be0180d18be7c1addbe
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 20:14:54 2017 -0500
12308: Fix seek/read/write race.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 331c3cb..8e488f5 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -10,6 +10,14 @@ import (
"github.com/curoverse/cgofuse/fuse"
)
+// sharedFile wraps arvados.File with a sync.Mutex, so fuse can safely
+// use a single filehandle concurrently on behalf of multiple
+// threads/processes.
+type sharedFile struct {
+ arvados.File
+ sync.Mutex
+}
+
type keepFS struct {
fuse.FileSystemBase
Client *arvados.Client
@@ -19,7 +27,7 @@ type keepFS struct {
Gid int
root arvados.FileSystem
- open map[uint64]arvados.File
+ open map[uint64]*sharedFile
lastFH uint64
sync.Mutex
}
@@ -32,14 +40,20 @@ func (fs *keepFS) newFH(f arvados.File) uint64 {
fs.Lock()
defer fs.Unlock()
if fs.open == nil {
- fs.open = make(map[uint64]arvados.File)
+ fs.open = make(map[uint64]*sharedFile)
}
fs.lastFH++
fh := fs.lastFH
- fs.open[fh] = f
+ fs.open[fh] = &sharedFile{File: f}
return fh
}
+func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
+ fs.Lock()
+ defer fs.Unlock()
+ return fs.open[fh]
+}
+
func (fs *keepFS) Init() {
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
}
@@ -165,14 +179,19 @@ func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
if fs.ReadOnly {
return -fuse.EROFS
}
- f := fs.lookupFH(fh)
- if f == nil {
- var err error
- if f, err = fs.root.OpenFile(path, os.O_RDWR, 0); err != nil {
- return fs.errCode(err)
- }
- defer f.Close()
+
+ // Sometimes fh is a valid filehandle and we don't need to
+ // waste a name lookup.
+ if f := fs.lookupFH(fh); f != nil {
+ return fs.errCode(f.Truncate(size))
}
+
+ // Other times, fh is invalid and we need to lookup path.
+ f, err := fs.root.OpenFile(path, os.O_RDWR, 0)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ defer f.Close()
return fs.errCode(f.Truncate(size))
}
@@ -180,8 +199,10 @@ func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int)
var fi os.FileInfo
var err error
if f := fs.lookupFH(fh); f != nil {
+ // Valid filehandle -- ignore path.
fi, err = f.Stat()
} else {
+ // Invalid filehandle -- lookup path.
fi, err = fs.root.Stat(path)
}
if err != nil {
@@ -233,9 +254,14 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
if fs.ReadOnly {
return -fuse.EROFS
- } else if f := fs.lookupFH(fh); f == nil {
+ }
+ f := fs.lookupFH(fh)
+ if f == nil {
return -fuse.EBADF
- } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
} else {
n, _ = f.Write(buf)
@@ -244,9 +270,13 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
- if f := fs.lookupFH(fh); f == nil {
+ f := fs.lookupFH(fh)
+ if f == nil {
return -fuse.EBADF
- } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
} else {
n, _ = f.Read(buf)
@@ -275,9 +305,3 @@ func (fs *keepFS) Readdir(path string,
}
return 0
}
-
-func (fs *keepFS) lookupFH(fh uint64) arvados.File {
- fs.Lock()
- defer fs.Unlock()
- return fs.open[fh]
-}
commit fd1e7a3998f4a60e9976010e13ab822456997143
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 17:54:52 2017 -0500
12308: Implement rm, mv, chmod, truncate.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index aa81a2a..331c3cb 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -96,6 +96,8 @@ func (fs *keepFS) errCode(err error) int {
return -fuse.EINVAL
case arvados.ErrInvalidOperation:
return -fuse.ENOSYS
+ case arvados.ErrDirectoryNotEmpty:
+ return -fuse.ENOTEMPTY
case nil:
return 0
default:
@@ -128,11 +130,11 @@ func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
return 0, fs.newFH(f)
}
-func (fs *keepFS) Releasedir(path string, fh uint64) int {
+func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
return fs.Release(path, fh)
}
-func (fs *keepFS) Release(path string, fh uint64) int {
+func (fs *keepFS) Release(path string, fh uint64) (errc int) {
fs.Lock()
defer fs.Unlock()
defer delete(fs.open, fh)
@@ -145,15 +147,63 @@ func (fs *keepFS) Release(path string, fh uint64) int {
return 0
}
+func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ return fs.errCode(fs.root.Rename(oldname, newname))
+}
+
+func (fs *keepFS) Unlink(path string) (errc int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ return fs.errCode(fs.root.Remove(path))
+}
+
+func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f := fs.lookupFH(fh)
+ if f == nil {
+ var err error
+ if f, err = fs.root.OpenFile(path, os.O_RDWR, 0); err != nil {
+ return fs.errCode(err)
+ }
+ defer f.Close()
+ }
+ return fs.errCode(f.Truncate(size))
+}
+
func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
- fi, err := fs.root.Stat(path)
+ var fi os.FileInfo
+ var err error
+ if f := fs.lookupFH(fh); f != nil {
+ fi, err = f.Stat()
+ } else {
+ fi, err = fs.root.Stat(path)
+ }
if err != nil {
- return -fuse.ENOENT
+ return fs.errCode(err)
}
fs.fillStat(stat, fi)
return 0
}
+func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ if fi, err := fs.root.Stat(path); err != nil {
+ return fs.errCode(err)
+ } else if (os.FileMode(mode)^fi.Mode())&os.ModePerm != 0 {
+ return -fuse.ENOSYS
+ } else {
+ return 0
+ }
+}
+
func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
var m uint32
if fi.IsDir() {
@@ -183,30 +233,25 @@ func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
if fs.ReadOnly {
return -fuse.EROFS
- }
- f := fs.lookupFH(fh)
- if f == nil {
+ } else if f := fs.lookupFH(fh); f == nil {
return -fuse.EBADF
+ } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ return fs.errCode(err)
+ } else {
+ n, _ = f.Write(buf)
+ return
}
- _, err := f.Seek(ofst, io.SeekStart)
- if err != nil {
- return -fuse.EINVAL
- }
- n, _ = f.Write(buf)
- return
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
- f := fs.lookupFH(fh)
- if f == nil {
- return 0
- }
- _, err := f.Seek(ofst, io.SeekStart)
- if err != nil {
- return 0
+ if f := fs.lookupFH(fh); f == nil {
+ return -fuse.EBADF
+ } else if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ return fs.errCode(err)
+ } else {
+ n, _ = f.Read(buf)
+ return
}
- n, _ = f.Read(buf)
- return
}
func (fs *keepFS) Readdir(path string,
@@ -222,12 +267,11 @@ func (fs *keepFS) Readdir(path string,
var stat fuse.Stat_t
fis, err := f.Readdir(-1)
if err != nil {
- return -fuse.ENOSYS // ???
+ return fs.errCode(err)
}
for _, fi := range fis {
fs.fillStat(&stat, fi)
- //fill(fi.Name(), &stat, 0)
- fill(fi.Name(), nil, 0)
+ fill(fi.Name(), &stat, 0)
}
return 0
}
commit 80900ad43ef0681d733d9f3b54cbbed1f0f24484
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 17:46:53 2017 -0500
12308: Fix inode not updated during rename().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index dcfd098..4647055 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -356,14 +356,22 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
if err != nil {
return oldinode
}
+ oldinode.Lock()
+ defer oldinode.Unlock()
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
+ n.treenode.fileinfo.name = newname
case *filenode:
- n.parent = newdirf.inode.(*dirnode)
+ n.parent = newdn
+ n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
+ olddn.treenode.fileinfo.modTime = time.Now()
+ newdn.treenode.fileinfo.modTime = time.Now()
return nil
})
return err
commit 014ef1b168b3f5f71eda4557353843f1ca759c75
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 00:43:30 2017 -0500
12308: Use mounting user/group as file owner/group.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/command.go b/lib/mount/command.go
index 9df7f7b..ed105e4 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -3,6 +3,7 @@ package mount
import (
"flag"
"log"
+ "os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -32,6 +33,8 @@ func Run(prog string, args []string) int {
Client: client,
KeepClient: kc,
ReadOnly: *ro,
+ Uid: os.Getuid(),
+ Gid: os.Getgid(),
})
notOK := host.Mount("", flags.Args())
if notOK {
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 3c01410..aa81a2a 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -15,6 +15,8 @@ type keepFS struct {
Client *arvados.Client
KeepClient *keepclient.KeepClient
ReadOnly bool
+ Uid int
+ Gid int
root arvados.FileSystem
open map[uint64]arvados.File
@@ -152,7 +154,7 @@ func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int)
return 0
}
-func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
var m uint32
if fi.IsDir() {
m = m | fuse.S_IFDIR
@@ -170,6 +172,12 @@ func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
stat.Birthtim = t
stat.Blksize = 1024
stat.Blocks = (stat.Size + stat.Blksize - 1) / stat.Blksize
+ if fs.Uid > 0 && int64(fs.Uid) < 1<<31 {
+ stat.Uid = uint32(fs.Uid)
+ }
+ if fs.Gid > 0 && int64(fs.Gid) < 1<<31 {
+ stat.Gid = uint32(fs.Gid)
+ }
}
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
commit 9ae4ad3ed94cd2aa07624aa228b0f872c4c2860b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 00:42:32 2017 -0500
12308: Split cli prog from mount lib.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/cmd/arvados-client/Makefile b/cmd/arvados-client/Makefile
new file mode 100644
index 0000000..d02fb78
--- /dev/null
+++ b/cmd/arvados-client/Makefile
@@ -0,0 +1,7 @@
+all:
+ go get .
+ docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" "$(GOPATH)"/src/github.com/curoverse/cgofuse
+ go get github.com/karalabe/xgo
+ xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
+ install arvados-* "$(GOPATH)"/bin/
+ rm --interactive=never arvados-*
diff --git a/services/mount/main.go b/lib/mount/command.go
similarity index 57%
rename from services/mount/main.go
rename to lib/mount/command.go
index c388368..9df7f7b 100644
--- a/services/mount/main.go
+++ b/lib/mount/command.go
@@ -1,4 +1,4 @@
-package main
+package mount
import (
"flag"
@@ -7,12 +7,17 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
- "github.com/billziss-gh/cgofuse/fuse"
+ "github.com/curoverse/cgofuse/fuse"
)
-func main() {
- ro := flag.Bool("ro", false, "read-only")
- flag.Parse()
+func Run(prog string, args []string) int {
+ flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
+ ro := flags.Bool("ro", false, "read-only")
+ err := flags.Parse(args)
+ if err != nil {
+ log.Print(err)
+ return 2
+ }
client := arvados.NewClientFromEnv()
ac, err := arvadosclient.New(client)
@@ -28,5 +33,10 @@ func main() {
KeepClient: kc,
ReadOnly: *ro,
})
- host.Mount("", flag.Args())
+ notOK := host.Mount("", flags.Args())
+ if notOK {
+ return 1
+ } else {
+ return 0
+ }
}
diff --git a/services/mount/fs.go b/lib/mount/fs.go
similarity index 98%
rename from services/mount/fs.go
rename to lib/mount/fs.go
index 5377803..3c01410 100644
--- a/services/mount/fs.go
+++ b/lib/mount/fs.go
@@ -1,4 +1,4 @@
-package main
+package mount
import (
"io"
@@ -7,7 +7,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
- "github.com/billziss-gh/cgofuse/fuse"
+ "github.com/curoverse/cgofuse/fuse"
)
type keepFS struct {
diff --git a/lib/mount/fs_test.go b/lib/mount/fs_test.go
new file mode 100644
index 0000000..a178d6a
--- /dev/null
+++ b/lib/mount/fs_test.go
@@ -0,0 +1,7 @@
+package mount
+
+import (
+ "github.com/curoverse/cgofuse/fuse"
+)
+
+var _ fuse.FileSystem = &keepFS{}
diff --git a/services/mount/.gitignore b/services/mount/.gitignore
deleted file mode 100644
index 8c9846d..0000000
--- a/services/mount/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-mount-*
diff --git a/services/mount/Makefile b/services/mount/Makefile
deleted file mode 100644
index 3730673..0000000
--- a/services/mount/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-all:
- go get .
- docker build --tag=cgofuse $(GOPATH)/src/github.com/billziss-gh/cgofuse
- go get github.com/karalabe/xgo
- xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
diff --git a/services/mount/fs_test.go b/services/mount/fs_test.go
deleted file mode 100644
index fadda1a..0000000
--- a/services/mount/fs_test.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package main
-
-import (
- "github.com/billziss-gh/cgofuse/fuse"
-)
-
-var _ fuse.FileSystem = &keepFS{}
commit d3a2efbf7bab9101d4e02369f7b4c22de5058b85
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Dec 16 04:51:39 2017 -0500
12308: read-only mode, mkdir, touch.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/mount/fs.go b/services/mount/fs.go
index 4789976..5377803 100644
--- a/services/mount/fs.go
+++ b/services/mount/fs.go
@@ -12,9 +12,9 @@ import (
type keepFS struct {
fuse.FileSystemBase
- Collection arvados.Collection
Client *arvados.Client
KeepClient *keepclient.KeepClient
+ ReadOnly bool
root arvados.FileSystem
open map[uint64]arvados.File
@@ -43,6 +43,9 @@ func (fs *keepFS) Init() {
}
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ if fs.ReadOnly {
+ return -fuse.EROFS, invalidFH
+ }
f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
if err == os.ErrExist {
return -fuse.EEXIST, invalidFH
@@ -53,6 +56,9 @@ func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint
}
func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
+ return -fuse.EROFS, invalidFH
+ }
f, err := fs.root.OpenFile(path, flags, 0)
if err != nil {
return -fuse.ENOENT, invalidFH
@@ -65,12 +71,54 @@ func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
return 0, fs.newFH(f)
}
+func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f, err := fs.root.OpenFile(path, 0, 0)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ f.Close()
+ return 0
+}
+
+func (fs *keepFS) errCode(err error) int {
+ if os.IsNotExist(err) {
+ return -fuse.ENOENT
+ }
+ switch err {
+ case os.ErrExist:
+ return -fuse.EEXIST
+ case arvados.ErrInvalidArgument:
+ return -fuse.EINVAL
+ case arvados.ErrInvalidOperation:
+ return -fuse.ENOSYS
+ case nil:
+ return 0
+ default:
+ return -fuse.EIO
+ }
+}
+
+func (fs *keepFS) Mkdir(path string, mode uint32) int {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f, err := fs.root.OpenFile(path, os.O_CREATE|os.O_EXCL, os.FileMode(mode)|os.ModeDir)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ f.Close()
+ return 0
+}
+
func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
f, err := fs.root.OpenFile(path, 0, 0)
if err != nil {
- return -fuse.ENOENT, invalidFH
+ return fs.errCode(err), invalidFH
} else if fi, err := f.Stat(); err != nil {
- return -fuse.EIO, invalidFH
+ return fs.errCode(err), invalidFH
} else if !fi.IsDir() {
f.Close()
return -fuse.ENOTDIR, invalidFH
@@ -125,13 +173,16 @@ func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
f := fs.lookupFH(fh)
if f == nil {
- return 0
+ return -fuse.EBADF
}
_, err := f.Seek(ofst, io.SeekStart)
if err != nil {
- return 0
+ return -fuse.EINVAL
}
n, _ = f.Write(buf)
return
diff --git a/services/mount/main.go b/services/mount/main.go
index c0ef380..c388368 100644
--- a/services/mount/main.go
+++ b/services/mount/main.go
@@ -1,8 +1,8 @@
package main
import (
+ "flag"
"log"
- "os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -11,6 +11,9 @@ import (
)
func main() {
+ ro := flag.Bool("ro", false, "read-only")
+ flag.Parse()
+
client := arvados.NewClientFromEnv()
ac, err := arvadosclient.New(client)
if err != nil {
@@ -23,6 +26,7 @@ func main() {
host := fuse.NewFileSystemHost(&keepFS{
Client: client,
KeepClient: kc,
+ ReadOnly: *ro,
})
- host.Mount("", os.Args[1:])
+ host.Mount("", flag.Args())
}
commit 2bae70dcf16f6306aa82ccec375ff94b46ec65f2
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 09:22:03 2017 -0500
12308: Mount collections on demand in mnt/by_id/.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/base_fs.go
new file mode 100644
index 0000000..4de1bf8
--- /dev/null
+++ b/sdk/go/arvados/base_fs.go
@@ -0,0 +1,97 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+)
+
+type nullnode struct{}
+
+func (*nullnode) Mkdir(string, os.FileMode) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) Read([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Write([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Truncate(int64) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) FileInfo() os.FileInfo {
+ return fileinfo{}
+}
+
+func (*nullnode) IsDir() bool {
+ return false
+}
+
+func (*nullnode) Readdir() []os.FileInfo {
+ return nil
+}
+
+func (*nullnode) Child(name string, replace func(inode) inode) inode {
+ return nil
+}
+
+type treenode struct {
+ parent inode
+ inodes map[string]inode
+ fileinfo fileinfo
+ sync.RWMutex
+ nullnode
+}
+
+func (n *treenode) Parent() inode {
+ n.RLock()
+ defer n.RUnlock()
+ return n.parent
+}
+
+func (n *treenode) IsDir() bool {
+ return true
+}
+
+func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
+ // TODO: special treatment for "", ".", ".."
+ child = n.inodes[name]
+ if replace != nil {
+ child = replace(child)
+ if child == nil {
+ delete(n.inodes, name)
+ } else {
+ n.inodes[name] = child
+ }
+ }
+ return
+}
+
+func (n *treenode) Size() int64 {
+ return n.FileInfo().Size()
+}
+
+func (n *treenode) FileInfo() os.FileInfo {
+ n.Lock()
+ defer n.Unlock()
+ n.fileinfo.size = int64(len(n.inodes))
+ return n.fileinfo
+}
+
+func (n *treenode) Readdir() (fi []os.FileInfo) {
+ n.RLock()
+ defer n.RUnlock()
+ fi = make([]os.FileInfo, 0, len(n.inodes))
+ for _, inode := range n.inodes {
+ fi = append(fi, inode.FileInfo())
+ }
+ return
+}
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index d8ee2a2..dcfd098 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -89,12 +89,14 @@ func (fi fileinfo) Sys() interface{} {
return nil
}
-// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files. All methods are safe to call
-// from multiple goroutines.
-type CollectionFileSystem interface {
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
http.FileSystem
+ inode
+
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -120,6 +122,12 @@ type CollectionFileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+}
+
+// A CollectionFileSystem is a FileSystem that can be serialized as a
+// manifest and stored as a collection.
+type CollectionFileSystem interface {
+ FileSystem
// Flush all file data to Keep and return a snapshot of the
// filesystem suitable for saving as (Collection)ManifestText.
@@ -129,39 +137,288 @@ type CollectionFileSystem interface {
}
type fileSystem struct {
- dirnode
+ inode
+}
+
+type collectionFileSystem struct {
+ fileSystem
}
+func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.inode.Lock()
+ defer fs.fileSystem.inode.Unlock()
+ return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+}
+
+// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.dirnode.OpenFile(name, flag, perm)
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
}
func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.dirnode.OpenFile(name, os.O_RDONLY, 0)
+ return fs.OpenFile(name, os.O_RDONLY, 0)
}
func (fs *fileSystem) Create(name string) (File, error) {
- return fs.dirnode.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := fs.dirnode.lookupPath(name)
+ node := rlookup(fs.inode, name)
if node == nil {
err = os.ErrNotExist
} else {
- fi = node.Stat()
+ fi = node.FileInfo()
}
return
}
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ case *filenode:
+ n.parent = newdirf.inode.(*dirnode)
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
+
type inode interface {
Parent() inode
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
+ IsDir() bool
Readdir() []os.FileInfo
Size() int64
- Stat() os.FileInfo
+ FileInfo() os.FileInfo
+ // Caller must have lock (or rlock if func is nil)
+ Child(string, func(inode) inode) inode
sync.Locker
RLock()
RUnlock()
@@ -177,6 +434,7 @@ type filenode struct {
repacked int64
memsize int64 // bytes in memSegments
sync.RWMutex
+ nullnode
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -264,10 +522,6 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
-func (fn *filenode) Readdir() []os.FileInfo {
- return nil
-}
-
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -302,7 +556,7 @@ func (fn *filenode) Size() int64 {
return fn.fileinfo.Size()
}
-func (fn *filenode) Stat() os.FileInfo {
+func (fn *filenode) FileInfo() os.FileInfo {
fn.RLock()
defer fn.RUnlock()
return fn.fileinfo
@@ -539,19 +793,22 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
} else {
modTime = *c.ModifiedAt
}
- fs := &fileSystem{dirnode: dirnode{
+ dn := &dirnode{
client: client,
kc: kc,
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
},
- parent: nil,
- inodes: make(map[string]inode),
- }}
- fs.dirnode.parent = &fs.dirnode
- if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
return fs, nil
@@ -622,7 +879,7 @@ func (f *filehandle) Write(p []byte) (n int, err error) {
}
func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.Stat().IsDir() {
+ if !f.inode.IsDir() {
return nil, ErrInvalidOperation
}
if count <= 0 {
@@ -643,7 +900,7 @@ func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
}
func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.Stat(), nil
+ return f.inode.FileInfo(), nil
}
func (f *filehandle) Close() error {
@@ -651,12 +908,9 @@ func (f *filehandle) Close() error {
}
type dirnode struct {
- fileinfo fileinfo
- parent *dirnode
- client *Client
- kc keepClient
- inodes map[string]inode
- sync.RWMutex
+ treenode
+ client *Client
+ kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -735,12 +989,6 @@ func (dn *dirnode) sync() error {
return flush(pending)
}
-func (dn *dirnode) MarshalManifest(prefix string) (string, error) {
- dn.Lock()
- defer dn.Unlock()
- return dn.marshalManifest(prefix)
-}
-
// caller must have read lock.
func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var streamLen int64
@@ -941,6 +1189,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
+ node := dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -950,336 +1199,111 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
for _, name := range names[:len(names)-1] {
switch name {
case "", ".":
+ continue
case "..":
- dn = dn.parent
- default:
- switch node := dn.inodes[name].(type) {
+ if node == dn {
+ // can't be sure parent will be a *dirnode
+ return nil, ErrInvalidArgument
+ }
+ node = node.Parent().(*dirnode)
+ continue
+ }
+ node.Child(name, func(child inode) inode {
+ switch child.(type) {
case nil:
- dn = dn.newDirnode(name, 0755, dn.fileinfo.modTime)
+ node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child = node
case *dirnode:
- dn = node
+ node = child.(*dirnode)
case *filenode:
err = ErrFileExists
- return
+ default:
+ err = ErrInvalidOperation
}
+ return child
+ })
+ if err != nil {
+ return
}
}
- switch node := dn.inodes[basename].(type) {
- case nil:
- fn = dn.newFilenode(basename, 0755, dn.fileinfo.modTime)
- case *filenode:
- fn = node
- case *dirnode:
- err = ErrIsDirectory
- }
- return
-}
-
-func (dn *dirnode) mkdir(name string) (*filehandle, error) {
- return dn.OpenFile(name, os.O_CREATE|os.O_EXCL, os.ModeDir|0755)
-}
-
-func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
- f, err := dn.mkdir(name)
- if err == nil {
- err = f.Close()
- }
- return err
-}
-
-func (dn *dirnode) Remove(name string) error {
- return dn.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (dn *dirnode) RemoveAll(name string) error {
- err := dn.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (dn *dirnode) remove(name string, recursive bool) error {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return os.ErrNotExist
- }
- dn.Lock()
- defer dn.Unlock()
- switch node := dn.inodes[name].(type) {
- case nil:
- return os.ErrNotExist
- case *dirnode:
- node.RLock()
- defer node.RUnlock()
- if !recursive && len(node.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ node.Child(basename, func(child inode) inode {
+ switch child := child.(type) {
+ case nil:
+ fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ return fn
+ case *filenode:
+ fn = child
+ return child
+ case *dirnode:
+ err = ErrIsDirectory
+ return child
+ default:
+ err = ErrInvalidOperation
+ return child
}
- }
- delete(dn.inodes, name)
- return nil
+ })
+ return
}
-func (dn *dirnode) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := dn.OpenFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := dn.OpenFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
}
- }
-
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- oldinode, ok := olddn.inodes[oldname]
- if !ok {
- return os.ErrNotExist
- }
- if locked[oldinode] {
- // oldinode cannot become a descendant of itself.
- return ErrInvalidArgument
- }
- if existing, ok := newdn.inodes[newname]; ok {
- // overwriting an existing file or dir
- if dn, ok := existing.(*dirnode); ok {
- if !oldinode.Stat().IsDir() {
- return ErrIsDirectory
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
}
- dn.RLock()
- defer dn.RUnlock()
- if len(dn.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ if name == ".." {
+ node = node.Parent()
+ continue
}
}
- } else {
- if newdn.inodes == nil {
- newdn.inodes = make(map[string]inode)
- }
- newdn.fileinfo.size++
- }
- newdn.inodes[newname] = oldinode
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdn
- case *filenode:
- n.parent = newdn
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- delete(olddn.inodes, oldname)
- olddn.fileinfo.size--
- return nil
-}
-
-func (dn *dirnode) Parent() inode {
- dn.RLock()
- defer dn.RUnlock()
- return dn.parent
-}
-
-func (dn *dirnode) Readdir() (fi []os.FileInfo) {
- dn.RLock()
- defer dn.RUnlock()
- fi = make([]os.FileInfo, 0, len(dn.inodes))
- for _, inode := range dn.inodes {
- fi = append(fi, inode.Stat())
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
}
return
}
-func (dn *dirnode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Write(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Size() int64 {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo.Size()
-}
-
-func (dn *dirnode) Stat() os.FileInfo {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo
-}
-
-func (dn *dirnode) Truncate(int64) error {
- return ErrInvalidOperation
-}
-
-// lookupPath returns the inode for the file/directory with the given
-// name (which may contain "/" separators), along with its parent
-// node. If no such file/directory exists, the returned node is nil.
-func (dn *dirnode) lookupPath(path string) (node inode) {
- node = dn
- for _, name := range strings.Split(path, "/") {
- dn, ok := node.(*dirnode)
- if !ok {
- return nil
- }
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- dn.RLock()
- node = dn.inodes[name]
- dn.RUnlock()
+// Caller must have lock, and must have already ensured
+// Children(name,nil) is nil.
+func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
}
- return
-}
-
-func (dn *dirnode) newDirnode(name string, perm os.FileMode, modTime time.Time) *dirnode {
- child := &dirnode{
- parent: dn,
+ return &dirnode{
client: dn.client,
kc: dn.kc,
- fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | perm,
- modTime: modTime,
+ treenode: treenode{
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
+ }, nil
}
-func (dn *dirnode) newFilenode(name string, perm os.FileMode, modTime time.Time) *filenode {
- child := &filenode{
- parent: dn,
+func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ parent: parent,
fileinfo: fileinfo{
name: name,
- mode: perm,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
-}
-
-// OpenFile is analogous to os.OpenFile().
-func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: dn}, nil
- case "..":
- return &filehandle{inode: dn.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- dn.Lock()
- defer dn.Unlock()
- } else {
- dn.RLock()
- defer dn.RUnlock()
- }
- n, ok := dn.inodes[name]
- if !ok {
- if !createMode {
- return nil, os.ErrNotExist
- }
- if perm.IsDir() {
- n = dn.newDirnode(name, 0755, time.Now())
- } else {
- n = dn.newFilenode(name, 0755, time.Now())
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
}, nil
}
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index bd5d08b..023226f 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -465,6 +465,10 @@ func (s *CollectionFSSuite) TestMkdir(c *check.C) {
}
func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
maxBlockSize = 8
defer func() { maxBlockSize = 2 << 26 }()
@@ -1014,6 +1018,10 @@ var _ = check.Suite(&CollectionFSUnitSuite{})
// expect ~2 seconds to load a manifest with 256K files
func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
const (
dirCount = 512
fileCount = 512
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/site_fs.go
new file mode 100644
index 0000000..853bd32
--- /dev/null
+++ b/sdk/go/arvados/site_fs.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "time"
+)
+
+func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ root := &treenode{
+ fileinfo: fileinfo{
+ name: "/",
+ mode: os.ModeDir | 0755,
+ modTime: time.Now(),
+ },
+ inodes: make(map[string]inode),
+ }
+ root.parent = root
+ root.Child("by_id", func(inode) inode {
+ return &vdirnode{
+ treenode: treenode{
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: "by_id",
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ create: func(name string) inode {
+ return newEntByID(c, kc, name)
+ },
+ }
+ })
+ return &fileSystem{inode: root}
+}
+
+func newEntByID(c *Client, kc keepClient, id string) inode {
+ var coll Collection
+ err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ if err != nil {
+ return nil
+ }
+ fs, err := coll.FileSystem(c, kc)
+ fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ if err != nil {
+ return nil
+ }
+ return fs
+}
+
+type vdirnode struct {
+ treenode
+ create func(string) inode
+}
+
+func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
+ return vn.treenode.Child(name, func(existing inode) inode {
+ if existing != nil {
+ return existing
+ } else {
+ return vn.create(name)
+ }
+ })
+}
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/site_fs_test.go
new file mode 100644
index 0000000..a8c369f
--- /dev/null
+++ b/sdk/go/arvados/site_fs_test.go
@@ -0,0 +1,58 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&SiteFSSuite{})
+
+type SiteFSSuite struct {
+ client *Client
+ fs FileSystem
+ kc keepClient
+}
+
+func (s *SiteFSSuite) SetUpTest(c *check.C) {
+ s.client = NewClientFromEnv()
+ s.kc = &keepClientStub{
+ blocks: map[string][]byte{
+ "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
+ }}
+ s.fs = s.client.SiteFileSystem(s.kc)
+}
+
+func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
+ _, ok := s.fs.(http.FileSystem)
+ c.Check(ok, check.Equals, true)
+}
+
+func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Equals, 0)
+}
+
+func (s *SiteFSSuite) TestByUUID(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 0)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.FooCollection)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(-1)
+ var names []string
+ for _, fi := range fis {
+ names = append(names, fi.Name())
+ }
+ c.Check(names, check.DeepEquals, []string{"foo"})
+}
diff --git a/services/mount/fs.go b/services/mount/fs.go
index a5f4082..4789976 100644
--- a/services/mount/fs.go
+++ b/services/mount/fs.go
@@ -6,12 +6,17 @@ import (
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
"github.com/billziss-gh/cgofuse/fuse"
)
type keepFS struct {
fuse.FileSystemBase
- root arvados.CollectionFileSystem
+ Collection arvados.Collection
+ Client *arvados.Client
+ KeepClient *keepclient.KeepClient
+
+ root arvados.FileSystem
open map[uint64]arvados.File
lastFH uint64
sync.Mutex
@@ -33,10 +38,16 @@ func (fs *keepFS) newFH(f arvados.File) uint64 {
return fh
}
+func (fs *keepFS) Init() {
+ fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
+}
+
func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
- if err != nil {
- return -fuse.EPERM, invalidFH
+ if err == os.ErrExist {
+ return -fuse.EEXIST, invalidFH
+ } else if err != nil {
+ return -fuse.EINVAL, invalidFH
}
return 0, fs.newFH(f)
}
@@ -102,7 +113,28 @@ func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
}
m = m | uint32(fi.Mode()&os.ModePerm)
stat.Mode = m
+ stat.Nlink = 1
stat.Size = fi.Size()
+ t := fuse.NewTimespec(fi.ModTime())
+ stat.Mtim = t
+ stat.Ctim = t
+ stat.Atim = t
+ stat.Birthtim = t
+ stat.Blksize = 1024
+ stat.Blocks = (stat.Size + stat.Blksize - 1) / stat.Blksize
+}
+
+func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return 0
+ }
+ _, err := f.Seek(ofst, io.SeekStart)
+ if err != nil {
+ return 0
+ }
+ n, _ = f.Write(buf)
+ return
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
@@ -114,7 +146,7 @@ func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
if err != nil {
return 0
}
- n, err = f.Read(buf)
+ n, _ = f.Read(buf)
return
}
diff --git a/services/mount/main.go b/services/mount/main.go
index 6f31739..c0ef380 100644
--- a/services/mount/main.go
+++ b/services/mount/main.go
@@ -1,8 +1,8 @@
package main
import (
- "flag"
"log"
+ "os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -11,14 +11,7 @@ import (
)
func main() {
- var coll arvados.Collection
- flag.StringVar(&coll.UUID, "id", "", "collection `uuid` or pdh")
- flag.Parse()
client := arvados.NewClientFromEnv()
- err := client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
- if err != nil {
- log.Fatal(err)
- }
ac, err := arvadosclient.New(client)
if err != nil {
log.Fatal(err)
@@ -27,10 +20,9 @@ func main() {
if err != nil {
log.Fatal(err)
}
- fs, err := coll.FileSystem(client, kc)
- if err != nil {
- log.Fatal(err)
- }
- host := fuse.NewFileSystemHost(&keepFS{root: fs})
- host.Mount("", flag.Args())
+ host := fuse.NewFileSystemHost(&keepFS{
+ Client: client,
+ KeepClient: kc,
+ })
+ host.Mount("", os.Args[1:])
}
commit e2237253c78efad1bda6b9a7d4c7a162fe4612ee
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 01:14:43 2017 -0500
12308: Mount a single collection readonly.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/mount/.gitignore b/services/mount/.gitignore
new file mode 100644
index 0000000..8c9846d
--- /dev/null
+++ b/services/mount/.gitignore
@@ -0,0 +1 @@
+mount-*
diff --git a/services/mount/Makefile b/services/mount/Makefile
new file mode 100644
index 0000000..3730673
--- /dev/null
+++ b/services/mount/Makefile
@@ -0,0 +1,5 @@
+all:
+ go get .
+ docker build --tag=cgofuse $(GOPATH)/src/github.com/billziss-gh/cgofuse
+ go get github.com/karalabe/xgo
+ xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
diff --git a/services/mount/fs.go b/services/mount/fs.go
new file mode 100644
index 0000000..a5f4082
--- /dev/null
+++ b/services/mount/fs.go
@@ -0,0 +1,148 @@
+package main
+
+import (
+ "io"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+type keepFS struct {
+ fuse.FileSystemBase
+ root arvados.CollectionFileSystem
+ open map[uint64]arvados.File
+ lastFH uint64
+ sync.Mutex
+}
+
+var (
+ invalidFH = ^uint64(0)
+)
+
+func (fs *keepFS) newFH(f arvados.File) uint64 {
+ fs.Lock()
+ defer fs.Unlock()
+ if fs.open == nil {
+ fs.open = make(map[uint64]arvados.File)
+ }
+ fs.lastFH++
+ fh := fs.lastFH
+ fs.open[fh] = f
+ return fh
+}
+
+func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
+ if err != nil {
+ return -fuse.EPERM, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, flags, 0)
+ if err != nil {
+ return -fuse.ENOENT, invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return -fuse.EIO, invalidFH
+ } else if fi.IsDir() {
+ f.Close()
+ return -fuse.EISDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ f, err := fs.root.OpenFile(path, 0, 0)
+ if err != nil {
+ return -fuse.ENOENT, invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return -fuse.EIO, invalidFH
+ } else if !fi.IsDir() {
+ f.Close()
+ return -fuse.ENOTDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Releasedir(path string, fh uint64) int {
+ return fs.Release(path, fh)
+}
+
+func (fs *keepFS) Release(path string, fh uint64) int {
+ fs.Lock()
+ defer fs.Unlock()
+ defer delete(fs.open, fh)
+ if f := fs.open[fh]; f != nil {
+ err := f.Close()
+ if err != nil {
+ return -fuse.EIO
+ }
+ }
+ return 0
+}
+
+func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ fi, err := fs.root.Stat(path)
+ if err != nil {
+ return -fuse.ENOENT
+ }
+ fs.fillStat(stat, fi)
+ return 0
+}
+
+func (*keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ var m uint32
+ if fi.IsDir() {
+ m = m | fuse.S_IFDIR
+ } else {
+ m = m | fuse.S_IFREG
+ }
+ m = m | uint32(fi.Mode()&os.ModePerm)
+ stat.Mode = m
+ stat.Size = fi.Size()
+}
+
+func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return 0
+ }
+ _, err := f.Seek(ofst, io.SeekStart)
+ if err != nil {
+ return 0
+ }
+ n, err = f.Read(buf)
+ return
+}
+
+func (fs *keepFS) Readdir(path string,
+ fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
+ ofst int64,
+ fh uint64) (errc int) {
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ fill(".", nil, 0)
+ fill("..", nil, 0)
+ var stat fuse.Stat_t
+ fis, err := f.Readdir(-1)
+ if err != nil {
+ return -fuse.ENOSYS // ???
+ }
+ for _, fi := range fis {
+ fs.fillStat(&stat, fi)
+ //fill(fi.Name(), &stat, 0)
+ fill(fi.Name(), nil, 0)
+ }
+ return 0
+}
+
+func (fs *keepFS) lookupFH(fh uint64) arvados.File {
+ fs.Lock()
+ defer fs.Unlock()
+ return fs.open[fh]
+}
diff --git a/services/mount/fs_test.go b/services/mount/fs_test.go
new file mode 100644
index 0000000..fadda1a
--- /dev/null
+++ b/services/mount/fs_test.go
@@ -0,0 +1,7 @@
+package main
+
+import (
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+var _ fuse.FileSystem = &keepFS{}
diff --git a/services/mount/main.go b/services/mount/main.go
new file mode 100644
index 0000000..6f31739
--- /dev/null
+++ b/services/mount/main.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+ "flag"
+ "log"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/billziss-gh/cgofuse/fuse"
+)
+
+func main() {
+ var coll arvados.Collection
+ flag.StringVar(&coll.UUID, "id", "", "collection `uuid` or pdh")
+ flag.Parse()
+ client := arvados.NewClientFromEnv()
+ err := client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ log.Fatal(err)
+ }
+ kc, err := keepclient.MakeKeepClient(ac)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fs, err := coll.FileSystem(client, kc)
+ if err != nil {
+ log.Fatal(err)
+ }
+ host := fuse.NewFileSystemHost(&keepFS{root: fs})
+ host.Mount("", flag.Args())
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list