[ARVADOS] updated: 1.1.0-154-gc03306b
Git user
git at public.curoverse.com
Fri Nov 17 03:04:26 EST 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 87 ++++++++++++++++++++++++++++----
sdk/go/arvados/collection_fs_test.go | 97 +++++++++++++++++++++++++++++++++++-
2 files changed, 172 insertions(+), 12 deletions(-)
via c03306b7c5258f67b7749aded028ca4fe24c2d82 (commit)
via a9755547bf6a083837ac3fd03b8b804bcb018b84 (commit)
via 7399705ec4c9834d4553d2de2a323c16ab9945f8 (commit)
via 0d29dc467a605bd70d1a95a5d32486e756685357 (commit)
from 9cee2680c3b1f74d2a43d9a0e2b572168e3b488d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c03306b7c5258f67b7749aded028ca4fe24c2d82
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Nov 17 03:04:00 2017 -0500
12483: Add comments.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index e720b93..6628a75 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -86,15 +86,35 @@ func (fi fileinfo) Sys() interface{} {
}
// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files.
+// support for opening writable files. All methods are safe to call
+// from multiple goroutines.
type CollectionFileSystem interface {
http.FileSystem
+
+ // analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
OpenFile(name string, flag int, perm os.FileMode) (File, error)
+
Mkdir(name string, perm os.FileMode) error
Remove(name string) error
- MarshalManifest(string) (string, error)
+ MarshalManifest(prefix string) (string, error)
}
type fileSystem struct {
@@ -991,6 +1011,7 @@ func (dn *dirnode) lookupPath(path string) (node inode) {
return
}
+// OpenFile is analogous to os.OpenFile().
func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, error) {
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
commit a9755547bf6a083837ac3fd03b8b804bcb018b84
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Nov 16 16:58:18 2017 -0500
12483: Support O_TRUNC flag.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index beddb92..e720b93 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -1061,6 +1061,14 @@ func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, err
dn.fileinfo.size++
} else if flag&os.O_EXCL != 0 {
return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
}
return &file{
inode: n,
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 6e8e615..fc3af34 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -641,12 +641,26 @@ func (s *CollectionFSSuite) TestFileModes(c *check.C) {
c.Check(n, check.Equals, 0)
c.Check(err, check.ErrorMatches, `read-only file`)
n, err = f.Read(make([]byte, 1))
+ c.Check(n, check.Equals, 0)
c.Check(err, check.Equals, io.EOF)
f, err = fs.OpenFile("new", os.O_RDWR, 0)
c.Assert(err, check.IsNil)
defer f.Close()
_, err = f.Write([]byte{4, 5, 6})
c.Check(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, int64(3))
+
+ f, err = fs.OpenFile("new", os.O_TRUNC|os.O_RDWR, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ pos, err := f.Seek(0, os.SEEK_END)
+ c.Check(pos, check.Equals, int64(0))
+ c.Check(err, check.IsNil)
+ fi, err = f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.Size(), check.Equals, int64(0))
fs.Remove("new")
buf := make([]byte, 64)
@@ -657,7 +671,7 @@ func (s *CollectionFSSuite) TestFileModes(c *check.C) {
n, _ = f.Read(buf[:1])
c.Check(n, check.Equals, 1)
c.Check(buf[:1], check.DeepEquals, []byte{1})
- pos, err := f.Seek(0, os.SEEK_CUR)
+ pos, err = f.Seek(0, os.SEEK_CUR)
c.Check(pos, check.Equals, int64(1))
f.Write([]byte{4, 5, 6})
pos, err = f.Seek(0, os.SEEK_CUR)
commit 7399705ec4c9834d4553d2de2a323c16ab9945f8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Nov 16 16:19:58 2017 -0500
12483: Compress adjacent segments when writing filetokens.
"0:100:foo 100:100:foo 200:100:foo" --> "0:300:foo"
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 3cae9cb..beddb92 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -725,11 +725,18 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
} else {
blocks = append(blocks, e.locator)
}
- segments = append(segments, m1segment{
+ next := m1segment{
name: name,
offset: streamLen + int64(e.offset),
length: int64(e.length),
- })
+ }
+ if prev := len(segments) - 1; prev >= 0 &&
+ segments[prev].name == name &&
+ segments[prev].offset+segments[prev].length == next.offset {
+ segments[prev].length += next.length
+ } else {
+ segments = append(segments, next)
+ }
streamLen += int64(e.size)
default:
// This can't happen: we
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 324ece1..6e8e615 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -738,7 +738,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
c.Check(currentMemExtents(), check.HasLen, 1)
m, err := fs.MarshalManifest(".")
- c.Check(m, check.Not(check.Equals), "")
+ c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
c.Check(err, check.IsNil)
c.Check(currentMemExtents(), check.HasLen, 0)
}
commit 0d29dc467a605bd70d1a95a5d32486e756685357
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Nov 16 16:06:20 2017 -0500
12483: Support O_APPEND. Check for invalid/unsupported file modes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 328d68f..3cae9cb 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -25,6 +25,8 @@ var (
ErrFileExists = errors.New("file exists")
ErrInvalidOperation = errors.New("invalid operation")
ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
ErrPermission = os.ErrPermission
maxBlockSize = 1 << 26
@@ -231,8 +233,6 @@ func (fn *filenode) Readdir() []os.FileInfo {
}
func (fn *filenode) Read(p []byte, startPtr filenodePtr) (n int, ptr filenodePtr, err error) {
- fn.RLock()
- defer fn.RUnlock()
ptr = fn.seek(startPtr)
if ptr.off < 0 {
err = ErrNegativeOffset
@@ -319,8 +319,6 @@ func (fn *filenode) Truncate(size int64) error {
}
func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePtr, err error) {
- fn.Lock()
- defer fn.Unlock()
ptr = fn.seek(startPtr)
if ptr.off < 0 {
err = ErrNegativeOffset
@@ -497,11 +495,17 @@ type file struct {
inode
ptr filenodePtr
append bool
+ readable bool
writable bool
unreaddirs []os.FileInfo
}
func (f *file) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
n, f.ptr, err = f.inode.Read(p, f.ptr)
return
}
@@ -540,6 +544,16 @@ func (f *file) Write(p []byte) (n int, err error) {
if !f.writable {
return 0, ErrReadOnlyFile
}
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ extentIdx: len(fn.extents),
+ extentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
n, f.ptr, err = f.inode.Write(p, f.ptr)
return
}
@@ -971,13 +985,27 @@ func (dn *dirnode) lookupPath(path string) (node inode) {
}
func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, error) {
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
dirname, name := path.Split(name)
dn, ok := dn.lookupPath(dirname).(*dirnode)
if !ok {
return nil, os.ErrNotExist
}
- writeMode := flag&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0
- if !writeMode {
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable {
// A directory can be opened via "foo/", "foo/.", or
// "foo/..".
switch name {
@@ -1030,7 +1058,8 @@ func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, err
return &file{
inode: n,
append: flag&os.O_APPEND != 0,
- writable: flag&(os.O_WRONLY|os.O_RDWR) != 0,
+ readable: readable,
+ writable: writable,
}, nil
}
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index ecb1f89..324ece1 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -626,6 +626,87 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
}
}
+func (s *CollectionFSSuite) TestFileModes(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.ErrorMatches, `file does not exist`)
+
+ f, err = fs.OpenFile("new", os.O_CREATE|os.O_RDONLY, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ n, err := f.Write([]byte{1, 2, 3})
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.ErrorMatches, `read-only file`)
+ n, err = f.Read(make([]byte, 1))
+ c.Check(err, check.Equals, io.EOF)
+ f, err = fs.OpenFile("new", os.O_RDWR, 0)
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ _, err = f.Write([]byte{4, 5, 6})
+ c.Check(err, check.IsNil)
+ fs.Remove("new")
+
+ buf := make([]byte, 64)
+ f, err = fs.OpenFile("append", os.O_EXCL|os.O_CREATE|os.O_RDWR|os.O_APPEND, 0)
+ c.Assert(err, check.IsNil)
+ f.Write([]byte{1, 2, 3})
+ f.Seek(0, os.SEEK_SET)
+ n, _ = f.Read(buf[:1])
+ c.Check(n, check.Equals, 1)
+ c.Check(buf[:1], check.DeepEquals, []byte{1})
+ pos, err := f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(1))
+ f.Write([]byte{4, 5, 6})
+ pos, err = f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(6))
+ f.Seek(0, os.SEEK_SET)
+ n, err = f.Read(buf)
+ c.Check(buf[:n], check.DeepEquals, []byte{1, 2, 3, 4, 5, 6})
+ c.Check(err, check.Equals, io.EOF)
+ f.Close()
+
+ f, err = fs.OpenFile("append", os.O_RDWR|os.O_APPEND, 0)
+ c.Assert(err, check.IsNil)
+ pos, err = f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(0))
+ c.Check(err, check.IsNil)
+ f.Read(buf[:3])
+ pos, _ = f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(3))
+ f.Write([]byte{7, 8, 9})
+ pos, err = f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(9))
+ f.Close()
+
+ f, err = fs.OpenFile("wronly", os.O_CREATE|os.O_WRONLY, 0)
+ c.Assert(err, check.IsNil)
+ n, err = f.Write([]byte{3, 2, 1})
+ c.Check(n, check.Equals, 3)
+ c.Check(err, check.IsNil)
+ pos, _ = f.Seek(0, os.SEEK_CUR)
+ c.Check(pos, check.Equals, int64(3))
+ pos, _ = f.Seek(0, os.SEEK_SET)
+ c.Check(pos, check.Equals, int64(0))
+ n, err = f.Read(buf)
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.ErrorMatches, `.*O_WRONLY.*`)
+ f, err = fs.OpenFile("wronly", os.O_RDONLY, 0)
+ c.Assert(err, check.IsNil)
+ n, _ = f.Read(buf)
+ c.Check(buf[:n], check.DeepEquals, []byte{3, 2, 1})
+
+ f, err = fs.OpenFile("unsupported", os.O_CREATE|os.O_SYNC, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.NotNil)
+
+ f, err = fs.OpenFile("append", os.O_RDWR|os.O_WRONLY, 0)
+ c.Check(f, check.IsNil)
+ c.Check(err, check.ErrorMatches, `invalid flag.*`)
+}
+
func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list