[ARVADOS] updated: 1.1.3-398-g4002c45
Git user
git at public.curoverse.com
Fri Apr 13 15:20:46 EDT 2018
Summary of changes:
lib/mount/fs.go | 18 +++--
sdk/go/arvados/fs_lookup.go | 70 ++++++++++++++++++
sdk/go/arvados/fs_project.go | 146 ++++++++++++++++---------------------
sdk/go/arvados/fs_project_test.go | 39 ++++++++--
sdk/go/arvados/fs_site.go | 27 ++-----
sdk/go/arvados/fs_users.go | 58 ++++++---------
sdk/go/keepclient/discover_test.go | 5 +-
sdk/go/keepclient/keepclient.go | 48 ++++++++----
8 files changed, 243 insertions(+), 168 deletions(-)
create mode 100644 sdk/go/arvados/fs_lookup.go
via 4002c4558e61f13cc1ba5ab0cad6d9bb6c41299a (commit)
via eb68ebd581c86ac34db4ef235fb02a733341b4ce (commit)
via 4ff4e03cfd8edfa946e340bc9937bca88348d400 (commit)
from e6c9fc68cd6d4d281d81928729a2bd18007a96a4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4002c4558e61f13cc1ba5ab0cad6d9bb6c41299a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:35:48 2018 -0400
13111: Accept chunked responses to GET requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index dbdda60..95a84c0 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -19,13 +19,14 @@ import (
func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
- hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ data := []byte("TestOverrideDiscovery")
+ hash := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
st := StubGetHandler{
c,
hash,
arvadostest.ActiveToken,
http.StatusOK,
- []byte("TestOverrideDiscovery")}
+ data}
ks := RunSomeFakeKeepServers(st, 2)
os.Setenv("ARVADOS_KEEP_SERVICES", "")
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 54a4a37..620bdbe 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -200,6 +200,15 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ var expectLength int64
+ if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
+ expectLength = -1
+ } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil {
+ expectLength = -1
+ } else {
+ expectLength = n
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
@@ -230,7 +239,9 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
// can try again.
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
retryList = append(retryList, host)
- } else if resp.StatusCode != http.StatusOK {
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
var respbody []byte
respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
@@ -247,24 +258,29 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
} else if resp.StatusCode == 404 {
count404++
}
- } else if resp.ContentLength < 0 {
- // Missing Content-Length
- resp.Body.Close()
- return nil, 0, "", fmt.Errorf("Missing Content-Length of block")
- } else {
- // Success.
- if method == "GET" {
- return HashCheckingReader{
- Reader: resp.Body,
- Hash: md5.New(),
- Check: locator[0:32],
- }, resp.ContentLength, url, nil
- } else {
+ continue
+ }
+ if expectLength < 0 {
+ if resp.ContentLength < 0 {
resp.Body.Close()
- return nil, resp.ContentLength, url, nil
+ return nil, 0, "", fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator)
}
+ expectLength = resp.ContentLength
+ } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength {
+ resp.Body.Close()
+ return nil, 0, "", fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength)
+ }
+ // Success
+ if method == "GET" {
+ return HashCheckingReader{
+ Reader: resp.Body,
+ Hash: md5.New(),
+ Check: locator[0:32],
+ }, expectLength, url, nil
+ } else {
+ resp.Body.Close()
+ return nil, expectLength, url, nil
}
-
}
serversToTry = retryList
}
commit eb68ebd581c86ac34db4ef235fb02a733341b4ce
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:34:30 2018 -0400
13111: Fix unhandled errors in cgofuse adapter.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 1633296..68ad0b4 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -299,10 +299,13 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
defer f.Unlock()
if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
- } else {
- n, _ = f.Write(buf)
- return
}
+ n, err := f.Write(buf)
+ if err != nil {
+ log.Printf("error writing %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
@@ -315,10 +318,13 @@ func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
defer f.Unlock()
if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
- } else {
- n, _ = f.Read(buf)
- return
}
+ n, err := f.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Printf("error reading %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
}
func (fs *keepFS) Readdir(path string,
commit 4ff4e03cfd8edfa946e340bc9937bca88348d400
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 16:30:17 2018 -0400
13111: Avoid multi-page API reqs when looking up entries by name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs_lookup.go
new file mode 100644
index 0000000..ff0e3b6
--- /dev/null
+++ b/sdk/go/arvados/fs_lookup.go
@@ -0,0 +1,70 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+ "time"
+)
+
+// lookupnode is a caching tree node that is initially empty and calls
+// loadOne and loadAll to load/update child nodes as needed.
+//
+// See (*customFileSystem)MountUsers for example usage.
+type lookupnode struct {
+ inode
+ loadOne func(parent inode, name string) (inode, error)
+ loadAll func(parent inode) ([]inode, error)
+ stale func(time.Time) bool
+
+ // internal fields
+ staleLock sync.Mutex
+ staleAll time.Time
+ staleOne map[string]time.Time
+}
+
+func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) {
+ all, err := ln.loadAll(ln)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range all {
+ _, err = ln.inode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
+ return child, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ ln.staleAll = checkTime
+ ln.staleOne = nil
+ }
+ return ln.inode.Readdir()
+}
+
+func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) && ln.stale(ln.staleOne[name]) {
+ _, err := ln.inode.Child(name, func(inode) (inode, error) {
+ return ln.loadOne(ln, name)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ln.staleOne == nil {
+ ln.staleOne = map[string]time.Time{name: checkTime}
+ } else {
+ ln.staleOne[name] = checkTime
+ }
+ }
+ return ln.inode.Child(name, replace)
+}
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 827a44b..9299551 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,58 +5,81 @@
package arvados
import (
+ "log"
"os"
- "sync"
- "time"
+ "strings"
)
-type staleChecker struct {
- mtx sync.Mutex
- last time.Time
+func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
+ if uuid != "" {
+ return uuid, nil
+ }
+ var resp User
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return "", err
+ }
+ return resp.UUID, nil
}
-func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
- sc.mtx.Lock()
- defer sc.mtx.Unlock()
- if !staleFunc(sc.last) {
- return
+// loadOneChild loads only the named child, if it exists.
+func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
- sc.last = time.Now()
- fn()
-}
-// projectnode exposes an Arvados project as a filesystem directory.
-type projectnode struct {
- inode
- staleChecker
- uuid string
- err error
-}
+ var contents CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{
+ {"name", "=", name},
+ {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
+ {"groups.group_class", "=", "project"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if len(contents.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ coll := contents.Items[0]
-func (pn *projectnode) load() {
- fs := pn.FS().(*customFileSystem)
+ if strings.Contains(coll.UUID, "-j7d0g-") {
+ // Group item was loaded into a Collection var -- but
+ // we only need the Name and UUID anyway, so it's OK.
+ return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ } else if strings.Contains(coll.UUID, "-4zz18-") {
+ return deferredCollectionFS(fs, parent, coll), nil
+ } else {
+ log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID)
+ return nil, ErrInvalidArgument
+ }
+}
- if pn.uuid == "" {
- var resp User
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
- if pn.err != nil {
- return
- }
- pn.uuid = resp.UUID
+func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
+
+ var inodes []inode
+
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ filters := []Filter{{"owner_uuid", "=", uuid}}
params := ResourceListParams{
+ Count: "none",
Filters: filters,
Order: "uuid",
}
for {
var resp CollectionList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -66,9 +89,7 @@ func (pn *projectnode) load() {
if !permittedName(coll.Name) {
continue
}
- pn.inode.Child(coll.Name, func(inode) (inode, error) {
- return deferredCollectionFS(fs, pn, coll), nil
- })
+ inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
@@ -77,9 +98,9 @@ func (pn *projectnode) load() {
params.Filters = filters
for {
var resp GroupList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -88,52 +109,9 @@ func (pn *projectnode) load() {
if !permittedName(group.Name) {
continue
}
- pn.inode.Child(group.Name, func(inode) (inode, error) {
- return fs.newProjectNode(pn, group.Name, group.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- pn.err = nil
-}
-
-func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- return pn.inode.Readdir()
-}
-
-func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- if replace == nil {
- // lookup
- return pn.inode.Child(name, nil)
- }
- return pn.inode.Child(name, func(existing inode) (inode, error) {
- if repl, err := replace(existing); err != nil {
- return existing, err
- } else if repl == nil {
- if existing == nil {
- return nil, nil
- }
- // rmdir
- // (TODO)
- return existing, ErrInvalidArgument
- } else if existing != nil {
- // clobber
- return existing, ErrInvalidArgument
- } else if repl.FileInfo().IsDir() {
- // mkdir
- // TODO: repl.SetParent(pn, name), etc.
- return existing, ErrInvalidArgument
- } else {
- // create file
- return existing, ErrInvalidArgument
- }
- })
+ return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 058eb40..1a06ce1 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -67,15 +67,15 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
_, basename := filepath.Split(path)
c.Check(fi.Name(), check.Equals, basename)
f, err = s.fs.Open(path + "/A Project/A Subproject")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fi, err = f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
@@ -90,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
+ f, err := s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project/Project does not exist")
+ c.Assert(err, check.NotNil)
+ f, err = s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fis)
+ var foundSubproject, foundCollection bool
+ for _, fi := range fis {
+ switch fi.Name() {
+ case "A Subproject":
+ foundSubproject = true
+ case "collection_to_move_around":
+ foundCollection = true
+ }
+ }
+ c.Check(foundSubproject, check.Equals, true)
+ c.Check(foundCollection, check.Equals, true)
+}
+
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
badCollection := Collection{
Name: "bad/collection",
@@ -109,7 +137,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
dir, err := s.fs.Open("/users/active/A Project")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fis, err := dir.Readdir(-1)
c.Check(err, check.IsNil)
for _, fi := range fis {
@@ -122,7 +150,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
@@ -140,6 +168,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
f, err := s.fs.Open("/home/A Project/oob")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
f.Close()
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index e9c8387..b5daf7b 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -73,7 +73,10 @@ func (fs *customFileSystem) MountProject(mount, uuid string) {
func (fs *customFileSystem) MountUsers(mount string) {
fs.root.inode.Child(mount, func(inode) (inode, error) {
- return &usersnode{
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: fs.usersLoadOne,
+ loadAll: fs.usersLoadAll,
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -136,24 +139,10 @@ func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
}
func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
- inode: &treenode{
- fs: fs,
- parent: root,
- inodes: make(map[string]inode),
- fileinfo: fileinfo{
- name: name,
- modTime: time.Now(),
- mode: 0755 | os.ModeDir,
- },
- },
- }
-}
-
-func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+ loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
inode: &treenode{
fs: fs,
parent: root,
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
index ccfe2c5..00f7036 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs_users.go
@@ -8,55 +8,41 @@ import (
"os"
)
-// usersnode is a virtual directory with an entry for each visible
-// Arvados username, each showing the respective user's "home
-// projects".
-type usersnode struct {
- inode
- staleChecker
- err error
+func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{{"username", "=", name}},
+ })
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ user := resp.Items[0]
+ return fs.newProjectNode(parent, user.Username, user.UUID), nil
}
-func (un *usersnode) load() {
- fs := un.FS().(*customFileSystem)
-
+func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
params := ResourceListParams{
+ Count: "none",
Order: "uuid",
}
+ var inodes []inode
for {
var resp UserList
- un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
- if un.err != nil {
- return
- }
- if len(resp.Items) == 0 {
- break
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return inodes, nil
}
for _, user := range resp.Items {
if user.Username == "" {
continue
}
- un.inode.Child(user.Username, func(inode) (inode, error) {
- return fs.newProjectNode(un, user.Username, user.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
- un.err = nil
-}
-
-func (un *usersnode) Readdir() ([]os.FileInfo, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Readdir()
-}
-
-func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Child(name, nil)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list