[ARVADOS] updated: 1.1.3-398-g73b36db
Git user
git at public.curoverse.com
Fri Apr 13 15:32:03 EDT 2018
Summary of changes:
sdk/go/arvados/fs_lookup.go | 3 +++
1 file changed, 3 insertions(+)
discards 4002c4558e61f13cc1ba5ab0cad6d9bb6c41299a (commit)
discards eb68ebd581c86ac34db4ef235fb02a733341b4ce (commit)
discards 4ff4e03cfd8edfa946e340bc9937bca88348d400 (commit)
via 73b36db80121301aaf3c3a38e313e1cbc62b3fdd (commit)
via e9f6f853e5d121ea43d7b97d8b87ab7990de3397 (commit)
via f46fe9f98b3ea1fb61e24dac59b684b166045b56 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (4002c4558e61f13cc1ba5ab0cad6d9bb6c41299a)
\
N -- N -- N (73b36db80121301aaf3c3a38e313e1cbc62b3fdd)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 73b36db80121301aaf3c3a38e313e1cbc62b3fdd
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:35:48 2018 -0400
13111: Accept chunked responses to GET requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index dbdda60..95a84c0 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -19,13 +19,14 @@ import (
func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
- hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ data := []byte("TestOverrideDiscovery")
+ hash := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
st := StubGetHandler{
c,
hash,
arvadostest.ActiveToken,
http.StatusOK,
- []byte("TestOverrideDiscovery")}
+ data}
ks := RunSomeFakeKeepServers(st, 2)
os.Setenv("ARVADOS_KEEP_SERVICES", "")
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 54a4a37..620bdbe 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -200,6 +200,15 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ var expectLength int64
+ if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
+ expectLength = -1
+ } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil {
+ expectLength = -1
+ } else {
+ expectLength = n
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
@@ -230,7 +239,9 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
// can try again.
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
retryList = append(retryList, host)
- } else if resp.StatusCode != http.StatusOK {
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
var respbody []byte
respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
@@ -247,24 +258,29 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
} else if resp.StatusCode == 404 {
count404++
}
- } else if resp.ContentLength < 0 {
- // Missing Content-Length
- resp.Body.Close()
- return nil, 0, "", fmt.Errorf("Missing Content-Length of block")
- } else {
- // Success.
- if method == "GET" {
- return HashCheckingReader{
- Reader: resp.Body,
- Hash: md5.New(),
- Check: locator[0:32],
- }, resp.ContentLength, url, nil
- } else {
+ continue
+ }
+ if expectLength < 0 {
+ if resp.ContentLength < 0 {
resp.Body.Close()
- return nil, resp.ContentLength, url, nil
+ return nil, 0, "", fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator)
}
+ expectLength = resp.ContentLength
+ } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength {
+ resp.Body.Close()
+ return nil, 0, "", fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength)
+ }
+ // Success
+ if method == "GET" {
+ return HashCheckingReader{
+ Reader: resp.Body,
+ Hash: md5.New(),
+ Check: locator[0:32],
+ }, expectLength, url, nil
+ } else {
+ resp.Body.Close()
+ return nil, expectLength, url, nil
}
-
}
serversToTry = retryList
}
commit e9f6f853e5d121ea43d7b97d8b87ab7990de3397
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:34:30 2018 -0400
13111: Fix unhandled errors in cgofuse adapter.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 1633296..68ad0b4 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -299,10 +299,13 @@ func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int)
defer f.Unlock()
if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
- } else {
- n, _ = f.Write(buf)
- return
}
+ n, err := f.Write(buf)
+ if err != nil {
+ log.Printf("error writing %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
}
func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
@@ -315,10 +318,13 @@ func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
defer f.Unlock()
if _, err := f.Seek(ofst, io.SeekStart); err != nil {
return fs.errCode(err)
- } else {
- n, _ = f.Read(buf)
- return
}
+ n, err := f.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Printf("error reading %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
}
func (fs *keepFS) Readdir(path string,
commit f46fe9f98b3ea1fb61e24dac59b684b166045b56
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 16:30:17 2018 -0400
13111: Avoid multi-page API reqs when looking up entries by name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs_lookup.go
new file mode 100644
index 0000000..42322a1
--- /dev/null
+++ b/sdk/go/arvados/fs_lookup.go
@@ -0,0 +1,73 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+ "time"
+)
+
+// lookupnode is a caching tree node that is initially empty and calls
+// loadOne and loadAll to load/update child nodes as needed.
+//
+// See (*customFileSystem)MountUsers for example usage.
+type lookupnode struct {
+ inode
+ loadOne func(parent inode, name string) (inode, error)
+ loadAll func(parent inode) ([]inode, error)
+ stale func(time.Time) bool
+
+ // internal fields
+ staleLock sync.Mutex
+ staleAll time.Time
+ staleOne map[string]time.Time
+}
+
+func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) {
+ all, err := ln.loadAll(ln)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range all {
+ _, err = ln.inode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
+ return child, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ ln.staleAll = checkTime
+ // No value in ln.staleOne can make a difference to an
+ // "entry is stale?" test now, because no value is
+ // newer than ln.staleAll. Reclaim memory.
+ ln.staleOne = nil
+ }
+ return ln.inode.Readdir()
+}
+
+func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) && ln.stale(ln.staleOne[name]) {
+ _, err := ln.inode.Child(name, func(inode) (inode, error) {
+ return ln.loadOne(ln, name)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ln.staleOne == nil {
+ ln.staleOne = map[string]time.Time{name: checkTime}
+ } else {
+ ln.staleOne[name] = checkTime
+ }
+ }
+ return ln.inode.Child(name, replace)
+}
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 827a44b..9299551 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,58 +5,81 @@
package arvados
import (
+ "log"
"os"
- "sync"
- "time"
+ "strings"
)
-type staleChecker struct {
- mtx sync.Mutex
- last time.Time
+func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
+ if uuid != "" {
+ return uuid, nil
+ }
+ var resp User
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return "", err
+ }
+ return resp.UUID, nil
}
-func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
- sc.mtx.Lock()
- defer sc.mtx.Unlock()
- if !staleFunc(sc.last) {
- return
+// loadOneChild loads only the named child, if it exists.
+func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
- sc.last = time.Now()
- fn()
-}
-// projectnode exposes an Arvados project as a filesystem directory.
-type projectnode struct {
- inode
- staleChecker
- uuid string
- err error
-}
+ var contents CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{
+ {"name", "=", name},
+ {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
+ {"groups.group_class", "=", "project"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if len(contents.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ coll := contents.Items[0]
-func (pn *projectnode) load() {
- fs := pn.FS().(*customFileSystem)
+ if strings.Contains(coll.UUID, "-j7d0g-") {
+ // Group item was loaded into a Collection var -- but
+ // we only need the Name and UUID anyway, so it's OK.
+ return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ } else if strings.Contains(coll.UUID, "-4zz18-") {
+ return deferredCollectionFS(fs, parent, coll), nil
+ } else {
+ log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID)
+ return nil, ErrInvalidArgument
+ }
+}
- if pn.uuid == "" {
- var resp User
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
- if pn.err != nil {
- return
- }
- pn.uuid = resp.UUID
+func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
+
+ var inodes []inode
+
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ filters := []Filter{{"owner_uuid", "=", uuid}}
params := ResourceListParams{
+ Count: "none",
Filters: filters,
Order: "uuid",
}
for {
var resp CollectionList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -66,9 +89,7 @@ func (pn *projectnode) load() {
if !permittedName(coll.Name) {
continue
}
- pn.inode.Child(coll.Name, func(inode) (inode, error) {
- return deferredCollectionFS(fs, pn, coll), nil
- })
+ inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
@@ -77,9 +98,9 @@ func (pn *projectnode) load() {
params.Filters = filters
for {
var resp GroupList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -88,52 +109,9 @@ func (pn *projectnode) load() {
if !permittedName(group.Name) {
continue
}
- pn.inode.Child(group.Name, func(inode) (inode, error) {
- return fs.newProjectNode(pn, group.Name, group.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- pn.err = nil
-}
-
-func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- return pn.inode.Readdir()
-}
-
-func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- if replace == nil {
- // lookup
- return pn.inode.Child(name, nil)
- }
- return pn.inode.Child(name, func(existing inode) (inode, error) {
- if repl, err := replace(existing); err != nil {
- return existing, err
- } else if repl == nil {
- if existing == nil {
- return nil, nil
- }
- // rmdir
- // (TODO)
- return existing, ErrInvalidArgument
- } else if existing != nil {
- // clobber
- return existing, ErrInvalidArgument
- } else if repl.FileInfo().IsDir() {
- // mkdir
- // TODO: repl.SetParent(pn, name), etc.
- return existing, ErrInvalidArgument
- } else {
- // create file
- return existing, ErrInvalidArgument
- }
- })
+ return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 058eb40..1a06ce1 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -67,15 +67,15 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
_, basename := filepath.Split(path)
c.Check(fi.Name(), check.Equals, basename)
f, err = s.fs.Open(path + "/A Project/A Subproject")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fi, err = f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
@@ -90,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
+ f, err := s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project/Project does not exist")
+ c.Assert(err, check.NotNil)
+ f, err = s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fis)
+ var foundSubproject, foundCollection bool
+ for _, fi := range fis {
+ switch fi.Name() {
+ case "A Subproject":
+ foundSubproject = true
+ case "collection_to_move_around":
+ foundCollection = true
+ }
+ }
+ c.Check(foundSubproject, check.Equals, true)
+ c.Check(foundCollection, check.Equals, true)
+}
+
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
badCollection := Collection{
Name: "bad/collection",
@@ -109,7 +137,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
dir, err := s.fs.Open("/users/active/A Project")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fis, err := dir.Readdir(-1)
c.Check(err, check.IsNil)
for _, fi := range fis {
@@ -122,7 +150,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
@@ -140,6 +168,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
f, err := s.fs.Open("/home/A Project/oob")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
f.Close()
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index e9c8387..b5daf7b 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -73,7 +73,10 @@ func (fs *customFileSystem) MountProject(mount, uuid string) {
func (fs *customFileSystem) MountUsers(mount string) {
fs.root.inode.Child(mount, func(inode) (inode, error) {
- return &usersnode{
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: fs.usersLoadOne,
+ loadAll: fs.usersLoadAll,
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -136,24 +139,10 @@ func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
}
func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
- inode: &treenode{
- fs: fs,
- parent: root,
- inodes: make(map[string]inode),
- fileinfo: fileinfo{
- name: name,
- modTime: time.Now(),
- mode: 0755 | os.ModeDir,
- },
- },
- }
-}
-
-func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+ loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
inode: &treenode{
fs: fs,
parent: root,
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
index ccfe2c5..00f7036 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs_users.go
@@ -8,55 +8,41 @@ import (
"os"
)
-// usersnode is a virtual directory with an entry for each visible
-// Arvados username, each showing the respective user's "home
-// projects".
-type usersnode struct {
- inode
- staleChecker
- err error
+func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{{"username", "=", name}},
+ })
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ user := resp.Items[0]
+ return fs.newProjectNode(parent, user.Username, user.UUID), nil
}
-func (un *usersnode) load() {
- fs := un.FS().(*customFileSystem)
-
+func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
params := ResourceListParams{
+ Count: "none",
Order: "uuid",
}
+ var inodes []inode
for {
var resp UserList
- un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
- if un.err != nil {
- return
- }
- if len(resp.Items) == 0 {
- break
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return inodes, nil
}
for _, user := range resp.Items {
if user.Username == "" {
continue
}
- un.inode.Child(user.Username, func(inode) (inode, error) {
- return fs.newProjectNode(un, user.Username, user.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
- un.err = nil
-}
-
-func (un *usersnode) Readdir() ([]os.FileInfo, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Readdir()
-}
-
-func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Child(name, nil)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list