[ARVADOS] updated: 1.1.4-93-g98911cf
Git user
git at public.curoverse.com
Fri Apr 13 22:19:02 EDT 2018
Summary of changes:
vendor/vendor.json | 6 ------
1 file changed, 6 deletions(-)
discards 45de5a83c8afd586036339921cf3dd959edb85b2 (commit)
discards a9916f5648ad8486812c3ec8ef6f627e2f0542e4 (commit)
discards 973d9760a020b5a1f7b421e620d5143e5c4010cf (commit)
discards e618db07e97783e6ab588332d0f2872842677f03 (commit)
discards 1f397d05c069e75ebbd74b59ab2133df4f7389c7 (commit)
discards a78cdf7dfd18c32431e9d2a26f08b4a35cd4d444 (commit)
discards 3b06a75d2756e3c7b2997ba38cfd54668096ee7e (commit)
discards 309e6cb3d00a2b731ffe66d89102caacf2cd81aa (commit)
discards e7fe3ce516f71bc14c5ff0e22dcd19772ab8c72a (commit)
discards 335ee474af3cf03865bc107478da6a51e362c2ab (commit)
discards aadbd550d97c2cbf47563ceb1a9004932fa5e28f (commit)
discards a6c49634025095e9b4d29f5971fe5ecb720f9539 (commit)
discards cd0945bb8591fd97fe37a77dc9057c29dc5d7558 (commit)
discards 04d11c9cf24f358a0780edc87c910c50130fd71b (commit)
discards c5d6ae2fc5f1c382f2011a128230d893a5b2103c (commit)
discards 3e2aed249f9ae1d99fb654b19d3ae49788f8fdaa (commit)
discards 8f0b7529cda14a10cd953819af9c1b76201ae4f5 (commit)
discards 282394bf9076af3387755979e4fd9eb360291861 (commit)
discards cedd5efb0f3813125931c64b754dddabd875d537 (commit)
discards 4c46038d3726e1ff7006cc30bbd6b71b957d0215 (commit)
discards ab1abbac7ec08c7a7599670459f04b8a35dcbe68 (commit)
discards 0c7a458fc70791a5f395dad171a272b92b6ffb7f (commit)
discards efd536fbbfca8473e7df444dd1d97ce916bfe345 (commit)
discards 541e6b47d90dbdadca287bea0cc303026f20facb (commit)
discards f09c8facc9a607f3b5e9171e66170536f1ba83c7 (commit)
discards 959335d561a3882b391c88c4b2106f263e827b51 (commit)
discards 9424d875b67ddda957b2ef705a1619394b57db70 (commit)
discards 14fadeaed3dd4f01778eae8342ca0b5e190e3429 (commit)
discards 3c230a24be3e99f89805199311ec8b36665c2b74 (commit)
discards f67ede4c778f5ac6f3bd0e04d1f47993603b3375 (commit)
discards e1bdb7bb963f3754f3aebced8810b062ee4fecf8 (commit)
discards 0ff4662fab347279897b285e3ef3320497ced8c9 (commit)
discards 649c27a98f398984b6aa1f806992cfdd9831c904 (commit)
discards c2e623cbcf93e996b4fb1d29fcf99d954f7f2ad3 (commit)
discards 183cf741009a42fa3dd2af911c416755e4c5cb0e (commit)
discards 4668cf881aef0b21245a419b661b010720035f3c (commit)
via 98911cfe4792b20798858cefb353c451460e1a80 (commit)
via fd95e7a933b1534b76e4820838e278595b4e1220 (commit)
via 1c19ed2dd05a023be541e369130910eb277b2816 (commit)
via 31a3b4f34aa56ade25def7cdd59b026f5a59ee13 (commit)
via f22f1e3c92e9a69e9e7f82fa6226ad100a8025da (commit)
via 6522329120f0e2a73bf9c55f4f937fa3d109803d (commit)
via 76cc598a22d58cd889ad41b47ef061bde90c9f52 (commit)
via 78eedab0a2feadc7877529ac4ce65ccaaf4db768 (commit)
via 4dda5d7b0ea74103c07617cc2ab9e5c97682c85d (commit)
via cd9626d625f72b31054204c68cb2bb32e5dd3111 (commit)
via b9e031258dc079151d815167d1d6665bc63e2318 (commit)
via 9a8b51c6c2468162ee9748514141a94d24e5f663 (commit)
via db30db0c7a441eec9d5dd2ca2b74cfde9966d6ce (commit)
via 85736e92085b10d58141ef66c1354fb1cf16c562 (commit)
via b29442ebd118d522f9be9508f5111f2a68eae1a4 (commit)
via 3cf6c6d8dcd3616d0075f9af1732d5ed4231b861 (commit)
via 418c57bce3aac1a22548e53e1018a1547d9efee4 (commit)
via 89c42b3d5d978de79797be9c9c142330d23e28c4 (commit)
via c69d271455a0550cda8e56ab6a4685fe3d7d91ea (commit)
via b5367444a51cc91e7b589bfdb68eb12e8a1f4937 (commit)
via 1b7ed029c3d9d50b275573b65e8fbf4943e76bcb (commit)
via 6b17b8e5aaf08b3124d0d7536b3c73581894d70a (commit)
via 11f2a3542e8bef42edcb413405efb43bd2e30be7 (commit)
via df591042778408d03d410d5c22a669d85652d1ea (commit)
via 29a6622585581b5e4f519968bbd291939bf49392 (commit)
via 8296c0784c70660e0a7247ba000741d37bda38fd (commit)
via d1ae12cad34862d063a1235bfe53459eef7ae589 (commit)
via 1012f12d29be01b56f2bbbe2e9bd5969d69f7b89 (commit)
via caf56e922b9caaa9a65332dc6bf3a36bd8fea48f (commit)
via 2f03d4d926870a93cb880b389519a05c97de73b3 (commit)
via f5d919c7e7bcf46e245a4459f3393022ff471db0 (commit)
via 47eb67e4c084abde49d5463d4ced8b4436a59dfd (commit)
via 7c32daf9b5b1dcb8a003ac30bfc0ed2a9ef0eb74 (commit)
via 07cb2b1d22be82abb87fd2a5f95ae86e760c87e6 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (45de5a83c8afd586036339921cf3dd959edb85b2)
\
N -- N -- N (98911cfe4792b20798858cefb353c451460e1a80)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 98911cfe4792b20798858cefb353c451460e1a80
Merge: fd95e7a b1160af
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 16:49:27 2018 -0400
13111: Merge branch 'master' into 13111-webdav-projects
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit fd95e7a933b1534b76e4820838e278595b4e1220
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:35:48 2018 -0400
13111: Accept chunked responses to GET requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index dbdda60..95a84c0 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -19,13 +19,14 @@ import (
func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
- hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ data := []byte("TestOverrideDiscovery")
+ hash := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
st := StubGetHandler{
c,
hash,
arvadostest.ActiveToken,
http.StatusOK,
- []byte("TestOverrideDiscovery")}
+ data}
ks := RunSomeFakeKeepServers(st, 2)
os.Setenv("ARVADOS_KEEP_SERVICES", "")
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 54a4a37..620bdbe 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -200,6 +200,15 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ var expectLength int64
+ if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
+ expectLength = -1
+ } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil {
+ expectLength = -1
+ } else {
+ expectLength = n
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
@@ -230,7 +239,9 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
// can try again.
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
retryList = append(retryList, host)
- } else if resp.StatusCode != http.StatusOK {
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
var respbody []byte
respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
@@ -247,24 +258,29 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
} else if resp.StatusCode == 404 {
count404++
}
- } else if resp.ContentLength < 0 {
- // Missing Content-Length
- resp.Body.Close()
- return nil, 0, "", fmt.Errorf("Missing Content-Length of block")
- } else {
- // Success.
- if method == "GET" {
- return HashCheckingReader{
- Reader: resp.Body,
- Hash: md5.New(),
- Check: locator[0:32],
- }, resp.ContentLength, url, nil
- } else {
+ continue
+ }
+ if expectLength < 0 {
+ if resp.ContentLength < 0 {
resp.Body.Close()
- return nil, resp.ContentLength, url, nil
+ return nil, 0, "", fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator)
}
+ expectLength = resp.ContentLength
+ } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength {
+ resp.Body.Close()
+ return nil, 0, "", fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength)
+ }
+ // Success
+ if method == "GET" {
+ return HashCheckingReader{
+ Reader: resp.Body,
+ Hash: md5.New(),
+ Check: locator[0:32],
+ }, expectLength, url, nil
+ } else {
+ resp.Body.Close()
+ return nil, expectLength, url, nil
}
-
}
serversToTry = retryList
}
commit 1c19ed2dd05a023be541e369130910eb277b2816
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 16:30:17 2018 -0400
13111: Avoid multi-page API reqs when looking up entries by name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs_lookup.go
new file mode 100644
index 0000000..42322a1
--- /dev/null
+++ b/sdk/go/arvados/fs_lookup.go
@@ -0,0 +1,73 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+ "time"
+)
+
+// lookupnode is a caching tree node that is initially empty and calls
+// loadOne and loadAll to load/update child nodes as needed.
+//
+// See (*customFileSystem)MountUsers for example usage.
+type lookupnode struct {
+ inode
+ loadOne func(parent inode, name string) (inode, error)
+ loadAll func(parent inode) ([]inode, error)
+ stale func(time.Time) bool
+
+ // internal fields
+ staleLock sync.Mutex
+ staleAll time.Time
+ staleOne map[string]time.Time
+}
+
+func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) {
+ all, err := ln.loadAll(ln)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range all {
+ _, err = ln.inode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
+ return child, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ ln.staleAll = checkTime
+ // No value in ln.staleOne can make a difference to an
+ // "entry is stale?" test now, because no value is
+ // newer than ln.staleAll. Reclaim memory.
+ ln.staleOne = nil
+ }
+ return ln.inode.Readdir()
+}
+
+func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) && ln.stale(ln.staleOne[name]) {
+ _, err := ln.inode.Child(name, func(inode) (inode, error) {
+ return ln.loadOne(ln, name)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ln.staleOne == nil {
+ ln.staleOne = map[string]time.Time{name: checkTime}
+ } else {
+ ln.staleOne[name] = checkTime
+ }
+ }
+ return ln.inode.Child(name, replace)
+}
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 827a44b..9299551 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,58 +5,81 @@
package arvados
import (
+ "log"
"os"
- "sync"
- "time"
+ "strings"
)
-type staleChecker struct {
- mtx sync.Mutex
- last time.Time
+func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
+ if uuid != "" {
+ return uuid, nil
+ }
+ var resp User
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return "", err
+ }
+ return resp.UUID, nil
}
-func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
- sc.mtx.Lock()
- defer sc.mtx.Unlock()
- if !staleFunc(sc.last) {
- return
+// loadOneChild loads only the named child, if it exists.
+func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
- sc.last = time.Now()
- fn()
-}
-// projectnode exposes an Arvados project as a filesystem directory.
-type projectnode struct {
- inode
- staleChecker
- uuid string
- err error
-}
+ var contents CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{
+ {"name", "=", name},
+ {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
+ {"groups.group_class", "=", "project"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if len(contents.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ coll := contents.Items[0]
-func (pn *projectnode) load() {
- fs := pn.FS().(*customFileSystem)
+ if strings.Contains(coll.UUID, "-j7d0g-") {
+ // Group item was loaded into a Collection var -- but
+ // we only need the Name and UUID anyway, so it's OK.
+ return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ } else if strings.Contains(coll.UUID, "-4zz18-") {
+ return deferredCollectionFS(fs, parent, coll), nil
+ } else {
+ log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID)
+ return nil, ErrInvalidArgument
+ }
+}
- if pn.uuid == "" {
- var resp User
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
- if pn.err != nil {
- return
- }
- pn.uuid = resp.UUID
+func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
+
+ var inodes []inode
+
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ filters := []Filter{{"owner_uuid", "=", uuid}}
params := ResourceListParams{
+ Count: "none",
Filters: filters,
Order: "uuid",
}
for {
var resp CollectionList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -66,9 +89,7 @@ func (pn *projectnode) load() {
if !permittedName(coll.Name) {
continue
}
- pn.inode.Child(coll.Name, func(inode) (inode, error) {
- return deferredCollectionFS(fs, pn, coll), nil
- })
+ inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
@@ -77,9 +98,9 @@ func (pn *projectnode) load() {
params.Filters = filters
for {
var resp GroupList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -88,52 +109,9 @@ func (pn *projectnode) load() {
if !permittedName(group.Name) {
continue
}
- pn.inode.Child(group.Name, func(inode) (inode, error) {
- return fs.newProjectNode(pn, group.Name, group.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- pn.err = nil
-}
-
-func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- return pn.inode.Readdir()
-}
-
-func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- if replace == nil {
- // lookup
- return pn.inode.Child(name, nil)
- }
- return pn.inode.Child(name, func(existing inode) (inode, error) {
- if repl, err := replace(existing); err != nil {
- return existing, err
- } else if repl == nil {
- if existing == nil {
- return nil, nil
- }
- // rmdir
- // (TODO)
- return existing, ErrInvalidArgument
- } else if existing != nil {
- // clobber
- return existing, ErrInvalidArgument
- } else if repl.FileInfo().IsDir() {
- // mkdir
- // TODO: repl.SetParent(pn, name), etc.
- return existing, ErrInvalidArgument
- } else {
- // create file
- return existing, ErrInvalidArgument
- }
- })
+ return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 058eb40..1a06ce1 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -67,15 +67,15 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
_, basename := filepath.Split(path)
c.Check(fi.Name(), check.Equals, basename)
f, err = s.fs.Open(path + "/A Project/A Subproject")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fi, err = f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
@@ -90,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
+ f, err := s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project/Project does not exist")
+ c.Assert(err, check.NotNil)
+ f, err = s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fis)
+ var foundSubproject, foundCollection bool
+ for _, fi := range fis {
+ switch fi.Name() {
+ case "A Subproject":
+ foundSubproject = true
+ case "collection_to_move_around":
+ foundCollection = true
+ }
+ }
+ c.Check(foundSubproject, check.Equals, true)
+ c.Check(foundCollection, check.Equals, true)
+}
+
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
badCollection := Collection{
Name: "bad/collection",
@@ -109,7 +137,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
dir, err := s.fs.Open("/users/active/A Project")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fis, err := dir.Readdir(-1)
c.Check(err, check.IsNil)
for _, fi := range fis {
@@ -122,7 +150,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
@@ -140,6 +168,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
f, err := s.fs.Open("/home/A Project/oob")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
f.Close()
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index e9c8387..b5daf7b 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -73,7 +73,10 @@ func (fs *customFileSystem) MountProject(mount, uuid string) {
func (fs *customFileSystem) MountUsers(mount string) {
fs.root.inode.Child(mount, func(inode) (inode, error) {
- return &usersnode{
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: fs.usersLoadOne,
+ loadAll: fs.usersLoadAll,
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -136,24 +139,10 @@ func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
}
func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
- inode: &treenode{
- fs: fs,
- parent: root,
- inodes: make(map[string]inode),
- fileinfo: fileinfo{
- name: name,
- modTime: time.Now(),
- mode: 0755 | os.ModeDir,
- },
- },
- }
-}
-
-func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+ loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
inode: &treenode{
fs: fs,
parent: root,
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
index ccfe2c5..00f7036 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs_users.go
@@ -8,55 +8,41 @@ import (
"os"
)
-// usersnode is a virtual directory with an entry for each visible
-// Arvados username, each showing the respective user's "home
-// projects".
-type usersnode struct {
- inode
- staleChecker
- err error
+func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{{"username", "=", name}},
+ })
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ user := resp.Items[0]
+ return fs.newProjectNode(parent, user.Username, user.UUID), nil
}
-func (un *usersnode) load() {
- fs := un.FS().(*customFileSystem)
-
+func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
params := ResourceListParams{
+ Count: "none",
Order: "uuid",
}
+ var inodes []inode
for {
var resp UserList
- un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
- if un.err != nil {
- return
- }
- if len(resp.Items) == 0 {
- break
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return inodes, nil
}
for _, user := range resp.Items {
if user.Username == "" {
continue
}
- un.inode.Child(user.Username, func(inode) (inode, error) {
- return fs.newProjectNode(un, user.Username, user.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
- un.err = nil
-}
-
-func (un *usersnode) Readdir() ([]os.FileInfo, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Readdir()
-}
-
-func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Child(name, nil)
}
commit 31a3b4f34aa56ade25def7cdd59b026f5a59ee13
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 13:29:20 2018 -0400
13111: Update docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index b7da3b0..6c867f0 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -133,8 +133,21 @@
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
+//
+// The following URLs are read-only, but otherwise interchangeable
+// with the above:
+//
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
+// http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt
+// http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt
+//
+// If the collection is named "My Collection" and located in a project
+// called "My Project" which is in the home project of a user with
+// username is "bob", the following read-only URL is also available
+// when authenticating as bob:
+//
+// http://collections.example.com/users/bob/My+Project/My+Collection/foo/bar.txt
//
// An additional form is supported specifically to make it more
// convenient to maintain support for existing Workbench download
@@ -148,6 +161,9 @@
//
// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
//
+// Collections can also be accessed (read-only) via "/by_id/X" where X
+// is a UUID or portable data hash.
+//
// Authorization mechanisms
//
// A token can be provided in an Authorization header:
@@ -178,11 +194,10 @@
//
// Indexes
//
-// Currently, keep-web does not generate HTML index listings, nor does
-// it serve a default file like "index.html" when a directory is
-// requested. These features are likely to be added in future
-// versions. Until then, keep-web responds with 404 if a directory
-// name (or any path ending with "/") is requested.
+// Keep-web returns a generic HTML index listing when a directory is
+// requested with the GET method. It does not serve a default file
+// like "index.html". Directory listings are also returned for WebDAV
+// PROPFIND requests.
//
// Compatibility
//
commit f22f1e3c92e9a69e9e7f82fa6226ad100a8025da
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 13:28:31 2018 -0400
13111: Follow attachment-only policy at /by_id/ and /users/ paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 0618588..8b61b54 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -14,6 +14,7 @@ import (
"net/http"
"net/url"
"os"
+ "path/filepath"
"sort"
"strconv"
"strings"
@@ -335,7 +336,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if useSiteFS {
- h.serveSiteFS(w, r, tokens, credentialsOK)
+ h.serveSiteFS(w, r, tokens, credentialsOK, attachment)
return
}
@@ -505,7 +506,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
-func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK bool) {
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
@@ -551,6 +552,10 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
}
return
}
+ if r.Method == "GET" {
+ _, basename := filepath.Split(r.URL.Path)
+ applyContentDispositionHdr(w, r, basename, attachment)
+ }
wh := webdav.Handler{
Prefix: "/",
FileSystem: &webdavFS{
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 15f32f1..4894ceb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -333,7 +333,20 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
http.StatusOK,
"foo",
)
- c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "foo",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
commit 6522329120f0e2a73bf9c55f4f937fa3d109803d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 10:14:31 2018 -0400
13111: Serve /by_id/ dir.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 22ddd84..db5d373 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -265,6 +265,22 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
}
}
+func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
+ for _, path := range []string{"/by_id", "/by_id/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
+ }
+ for _, path := range []string{
+ "/by_id/" + arvadostest.FooPdh,
+ "/by_id/" + arvadostest.FooPdh + "/",
+ "/by_id/" + arvadostest.FooCollection,
+ "/by_id/" + arvadostest.FooCollection + "/",
+ } {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
+ }
+}
+
func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
for _, path := range []string{"/"} {
stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 389ab73..0618588 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -163,6 +163,12 @@ var (
"HEAD": true,
"POST": true,
}
+ // top-level dirs to serve with siteFS
+ siteFSDir = map[string]bool{
+ "": true, // root directory
+ "by_id": true,
+ "users": true,
+ }
)
// ServeHTTP implements http.Handler.
@@ -250,7 +256,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if r.URL.Path == "/" || (len(pathParts) >= 1 && pathParts[0] == "users") {
+ } else if siteFSDir[pathParts[0]] {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
commit 76cc598a22d58cd889ad41b47ef061bde90c9f52
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 09:14:52 2018 -0400
13111: Fix skipped error check when Write called before WriteHeader.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5e14cb4..389ab73 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -112,12 +112,12 @@ type updateOnSuccess struct {
}
func (uos *updateOnSuccess) Write(p []byte) (int, error) {
- if uos.err != nil {
- return 0, uos.err
- }
if !uos.sentHeader {
uos.WriteHeader(http.StatusOK)
}
+ if uos.err != nil {
+ return 0, uos.err
+ }
return uos.ResponseWriter.Write(p)
}
commit 78eedab0a2feadc7877529ac4ce65ccaaf4db768
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 11 19:59:18 2018 -0400
13111: Avoid double WriteHeader after implicit 200 on first Write.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/httpserver/responsewriter.go b/sdk/go/httpserver/responsewriter.go
index d37822f..8dea759 100644
--- a/sdk/go/httpserver/responsewriter.go
+++ b/sdk/go/httpserver/responsewriter.go
@@ -41,6 +41,9 @@ func (w *responseWriter) WriteHeader(s int) {
}
func (w *responseWriter) Write(data []byte) (n int, err error) {
+ if w.wroteStatus == 0 {
+ w.WriteHeader(http.StatusOK)
+ }
n, err = w.ResponseWriter.Write(data)
w.wroteBodyBytes += n
w.err = err
commit 4dda5d7b0ea74103c07617cc2ab9e5c97682c85d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 11 09:07:09 2018 -0400
13111: Ignore projects and collections with "/" in name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 45beff6..3058a76 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -589,3 +589,7 @@ func rlookup(start inode, path string) (node inode, err error) {
}
return
}
+
+func permittedName(name string) bool {
+ return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 923615b..7ce37aa 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -821,8 +821,8 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
- if basename == "" || basename == "." || basename == ".." {
- err = fmt.Errorf("invalid filename")
+ if !permittedName(basename) {
+ err = fmt.Errorf("invalid file part %q in path %q", basename, path)
return
}
for _, name := range names[:len(names)-1] {
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 071cc0a..827a44b 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -63,7 +63,7 @@ func (pn *projectnode) load() {
}
for _, i := range resp.Items {
coll := i
- if coll.Name == "" || coll.Name == "." || coll.Name == ".." {
+ if !permittedName(coll.Name) {
continue
}
pn.inode.Child(coll.Name, func(inode) (inode, error) {
@@ -85,7 +85,7 @@ func (pn *projectnode) load() {
break
}
for _, group := range resp.Items {
- if group.Name == "" || group.Name == "." || group.Name == ".." {
+ if !permittedName(group.Name) {
continue
}
pn.inode.Child(group.Name, func(inode) (inode, error) {
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 69bae89..058eb40 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
"path/filepath"
+ "strings"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -89,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestSlashInName(c *check.C) {
+ badCollection := Collection{
+ Name: "bad/collection",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", s.client.UpdateBody(&badCollection), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
+
+ badProject := Group{
+ Name: "bad/project",
+ GroupClass: "project",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", s.client.UpdateBody(&badProject), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
+
+ dir, err := s.fs.Open("/users/active/A Project")
+ c.Check(err, check.IsNil)
+ fis, err := dir.Readdir(-1)
+ c.Check(err, check.IsNil)
+ for _, fi := range fis {
+ c.Logf("fi.Name() == %q", fi.Name())
+ c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
+ }
+}
+
func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
diff --git a/sdk/go/arvados/group.go b/sdk/go/arvados/group.go
index b00809f..6b5718a 100644
--- a/sdk/go/arvados/group.go
+++ b/sdk/go/arvados/group.go
@@ -6,9 +6,10 @@ package arvados
// Group is an arvados#group record
type Group struct {
- UUID string `json:"uuid,omitempty"`
- Name string `json:"name,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
+ UUID string `json:"uuid,omitempty"`
+ Name string `json:"name,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+ GroupClass string `json:"group_class"`
}
// GroupList is an arvados#groupList resource.
@@ -18,3 +19,7 @@ type GroupList struct {
Offset int `json:"offset"`
Limit int `json:"limit"`
}
+
+func (g Group) resourceName() string {
+ return "group"
+}
commit cd9626d625f72b31054204c68cb2bb32e5dd3111
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:41:11 2018 -0400
13111: Note safe use of append().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 300c77a..071cc0a 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -44,6 +44,9 @@ func (pn *projectnode) load() {
}
pn.uuid = resp.UUID
}
+ // Note: the "filters" slice's backing array might be reused
+ // by append(filters,...) below. This isn't goroutine safe,
+ // but all accesses are in the same goroutine, so it's OK.
filters := []Filter{{"owner_uuid", "=", pn.uuid}}
params := ResourceListParams{
Filters: filters,
commit b9e031258dc079151d815167d1d6665bc63e2318
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:41:00 2018 -0400
13111: Update comment.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fa43cca..45beff6 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -452,7 +452,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// call nca.FS().Rename() instead of proceeding. Until then
// it's awkward for filesystems to implement their own Rename
// methods effectively: the only one that runs is the one on
- // the root filesystem exposed to the caller (webdav, fuse,
+ // the root FileSystem exposed to the caller (webdav, fuse,
// etc).
// When acquiring locks on multiple inodes, avoid deadlock by
commit 9a8b51c6c2468162ee9748514141a94d24e5f663
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:08:35 2018 -0400
13111: Redirect /dir to /dir/ at siteFS paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 446eeb4..5e14cb4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -329,7 +329,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if useSiteFS {
- h.serveSiteFS(w, r, tokens)
+ h.serveSiteFS(w, r, tokens, credentialsOK)
return
}
@@ -499,7 +499,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
-func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
@@ -528,18 +528,22 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
Insecure: arv.ApiInsecure,
}
fs := client.SiteFileSystem(kc)
- if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+ f, err := fs.Open(r.URL.Path)
+ if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
} else if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
- } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
-
- h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ }
+ defer f.Close()
+ if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+ if !strings.HasSuffix(r.URL.Path, "/") {
+ h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
+ } else {
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ }
return
- } else {
- f.Close()
}
wh := webdav.Handler{
Prefix: "/",
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 7fed6fb..15f32f1 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -493,10 +493,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
for _, trial := range []struct {
- uri string
- header http.Header
- expect []string
- cutDirs int
+ uri string
+ header http.Header
+ expect []string
+ redirect string
+ cutDirs int
}{
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
@@ -529,12 +530,32 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 4,
},
{
+ uri: "download.example.com/",
+ header: authHeader,
+ expect: []string{"users/"},
+ cutDirs: 0,
+ },
+ {
+ uri: "download.example.com/users",
+ header: authHeader,
+ redirect: "/users/",
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
uri: "download.example.com/users/",
header: authHeader,
expect: []string{"active/"},
cutDirs: 1,
},
{
+ uri: "download.example.com/users/active",
+ header: authHeader,
+ redirect: "/users/active/",
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
uri: "download.example.com/users/active/",
header: authHeader,
expect: []string{"foo_file_in_dir/"},
@@ -565,10 +586,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 1,
},
{
- uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 2,
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
+ header: authHeader,
+ redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
@@ -577,10 +599,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 3,
},
{
- uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 1,
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
+ redirect: "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
@@ -616,6 +639,9 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
resp = httptest.NewRecorder()
s.testServer.Handler.ServeHTTP(resp, req)
}
+ if trial.redirect != "" {
+ c.Check(req.URL.Path, check.Equals, trial.redirect)
+ }
if trial.expect == nil {
c.Check(resp.Code, check.Equals, http.StatusNotFound)
} else {
commit db30db0c7a441eec9d5dd2ca2b74cfde9966d6ce
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:54:40 2018 -0400
13111: Serve "site filesystem" at keep-web root URL.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 0fc6e47..22ddd84 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -266,6 +266,11 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
}
func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
+ for _, path := range []string{"/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
+ }
for _, path := range []string{"/users", "/users/"} {
stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 00af0f4..446eeb4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -250,7 +250,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+ } else if r.URL.Path == "/" || (len(pathParts) >= 1 && pathParts[0] == "users") {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 02f03d0..ee585ad 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -59,7 +59,6 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
func (s *IntegrationSuite) Test404(c *check.C) {
for _, uri := range []string{
// Routing errors (always 404 regardless of what's stored in Keep)
- "/",
"/foo",
"/download",
"/collections",
commit 85736e92085b10d58141ef66c1354fb1cf16c562
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:22:08 2018 -0400
13111: Skip collections named "." or "..".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 7f34a42..300c77a 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -60,7 +60,7 @@ func (pn *projectnode) load() {
}
for _, i := range resp.Items {
coll := i
- if coll.Name == "" {
+ if coll.Name == "" || coll.Name == "." || coll.Name == ".." {
continue
}
pn.inode.Child(coll.Name, func(inode) (inode, error) {
commit b29442ebd118d522f9be9508f5111f2a68eae1a4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:21:53 2018 -0400
13111: Clarify Child() comment.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 369b4bb..fa43cca 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -103,6 +103,9 @@ type inode interface {
// Child() performs lookups and updates of named child nodes.
//
+ // (The term "child" here is used strictly. This means name is
+ // not "." or "..", and name does not contain "/".)
+ //
// If replace is non-nil, Child calls replace(x) where x is
// the current child inode with the given name. If possible,
// the child inode is replaced with the one returned by
commit 3cf6c6d8dcd3616d0075f9af1732d5ed4231b861
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:20:49 2018 -0400
13111: Test /users/name and /users paths with cadaver.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 843a8e8..0fc6e47 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -250,35 +250,8 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
os.Remove(checkfile.Name())
- cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
- if password != "" {
- // cadaver won't try username/password
- // authentication unless the server responds
- // 401 to an unauthenticated request, which it
- // only does in AttachmentOnlyHost,
- // TrustAllContent, and per-collection vhost
- // cases.
- s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
-
- cmd.Env = append(os.Environ(), "HOME="+tempdir)
- f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
- c.Assert(err, check.IsNil)
- _, err = fmt.Fprintf(f, "default login none password %s\n", password)
- c.Assert(err, check.IsNil)
- c.Assert(f.Close(), check.IsNil)
- }
- cmd.Stdin = bytes.NewBufferString(trial.cmd)
- stdout, err := cmd.StdoutPipe()
- c.Assert(err, check.Equals, nil)
- cmd.Stderr = cmd.Stdout
- go cmd.Start()
-
- var buf bytes.Buffer
- _, err = io.Copy(&buf, stdout)
- c.Check(err, check.Equals, nil)
- err = cmd.Wait()
- c.Check(err, check.Equals, nil)
- c.Check(buf.String(), check.Matches, trial.match)
+ stdout := s.runCadaver(c, password, trial.path, trial.cmd)
+ c.Check(stdout, check.Matches, trial.match)
if trial.data == nil {
continue
@@ -291,3 +264,54 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
c.Check(err, check.IsNil)
}
}
+
+func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
+ for _, path := range []string{"/users", "/users/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
+ }
+ for _, path := range []string{"/users/active", "/users/active/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
+ }
+ for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
+ }
+}
+
+func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
+ if password != "" {
+ // cadaver won't try username/password authentication
+ // unless the server responds 401 to an
+ // unauthenticated request, which it only does in
+ // AttachmentOnlyHost, TrustAllContent, and
+ // per-collection vhost cases.
+ s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
+
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ cmd.Stdin = bytes.NewBufferString(stdin)
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ return buf.String()
+}
commit 418c57bce3aac1a22548e53e1018a1547d9efee4
Merge: 89c42b3 c4bd314
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 16:14:35 2018 -0400
13111: Merge branch 'master' into 13111-webdav-projects
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc services/keep-web/webdav.go
index 3e62b19,432c6af..5b23c9c
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@@ -47,10 -47,7 +47,10 @@@ type webdavFS struct
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
- dir, name := path.Split(name)
+ dir, _ := path.Split(name)
if dir == "" || dir == "/" {
return
}
commit 89c42b3d5d978de79797be9c9c142330d23e28c4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Mar 28 09:10:32 2018 -0400
13111: Update tests to accept username in API response.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index a506486..39c905a 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -817,7 +817,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
- "last_name"].sort
+ "last_name", "username"].sort
def check_non_admin_index
assert_response :success
commit c69d271455a0550cda8e56ab6a4685fe3d7d91ea
Merge: b536744 9e92ae4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 27 09:28:05 2018 -0400
13111: Merge branch 'master' into 13111-webdav-projects
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit b5367444a51cc91e7b589bfdb68eb12e8a1f4937
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Mar 23 17:25:27 2018 -0400
13111: Expose read-only /users/ tree.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 2d6dd2b..6a7ce2a 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -5,7 +5,6 @@
package main
import (
- _ "encoding/json"
"fmt"
"io"
"io/ioutil"
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 5f5f69a..843a8e8 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -27,7 +27,7 @@ func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
@@ -36,19 +36,23 @@ func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ rpath := "/users/active/foo_file_in_dir/"
s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
- r := "/users/active/foo_file_in_dir/"
- w := "/users/active/" + newCollection.Name
+ wpath := "/users/active/" + newCollection.Name
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
- return r, w, pdh
+ return rpath, wpath, pdh
+ }, func(path string) bool {
+ // Skip tests that rely on writes, because /users/
+ // tree is read-only.
+ return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -239,6 +243,10 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
},
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ if skip != nil && skip(trial.path) {
+ c.Log("(skip)")
+ continue
+ }
os.Remove(checkfile.Name())
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5ab4f70..00af0f4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -226,13 +226,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
}
- arv := h.clientPool.Get()
- if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
- return
- }
- defer h.clientPool.Put(arv)
-
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
@@ -241,6 +234,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
var attachment bool
+ var useSiteFS bool
credentialsOK := h.Config.TrustAllContent
if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
@@ -256,6 +250,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
+ } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+ useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
collectionID = parseCollectionIDFromURL(pathParts[0][2:])
@@ -275,6 +271,16 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+ if collectionID == "" && !useSiteFS {
+ statusCode = http.StatusNotFound
+ return
+ }
+
+ forceReload := false
+ if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
+ forceReload = true
+ }
+
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -322,6 +328,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokens = append(reqTokens, h.Config.AnonymousTokens...)
}
+ if useSiteFS {
+ h.serveSiteFS(w, r, tokens)
+ return
+ }
+
if len(targetPath) > 0 && targetPath[0] == "_" {
// If a collection has a directory called "t=foo" or
// "_", it can be served at
@@ -333,10 +344,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
stripParts++
}
- forceReload := false
- if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
- forceReload = true
+ arv := h.clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
+ return
}
+ defer h.clientPool.Put(arv)
var collection *arvados.Collection
tokenResult := make(map[string]int)
@@ -410,12 +423,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}
- var fs arvados.FileSystem
- if collectionID == "" {
- fs = client.SiteFileSystem(kc)
- } else {
- fs, err = collection.FileSystem(client, kc)
- }
+ fs, err := collection.FileSystem(client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
@@ -475,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
+ h.serveDirectory(w, r, collection.Name, fs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -491,10 +499,69 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+ if len(tokens) == 0 {
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+ http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ return
+ }
+ if writeMethod[r.Method] {
+ http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
+ return
+ }
+ arv := h.clientPool.Get()
+ if arv == nil {
+ http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ return
+ }
+ defer h.clientPool.Put(arv)
+ arv.ApiToken = tokens[0]
+
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ client := &arvados.Client{
+ APIHost: arv.ApiServer,
+ AuthToken: arv.ApiToken,
+ Insecure: arv.ApiInsecure,
+ }
+ fs := client.SiteFileSystem(kc)
+ if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ return
+ } else {
+ f.Close()
+ }
+ wh := webdav.Handler{
+ Prefix: "/",
+ FileSystem: &webdavFS{
+ collfs: fs,
+ writing: writeMethod[r.Method],
+ alwaysReadEOF: r.Method == "PROPFIND",
+ },
+ LockSystem: h.webdavLS,
+ Logger: func(_ *http.Request, err error) {
+ if err != nil {
+ log.Printf("error from webdav handler: %q", err)
+ }
+ },
+ }
+ wh.ServeHTTP(w, r)
+}
+
var dirListingTemplate = `<!DOCTYPE HTML>
<HTML><HEAD>
<META name="robots" content="NOINDEX">
- <TITLE>{{ .Collection.Name }}</TITLE>
+ <TITLE>{{ .CollectionName }}</TITLE>
<STYLE type="text/css">
body {
margin: 1.5em;
@@ -518,19 +585,26 @@ var dirListingTemplate = `<!DOCTYPE HTML>
</STYLE>
</HEAD>
<BODY>
+
<H1>{{ .CollectionName }}</H1>
<P>This collection of data files is being shared with you through
Arvados. You can download individual files listed below. To download
-the entire collection with wget, try:</P>
+the entire directory tree with wget, try:</P>
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
<H2>File Listing</H2>
{{if .Files}}
<UL>
-{{range .Files}} <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
+{{range .Files}}
+{{if .IsDir }}
+ <LI>{{" " | printf "%15s " | nbsp}}<A href="{{.Name}}/">{{.Name}}/</A></LI>
+{{else}}
+ <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>
+{{end}}
+{{end}}
</UL>
{{else}}
<P>(No files; this collection is empty.)</P>
@@ -550,11 +624,12 @@ the entire collection with wget, try:</P>
`
type fileListEnt struct {
- Name string
- Size int64
+ Name string
+ Size int64
+ IsDir bool
}
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
var files []fileListEnt
var walk func(string) error
if !strings.HasSuffix(base, "/") {
@@ -574,15 +649,16 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
return err
}
for _, ent := range ents {
- if ent.IsDir() {
+ if recurse && ent.IsDir() {
err = walk(path + ent.Name() + "/")
if err != nil {
return err
}
} else {
files = append(files, fileListEnt{
- Name: path + ent.Name(),
- Size: ent.Size(),
+ Name: path + ent.Name(),
+ Size: ent.Size(),
+ IsDir: ent.IsDir(),
})
}
}
@@ -611,7 +687,7 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
"CollectionName": collectionName,
"Files": files,
"Request": r,
- "StripParts": stripParts,
+ "StripParts": strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
})
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 3e7ae5f..7fed6fb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -508,7 +508,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
@@ -517,12 +517,30 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
- uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ uri: "download.example.com/users/active/foo_file_in_dir/",
header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
+ expect: []string{"dir1/"},
cutDirs: 3,
},
{
+ uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
+ header: authHeader,
+ expect: []string{"bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/users/",
+ header: authHeader,
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/active/",
+ header: authHeader,
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
@@ -550,19 +568,19 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 1,
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 2,
+ cutDirs: 3,
},
{
uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 941090a..3e62b19 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -47,6 +47,9 @@ type webdavFS struct {
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
dir, name := path.Split(name)
if dir == "" || dir == "/" {
return
@@ -66,7 +69,7 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
}
func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
- writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+ writing := flag&(os.O_WRONLY|os.O_RDWR|os.O_TRUNC) != 0
if writing {
fs.makeparents(name)
}
@@ -75,8 +78,13 @@ func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os
// webdav module returns 404 on all OpenFile errors,
// but returns 405 Method Not Allowed if OpenFile()
// succeeds but Write() or Close() fails. We'd rather
- // have 405.
- f = writeFailer{File: f, err: errReadOnly}
+ // have 405. writeFailer ensures Close() fails if the
+ // file is opened for writing *or* Write() is called.
+ var err error
+ if writing {
+ err = errReadOnly
+ }
+ f = writeFailer{File: f, err: err}
}
if fs.alwaysReadEOF {
f = readEOF{File: f}
@@ -109,10 +117,15 @@ type writeFailer struct {
}
func (wf writeFailer) Write([]byte) (int, error) {
+ wf.err = errReadOnly
return 0, wf.err
}
func (wf writeFailer) Close() error {
+ err := wf.File.Close()
+ if err != nil {
+ wf.err = err
+ }
return wf.err
}
commit 1b7ed029c3d9d50b275573b65e8fbf4943e76bcb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 16:56:16 2018 -0400
13111: Add /users virtual dir to siteFS.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 776c8a3..7f34a42 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -10,25 +10,31 @@ import (
"time"
)
+type staleChecker struct {
+ mtx sync.Mutex
+ last time.Time
+}
+
+func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
+ sc.mtx.Lock()
+ defer sc.mtx.Unlock()
+ if !staleFunc(sc.last) {
+ return
+ }
+ sc.last = time.Now()
+ fn()
+}
+
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
+ staleChecker
uuid string
err error
-
- loadLock sync.Mutex
- loadStart time.Time
}
func (pn *projectnode) load() {
- fs := pn.FS().(*siteFileSystem)
-
- pn.loadLock.Lock()
- defer pn.loadLock.Unlock()
- if !fs.Stale(pn.loadStart) {
- return
- }
- pn.loadStart = time.Now()
+ fs := pn.FS().(*customFileSystem)
if pn.uuid == "" {
var resp User
@@ -89,7 +95,7 @@ func (pn *projectnode) load() {
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
@@ -97,7 +103,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index b7dc08e..69bae89 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"io"
"os"
+ "path/filepath"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -38,8 +39,17 @@ func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, b
return sc.Client.RequestAndDecode(dst, method, path, body, params)
}
-func (s *SiteFSSuite) TestHomeProject(c *check.C) {
- f, err := s.fs.Open("/home")
+func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
+ s.fs.MountProject("home", "")
+ s.testHomeProject(c, "/home")
+}
+
+func (s *SiteFSSuite) TestUsersDir(c *check.C) {
+ s.testHomeProject(c, "/users/active")
+}
+
+func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
+ f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
c.Check(len(fis), check.Not(check.Equals), 0)
@@ -53,23 +63,24 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/A Project/..")
+ f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
- c.Check(fi.Name(), check.Equals, "home")
+ _, basename := filepath.Split(path)
+ c.Check(fi.Name(), check.Equals, basename)
- f, err = s.fs.Open("/home/A Project/A Subproject")
+ f, err = s.fs.Open(path + "/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/Unrestricted public data",
- "/home/Unrestricted public data/does not exist",
- "/home/A Project/does not exist",
+ path + "/Unrestricted public data",
+ path + "/Unrestricted public data/does not exist",
+ path + "/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -79,6 +90,8 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ s.fs.MountProject("home", "")
+
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
c.Check(err, check.IsNil)
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index cdcf40e..e9c8387 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -10,22 +10,25 @@ import (
"time"
)
-type siteFileSystem struct {
+type CustomFileSystem interface {
+ FileSystem
+ MountByID(mount string)
+ MountProject(mount, uuid string)
+ MountUsers(mount string)
+}
+
+type customFileSystem struct {
fileSystem
+ root *vdirnode
staleThreshold time.Time
staleLock sync.Mutex
}
-// SiteFileSystem returns a FileSystem that maps collections and other
-// Arvados objects onto a filesystem layout.
-//
-// This is experimental: the filesystem layout is not stable, and
-// there are significant known bugs and shortcomings. For example,
-// writes are not persisted until Sync() is called.
-func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
root := &vdirnode{}
- fs := &siteFileSystem{
+ fs := &customFileSystem{
+ root: root,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: c, keepClient: kc},
root: root,
@@ -41,14 +44,18 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.inode.Child("by_id", func(inode) (inode, error) {
+ return fs
+}
+
+func (fs *customFileSystem) MountByID(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
- name: "by_id",
+ name: mount,
modTime: time.Now(),
mode: 0755 | os.ModeDir,
},
@@ -56,13 +63,45 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
create: fs.mountCollection,
}, nil
})
- root.inode.Child("home", func(inode) (inode, error) {
- return fs.newProjectNode(fs.root, "home", ""), nil
+}
+
+func (fs *customFileSystem) MountProject(mount, uuid string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, mount, uuid), nil
})
+}
+
+func (fs *customFileSystem) MountUsers(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return &usersnode{
+ inode: &treenode{
+ fs: fs,
+ parent: fs.root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: mount,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }, nil
+ })
+}
+
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// writes are not persisted until Sync() is called.
+func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
+ fs := c.CustomFileSystem(kc)
+ fs.MountByID("by_id")
+ fs.MountUsers("users")
return fs
}
-func (fs *siteFileSystem) Sync() error {
+func (fs *customFileSystem) Sync() error {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
fs.staleThreshold = time.Now()
@@ -71,17 +110,17 @@ func (fs *siteFileSystem) Sync() error {
// Stale returns true if information obtained at time t should be
// considered stale.
-func (fs *siteFileSystem) Stale(t time.Time) bool {
+func (fs *customFileSystem) Stale(t time.Time) bool {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
return !fs.staleThreshold.Before(t)
}
-func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
-func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
+func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
@@ -96,7 +135,23 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
-func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
+func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
return &projectnode{
uuid: uuid,
inode: &treenode{
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index a3a9712..e35ae48 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -16,7 +16,7 @@ var _ = check.Suite(&SiteFSSuite{})
type SiteFSSuite struct {
client *Client
- fs FileSystem
+ fs CustomFileSystem
kc keepClient
}
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
new file mode 100644
index 0000000..ccfe2c5
--- /dev/null
+++ b/sdk/go/arvados/fs_users.go
@@ -0,0 +1,62 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+)
+
+// usersnode is a virtual directory with an entry for each visible
+// Arvados username, each showing the respective user's "home
+// projects".
+type usersnode struct {
+ inode
+ staleChecker
+ err error
+}
+
+func (un *usersnode) load() {
+ fs := un.FS().(*customFileSystem)
+
+ params := ResourceListParams{
+ Order: "uuid",
+ }
+ for {
+ var resp UserList
+ un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if un.err != nil {
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, user := range resp.Items {
+ if user.Username == "" {
+ continue
+ }
+ un.inode.Child(user.Username, func(inode) (inode, error) {
+ return fs.newProjectNode(un, user.Username, user.UUID), nil
+ })
+ }
+ params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ }
+ un.err = nil
+}
+
+func (un *usersnode) Readdir() ([]os.FileInfo, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Readdir()
+}
+
+func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Child(name, nil)
+}
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index dc7e62f..587f4db 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -159,7 +159,7 @@ class Arvados::V1::UsersController < ApplicationController
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name"]
+ safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username"]
if @select
@select = @select & safe_attrs
else
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 19a2040..5ab4f70 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -236,7 +236,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
- var targetID string
+ var collectionID string
var tokens []string
var reqTokens []string
var pathToken bool
@@ -250,7 +250,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
attachment = true
}
- if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
+ if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
@@ -258,28 +258,23 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
- targetID = parseCollectionIDFromURL(pathParts[0][2:])
+ collectionID = parseCollectionIDFromURL(pathParts[0][2:])
stripParts = 1
} else if len(pathParts) >= 2 && pathParts[0] == "collections" {
if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetID = parseCollectionIDFromURL(pathParts[2])
+ collectionID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
stripParts = 4
pathToken = true
} else {
// /collections/ID/PATH...
- targetID = parseCollectionIDFromURL(pathParts[1])
+ collectionID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
stripParts = 2
}
}
- if targetID == "" {
- statusCode = http.StatusNotFound
- return
- }
-
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -347,7 +342,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
+ collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
@@ -414,14 +409,21 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
}
- fs, err := collection.FileSystem(client, kc)
+
+ var fs arvados.FileSystem
+ if collectionID == "" {
+ fs = client.SiteFileSystem(kc)
+ } else {
+ fs, err = collection.FileSystem(client, kc)
+ }
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- targetIsPDH := arvadosclient.PDHMatch(targetID)
- if targetIsPDH && writeMethod[r.Method] {
+ writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ targetIsPDH := arvadosclient.PDHMatch(collectionID)
+ if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
return
}
@@ -435,7 +437,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w = &updateOnSuccess{
ResponseWriter: w,
update: func() error {
- return h.Config.Cache.Update(client, *collection, fs)
+ return h.Config.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index af83681..941090a 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -36,7 +36,7 @@ var (
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.CollectionFileSystem
+ collfs arvados.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
commit 6b17b8e5aaf08b3124d0d7536b3c73581894d70a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 10:52:09 2018 -0400
13111: Add tests for /users/ paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index b7ec1de..5f5f69a 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -22,24 +22,33 @@ import (
)
func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
- s.testCadaver(c, arvadostest.ActiveToken, func(newUUID string) (string, string, string) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
- w := "/c=" + newUUID + "/"
+ w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
})
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
- s.testCadaver(c, "", func(newUUID string) (string, string, string) {
+ s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
- w := "/c=" + newUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(string) (string, string, string)) {
+func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
+ r := "/users/active/foo_file_in_dir/"
+ w := "/users/active/" + newCollection.Name
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -62,7 +71,7 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- readPath, writePath, pdhPath := pathFunc(newCollection.UUID)
+ readPath, writePath, pdhPath := pathFunc(newCollection)
matchToday := time.Now().Format("Jan +2")
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 21e47c8..3e7ae5f 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -517,6 +517,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
+ uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 3,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
commit 11f2a3542e8bef42edcb413405efb43bd2e30be7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 10:00:36 2018 -0400
13111: Test webdav with http basic authentication.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index eb32367..b7ec1de 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -6,11 +6,13 @@ package main
import (
"bytes"
+ "fmt"
"io"
"io/ioutil"
"net/url"
"os"
"os/exec"
+ "path/filepath"
"strings"
"time"
@@ -19,34 +21,51 @@ import (
check "gopkg.in/check.v1"
)
-func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
+func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newUUID string) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
+ w := "/c=" + newUUID + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
+ s.testCadaver(c, "", func(newUUID string) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newUUID + "/t=" + arvadostest.ActiveToken + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(string) (string, string, string)) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
- localfile, err := ioutil.TempFile("", "localfile")
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ localfile, err := ioutil.TempFile(tempdir, "localfile")
c.Assert(err, check.IsNil)
- defer os.Remove(localfile.Name())
localfile.Write(testdata)
- emptyfile, err := ioutil.TempFile("", "emptyfile")
+ emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
c.Assert(err, check.IsNil)
- defer os.Remove(emptyfile.Name())
- checkfile, err := ioutil.TempFile("", "checkfile")
+ checkfile, err := ioutil.TempFile(tempdir, "checkfile")
c.Assert(err, check.IsNil)
- defer os.Remove(checkfile.Name())
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
- pdhPath := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arv.AuthToken + "/"
+ readPath, writePath, pdhPath := pathFunc(newCollection.UUID)
matchToday := time.Now().Format("Jan +2")
- readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
type testcase struct {
path string
cmd string
@@ -215,6 +234,22 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
os.Remove(checkfile.Name())
cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
+ if password != "" {
+ // cadaver won't try username/password
+ // authentication unless the server responds
+ // 401 to an unauthenticated request, which it
+ // only does in AttachmentOnlyHost,
+ // TrustAllContent, and per-collection vhost
+ // cases.
+ s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
+
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
cmd.Stdin = bytes.NewBufferString(trial.cmd)
stdout, err := cmd.StdoutPipe()
c.Assert(err, check.Equals, nil)
commit df591042778408d03d410d5c22a669d85652d1ea
Merge: 29a6622 f7fe9cc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Mar 19 14:39:16 2018 -0400
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 29a6622585581b5e4f519968bbd291939bf49392
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Feb 13 17:29:38 2018 -0500
13111: Reload project dir if fsync(2) was called since last load.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 999b4e9..aea0cc0 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -16,6 +16,7 @@ import (
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
TrashAt *time.Time `json:"trash_at,omitempty"`
ManifestText string `json:"manifest_text,omitempty"`
UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index a5e4710..776c8a3 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -7,18 +7,29 @@ package arvados
import (
"os"
"sync"
+ "time"
)
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
- uuid string
- setupOnce sync.Once
- err error
+ uuid string
+ err error
+
+ loadLock sync.Mutex
+ loadStart time.Time
}
-func (pn *projectnode) setup() {
+func (pn *projectnode) load() {
fs := pn.FS().(*siteFileSystem)
+
+ pn.loadLock.Lock()
+ defer pn.loadLock.Unlock()
+ if !fs.Stale(pn.loadStart) {
+ return
+ }
+ pn.loadStart = time.Now()
+
if pn.uuid == "" {
var resp User
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
@@ -36,7 +47,6 @@ func (pn *projectnode) setup() {
var resp CollectionList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -60,7 +70,6 @@ func (pn *projectnode) setup() {
var resp GroupList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -76,10 +85,11 @@ func (pn *projectnode) setup() {
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
+ pn.err = nil
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
@@ -87,7 +97,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 1de0714..b7dc08e 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -46,29 +47,29 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
ok := false
for _, fi := range fis {
c.Check(fi.Name(), check.Not(check.Equals), "")
- if fi.Name() == "Unrestricted public data" {
+ if fi.Name() == "A Project" {
ok = true
}
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/Unrestricted public data/..")
+ f, err = s.fs.Open("/home/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
c.Check(fi.Name(), check.Equals, "home")
- f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+ f, err = s.fs.Open("/home/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/A Project",
- "/home/A Project/does not exist",
+ "/home/Unrestricted public data",
"/home/Unrestricted public data/does not exist",
+ "/home/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -76,3 +77,54 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
c.Check(os.IsNotExist(err), check.Equals, true)
}
}
+
+func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ project, err := s.fs.OpenFile("/home/A Project", 0, 0)
+ c.Check(err, check.IsNil)
+
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+
+ oob := Collection{
+ Name: "oob",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ f, err := s.fs.Open("/home/A Project/oob")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Check(fi.IsDir(), check.Equals, true)
+ f.Close()
+
+ wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
+ c.Assert(err, check.IsNil)
+ _, err = wf.Write([]byte("hello oob\n"))
+ c.Check(err, check.IsNil)
+ err = wf.Close()
+ c.Check(err, check.IsNil)
+
+ // Delete test.txt behind s.fs's back by updating the
+ // collection record with the old (empty) ManifestText.
+ err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob/test.txt")
+ c.Check(err, check.NotNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.IsNil)
+
+ err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 701711e..cdcf40e 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -6,11 +6,15 @@ package arvados
import (
"os"
+ "sync"
"time"
)
type siteFileSystem struct {
fileSystem
+
+ staleThreshold time.Time
+ staleLock sync.Mutex
}
// SiteFileSystem returns a FileSystem that maps collections and other
@@ -58,6 +62,21 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
+func (fs *siteFileSystem) Sync() error {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ fs.staleThreshold = time.Now()
+ return nil
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *siteFileSystem) Stale(t time.Time) bool {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ return !fs.staleThreshold.Before(t)
+}
+
func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 26a2212..a3a9712 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -21,7 +21,11 @@ type SiteFSSuite struct {
}
func (s *SiteFSSuite) SetUpTest(c *check.C) {
- s.client = NewClientFromEnv()
+ s.client = &Client{
+ APIHost: os.Getenv("ARVADOS_API_HOST"),
+ AuthToken: arvadostest.ActiveToken,
+ Insecure: true,
+ }
s.kc = &keepClientStub{
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index d057c09..5fccfb3 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -25,6 +25,9 @@ const (
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
+ AProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
+
FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
commit 8296c0784c70660e0a7247ba000741d37bda38fd
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 18:16:53 2018 -0500
13111: Propagate errors in Child().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 4d22c1e..369b4bb 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -27,6 +27,7 @@ var (
ErrWriteOnlyMode = errors.New("file is O_WRONLY")
ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrNotADirectory = errors.New("not a directory")
ErrPermission = os.ErrPermission
)
@@ -128,7 +129,7 @@ type inode interface {
// a child was added or changed, the new child is returned.
//
// Caller must have lock (or rlock if replace is nil).
- Child(name string, replace func(inode) inode) inode
+ Child(name string, replace func(inode) (inode, error)) (inode, error)
sync.Locker
RLock()
@@ -202,8 +203,8 @@ func (*nullnode) Readdir() ([]os.FileInfo, error) {
return nil, ErrInvalidOperation
}
-func (*nullnode) Child(name string, replace func(inode) inode) inode {
- return nil
+func (*nullnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return nil, ErrNotADirectory
}
type treenode struct {
@@ -236,18 +237,25 @@ func (n *treenode) IsDir() bool {
return true
}
-func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
- // TODO: special treatment for "", ".", ".."
+func (n *treenode) Child(name string, replace func(inode) (inode, error)) (child inode, err error) {
child = n.inodes[name]
- if replace != nil {
- newchild := replace(child)
- if newchild == nil {
- delete(n.inodes, name)
- } else if newchild != child {
- n.inodes[name] = newchild
- n.fileinfo.modTime = time.Now()
- child = newchild
- }
+ if name == "" || name == "." || name == ".." {
+ err = ErrInvalidArgument
+ return
+ }
+ if replace == nil {
+ return
+ }
+ newchild, err := replace(child)
+ if err != nil {
+ return
+ }
+ if newchild == nil {
+ delete(n.inodes, name)
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ n.fileinfo.modTime = time.Now()
+ child = newchild
}
return
}
@@ -297,9 +305,9 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(fs.root, dirname)
- if parent == nil {
- return nil, os.ErrNotExist
+ parent, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return nil, err
}
var readable, writable bool
switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
@@ -331,22 +339,26 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
parent.RLock()
defer parent.RUnlock()
}
- n := parent.Child(name, nil)
- if n == nil {
+ n, err := parent.Child(name, nil)
+ if err != nil {
+ return nil, err
+ } else if n == nil {
if !createMode {
return nil, os.ErrNotExist
}
- var err error
- n = parent.Child(name, func(inode) inode {
- n, err = parent.FS().newNode(name, perm|0755, time.Now())
- n.SetParent(parent, name)
- return n
+ n, err = parent.Child(name, func(inode) (repl inode, err error) {
+ repl, err = parent.FS().newNode(name, perm|0755, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(parent, name)
+ return
})
if err != nil {
return nil, err
} else if n == nil {
- // parent rejected new child
- return nil, ErrInvalidOperation
+ // Parent rejected new child, but returned no error
+ return nil, ErrInvalidArgument
}
} else if flag&os.O_EXCL != 0 {
return nil, ErrFileExists
@@ -375,38 +387,37 @@ func (fs *fileSystem) Create(name string) (File, error) {
return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
}
-func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) error {
dirname, name := path.Split(name)
- n := rlookup(fs.root, dirname)
- if n == nil {
- return os.ErrNotExist
+ n, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
}
n.Lock()
defer n.Unlock()
- if n.Child(name, nil) != nil {
+ if child, err := n.Child(name, nil); err != nil {
+ return err
+ } else if child != nil {
return os.ErrExist
}
- child := n.Child(name, func(inode) (child inode) {
- child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
- child.SetParent(n, name)
+
+ _, err = n.Child(name, func(inode) (repl inode, err error) {
+ repl, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(n, name)
return
})
- if err != nil {
- return err
- } else if child == nil {
- return ErrInvalidArgument
- }
- return nil
+ return err
}
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.root, name)
- if node == nil {
- err = os.ErrNotExist
- } else {
- fi = node.FileInfo()
+func (fs *fileSystem) Stat(name string) (os.FileInfo, error) {
+ node, err := rlookup(fs.root, name)
+ if err != nil {
+ return nil, err
}
- return
+ return node.FileInfo(), nil
}
func (fs *fileSystem) Rename(oldname, newname string) error {
@@ -475,43 +486,31 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- // Return ErrInvalidOperation if olddirf.inode doesn't even
- // bother calling our "remove oldname entry" replacer func.
- err = ErrInvalidArgument
- olddirf.inode.Child(oldname, func(oldinode inode) inode {
- err = nil
+ _, err = olddirf.inode.Child(oldname, func(oldinode inode) (inode, error) {
if oldinode == nil {
- err = os.ErrNotExist
- return nil
+ return oldinode, os.ErrNotExist
}
if locked[oldinode] {
// oldinode cannot become a descendant of itself.
- err = ErrInvalidArgument
- return oldinode
+ return oldinode, ErrInvalidArgument
}
if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
// moving a mount point to a different parent
// is not (yet) supported.
- err = ErrInvalidArgument
- return oldinode
+ return oldinode, ErrInvalidArgument
}
- accepted := newdirf.inode.Child(newname, func(existing inode) inode {
+ accepted, err := newdirf.inode.Child(newname, func(existing inode) (inode, error) {
if existing != nil && existing.IsDir() {
- err = ErrIsDirectory
- return existing
+ return existing, ErrIsDirectory
}
- return oldinode
+ return oldinode, nil
})
- if accepted != oldinode {
- if err == nil {
- // newdirf didn't accept oldinode.
- err = ErrInvalidArgument
- }
+ if err != nil {
// Leave oldinode in olddir.
- return oldinode
+ return oldinode, err
}
accepted.SetParent(newdirf.inode, newname)
- return nil
+ return nil, nil
})
return err
}
@@ -530,27 +529,25 @@ func (fs *fileSystem) RemoveAll(name string) error {
return err
}
-func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+func (fs *fileSystem) remove(name string, recursive bool) error {
dirname, name := path.Split(name)
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs.root, dirname)
- if dir == nil {
- return os.ErrNotExist
+ dir, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
}
dir.Lock()
defer dir.Unlock()
- dir.Child(name, func(node inode) inode {
+ _, err = dir.Child(name, func(node inode) (inode, error) {
if node == nil {
- err = os.ErrNotExist
- return nil
+ return nil, os.ErrNotExist
}
if !recursive && node.IsDir() && node.Size() > 0 {
- err = ErrDirectoryNotEmpty
- return node
+ return node, ErrDirectoryNotEmpty
}
- return nil
+ return nil, nil
})
return err
}
@@ -563,12 +560,9 @@ func (fs *fileSystem) Sync() error {
// rlookup (recursive lookup) returns the inode for the file/directory
// with the given name (which may contain "/" separators). If no such
// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
+func rlookup(start inode, path string) (node inode, err error) {
node = start
for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
if node.IsDir() {
if name == "." || name == "" {
continue
@@ -578,11 +572,17 @@ func rlookup(start inode, path string) (node inode) {
continue
}
}
- node = func() inode {
+ node, err = func() (inode, error) {
node.RLock()
defer node.RUnlock()
return node.Child(name, nil)
}()
+ if node == nil || err != nil {
+ break
+ }
+ }
+ if node == nil && err == nil {
+ err = os.ErrNotExist
}
return
}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fbd9775..923615b 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -521,7 +521,7 @@ func (dn *dirnode) FS() FileSystem {
return dn.fs
}
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -537,7 +537,7 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
return data, err
}}
gn.SetParent(dn, name)
- return gn
+ return gn, nil
}
return dn.treenode.Child(name, replace)
}
@@ -837,38 +837,41 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node = node.Parent()
continue
}
- node.Child(name, func(child inode) inode {
+ node, err = node.Child(name, func(child inode) (inode, error) {
if child == nil {
- child, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child, err := node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
child.SetParent(node, name)
- node = child
+ return child, nil
} else if !child.IsDir() {
- err = ErrFileExists
+ return child, ErrFileExists
} else {
- node = child
+ return child, nil
}
- return child
})
if err != nil {
return
}
}
- node.Child(basename, func(child inode) inode {
+ _, err = node.Child(basename, func(child inode) (inode, error) {
switch child := child.(type) {
case nil:
child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
child.SetParent(node, basename)
fn = child.(*filenode)
- return child
+ return child, nil
case *filenode:
fn = child
- return child
+ return child, nil
case *dirnode:
- err = ErrIsDirectory
- return child
+ return child, ErrIsDirectory
default:
- err = ErrInvalidArgument
- return child
+ return child, ErrInvalidArgument
}
})
return
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 023226f..8d32eb2 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -688,13 +688,13 @@ func (s *CollectionFSSuite) TestRename(c *check.C) {
err = fs.Rename(
fmt.Sprintf("dir%d/file%d/patherror", i, j),
fmt.Sprintf("dir%d/irrelevant", i))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
// newname parent dir is a file
err = fs.Rename(
fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
}(i, j)
}
}
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
index 97fe68b..a84f64f 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs_deferred.go
@@ -85,7 +85,7 @@ func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, erro
return dn.realinode().Write(p, pos)
}
-func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
+func (dn *deferrednode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
return dn.realinode().Child(name, replace)
}
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs_getternode.go
index c9ffb38..966fe9d 100644
--- a/sdk/go/arvados/fs_getternode.go
+++ b/sdk/go/arvados/fs_getternode.go
@@ -23,8 +23,8 @@ func (*getternode) IsDir() bool {
return false
}
-func (*getternode) Child(string, func(inode) inode) inode {
- return nil
+func (*getternode) Child(string, func(inode) (inode, error)) (inode, error) {
+ return nil, ErrInvalidArgument
}
func (gn *getternode) get() error {
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index f9cb799..a5e4710 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,7 +5,6 @@
package arvados
import (
- "log"
"os"
"sync"
)
@@ -48,8 +47,8 @@ func (pn *projectnode) setup() {
if coll.Name == "" {
continue
}
- pn.inode.Child(coll.Name, func(inode) inode {
- return deferredCollectionFS(fs, pn, coll)
+ pn.inode.Child(coll.Name, func(inode) (inode, error) {
+ return deferredCollectionFS(fs, pn, coll), nil
})
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
@@ -71,8 +70,8 @@ func (pn *projectnode) setup() {
if group.Name == "" || group.Name == "." || group.Name == ".." {
continue
}
- pn.inode.Child(group.Name, func(inode) inode {
- return fs.newProjectNode(pn, group.Name, group.UUID)
+ pn.inode.Child(group.Name, func(inode) (inode, error) {
+ return fs.newProjectNode(pn, group.Name, group.UUID), nil
})
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
@@ -87,30 +86,35 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
return pn.inode.Readdir()
}
-func (pn *projectnode) Child(name string, replace func(inode) inode) inode {
+func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
pn.setupOnce.Do(pn.setup)
if pn.err != nil {
- log.Printf("BUG: not propagating error setting up %T %v: %s", pn, pn, pn.err)
- // TODO: propagate error, instead of just being empty
- return nil
+ return nil, pn.err
}
if replace == nil {
// lookup
return pn.inode.Child(name, nil)
}
- return pn.inode.Child(name, func(existing inode) inode {
- if repl := replace(existing); repl == nil {
- // delete
+ return pn.inode.Child(name, func(existing inode) (inode, error) {
+ if repl, err := replace(existing); err != nil {
+ return existing, err
+ } else if repl == nil {
+ if existing == nil {
+ return nil, nil
+ }
+ // rmdir
// (TODO)
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
+ } else if existing != nil {
+ // clobber
+ return existing, ErrInvalidArgument
} else if repl.FileInfo().IsDir() {
// mkdir
// TODO: repl.SetParent(pn, name), etc.
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
} else {
// create file
- // TODO: repl.SetParent(pn, name), etc.
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
}
})
}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index c8d7360..701711e 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -37,7 +37,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.inode.Child("by_id", func(inode) inode {
+ root.inode.Child("by_id", func(inode) (inode, error) {
return &vdirnode{
inode: &treenode{
fs: fs,
@@ -50,10 +50,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: fs.mountCollection,
- }
+ }, nil
})
- root.inode.Child("home", func(inode) inode {
- return fs.newProjectNode(fs.root, "home", "")
+ root.inode.Child("home", func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, "home", ""), nil
})
return fs
}
@@ -104,18 +104,23 @@ type vdirnode struct {
create func(parent inode, name string) inode
}
-func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
- return vn.inode.Child(name, func(existing inode) inode {
- if existing != nil {
- return existing
- } else if vn.create == nil {
- return nil
+func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return vn.inode.Child(name, func(existing inode) (inode, error) {
+ if existing == nil && vn.create != nil {
+ existing = vn.create(vn, name)
+ if existing != nil {
+ existing.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ }
}
- n := vn.create(vn, name)
- if n != nil {
- n.SetParent(vn, name)
- vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ if replace == nil {
+ return existing, nil
+ } else if tryRepl, err := replace(existing); err != nil {
+ return existing, err
+ } else if tryRepl != existing {
+ return existing, ErrInvalidArgument
+ } else {
+ return existing, nil
}
- return n
})
}
commit d1ae12cad34862d063a1235bfe53459eef7ae589
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 16:29:40 2018 -0500
13111: Propagate errors in Readdir().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 6402a8e..4d22c1e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -96,7 +96,7 @@ type inode interface {
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
IsDir() bool
- Readdir() []os.FileInfo
+ Readdir() ([]os.FileInfo, error)
Size() int64
FileInfo() os.FileInfo
@@ -198,8 +198,8 @@ func (*nullnode) IsDir() bool {
return false
}
-func (*nullnode) Readdir() []os.FileInfo {
- return nil
+func (*nullnode) Readdir() ([]os.FileInfo, error) {
+ return nil, ErrInvalidOperation
}
func (*nullnode) Child(name string, replace func(inode) inode) inode {
@@ -263,7 +263,7 @@ func (n *treenode) FileInfo() os.FileInfo {
return n.fileinfo
}
-func (n *treenode) Readdir() (fi []os.FileInfo) {
+func (n *treenode) Readdir() (fi []os.FileInfo, err error) {
n.RLock()
defer n.RUnlock()
fi = make([]os.FileInfo, 0, len(n.inodes))
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
index e638838..97fe68b 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs_deferred.go
@@ -89,15 +89,15 @@ func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
return dn.realinode().Child(name, replace)
}
-func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
-func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
-func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
-func (dn *deferrednode) Readdir() []os.FileInfo { return dn.realinode().Readdir() }
-func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
-func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
-func (dn *deferrednode) Lock() { dn.realinode().Lock() }
-func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
-func (dn *deferrednode) RLock() { dn.realinode().RLock() }
-func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
-func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
-func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() ([]os.FileInfo, error) { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index d586531..127bee8 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -74,10 +74,14 @@ func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
return nil, ErrInvalidOperation
}
if count <= 0 {
- return f.inode.Readdir(), nil
+ return f.inode.Readdir()
}
if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
+ var err error
+ f.unreaddirs, err = f.inode.Readdir()
+ if err != nil {
+ return nil, err
+ }
}
if len(f.unreaddirs) == 0 {
return nil, io.EOF
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 4dd8699..f9cb799 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -79,8 +79,11 @@ func (pn *projectnode) setup() {
}
}
-func (pn *projectnode) Readdir() []os.FileInfo {
+func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
pn.setupOnce.Do(pn.setup)
+ if pn.err != nil {
+ return nil, pn.err
+ }
return pn.inode.Readdir()
}
commit 1012f12d29be01b56f2bbbe2e9bd5969d69f7b89
Merge: caf56e9 acb392d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 14:41:05 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit caf56e922b9caaa9a65332dc6bf3a36bd8fea48f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 1 09:45:47 2018 -0500
13111: Add spying client for testing deferred loading.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 92d8672..1de0714 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -5,11 +5,38 @@
package arvados
import (
+ "bytes"
+ "encoding/json"
+ "io"
"os"
check "gopkg.in/check.v1"
)
+type spiedRequest struct {
+ method string
+ path string
+ params map[string]interface{}
+}
+
+type spyingClient struct {
+ *Client
+ calls []spiedRequest
+}
+
+func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
+ var paramsCopy map[string]interface{}
+ var buf bytes.Buffer
+ json.NewEncoder(&buf).Encode(params)
+ json.NewDecoder(&buf).Decode(¶msCopy)
+ sc.calls = append(sc.calls, spiedRequest{
+ method: method,
+ path: path,
+ params: paramsCopy,
+ })
+ return sc.Client.RequestAndDecode(dst, method, path, body, params)
+}
+
func (s *SiteFSSuite) TestHomeProject(c *check.C) {
f, err := s.fs.Open("/home")
c.Assert(err, check.IsNil)
commit 2f03d4d926870a93cb880b389519a05c97de73b3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 1 09:37:31 2018 -0500
13111: Add /home test case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
new file mode 100644
index 0000000..92d8672
--- /dev/null
+++ b/sdk/go/arvados/fs_project_test.go
@@ -0,0 +1,51 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+func (s *SiteFSSuite) TestHomeProject(c *check.C) {
+ f, err := s.fs.Open("/home")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Not(check.Equals), 0)
+
+ ok := false
+ for _, fi := range fis {
+ c.Check(fi.Name(), check.Not(check.Equals), "")
+ if fi.Name() == "Unrestricted public data" {
+ ok = true
+ }
+ }
+ c.Check(ok, check.Equals, true)
+
+ f, err = s.fs.Open("/home/Unrestricted public data/..")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Check(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+ c.Check(fi.Name(), check.Equals, "home")
+
+ f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+ c.Check(err, check.IsNil)
+ fi, err = f.Stat()
+ c.Check(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+
+ for _, nx := range []string{
+ "/home/A Project",
+ "/home/A Project/does not exist",
+ "/home/Unrestricted public data/does not exist",
+ } {
+ c.Log(nx)
+ f, err = s.fs.Open(nx)
+ c.Check(err, check.NotNil)
+ c.Check(os.IsNotExist(err), check.Equals, true)
+ }
+}
commit f5d919c7e7bcf46e245a4459f3393022ff471db0
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 16:57:30 2018 -0500
13111: Add projects hierarchy under "home".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
new file mode 100644
index 0000000..e638838
--- /dev/null
+++ b/sdk/go/arvados/fs_deferred.go
@@ -0,0 +1,103 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+ "time"
+)
+
+func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+ var modTime time.Time
+ if coll.ModifiedAt != nil {
+ modTime = *coll.ModifiedAt
+ } else {
+ modTime = time.Now()
+ }
+ placeholder := &treenode{
+ fs: fs,
+ parent: parent,
+ inodes: nil,
+ fileinfo: fileinfo{
+ name: coll.Name,
+ modTime: modTime,
+ mode: 0755 | os.ModeDir,
+ },
+ }
+ return &deferrednode{wrapped: placeholder, create: func() inode {
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ cfs, err := coll.FileSystem(fs, fs)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ root := cfs.rootnode()
+ root.SetParent(parent, coll.Name)
+ return root
+ }}
+}
+
+// A deferrednode wraps an inode that's expensive to build. Initially,
+// it responds to basic directory functions by proxying to the given
+// placeholder. If a caller uses a read/write/lock operation,
+// deferrednode calls the create() func to create the real inode, and
+// proxies to the real inode from then on.
+//
+// In practice, this means a deferrednode's parent's directory listing
+// can be generated using only the placeholder, instead of waiting for
+// create().
+type deferrednode struct {
+ wrapped inode
+ create func() inode
+ mtx sync.Mutex
+ created bool
+}
+
+func (dn *deferrednode) realinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ if !dn.created {
+ dn.wrapped = dn.create()
+ dn.created = true
+ }
+ return dn.wrapped
+}
+
+func (dn *deferrednode) currentinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ return dn.wrapped
+}
+
+func (dn *deferrednode) Read(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Read(p, pos)
+}
+
+func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Write(p, pos)
+}
+
+func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
+ return dn.realinode().Child(name, replace)
+}
+
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() []os.FileInfo { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
new file mode 100644
index 0000000..4dd8699
--- /dev/null
+++ b/sdk/go/arvados/fs_project.go
@@ -0,0 +1,113 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+)
+
+// projectnode exposes an Arvados project as a filesystem directory.
+type projectnode struct {
+ inode
+ uuid string
+ setupOnce sync.Once
+ err error
+}
+
+func (pn *projectnode) setup() {
+ fs := pn.FS().(*siteFileSystem)
+ if pn.uuid == "" {
+ var resp User
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if pn.err != nil {
+ return
+ }
+ pn.uuid = resp.UUID
+ }
+ filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ params := ResourceListParams{
+ Filters: filters,
+ Order: "uuid",
+ }
+ for {
+ var resp CollectionList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, i := range resp.Items {
+ coll := i
+ if coll.Name == "" {
+ continue
+ }
+ pn.inode.Child(coll.Name, func(inode) inode {
+ return deferredCollectionFS(fs, pn, coll)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+
+ filters = append(filters, Filter{"group_class", "=", "project"})
+ params.Filters = filters
+ for {
+ var resp GroupList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, group := range resp.Items {
+ if group.Name == "" || group.Name == "." || group.Name == ".." {
+ continue
+ }
+ pn.inode.Child(group.Name, func(inode) inode {
+ return fs.newProjectNode(pn, group.Name, group.UUID)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+}
+
+func (pn *projectnode) Readdir() []os.FileInfo {
+ pn.setupOnce.Do(pn.setup)
+ return pn.inode.Readdir()
+}
+
+func (pn *projectnode) Child(name string, replace func(inode) inode) inode {
+ pn.setupOnce.Do(pn.setup)
+ if pn.err != nil {
+ log.Printf("BUG: not propagating error setting up %T %v: %s", pn, pn, pn.err)
+ // TODO: propagate error, instead of just being empty
+ return nil
+ }
+ if replace == nil {
+ // lookup
+ return pn.inode.Child(name, nil)
+ }
+ return pn.inode.Child(name, func(existing inode) inode {
+ if repl := replace(existing); repl == nil {
+ // delete
+ // (TODO)
+ return pn.Child(name, nil) // not implemented
+ } else if repl.FileInfo().IsDir() {
+ // mkdir
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ } else {
+ // create file
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ }
+ })
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 37ec8a3..c8d7360 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -38,8 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
inodes: make(map[string]inode),
}
root.inode.Child("by_id", func(inode) inode {
- var vn inode
- vn = &vdirnode{
+ return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -52,7 +51,9 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
create: fs.mountCollection,
}
- return vn
+ })
+ root.inode.Child("home", func(inode) inode {
+ return fs.newProjectNode(fs.root, "home", "")
})
return fs
}
@@ -76,6 +77,22 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
+func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
// vdirnode wraps an inode by ignoring any requests to add/replace
// children, and calling a create() func when a non-existing child is
// looked up.
commit 47eb67e4c084abde49d5463d4ced8b4436a59dfd
Merge: 7c32daf cd3bab6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 13:13:36 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 7c32daf9b5b1dcb8a003ac30bfc0ed2a9ef0eb74
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Jan 16 16:40:20 2018 -0500
13111: Fix nil pointer dereference at sitefs root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 9f3dbce..37ec8a3 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -91,13 +91,14 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
return vn.inode.Child(name, func(existing inode) inode {
if existing != nil {
return existing
- } else {
- n := vn.create(vn, name)
- if n != nil {
- n.SetParent(vn, name)
- vn.inode.(*treenode).fileinfo.modTime = time.Now()
- }
- return n
+ } else if vn.create == nil {
+ return nil
}
+ n := vn.create(vn, name)
+ if n != nil {
+ n.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ }
+ return n
})
}
commit 07cb2b1d22be82abb87fd2a5f95ae86e760c87e6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Jan 13 15:09:40 2018 -0500
13111: Improve deadlock prevention. Prevent unsupported renames.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fd8c18b..6402a8e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -53,6 +53,10 @@ type FileSystem interface {
rootnode() inode
+ // filesystem-wide lock: used by Rename() to prevent deadlock
+ // while locking multiple inodes.
+ locker() sync.Locker
+
// create a new node with nil parent.
newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
@@ -272,12 +276,17 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
type fileSystem struct {
root inode
fsBackend
+ mutex sync.Mutex
}
func (fs *fileSystem) rootnode() inode {
return fs.root
}
+func (fs *fileSystem) locker() sync.Locker {
+ return &fs.mutex
+}
+
// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.openFile(name, flag, perm)
@@ -424,12 +433,31 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
defer newdirf.Close()
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from
- // filesystem root to newdir, then locking the path from
- // filesystem root to olddir, skipping any already-locked
- // nodes.
+ // TODO: If the nearest common ancestor ("nca") of olddirf and
+ // newdirf is on a different filesystem than fs, we should
+ // call nca.FS().Rename() instead of proceeding. Until then
+ // it's awkward for filesystems to implement their own Rename
+ // methods effectively: the only one that runs is the one on
+ // the root filesystem exposed to the caller (webdav, fuse,
+ // etc).
+
+ // When acquiring locks on multiple inodes, avoid deadlock by
+ // locking the entire containing filesystem first.
+ cfs := olddirf.inode.FS()
+ cfs.locker().Lock()
+ defer cfs.locker().Unlock()
+
+ if cfs != newdirf.inode.FS() {
+ // Moving inodes across filesystems is not (yet)
+ // supported. Locking inodes from different
+ // filesystems could deadlock, so we must error out
+ // now.
+ return ErrInvalidArgument
+ }
+
+ // To ensure we can test reliably whether we're about to move
+ // a directory into itself, lock all potential common
+ // ancestors of olddir and newdir.
needLock := []sync.Locker{}
for _, node := range []inode{olddirf.inode, newdirf.inode} {
needLock = append(needLock, node)
@@ -447,8 +475,11 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- err = nil
+ // Return ErrInvalidOperation if olddirf.inode doesn't even
+ // bother calling our "remove oldname entry" replacer func.
+ err = ErrInvalidArgument
olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ err = nil
if oldinode == nil {
err = os.ErrNotExist
return nil
@@ -458,6 +489,12 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = ErrInvalidArgument
return oldinode
}
+ if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
+ // moving a mount point to a different parent
+ // is not (yet) supported.
+ err = ErrInvalidArgument
+ return oldinode
+ }
accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index d3ca510..9f3dbce 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -20,13 +20,16 @@ type siteFileSystem struct {
// there are significant known bugs and shortcomings. For example,
// writes are not persisted until Sync() is called.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ root := &vdirnode{}
fs := &siteFileSystem{
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ root: root,
},
}
- root := &treenode{
- fs: fs,
+ root.inode = &treenode{
+ fs: fs,
+ parent: root,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -34,13 +37,12 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.parent = root
- root.Child("by_id", func(inode) inode {
+ root.inode.Child("by_id", func(inode) inode {
var vn inode
vn = &vdirnode{
inode: &treenode{
fs: fs,
- parent: root,
+ parent: fs.root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
name: "by_id",
@@ -52,7 +54,6 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
return vn
})
- fs.root = root
return fs
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index a8c369f..26a2212 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -6,6 +6,7 @@ package arvados
import (
"net/http"
+ "os"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -47,6 +48,12 @@ func (s *SiteFSSuite) TestByUUID(c *check.C) {
c.Check(err, check.IsNil)
c.Check(len(fis), check.Equals, 0)
+ err = s.fs.Mkdir("/by_id/"+arvadostest.FooCollection, 0755)
+ c.Check(err, check.Equals, os.ErrExist)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.NonexistentCollection)
+ c.Assert(err, check.Equals, os.ErrNotExist)
+
f, err = s.fs.Open("/by_id/" + arvadostest.FooCollection)
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
@@ -55,4 +62,18 @@ func (s *SiteFSSuite) TestByUUID(c *check.C) {
names = append(names, fi.Name())
}
c.Check(names, check.DeepEquals, []string{"foo"})
+
+ _, err = s.fs.OpenFile("/by_id/"+arvadostest.NonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
+ c.Check(err, check.Equals, ErrInvalidOperation)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection, "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ _, err = s.fs.Stat("/by_id/beep")
+ c.Check(err, check.Equals, os.ErrNotExist)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/"+arvadostest.FooCollection+"/bar")
+ c.Check(err, check.IsNil)
+
+ err = s.fs.Rename("/by_id", "/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list