[ARVADOS] updated: 1.1.3-386-ga9916f5
Git user
git at public.curoverse.com
Fri Apr 13 16:41:57 EDT 2018
Summary of changes:
build/run-tests.sh | 2 -
cmd/arvados-client/Makefile | 11 --
cmd/arvados-client/cmd.go | 3 -
lib/mount/command.go | 76 ---------
lib/mount/command_test.go | 59 -------
lib/mount/fs.go | 375 --------------------------------------------
lib/mount/fs_test.go | 49 ------
7 files changed, 575 deletions(-)
delete mode 100644 cmd/arvados-client/Makefile
delete mode 100644 lib/mount/command.go
delete mode 100644 lib/mount/command_test.go
delete mode 100644 lib/mount/fs.go
delete mode 100644 lib/mount/fs_test.go
discards 73b36db80121301aaf3c3a38e313e1cbc62b3fdd (commit)
discards e9f6f853e5d121ea43d7b97d8b87ab7990de3397 (commit)
discards f46fe9f98b3ea1fb61e24dac59b684b166045b56 (commit)
discards e6c9fc68cd6d4d281d81928729a2bd18007a96a4 (commit)
discards 3e16165a984135149ae9d87d29afa40e691c83b9 (commit)
discards 56d62dcc77f87af6d7f1403f510639740d87e6c2 (commit)
discards 57ecd452751d0f6efd2ecdb6add67890d2e36964 (commit)
discards 6ae864435a2c185b373703b3d6fc696211c3b79c (commit)
discards cf79bd5e4f3cb2176a9d8939c720f95b26caea91 (commit)
discards 61a5f1cab58f4652a9f9bdd4f6bc26c887eae75b (commit)
discards c4d189853805f53b70e8422820a69fe77fdaa11f (commit)
discards 518f6844fe58425695dad9a27bedea506aa2d709 (commit)
discards dab585f6a010428867c33887a2501f0b6f6a6273 (commit)
discards 9e95cfe059f31c0a7aa5dda415e144f1b317626b (commit)
discards 8dd1a09d1b8622f93ee0c46e032a9eec17dc7482 (commit)
discards f0079947e70efe61e98f8b0e58c894f7b045c306 (commit)
discards 0f0e585847ab5dae34bc66f981e361096a70e15e (commit)
discards ee4cf1187d0dd44608e501df939d81b35d37772e (commit)
discards ff5d3339957428c7a0d614d35cad2dbb206a9f88 (commit)
discards 127c8584fa1a2336fd6569d9d6013f3f2b166088 (commit)
discards c3cb135e3ff0617956608fbeadf64de0528dc9dd (commit)
discards 8c7d6a0a98c0bb197ff5384d5bb3da1f3d4d8007 (commit)
discards ef0356b856c06ba07b357dcd52c6cb723f63cf19 (commit)
discards 0d5f2f7c474724871f05bad1308c66e68c9d7473 (commit)
discards 177a91367f6c547dce717736bae38170d3f0961f (commit)
discards 2c3b9beb3345ac31722bcd04cb41ed318469bb79 (commit)
discards 9451dec1ace2370015b4c75367076cb5d4901f7e (commit)
discards d1ac0920b9f8c4b793cf33a7016c1dfad4ffba10 (commit)
discards 50b3e54b41ada32f84b154b35f7b02b6c0217c8e (commit)
discards 0faa78426b42eb7cb40822c0963b15f07c1aa9cf (commit)
discards bb141976709b75a8b8cdf3135fc1d73e22e7ba72 (commit)
discards 8843744ac700dd080333e8e1bcf6273e7fe273c3 (commit)
discards dbcb7c65b6324273fabc7dc2507a1978d8a538b8 (commit)
discards ceca47732d7c6f768e8e6f54e6fa96a49f07a553 (commit)
discards 25e9311a9f958e3e0e42929a90f54b15997b22c0 (commit)
discards c65eed0dd97fe3adb58b1029db68db58c6ed6989 (commit)
discards e509c5f6c1c9f9b212705da401713d49c9f78a71 (commit)
discards e585fcd60cdc0bf9f846e22241af7043364c9e47 (commit)
discards c549ff8ad5a98d5f12f5081881fcb4e1ae392c1c (commit)
discards 8467390d785c59b4ab219e901ba3df63baf788e8 (commit)
discards cda0f329a1210b23fda9cd9fefc52ae925adb3cf (commit)
discards f805036cee9d8316b609fa4d73b46d3e1a14ab16 (commit)
discards 56ff8f92f0028b1274db9f66c291f52b6492ccb1 (commit)
discards 9c6b4501a9c1c686ecac72fed5026686171d9d29 (commit)
discards d9ec8063857543da420907319d9a4f1b990f3dcb (commit)
discards 67ad6de632f9390d593dc289eca9e138bac563b1 (commit)
discards ac7d4334a2be25126e277529b0644a97ec8927cb (commit)
discards db551f2d600ded8ee33b02fd3bcc3e0306ec3cff (commit)
discards a3994591d74a61efab6cdb76af191a37a8e32399 (commit)
discards 961c50a5a808bafab4a975af5e5bf8a3e04bb448 (commit)
discards 8ba6b8bf39075278c1e5f823e4defcc1131973a7 (commit)
discards eca8dff2d88265aac1d6f366b8c8c9759648a5b3 (commit)
discards aa0ad923e691341b0dbcf16e157a1796efb56ea9 (commit)
discards 28f6431aa5cdd2fa09dfd226e741c73c4c040e2f (commit)
discards 721ad80e9f349cea32415d6d8b2643b3478429fb (commit)
discards cd48b440cbd6aa3f46a16be0180d18be7c1addbe (commit)
discards fd1e7a3998f4a60e9976010e13ab822456997143 (commit)
discards 80900ad43ef0681d733d9f3b54cbbed1f0f24484 (commit)
discards 014ef1b168b3f5f71eda4557353843f1ca759c75 (commit)
discards 9ae4ad3ed94cd2aa07624aa228b0f872c4c2860b (commit)
discards d3a2efbf7bab9101d4e02369f7b4c22de5058b85 (commit)
discards 2bae70dcf16f6306aa82ccec375ff94b46ec65f2 (commit)
discards e2237253c78efad1bda6b9a7d4c7a162fe4612ee (commit)
via a9916f5648ad8486812c3ec8ef6f627e2f0542e4 (commit)
via 973d9760a020b5a1f7b421e620d5143e5c4010cf (commit)
via e618db07e97783e6ab588332d0f2872842677f03 (commit)
via 1f397d05c069e75ebbd74b59ab2133df4f7389c7 (commit)
via a78cdf7dfd18c32431e9d2a26f08b4a35cd4d444 (commit)
via 3b06a75d2756e3c7b2997ba38cfd54668096ee7e (commit)
via 309e6cb3d00a2b731ffe66d89102caacf2cd81aa (commit)
via e7fe3ce516f71bc14c5ff0e22dcd19772ab8c72a (commit)
via 335ee474af3cf03865bc107478da6a51e362c2ab (commit)
via aadbd550d97c2cbf47563ceb1a9004932fa5e28f (commit)
via a6c49634025095e9b4d29f5971fe5ecb720f9539 (commit)
via cd0945bb8591fd97fe37a77dc9057c29dc5d7558 (commit)
via 04d11c9cf24f358a0780edc87c910c50130fd71b (commit)
via c5d6ae2fc5f1c382f2011a128230d893a5b2103c (commit)
via 3e2aed249f9ae1d99fb654b19d3ae49788f8fdaa (commit)
via 8f0b7529cda14a10cd953819af9c1b76201ae4f5 (commit)
via 282394bf9076af3387755979e4fd9eb360291861 (commit)
via cedd5efb0f3813125931c64b754dddabd875d537 (commit)
via 4c46038d3726e1ff7006cc30bbd6b71b957d0215 (commit)
via ab1abbac7ec08c7a7599670459f04b8a35dcbe68 (commit)
via 0c7a458fc70791a5f395dad171a272b92b6ffb7f (commit)
via efd536fbbfca8473e7df444dd1d97ce916bfe345 (commit)
via 541e6b47d90dbdadca287bea0cc303026f20facb (commit)
via f09c8facc9a607f3b5e9171e66170536f1ba83c7 (commit)
via 959335d561a3882b391c88c4b2106f263e827b51 (commit)
via 9424d875b67ddda957b2ef705a1619394b57db70 (commit)
via 14fadeaed3dd4f01778eae8342ca0b5e190e3429 (commit)
via 3c230a24be3e99f89805199311ec8b36665c2b74 (commit)
via f67ede4c778f5ac6f3bd0e04d1f47993603b3375 (commit)
via e1bdb7bb963f3754f3aebced8810b062ee4fecf8 (commit)
via 0ff4662fab347279897b285e3ef3320497ced8c9 (commit)
via 649c27a98f398984b6aa1f806992cfdd9831c904 (commit)
via c2e623cbcf93e996b4fb1d29fcf99d954f7f2ad3 (commit)
via 183cf741009a42fa3dd2af911c416755e4c5cb0e (commit)
via 4668cf881aef0b21245a419b661b010720035f3c (commit)
via 5d5af52a33ec8b10a9af6afd50141db3923441ec (commit)
via c02ceff00fce94ec5794b53fe890f681acf31121 (commit)
via 699039c46b728bf27013adeb19d8fe92eab76071 (commit)
via 5571e3a20a88a1d9b4dd61df7a5fa9be674b2562 (commit)
via a7f0a9c188fd9c2bfe52b421fd30691631e63875 (commit)
via 840e18117654b70c8cdb28888f227810369f82c1 (commit)
via af83c3a047bebc48db9490f3523208c4c3f87b6f (commit)
via 504e0eb8c75d79e5664897709935512f2c59ff95 (commit)
via 0f2575bc8348637a02bc0165456886ce7f3b57bf (commit)
via a0a02d2f4180f908a39293ef4adb00f927b7ad6f (commit)
via 18c8fa2337a2db04ee6060184885731f4b5c7d7b (commit)
via 67bd03fc4c2b3fbb25613c76a9470b53ebaa832a (commit)
via 2eb576727b1c9551141083b82e1165f9571e2d2f (commit)
via 10d551cbba521857f3967968348fd107f32651df (commit)
via dcba8d1ddddc578d97b210f32eb6879a4652039f (commit)
via 5aeb3af0666da32adce3fbc0c9cc38d9c67de8ec (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (73b36db80121301aaf3c3a38e313e1cbc62b3fdd)
\
N -- N -- N (a9916f5648ad8486812c3ec8ef6f627e2f0542e4)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a9916f5648ad8486812c3ec8ef6f627e2f0542e4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 14:35:48 2018 -0400
13111: Accept chunked responses to GET requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index dbdda60..95a84c0 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -19,13 +19,14 @@ import (
func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
- hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ data := []byte("TestOverrideDiscovery")
+ hash := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
st := StubGetHandler{
c,
hash,
arvadostest.ActiveToken,
http.StatusOK,
- []byte("TestOverrideDiscovery")}
+ data}
ks := RunSomeFakeKeepServers(st, 2)
os.Setenv("ARVADOS_KEEP_SERVICES", "")
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 54a4a37..620bdbe 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -200,6 +200,15 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ var expectLength int64
+ if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
+ expectLength = -1
+ } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil {
+ expectLength = -1
+ } else {
+ expectLength = n
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
@@ -230,7 +239,9 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
// can try again.
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
retryList = append(retryList, host)
- } else if resp.StatusCode != http.StatusOK {
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
var respbody []byte
respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
@@ -247,24 +258,29 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
} else if resp.StatusCode == 404 {
count404++
}
- } else if resp.ContentLength < 0 {
- // Missing Content-Length
- resp.Body.Close()
- return nil, 0, "", fmt.Errorf("Missing Content-Length of block")
- } else {
- // Success.
- if method == "GET" {
- return HashCheckingReader{
- Reader: resp.Body,
- Hash: md5.New(),
- Check: locator[0:32],
- }, resp.ContentLength, url, nil
- } else {
+ continue
+ }
+ if expectLength < 0 {
+ if resp.ContentLength < 0 {
resp.Body.Close()
- return nil, resp.ContentLength, url, nil
+ return nil, 0, "", fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator)
}
+ expectLength = resp.ContentLength
+ } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength {
+ resp.Body.Close()
+ return nil, 0, "", fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength)
+ }
+ // Success
+ if method == "GET" {
+ return HashCheckingReader{
+ Reader: resp.Body,
+ Hash: md5.New(),
+ Check: locator[0:32],
+ }, expectLength, url, nil
+ } else {
+ resp.Body.Close()
+ return nil, expectLength, url, nil
}
-
}
serversToTry = retryList
}
commit 973d9760a020b5a1f7b421e620d5143e5c4010cf
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 16:30:17 2018 -0400
13111: Avoid multi-page API reqs when looking up entries by name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs_lookup.go
new file mode 100644
index 0000000..42322a1
--- /dev/null
+++ b/sdk/go/arvados/fs_lookup.go
@@ -0,0 +1,73 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+ "time"
+)
+
+// lookupnode is a caching tree node that is initially empty and calls
+// loadOne and loadAll to load/update child nodes as needed.
+//
+// See (*customFileSystem)MountUsers for example usage.
+type lookupnode struct {
+ inode
+ loadOne func(parent inode, name string) (inode, error)
+ loadAll func(parent inode) ([]inode, error)
+ stale func(time.Time) bool
+
+ // internal fields
+ staleLock sync.Mutex
+ staleAll time.Time
+ staleOne map[string]time.Time
+}
+
+func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) {
+ all, err := ln.loadAll(ln)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range all {
+ _, err = ln.inode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
+ return child, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ ln.staleAll = checkTime
+ // No value in ln.staleOne can make a difference to an
+ // "entry is stale?" test now, because no value is
+ // newer than ln.staleAll. Reclaim memory.
+ ln.staleOne = nil
+ }
+ return ln.inode.Readdir()
+}
+
+func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) && ln.stale(ln.staleOne[name]) {
+ _, err := ln.inode.Child(name, func(inode) (inode, error) {
+ return ln.loadOne(ln, name)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ln.staleOne == nil {
+ ln.staleOne = map[string]time.Time{name: checkTime}
+ } else {
+ ln.staleOne[name] = checkTime
+ }
+ }
+ return ln.inode.Child(name, replace)
+}
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 827a44b..9299551 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,58 +5,81 @@
package arvados
import (
+ "log"
"os"
- "sync"
- "time"
+ "strings"
)
-type staleChecker struct {
- mtx sync.Mutex
- last time.Time
+func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
+ if uuid != "" {
+ return uuid, nil
+ }
+ var resp User
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return "", err
+ }
+ return resp.UUID, nil
}
-func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
- sc.mtx.Lock()
- defer sc.mtx.Unlock()
- if !staleFunc(sc.last) {
- return
+// loadOneChild loads only the named child, if it exists.
+func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
- sc.last = time.Now()
- fn()
-}
-// projectnode exposes an Arvados project as a filesystem directory.
-type projectnode struct {
- inode
- staleChecker
- uuid string
- err error
-}
+ var contents CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{
+ {"name", "=", name},
+ {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
+ {"groups.group_class", "=", "project"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if len(contents.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ coll := contents.Items[0]
-func (pn *projectnode) load() {
- fs := pn.FS().(*customFileSystem)
+ if strings.Contains(coll.UUID, "-j7d0g-") {
+ // Group item was loaded into a Collection var -- but
+ // we only need the Name and UUID anyway, so it's OK.
+ return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ } else if strings.Contains(coll.UUID, "-4zz18-") {
+ return deferredCollectionFS(fs, parent, coll), nil
+ } else {
+ log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID)
+ return nil, ErrInvalidArgument
+ }
+}
- if pn.uuid == "" {
- var resp User
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
- if pn.err != nil {
- return
- }
- pn.uuid = resp.UUID
+func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
}
+
+ var inodes []inode
+
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ filters := []Filter{{"owner_uuid", "=", uuid}}
params := ResourceListParams{
+ Count: "none",
Filters: filters,
Order: "uuid",
}
for {
var resp CollectionList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -66,9 +89,7 @@ func (pn *projectnode) load() {
if !permittedName(coll.Name) {
continue
}
- pn.inode.Child(coll.Name, func(inode) (inode, error) {
- return deferredCollectionFS(fs, pn, coll), nil
- })
+ inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
@@ -77,9 +98,9 @@ func (pn *projectnode) load() {
params.Filters = filters
for {
var resp GroupList
- pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
- if pn.err != nil {
- return
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if err != nil {
+ return nil, err
}
if len(resp.Items) == 0 {
break
@@ -88,52 +109,9 @@ func (pn *projectnode) load() {
if !permittedName(group.Name) {
continue
}
- pn.inode.Child(group.Name, func(inode) (inode, error) {
- return fs.newProjectNode(pn, group.Name, group.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- pn.err = nil
-}
-
-func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- return pn.inode.Readdir()
-}
-
-func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
- if pn.err != nil {
- return nil, pn.err
- }
- if replace == nil {
- // lookup
- return pn.inode.Child(name, nil)
- }
- return pn.inode.Child(name, func(existing inode) (inode, error) {
- if repl, err := replace(existing); err != nil {
- return existing, err
- } else if repl == nil {
- if existing == nil {
- return nil, nil
- }
- // rmdir
- // (TODO)
- return existing, ErrInvalidArgument
- } else if existing != nil {
- // clobber
- return existing, ErrInvalidArgument
- } else if repl.FileInfo().IsDir() {
- // mkdir
- // TODO: repl.SetParent(pn, name), etc.
- return existing, ErrInvalidArgument
- } else {
- // create file
- return existing, ErrInvalidArgument
- }
- })
+ return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 058eb40..1a06ce1 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -67,15 +67,15 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
_, basename := filepath.Split(path)
c.Check(fi.Name(), check.Equals, basename)
f, err = s.fs.Open(path + "/A Project/A Subproject")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fi, err = f.Stat()
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
@@ -90,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
+ f, err := s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project/Project does not exist")
+ c.Assert(err, check.NotNil)
+ f, err = s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fis)
+ var foundSubproject, foundCollection bool
+ for _, fi := range fis {
+ switch fi.Name() {
+ case "A Subproject":
+ foundSubproject = true
+ case "collection_to_move_around":
+ foundCollection = true
+ }
+ }
+ c.Check(foundSubproject, check.Equals, true)
+ c.Check(foundCollection, check.Equals, true)
+}
+
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
badCollection := Collection{
Name: "bad/collection",
@@ -109,7 +137,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
dir, err := s.fs.Open("/users/active/A Project")
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
fis, err := dir.Readdir(-1)
c.Check(err, check.IsNil)
for _, fi := range fis {
@@ -122,7 +150,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
@@ -140,6 +168,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
f, err := s.fs.Open("/home/A Project/oob")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
f.Close()
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index e9c8387..b5daf7b 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -73,7 +73,10 @@ func (fs *customFileSystem) MountProject(mount, uuid string) {
func (fs *customFileSystem) MountUsers(mount string) {
fs.root.inode.Child(mount, func(inode) (inode, error) {
- return &usersnode{
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: fs.usersLoadOne,
+ loadAll: fs.usersLoadAll,
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -136,24 +139,10 @@ func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
}
func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
- inode: &treenode{
- fs: fs,
- parent: root,
- inodes: make(map[string]inode),
- fileinfo: fileinfo{
- name: name,
- modTime: time.Now(),
- mode: 0755 | os.ModeDir,
- },
- },
- }
-}
-
-func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
- return &projectnode{
- uuid: uuid,
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+ loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
inode: &treenode{
fs: fs,
parent: root,
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
index ccfe2c5..00f7036 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs_users.go
@@ -8,55 +8,41 @@ import (
"os"
)
-// usersnode is a virtual directory with an entry for each visible
-// Arvados username, each showing the respective user's "home
-// projects".
-type usersnode struct {
- inode
- staleChecker
- err error
+func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{{"username", "=", name}},
+ })
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ user := resp.Items[0]
+ return fs.newProjectNode(parent, user.Username, user.UUID), nil
}
-func (un *usersnode) load() {
- fs := un.FS().(*customFileSystem)
-
+func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
params := ResourceListParams{
+ Count: "none",
Order: "uuid",
}
+ var inodes []inode
for {
var resp UserList
- un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
- if un.err != nil {
- return
- }
- if len(resp.Items) == 0 {
- break
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return inodes, nil
}
for _, user := range resp.Items {
if user.Username == "" {
continue
}
- un.inode.Child(user.Username, func(inode) (inode, error) {
- return fs.newProjectNode(un, user.Username, user.UUID), nil
- })
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
- un.err = nil
-}
-
-func (un *usersnode) Readdir() ([]os.FileInfo, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Readdir()
-}
-
-func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
- un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
- if un.err != nil {
- return nil, un.err
- }
- return un.inode.Child(name, nil)
}
commit e618db07e97783e6ab588332d0f2872842677f03
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 13:29:20 2018 -0400
13111: Update docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index b7da3b0..6c867f0 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -133,8 +133,21 @@
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
+//
+// The following URLs are read-only, but otherwise interchangeable
+// with the above:
+//
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
+// http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt
+// http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt
+//
+// If the collection is named "My Collection" and located in a project
+// called "My Project" which is in the home project of a user with
+// username is "bob", the following read-only URL is also available
+// when authenticating as bob:
+//
+// http://collections.example.com/users/bob/My+Project/My+Collection/foo/bar.txt
//
// An additional form is supported specifically to make it more
// convenient to maintain support for existing Workbench download
@@ -148,6 +161,9 @@
//
// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
//
+// Collections can also be accessed (read-only) via "/by_id/X" where X
+// is a UUID or portable data hash.
+//
// Authorization mechanisms
//
// A token can be provided in an Authorization header:
@@ -178,11 +194,10 @@
//
// Indexes
//
-// Currently, keep-web does not generate HTML index listings, nor does
-// it serve a default file like "index.html" when a directory is
-// requested. These features are likely to be added in future
-// versions. Until then, keep-web responds with 404 if a directory
-// name (or any path ending with "/") is requested.
+// Keep-web returns a generic HTML index listing when a directory is
+// requested with the GET method. It does not serve a default file
+// like "index.html". Directory listings are also returned for WebDAV
+// PROPFIND requests.
//
// Compatibility
//
commit 1f397d05c069e75ebbd74b59ab2133df4f7389c7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 13:28:31 2018 -0400
13111: Follow attachment-only policy at /by_id/ and /users/ paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 0618588..8b61b54 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -14,6 +14,7 @@ import (
"net/http"
"net/url"
"os"
+ "path/filepath"
"sort"
"strconv"
"strings"
@@ -335,7 +336,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if useSiteFS {
- h.serveSiteFS(w, r, tokens, credentialsOK)
+ h.serveSiteFS(w, r, tokens, credentialsOK, attachment)
return
}
@@ -505,7 +506,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
-func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK bool) {
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
@@ -551,6 +552,10 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
}
return
}
+ if r.Method == "GET" {
+ _, basename := filepath.Split(r.URL.Path)
+ applyContentDispositionHdr(w, r, basename, attachment)
+ }
wh := webdav.Handler{
Prefix: "/",
FileSystem: &webdavFS{
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 15f32f1..4894ceb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -333,7 +333,20 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
http.StatusOK,
"foo",
)
- c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "foo",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
commit a78cdf7dfd18c32431e9d2a26f08b4a35cd4d444
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 10:14:31 2018 -0400
13111: Serve /by_id/ dir.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 22ddd84..db5d373 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -265,6 +265,22 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
}
}
+func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
+ for _, path := range []string{"/by_id", "/by_id/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
+ }
+ for _, path := range []string{
+ "/by_id/" + arvadostest.FooPdh,
+ "/by_id/" + arvadostest.FooPdh + "/",
+ "/by_id/" + arvadostest.FooCollection,
+ "/by_id/" + arvadostest.FooCollection + "/",
+ } {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
+ }
+}
+
func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
for _, path := range []string{"/"} {
stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 389ab73..0618588 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -163,6 +163,12 @@ var (
"HEAD": true,
"POST": true,
}
+ // top-level dirs to serve with siteFS
+ siteFSDir = map[string]bool{
+ "": true, // root directory
+ "by_id": true,
+ "users": true,
+ }
)
// ServeHTTP implements http.Handler.
@@ -250,7 +256,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if r.URL.Path == "/" || (len(pathParts) >= 1 && pathParts[0] == "users") {
+ } else if siteFSDir[pathParts[0]] {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
commit 3b06a75d2756e3c7b2997ba38cfd54668096ee7e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 12 09:14:52 2018 -0400
13111: Fix skipped error check when Write called before WriteHeader.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5e14cb4..389ab73 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -112,12 +112,12 @@ type updateOnSuccess struct {
}
func (uos *updateOnSuccess) Write(p []byte) (int, error) {
- if uos.err != nil {
- return 0, uos.err
- }
if !uos.sentHeader {
uos.WriteHeader(http.StatusOK)
}
+ if uos.err != nil {
+ return 0, uos.err
+ }
return uos.ResponseWriter.Write(p)
}
commit 309e6cb3d00a2b731ffe66d89102caacf2cd81aa
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 11 19:59:18 2018 -0400
13111: Avoid double WriteHeader after implicit 200 on first Write.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/httpserver/responsewriter.go b/sdk/go/httpserver/responsewriter.go
index d37822f..8dea759 100644
--- a/sdk/go/httpserver/responsewriter.go
+++ b/sdk/go/httpserver/responsewriter.go
@@ -41,6 +41,9 @@ func (w *responseWriter) WriteHeader(s int) {
}
func (w *responseWriter) Write(data []byte) (n int, err error) {
+ if w.wroteStatus == 0 {
+ w.WriteHeader(http.StatusOK)
+ }
n, err = w.ResponseWriter.Write(data)
w.wroteBodyBytes += n
w.err = err
commit e7fe3ce516f71bc14c5ff0e22dcd19772ab8c72a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 11 09:07:09 2018 -0400
13111: Ignore projects and collections with "/" in name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 45beff6..3058a76 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -589,3 +589,7 @@ func rlookup(start inode, path string) (node inode, err error) {
}
return
}
+
+func permittedName(name string) bool {
+ return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 923615b..7ce37aa 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -821,8 +821,8 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
- if basename == "" || basename == "." || basename == ".." {
- err = fmt.Errorf("invalid filename")
+ if !permittedName(basename) {
+ err = fmt.Errorf("invalid file part %q in path %q", basename, path)
return
}
for _, name := range names[:len(names)-1] {
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 071cc0a..827a44b 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -63,7 +63,7 @@ func (pn *projectnode) load() {
}
for _, i := range resp.Items {
coll := i
- if coll.Name == "" || coll.Name == "." || coll.Name == ".." {
+ if !permittedName(coll.Name) {
continue
}
pn.inode.Child(coll.Name, func(inode) (inode, error) {
@@ -85,7 +85,7 @@ func (pn *projectnode) load() {
break
}
for _, group := range resp.Items {
- if group.Name == "" || group.Name == "." || group.Name == ".." {
+ if !permittedName(group.Name) {
continue
}
pn.inode.Child(group.Name, func(inode) (inode, error) {
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 69bae89..058eb40 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
"path/filepath"
+ "strings"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -89,6 +90,34 @@ func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
}
}
+func (s *SiteFSSuite) TestSlashInName(c *check.C) {
+ badCollection := Collection{
+ Name: "bad/collection",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", s.client.UpdateBody(&badCollection), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
+
+ badProject := Group{
+ Name: "bad/project",
+ GroupClass: "project",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", s.client.UpdateBody(&badProject), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
+
+ dir, err := s.fs.Open("/users/active/A Project")
+ c.Check(err, check.IsNil)
+ fis, err := dir.Readdir(-1)
+ c.Check(err, check.IsNil)
+ for _, fi := range fis {
+ c.Logf("fi.Name() == %q", fi.Name())
+ c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
+ }
+}
+
func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
s.fs.MountProject("home", "")
diff --git a/sdk/go/arvados/group.go b/sdk/go/arvados/group.go
index b00809f..6b5718a 100644
--- a/sdk/go/arvados/group.go
+++ b/sdk/go/arvados/group.go
@@ -6,9 +6,10 @@ package arvados
// Group is an arvados#group record
type Group struct {
- UUID string `json:"uuid,omitempty"`
- Name string `json:"name,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
+ UUID string `json:"uuid,omitempty"`
+ Name string `json:"name,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+ GroupClass string `json:"group_class"`
}
// GroupList is an arvados#groupList resource.
@@ -18,3 +19,7 @@ type GroupList struct {
Offset int `json:"offset"`
Limit int `json:"limit"`
}
+
+func (g Group) resourceName() string {
+ return "group"
+}
commit 335ee474af3cf03865bc107478da6a51e362c2ab
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:41:11 2018 -0400
13111: Note safe use of append().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 300c77a..071cc0a 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -44,6 +44,9 @@ func (pn *projectnode) load() {
}
pn.uuid = resp.UUID
}
+ // Note: the "filters" slice's backing array might be reused
+ // by append(filters,...) below. This isn't goroutine safe,
+ // but all accesses are in the same goroutine, so it's OK.
filters := []Filter{{"owner_uuid", "=", pn.uuid}}
params := ResourceListParams{
Filters: filters,
commit aadbd550d97c2cbf47563ceb1a9004932fa5e28f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:41:00 2018 -0400
13111: Update comment.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fa43cca..45beff6 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -452,7 +452,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// call nca.FS().Rename() instead of proceeding. Until then
// it's awkward for filesystems to implement their own Rename
// methods effectively: the only one that runs is the one on
- // the root filesystem exposed to the caller (webdav, fuse,
+ // the root FileSystem exposed to the caller (webdav, fuse,
// etc).
// When acquiring locks on multiple inodes, avoid deadlock by
commit a6c49634025095e9b4d29f5971fe5ecb720f9539
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 6 16:08:35 2018 -0400
13111: Redirect /dir to /dir/ at siteFS paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 446eeb4..5e14cb4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -329,7 +329,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if useSiteFS {
- h.serveSiteFS(w, r, tokens)
+ h.serveSiteFS(w, r, tokens, credentialsOK)
return
}
@@ -499,7 +499,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
-func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
@@ -528,18 +528,22 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
Insecure: arv.ApiInsecure,
}
fs := client.SiteFileSystem(kc)
- if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+ f, err := fs.Open(r.URL.Path)
+ if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
} else if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
- } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
-
- h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ }
+ defer f.Close()
+ if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+ if !strings.HasSuffix(r.URL.Path, "/") {
+ h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
+ } else {
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ }
return
- } else {
- f.Close()
}
wh := webdav.Handler{
Prefix: "/",
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 7fed6fb..15f32f1 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -493,10 +493,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
for _, trial := range []struct {
- uri string
- header http.Header
- expect []string
- cutDirs int
+ uri string
+ header http.Header
+ expect []string
+ redirect string
+ cutDirs int
}{
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
@@ -529,12 +530,32 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 4,
},
{
+ uri: "download.example.com/",
+ header: authHeader,
+ expect: []string{"users/"},
+ cutDirs: 0,
+ },
+ {
+ uri: "download.example.com/users",
+ header: authHeader,
+ redirect: "/users/",
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
uri: "download.example.com/users/",
header: authHeader,
expect: []string{"active/"},
cutDirs: 1,
},
{
+ uri: "download.example.com/users/active",
+ header: authHeader,
+ redirect: "/users/active/",
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
uri: "download.example.com/users/active/",
header: authHeader,
expect: []string{"foo_file_in_dir/"},
@@ -565,10 +586,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 1,
},
{
- uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 2,
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
+ header: authHeader,
+ redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
@@ -577,10 +599,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 3,
},
{
- uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 1,
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
+ redirect: "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
@@ -616,6 +639,9 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
resp = httptest.NewRecorder()
s.testServer.Handler.ServeHTTP(resp, req)
}
+ if trial.redirect != "" {
+ c.Check(req.URL.Path, check.Equals, trial.redirect)
+ }
if trial.expect == nil {
c.Check(resp.Code, check.Equals, http.StatusNotFound)
} else {
commit cd0945bb8591fd97fe37a77dc9057c29dc5d7558
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:54:40 2018 -0400
13111: Serve "site filesystem" at keep-web root URL.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 0fc6e47..22ddd84 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -266,6 +266,11 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
}
func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
+ for _, path := range []string{"/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
+ }
for _, path := range []string{"/users", "/users/"} {
stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 00af0f4..446eeb4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -250,7 +250,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+ } else if r.URL.Path == "/" || (len(pathParts) >= 1 && pathParts[0] == "users") {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 02f03d0..ee585ad 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -59,7 +59,6 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
func (s *IntegrationSuite) Test404(c *check.C) {
for _, uri := range []string{
// Routing errors (always 404 regardless of what's stored in Keep)
- "/",
"/foo",
"/download",
"/collections",
commit 04d11c9cf24f358a0780edc87c910c50130fd71b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:22:08 2018 -0400
13111: Skip collections named "." or "..".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 7f34a42..300c77a 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -60,7 +60,7 @@ func (pn *projectnode) load() {
}
for _, i := range resp.Items {
coll := i
- if coll.Name == "" {
+ if coll.Name == "" || coll.Name == "." || coll.Name == ".." {
continue
}
pn.inode.Child(coll.Name, func(inode) (inode, error) {
commit c5d6ae2fc5f1c382f2011a128230d893a5b2103c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:21:53 2018 -0400
13111: Clarify Child() comment.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 369b4bb..fa43cca 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -103,6 +103,9 @@ type inode interface {
// Child() performs lookups and updates of named child nodes.
//
+ // (The term "child" here is used strictly. This means name is
+ // not "." or "..", and name does not contain "/".)
+ //
// If replace is non-nil, Child calls replace(x) where x is
// the current child inode with the given name. If possible,
// the child inode is replaced with the one returned by
commit 3e2aed249f9ae1d99fb654b19d3ae49788f8fdaa
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 17:20:49 2018 -0400
13111: Test /users/name and /users paths with cadaver.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 843a8e8..0fc6e47 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -250,35 +250,8 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
os.Remove(checkfile.Name())
- cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
- if password != "" {
- // cadaver won't try username/password
- // authentication unless the server responds
- // 401 to an unauthenticated request, which it
- // only does in AttachmentOnlyHost,
- // TrustAllContent, and per-collection vhost
- // cases.
- s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
-
- cmd.Env = append(os.Environ(), "HOME="+tempdir)
- f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
- c.Assert(err, check.IsNil)
- _, err = fmt.Fprintf(f, "default login none password %s\n", password)
- c.Assert(err, check.IsNil)
- c.Assert(f.Close(), check.IsNil)
- }
- cmd.Stdin = bytes.NewBufferString(trial.cmd)
- stdout, err := cmd.StdoutPipe()
- c.Assert(err, check.Equals, nil)
- cmd.Stderr = cmd.Stdout
- go cmd.Start()
-
- var buf bytes.Buffer
- _, err = io.Copy(&buf, stdout)
- c.Check(err, check.Equals, nil)
- err = cmd.Wait()
- c.Check(err, check.Equals, nil)
- c.Check(buf.String(), check.Matches, trial.match)
+ stdout := s.runCadaver(c, password, trial.path, trial.cmd)
+ c.Check(stdout, check.Matches, trial.match)
if trial.data == nil {
continue
@@ -291,3 +264,54 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
c.Check(err, check.IsNil)
}
}
+
+func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
+ for _, path := range []string{"/users", "/users/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
+ }
+ for _, path := range []string{"/users/active", "/users/active/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
+ }
+ for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
+ }
+}
+
+func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
+ if password != "" {
+ // cadaver won't try username/password authentication
+ // unless the server responds 401 to an
+ // unauthenticated request, which it only does in
+ // AttachmentOnlyHost, TrustAllContent, and
+ // per-collection vhost cases.
+ s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
+
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ cmd.Stdin = bytes.NewBufferString(stdin)
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ return buf.String()
+}
commit 8f0b7529cda14a10cd953819af9c1b76201ae4f5
Merge: 282394b c4bd314
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 5 16:14:35 2018 -0400
13111: Merge branch 'master' into 13111-webdav-projects
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc services/keep-web/webdav.go
index 3e62b19,432c6af..5b23c9c
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@@ -47,10 -47,7 +47,10 @@@ type webdavFS struct
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
- dir, name := path.Split(name)
+ dir, _ := path.Split(name)
if dir == "" || dir == "/" {
return
}
commit 282394bf9076af3387755979e4fd9eb360291861
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Mar 28 09:10:32 2018 -0400
13111: Update tests to accept username in API response.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index a506486..39c905a 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -817,7 +817,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
- "last_name"].sort
+ "last_name", "username"].sort
def check_non_admin_index
assert_response :success
commit cedd5efb0f3813125931c64b754dddabd875d537
Merge: 4c46038 9e92ae4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 27 09:28:05 2018 -0400
13111: Merge branch 'master' into 13111-webdav-projects
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 4c46038d3726e1ff7006cc30bbd6b71b957d0215
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Mar 23 17:25:27 2018 -0400
13111: Expose read-only /users/ tree.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 2d6dd2b..6a7ce2a 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -5,7 +5,6 @@
package main
import (
- _ "encoding/json"
"fmt"
"io"
"io/ioutil"
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 5f5f69a..843a8e8 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -27,7 +27,7 @@ func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
@@ -36,19 +36,23 @@ func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ rpath := "/users/active/foo_file_in_dir/"
s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
- r := "/users/active/foo_file_in_dir/"
- w := "/users/active/" + newCollection.Name
+ wpath := "/users/active/" + newCollection.Name
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
- return r, w, pdh
+ return rpath, wpath, pdh
+ }, func(path string) bool {
+ // Skip tests that rely on writes, because /users/
+ // tree is read-only.
+ return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -239,6 +243,10 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
},
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ if skip != nil && skip(trial.path) {
+ c.Log("(skip)")
+ continue
+ }
os.Remove(checkfile.Name())
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5ab4f70..00af0f4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -226,13 +226,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
}
- arv := h.clientPool.Get()
- if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
- return
- }
- defer h.clientPool.Put(arv)
-
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
@@ -241,6 +234,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
var attachment bool
+ var useSiteFS bool
credentialsOK := h.Config.TrustAllContent
if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
@@ -256,6 +250,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
+ } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+ useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
collectionID = parseCollectionIDFromURL(pathParts[0][2:])
@@ -275,6 +271,16 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+ if collectionID == "" && !useSiteFS {
+ statusCode = http.StatusNotFound
+ return
+ }
+
+ forceReload := false
+ if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
+ forceReload = true
+ }
+
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -322,6 +328,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokens = append(reqTokens, h.Config.AnonymousTokens...)
}
+ if useSiteFS {
+ h.serveSiteFS(w, r, tokens)
+ return
+ }
+
if len(targetPath) > 0 && targetPath[0] == "_" {
// If a collection has a directory called "t=foo" or
// "_", it can be served at
@@ -333,10 +344,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
stripParts++
}
- forceReload := false
- if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
- forceReload = true
+ arv := h.clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
+ return
}
+ defer h.clientPool.Put(arv)
var collection *arvados.Collection
tokenResult := make(map[string]int)
@@ -410,12 +423,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}
- var fs arvados.FileSystem
- if collectionID == "" {
- fs = client.SiteFileSystem(kc)
- } else {
- fs, err = collection.FileSystem(client, kc)
- }
+ fs, err := collection.FileSystem(client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
@@ -475,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
+ h.serveDirectory(w, r, collection.Name, fs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -491,10 +499,69 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+ if len(tokens) == 0 {
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+ http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ return
+ }
+ if writeMethod[r.Method] {
+ http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
+ return
+ }
+ arv := h.clientPool.Get()
+ if arv == nil {
+ http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ return
+ }
+ defer h.clientPool.Put(arv)
+ arv.ApiToken = tokens[0]
+
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ client := &arvados.Client{
+ APIHost: arv.ApiServer,
+ AuthToken: arv.ApiToken,
+ Insecure: arv.ApiInsecure,
+ }
+ fs := client.SiteFileSystem(kc)
+ if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ return
+ } else {
+ f.Close()
+ }
+ wh := webdav.Handler{
+ Prefix: "/",
+ FileSystem: &webdavFS{
+ collfs: fs,
+ writing: writeMethod[r.Method],
+ alwaysReadEOF: r.Method == "PROPFIND",
+ },
+ LockSystem: h.webdavLS,
+ Logger: func(_ *http.Request, err error) {
+ if err != nil {
+ log.Printf("error from webdav handler: %q", err)
+ }
+ },
+ }
+ wh.ServeHTTP(w, r)
+}
+
var dirListingTemplate = `<!DOCTYPE HTML>
<HTML><HEAD>
<META name="robots" content="NOINDEX">
- <TITLE>{{ .Collection.Name }}</TITLE>
+ <TITLE>{{ .CollectionName }}</TITLE>
<STYLE type="text/css">
body {
margin: 1.5em;
@@ -518,19 +585,26 @@ var dirListingTemplate = `<!DOCTYPE HTML>
</STYLE>
</HEAD>
<BODY>
+
<H1>{{ .CollectionName }}</H1>
<P>This collection of data files is being shared with you through
Arvados. You can download individual files listed below. To download
-the entire collection with wget, try:</P>
+the entire directory tree with wget, try:</P>
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
<H2>File Listing</H2>
{{if .Files}}
<UL>
-{{range .Files}} <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
+{{range .Files}}
+{{if .IsDir }}
+ <LI>{{" " | printf "%15s " | nbsp}}<A href="{{.Name}}/">{{.Name}}/</A></LI>
+{{else}}
+ <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>
+{{end}}
+{{end}}
</UL>
{{else}}
<P>(No files; this collection is empty.)</P>
@@ -550,11 +624,12 @@ the entire collection with wget, try:</P>
`
type fileListEnt struct {
- Name string
- Size int64
+ Name string
+ Size int64
+ IsDir bool
}
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
var files []fileListEnt
var walk func(string) error
if !strings.HasSuffix(base, "/") {
@@ -574,15 +649,16 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
return err
}
for _, ent := range ents {
- if ent.IsDir() {
+ if recurse && ent.IsDir() {
err = walk(path + ent.Name() + "/")
if err != nil {
return err
}
} else {
files = append(files, fileListEnt{
- Name: path + ent.Name(),
- Size: ent.Size(),
+ Name: path + ent.Name(),
+ Size: ent.Size(),
+ IsDir: ent.IsDir(),
})
}
}
@@ -611,7 +687,7 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
"CollectionName": collectionName,
"Files": files,
"Request": r,
- "StripParts": stripParts,
+ "StripParts": strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
})
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 3e7ae5f..7fed6fb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -508,7 +508,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
@@ -517,12 +517,30 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
- uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ uri: "download.example.com/users/active/foo_file_in_dir/",
header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
+ expect: []string{"dir1/"},
cutDirs: 3,
},
{
+ uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
+ header: authHeader,
+ expect: []string{"bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/users/",
+ header: authHeader,
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/active/",
+ header: authHeader,
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
@@ -550,19 +568,19 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 1,
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 2,
+ cutDirs: 3,
},
{
uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 941090a..3e62b19 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -47,6 +47,9 @@ type webdavFS struct {
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
dir, name := path.Split(name)
if dir == "" || dir == "/" {
return
@@ -66,7 +69,7 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
}
func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
- writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+ writing := flag&(os.O_WRONLY|os.O_RDWR|os.O_TRUNC) != 0
if writing {
fs.makeparents(name)
}
@@ -75,8 +78,13 @@ func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os
// webdav module returns 404 on all OpenFile errors,
// but returns 405 Method Not Allowed if OpenFile()
// succeeds but Write() or Close() fails. We'd rather
- // have 405.
- f = writeFailer{File: f, err: errReadOnly}
+ // have 405. writeFailer ensures Close() fails if the
+ // file is opened for writing *or* Write() is called.
+ var err error
+ if writing {
+ err = errReadOnly
+ }
+ f = writeFailer{File: f, err: err}
}
if fs.alwaysReadEOF {
f = readEOF{File: f}
@@ -109,10 +117,15 @@ type writeFailer struct {
}
func (wf writeFailer) Write([]byte) (int, error) {
+ wf.err = errReadOnly
return 0, wf.err
}
func (wf writeFailer) Close() error {
+ err := wf.File.Close()
+ if err != nil {
+ wf.err = err
+ }
return wf.err
}
commit ab1abbac7ec08c7a7599670459f04b8a35dcbe68
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 16:56:16 2018 -0400
13111: Add /users virtual dir to siteFS.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 776c8a3..7f34a42 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -10,25 +10,31 @@ import (
"time"
)
+type staleChecker struct {
+ mtx sync.Mutex
+ last time.Time
+}
+
+func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
+ sc.mtx.Lock()
+ defer sc.mtx.Unlock()
+ if !staleFunc(sc.last) {
+ return
+ }
+ sc.last = time.Now()
+ fn()
+}
+
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
+ staleChecker
uuid string
err error
-
- loadLock sync.Mutex
- loadStart time.Time
}
func (pn *projectnode) load() {
- fs := pn.FS().(*siteFileSystem)
-
- pn.loadLock.Lock()
- defer pn.loadLock.Unlock()
- if !fs.Stale(pn.loadStart) {
- return
- }
- pn.loadStart = time.Now()
+ fs := pn.FS().(*customFileSystem)
if pn.uuid == "" {
var resp User
@@ -89,7 +95,7 @@ func (pn *projectnode) load() {
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
@@ -97,7 +103,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index b7dc08e..69bae89 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"io"
"os"
+ "path/filepath"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -38,8 +39,17 @@ func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, b
return sc.Client.RequestAndDecode(dst, method, path, body, params)
}
-func (s *SiteFSSuite) TestHomeProject(c *check.C) {
- f, err := s.fs.Open("/home")
+func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
+ s.fs.MountProject("home", "")
+ s.testHomeProject(c, "/home")
+}
+
+func (s *SiteFSSuite) TestUsersDir(c *check.C) {
+ s.testHomeProject(c, "/users/active")
+}
+
+func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
+ f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
c.Check(len(fis), check.Not(check.Equals), 0)
@@ -53,23 +63,24 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/A Project/..")
+ f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
- c.Check(fi.Name(), check.Equals, "home")
+ _, basename := filepath.Split(path)
+ c.Check(fi.Name(), check.Equals, basename)
- f, err = s.fs.Open("/home/A Project/A Subproject")
+ f, err = s.fs.Open(path + "/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/Unrestricted public data",
- "/home/Unrestricted public data/does not exist",
- "/home/A Project/does not exist",
+ path + "/Unrestricted public data",
+ path + "/Unrestricted public data/does not exist",
+ path + "/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -79,6 +90,8 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ s.fs.MountProject("home", "")
+
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
c.Check(err, check.IsNil)
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index cdcf40e..e9c8387 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -10,22 +10,25 @@ import (
"time"
)
-type siteFileSystem struct {
+type CustomFileSystem interface {
+ FileSystem
+ MountByID(mount string)
+ MountProject(mount, uuid string)
+ MountUsers(mount string)
+}
+
+type customFileSystem struct {
fileSystem
+ root *vdirnode
staleThreshold time.Time
staleLock sync.Mutex
}
-// SiteFileSystem returns a FileSystem that maps collections and other
-// Arvados objects onto a filesystem layout.
-//
-// This is experimental: the filesystem layout is not stable, and
-// there are significant known bugs and shortcomings. For example,
-// writes are not persisted until Sync() is called.
-func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
root := &vdirnode{}
- fs := &siteFileSystem{
+ fs := &customFileSystem{
+ root: root,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: c, keepClient: kc},
root: root,
@@ -41,14 +44,18 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.inode.Child("by_id", func(inode) (inode, error) {
+ return fs
+}
+
+func (fs *customFileSystem) MountByID(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
- name: "by_id",
+ name: mount,
modTime: time.Now(),
mode: 0755 | os.ModeDir,
},
@@ -56,13 +63,45 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
create: fs.mountCollection,
}, nil
})
- root.inode.Child("home", func(inode) (inode, error) {
- return fs.newProjectNode(fs.root, "home", ""), nil
+}
+
+func (fs *customFileSystem) MountProject(mount, uuid string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, mount, uuid), nil
})
+}
+
+func (fs *customFileSystem) MountUsers(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return &usersnode{
+ inode: &treenode{
+ fs: fs,
+ parent: fs.root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: mount,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }, nil
+ })
+}
+
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// writes are not persisted until Sync() is called.
+func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
+ fs := c.CustomFileSystem(kc)
+ fs.MountByID("by_id")
+ fs.MountUsers("users")
return fs
}
-func (fs *siteFileSystem) Sync() error {
+func (fs *customFileSystem) Sync() error {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
fs.staleThreshold = time.Now()
@@ -71,17 +110,17 @@ func (fs *siteFileSystem) Sync() error {
// Stale returns true if information obtained at time t should be
// considered stale.
-func (fs *siteFileSystem) Stale(t time.Time) bool {
+func (fs *customFileSystem) Stale(t time.Time) bool {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
return !fs.staleThreshold.Before(t)
}
-func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
-func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
+func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
@@ -96,7 +135,23 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
-func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
+func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
return &projectnode{
uuid: uuid,
inode: &treenode{
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index a3a9712..e35ae48 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -16,7 +16,7 @@ var _ = check.Suite(&SiteFSSuite{})
type SiteFSSuite struct {
client *Client
- fs FileSystem
+ fs CustomFileSystem
kc keepClient
}
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
new file mode 100644
index 0000000..ccfe2c5
--- /dev/null
+++ b/sdk/go/arvados/fs_users.go
@@ -0,0 +1,62 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+)
+
+// usersnode is a virtual directory with an entry for each visible
+// Arvados username, each showing the respective user's "home
+// projects".
+type usersnode struct {
+ inode
+ staleChecker
+ err error
+}
+
+func (un *usersnode) load() {
+ fs := un.FS().(*customFileSystem)
+
+ params := ResourceListParams{
+ Order: "uuid",
+ }
+ for {
+ var resp UserList
+ un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if un.err != nil {
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, user := range resp.Items {
+ if user.Username == "" {
+ continue
+ }
+ un.inode.Child(user.Username, func(inode) (inode, error) {
+ return fs.newProjectNode(un, user.Username, user.UUID), nil
+ })
+ }
+ params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ }
+ un.err = nil
+}
+
+func (un *usersnode) Readdir() ([]os.FileInfo, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Readdir()
+}
+
+func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Child(name, nil)
+}
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index dc7e62f..587f4db 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -159,7 +159,7 @@ class Arvados::V1::UsersController < ApplicationController
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name"]
+ safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username"]
if @select
@select = @select & safe_attrs
else
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 19a2040..5ab4f70 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -236,7 +236,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
- var targetID string
+ var collectionID string
var tokens []string
var reqTokens []string
var pathToken bool
@@ -250,7 +250,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
attachment = true
}
- if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
+ if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
@@ -258,28 +258,23 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
- targetID = parseCollectionIDFromURL(pathParts[0][2:])
+ collectionID = parseCollectionIDFromURL(pathParts[0][2:])
stripParts = 1
} else if len(pathParts) >= 2 && pathParts[0] == "collections" {
if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetID = parseCollectionIDFromURL(pathParts[2])
+ collectionID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
stripParts = 4
pathToken = true
} else {
// /collections/ID/PATH...
- targetID = parseCollectionIDFromURL(pathParts[1])
+ collectionID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
stripParts = 2
}
}
- if targetID == "" {
- statusCode = http.StatusNotFound
- return
- }
-
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -347,7 +342,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
+ collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
@@ -414,14 +409,21 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
}
- fs, err := collection.FileSystem(client, kc)
+
+ var fs arvados.FileSystem
+ if collectionID == "" {
+ fs = client.SiteFileSystem(kc)
+ } else {
+ fs, err = collection.FileSystem(client, kc)
+ }
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- targetIsPDH := arvadosclient.PDHMatch(targetID)
- if targetIsPDH && writeMethod[r.Method] {
+ writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ targetIsPDH := arvadosclient.PDHMatch(collectionID)
+ if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
return
}
@@ -435,7 +437,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w = &updateOnSuccess{
ResponseWriter: w,
update: func() error {
- return h.Config.Cache.Update(client, *collection, fs)
+ return h.Config.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index af83681..941090a 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -36,7 +36,7 @@ var (
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.CollectionFileSystem
+ collfs arvados.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
commit 0c7a458fc70791a5f395dad171a272b92b6ffb7f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 10:52:09 2018 -0400
13111: Add tests for /users/ paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index b7ec1de..5f5f69a 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -22,24 +22,33 @@ import (
)
func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
- s.testCadaver(c, arvadostest.ActiveToken, func(newUUID string) (string, string, string) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
- w := "/c=" + newUUID + "/"
+ w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
})
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
- s.testCadaver(c, "", func(newUUID string) (string, string, string) {
+ s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
- w := "/c=" + newUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(string) (string, string, string)) {
+func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
+ r := "/users/active/foo_file_in_dir/"
+ w := "/users/active/" + newCollection.Name
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -62,7 +71,7 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- readPath, writePath, pdhPath := pathFunc(newCollection.UUID)
+ readPath, writePath, pdhPath := pathFunc(newCollection)
matchToday := time.Now().Format("Jan +2")
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 21e47c8..3e7ae5f 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -517,6 +517,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
+ uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 3,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
commit efd536fbbfca8473e7df444dd1d97ce916bfe345
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Mar 26 15:23:15 2018 -0400
13111: Sort vendor.json
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 1189ca8..7af81ba 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -84,18 +84,18 @@
"revisionTime": "2018-01-08T08:51:32Z"
},
{
- "checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
- "path": "github.com/dgrijalva/jwt-go",
- "revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
- "revisionTime": "2017-10-19T21:57:19Z"
- },
- {
"checksumSHA1": "gMBls0ytB5wHvZizUQE8Eivv9WQ=",
"path": "github.com/curoverse/cgofuse/fuse",
"revision": "d08d9e36b4ca1364eb7a4eb9db0b7fa76c9250a2",
"revisionTime": "2017-12-17T05:18:50Z"
},
{
+ "checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
+ "path": "github.com/dgrijalva/jwt-go",
+ "revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
+ "revisionTime": "2017-10-19T21:57:19Z"
+ },
+ {
"checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
"path": "github.com/docker/distribution/digestset",
"revision": "277ed486c948042cab91ad367c379524f3b25e18",
commit 541e6b47d90dbdadca287bea0cc303026f20facb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 10:00:36 2018 -0400
13111: Test webdav with http basic authentication.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index eb32367..b7ec1de 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -6,11 +6,13 @@ package main
import (
"bytes"
+ "fmt"
"io"
"io/ioutil"
"net/url"
"os"
"os/exec"
+ "path/filepath"
"strings"
"time"
@@ -19,34 +21,51 @@ import (
check "gopkg.in/check.v1"
)
-func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
+func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newUUID string) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
+ w := "/c=" + newUUID + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
+ s.testCadaver(c, "", func(newUUID string) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newUUID + "/t=" + arvadostest.ActiveToken + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(string) (string, string, string)) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
- localfile, err := ioutil.TempFile("", "localfile")
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ localfile, err := ioutil.TempFile(tempdir, "localfile")
c.Assert(err, check.IsNil)
- defer os.Remove(localfile.Name())
localfile.Write(testdata)
- emptyfile, err := ioutil.TempFile("", "emptyfile")
+ emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
c.Assert(err, check.IsNil)
- defer os.Remove(emptyfile.Name())
- checkfile, err := ioutil.TempFile("", "checkfile")
+ checkfile, err := ioutil.TempFile(tempdir, "checkfile")
c.Assert(err, check.IsNil)
- defer os.Remove(checkfile.Name())
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
- pdhPath := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arv.AuthToken + "/"
+ readPath, writePath, pdhPath := pathFunc(newCollection.UUID)
matchToday := time.Now().Format("Jan +2")
- readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
type testcase struct {
path string
cmd string
@@ -215,6 +234,22 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
os.Remove(checkfile.Name())
cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
+ if password != "" {
+ // cadaver won't try username/password
+ // authentication unless the server responds
+ // 401 to an unauthenticated request, which it
+ // only does in AttachmentOnlyHost,
+ // TrustAllContent, and per-collection vhost
+ // cases.
+ s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
+
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
cmd.Stdin = bytes.NewBufferString(trial.cmd)
stdout, err := cmd.StdoutPipe()
c.Assert(err, check.Equals, nil)
commit f09c8facc9a607f3b5e9171e66170536f1ba83c7
Merge: 959335d f7fe9cc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Mar 19 14:39:16 2018 -0400
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc vendor/vendor.json
index 1bff878,a4f750b..1189ca8
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@@ -42,18 -84,12 +84,18 @@@
"revisionTime": "2018-01-08T08:51:32Z"
},
{
- "checksumSHA1": "pAu+do4x7E5SFLfIqJeGwhcOd6E=",
- "path": "github.com/curoverse/azure-sdk-for-go/storage",
- "revision": "1620af6b32398bfc91827ceae54a8cc1f55df04d",
- "revisionTime": "2016-12-14T20:08:43Z"
+ "checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
+ "path": "github.com/dgrijalva/jwt-go",
+ "revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
+ "revisionTime": "2017-10-19T21:57:19Z"
},
{
+ "checksumSHA1": "gMBls0ytB5wHvZizUQE8Eivv9WQ=",
+ "path": "github.com/curoverse/cgofuse/fuse",
+ "revision": "d08d9e36b4ca1364eb7a4eb9db0b7fa76c9250a2",
+ "revisionTime": "2017-12-17T05:18:50Z"
+ },
+ {
"checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
"path": "github.com/docker/distribution/digestset",
"revision": "277ed486c948042cab91ad367c379524f3b25e18",
commit 959335d561a3882b391c88c4b2106f263e827b51
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Feb 13 17:29:38 2018 -0500
13111: Reload project dir if fsync(2) was called since last load.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 999b4e9..aea0cc0 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -16,6 +16,7 @@ import (
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
TrashAt *time.Time `json:"trash_at,omitempty"`
ManifestText string `json:"manifest_text,omitempty"`
UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index a5e4710..776c8a3 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -7,18 +7,29 @@ package arvados
import (
"os"
"sync"
+ "time"
)
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
- uuid string
- setupOnce sync.Once
- err error
+ uuid string
+ err error
+
+ loadLock sync.Mutex
+ loadStart time.Time
}
-func (pn *projectnode) setup() {
+func (pn *projectnode) load() {
fs := pn.FS().(*siteFileSystem)
+
+ pn.loadLock.Lock()
+ defer pn.loadLock.Unlock()
+ if !fs.Stale(pn.loadStart) {
+ return
+ }
+ pn.loadStart = time.Now()
+
if pn.uuid == "" {
var resp User
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
@@ -36,7 +47,6 @@ func (pn *projectnode) setup() {
var resp CollectionList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -60,7 +70,6 @@ func (pn *projectnode) setup() {
var resp GroupList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -76,10 +85,11 @@ func (pn *projectnode) setup() {
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
+ pn.err = nil
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
@@ -87,7 +97,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 1de0714..b7dc08e 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -46,29 +47,29 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
ok := false
for _, fi := range fis {
c.Check(fi.Name(), check.Not(check.Equals), "")
- if fi.Name() == "Unrestricted public data" {
+ if fi.Name() == "A Project" {
ok = true
}
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/Unrestricted public data/..")
+ f, err = s.fs.Open("/home/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
c.Check(fi.Name(), check.Equals, "home")
- f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+ f, err = s.fs.Open("/home/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/A Project",
- "/home/A Project/does not exist",
+ "/home/Unrestricted public data",
"/home/Unrestricted public data/does not exist",
+ "/home/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -76,3 +77,54 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
c.Check(os.IsNotExist(err), check.Equals, true)
}
}
+
+func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ project, err := s.fs.OpenFile("/home/A Project", 0, 0)
+ c.Check(err, check.IsNil)
+
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+
+ oob := Collection{
+ Name: "oob",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ f, err := s.fs.Open("/home/A Project/oob")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Check(fi.IsDir(), check.Equals, true)
+ f.Close()
+
+ wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
+ c.Assert(err, check.IsNil)
+ _, err = wf.Write([]byte("hello oob\n"))
+ c.Check(err, check.IsNil)
+ err = wf.Close()
+ c.Check(err, check.IsNil)
+
+ // Delete test.txt behind s.fs's back by updating the
+ // collection record with the old (empty) ManifestText.
+ err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob/test.txt")
+ c.Check(err, check.NotNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.IsNil)
+
+ err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 701711e..cdcf40e 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -6,11 +6,15 @@ package arvados
import (
"os"
+ "sync"
"time"
)
type siteFileSystem struct {
fileSystem
+
+ staleThreshold time.Time
+ staleLock sync.Mutex
}
// SiteFileSystem returns a FileSystem that maps collections and other
@@ -58,6 +62,21 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
+func (fs *siteFileSystem) Sync() error {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ fs.staleThreshold = time.Now()
+ return nil
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *siteFileSystem) Stale(t time.Time) bool {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ return !fs.staleThreshold.Before(t)
+}
+
func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 26a2212..a3a9712 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -21,7 +21,11 @@ type SiteFSSuite struct {
}
func (s *SiteFSSuite) SetUpTest(c *check.C) {
- s.client = NewClientFromEnv()
+ s.client = &Client{
+ APIHost: os.Getenv("ARVADOS_API_HOST"),
+ AuthToken: arvadostest.ActiveToken,
+ Insecure: true,
+ }
s.kc = &keepClientStub{
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index d057c09..5fccfb3 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -25,6 +25,9 @@ const (
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
+ AProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
+
FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
commit 9424d875b67ddda957b2ef705a1619394b57db70
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 18:16:53 2018 -0500
13111: Propagate errors in Child().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 4d22c1e..369b4bb 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -27,6 +27,7 @@ var (
ErrWriteOnlyMode = errors.New("file is O_WRONLY")
ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrNotADirectory = errors.New("not a directory")
ErrPermission = os.ErrPermission
)
@@ -128,7 +129,7 @@ type inode interface {
// a child was added or changed, the new child is returned.
//
// Caller must have lock (or rlock if replace is nil).
- Child(name string, replace func(inode) inode) inode
+ Child(name string, replace func(inode) (inode, error)) (inode, error)
sync.Locker
RLock()
@@ -202,8 +203,8 @@ func (*nullnode) Readdir() ([]os.FileInfo, error) {
return nil, ErrInvalidOperation
}
-func (*nullnode) Child(name string, replace func(inode) inode) inode {
- return nil
+func (*nullnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return nil, ErrNotADirectory
}
type treenode struct {
@@ -236,18 +237,25 @@ func (n *treenode) IsDir() bool {
return true
}
-func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
- // TODO: special treatment for "", ".", ".."
+func (n *treenode) Child(name string, replace func(inode) (inode, error)) (child inode, err error) {
child = n.inodes[name]
- if replace != nil {
- newchild := replace(child)
- if newchild == nil {
- delete(n.inodes, name)
- } else if newchild != child {
- n.inodes[name] = newchild
- n.fileinfo.modTime = time.Now()
- child = newchild
- }
+ if name == "" || name == "." || name == ".." {
+ err = ErrInvalidArgument
+ return
+ }
+ if replace == nil {
+ return
+ }
+ newchild, err := replace(child)
+ if err != nil {
+ return
+ }
+ if newchild == nil {
+ delete(n.inodes, name)
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ n.fileinfo.modTime = time.Now()
+ child = newchild
}
return
}
@@ -297,9 +305,9 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(fs.root, dirname)
- if parent == nil {
- return nil, os.ErrNotExist
+ parent, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return nil, err
}
var readable, writable bool
switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
@@ -331,22 +339,26 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
parent.RLock()
defer parent.RUnlock()
}
- n := parent.Child(name, nil)
- if n == nil {
+ n, err := parent.Child(name, nil)
+ if err != nil {
+ return nil, err
+ } else if n == nil {
if !createMode {
return nil, os.ErrNotExist
}
- var err error
- n = parent.Child(name, func(inode) inode {
- n, err = parent.FS().newNode(name, perm|0755, time.Now())
- n.SetParent(parent, name)
- return n
+ n, err = parent.Child(name, func(inode) (repl inode, err error) {
+ repl, err = parent.FS().newNode(name, perm|0755, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(parent, name)
+ return
})
if err != nil {
return nil, err
} else if n == nil {
- // parent rejected new child
- return nil, ErrInvalidOperation
+ // Parent rejected new child, but returned no error
+ return nil, ErrInvalidArgument
}
} else if flag&os.O_EXCL != 0 {
return nil, ErrFileExists
@@ -375,38 +387,37 @@ func (fs *fileSystem) Create(name string) (File, error) {
return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
}
-func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) error {
dirname, name := path.Split(name)
- n := rlookup(fs.root, dirname)
- if n == nil {
- return os.ErrNotExist
+ n, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
}
n.Lock()
defer n.Unlock()
- if n.Child(name, nil) != nil {
+ if child, err := n.Child(name, nil); err != nil {
+ return err
+ } else if child != nil {
return os.ErrExist
}
- child := n.Child(name, func(inode) (child inode) {
- child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
- child.SetParent(n, name)
+
+ _, err = n.Child(name, func(inode) (repl inode, err error) {
+ repl, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(n, name)
return
})
- if err != nil {
- return err
- } else if child == nil {
- return ErrInvalidArgument
- }
- return nil
+ return err
}
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.root, name)
- if node == nil {
- err = os.ErrNotExist
- } else {
- fi = node.FileInfo()
+func (fs *fileSystem) Stat(name string) (os.FileInfo, error) {
+ node, err := rlookup(fs.root, name)
+ if err != nil {
+ return nil, err
}
- return
+ return node.FileInfo(), nil
}
func (fs *fileSystem) Rename(oldname, newname string) error {
@@ -475,43 +486,31 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- // Return ErrInvalidOperation if olddirf.inode doesn't even
- // bother calling our "remove oldname entry" replacer func.
- err = ErrInvalidArgument
- olddirf.inode.Child(oldname, func(oldinode inode) inode {
- err = nil
+ _, err = olddirf.inode.Child(oldname, func(oldinode inode) (inode, error) {
if oldinode == nil {
- err = os.ErrNotExist
- return nil
+ return oldinode, os.ErrNotExist
}
if locked[oldinode] {
// oldinode cannot become a descendant of itself.
- err = ErrInvalidArgument
- return oldinode
+ return oldinode, ErrInvalidArgument
}
if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
// moving a mount point to a different parent
// is not (yet) supported.
- err = ErrInvalidArgument
- return oldinode
+ return oldinode, ErrInvalidArgument
}
- accepted := newdirf.inode.Child(newname, func(existing inode) inode {
+ accepted, err := newdirf.inode.Child(newname, func(existing inode) (inode, error) {
if existing != nil && existing.IsDir() {
- err = ErrIsDirectory
- return existing
+ return existing, ErrIsDirectory
}
- return oldinode
+ return oldinode, nil
})
- if accepted != oldinode {
- if err == nil {
- // newdirf didn't accept oldinode.
- err = ErrInvalidArgument
- }
+ if err != nil {
// Leave oldinode in olddir.
- return oldinode
+ return oldinode, err
}
accepted.SetParent(newdirf.inode, newname)
- return nil
+ return nil, nil
})
return err
}
@@ -530,27 +529,25 @@ func (fs *fileSystem) RemoveAll(name string) error {
return err
}
-func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+func (fs *fileSystem) remove(name string, recursive bool) error {
dirname, name := path.Split(name)
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs.root, dirname)
- if dir == nil {
- return os.ErrNotExist
+ dir, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
}
dir.Lock()
defer dir.Unlock()
- dir.Child(name, func(node inode) inode {
+ _, err = dir.Child(name, func(node inode) (inode, error) {
if node == nil {
- err = os.ErrNotExist
- return nil
+ return nil, os.ErrNotExist
}
if !recursive && node.IsDir() && node.Size() > 0 {
- err = ErrDirectoryNotEmpty
- return node
+ return node, ErrDirectoryNotEmpty
}
- return nil
+ return nil, nil
})
return err
}
@@ -563,12 +560,9 @@ func (fs *fileSystem) Sync() error {
// rlookup (recursive lookup) returns the inode for the file/directory
// with the given name (which may contain "/" separators). If no such
// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
+func rlookup(start inode, path string) (node inode, err error) {
node = start
for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
if node.IsDir() {
if name == "." || name == "" {
continue
@@ -578,11 +572,17 @@ func rlookup(start inode, path string) (node inode) {
continue
}
}
- node = func() inode {
+ node, err = func() (inode, error) {
node.RLock()
defer node.RUnlock()
return node.Child(name, nil)
}()
+ if node == nil || err != nil {
+ break
+ }
+ }
+ if node == nil && err == nil {
+ err = os.ErrNotExist
}
return
}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fbd9775..923615b 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -521,7 +521,7 @@ func (dn *dirnode) FS() FileSystem {
return dn.fs
}
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -537,7 +537,7 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
return data, err
}}
gn.SetParent(dn, name)
- return gn
+ return gn, nil
}
return dn.treenode.Child(name, replace)
}
@@ -837,38 +837,41 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node = node.Parent()
continue
}
- node.Child(name, func(child inode) inode {
+ node, err = node.Child(name, func(child inode) (inode, error) {
if child == nil {
- child, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child, err := node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
child.SetParent(node, name)
- node = child
+ return child, nil
} else if !child.IsDir() {
- err = ErrFileExists
+ return child, ErrFileExists
} else {
- node = child
+ return child, nil
}
- return child
})
if err != nil {
return
}
}
- node.Child(basename, func(child inode) inode {
+ _, err = node.Child(basename, func(child inode) (inode, error) {
switch child := child.(type) {
case nil:
child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
child.SetParent(node, basename)
fn = child.(*filenode)
- return child
+ return child, nil
case *filenode:
fn = child
- return child
+ return child, nil
case *dirnode:
- err = ErrIsDirectory
- return child
+ return child, ErrIsDirectory
default:
- err = ErrInvalidArgument
- return child
+ return child, ErrInvalidArgument
}
})
return
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 023226f..8d32eb2 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -688,13 +688,13 @@ func (s *CollectionFSSuite) TestRename(c *check.C) {
err = fs.Rename(
fmt.Sprintf("dir%d/file%d/patherror", i, j),
fmt.Sprintf("dir%d/irrelevant", i))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
// newname parent dir is a file
err = fs.Rename(
fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
}(i, j)
}
}
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
index 97fe68b..a84f64f 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs_deferred.go
@@ -85,7 +85,7 @@ func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, erro
return dn.realinode().Write(p, pos)
}
-func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
+func (dn *deferrednode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
return dn.realinode().Child(name, replace)
}
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs_getternode.go
index c9ffb38..966fe9d 100644
--- a/sdk/go/arvados/fs_getternode.go
+++ b/sdk/go/arvados/fs_getternode.go
@@ -23,8 +23,8 @@ func (*getternode) IsDir() bool {
return false
}
-func (*getternode) Child(string, func(inode) inode) inode {
- return nil
+func (*getternode) Child(string, func(inode) (inode, error)) (inode, error) {
+ return nil, ErrInvalidArgument
}
func (gn *getternode) get() error {
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index f9cb799..a5e4710 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -5,7 +5,6 @@
package arvados
import (
- "log"
"os"
"sync"
)
@@ -48,8 +47,8 @@ func (pn *projectnode) setup() {
if coll.Name == "" {
continue
}
- pn.inode.Child(coll.Name, func(inode) inode {
- return deferredCollectionFS(fs, pn, coll)
+ pn.inode.Child(coll.Name, func(inode) (inode, error) {
+ return deferredCollectionFS(fs, pn, coll), nil
})
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
@@ -71,8 +70,8 @@ func (pn *projectnode) setup() {
if group.Name == "" || group.Name == "." || group.Name == ".." {
continue
}
- pn.inode.Child(group.Name, func(inode) inode {
- return fs.newProjectNode(pn, group.Name, group.UUID)
+ pn.inode.Child(group.Name, func(inode) (inode, error) {
+ return fs.newProjectNode(pn, group.Name, group.UUID), nil
})
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
@@ -87,30 +86,35 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
return pn.inode.Readdir()
}
-func (pn *projectnode) Child(name string, replace func(inode) inode) inode {
+func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
pn.setupOnce.Do(pn.setup)
if pn.err != nil {
- log.Printf("BUG: not propagating error setting up %T %v: %s", pn, pn, pn.err)
- // TODO: propagate error, instead of just being empty
- return nil
+ return nil, pn.err
}
if replace == nil {
// lookup
return pn.inode.Child(name, nil)
}
- return pn.inode.Child(name, func(existing inode) inode {
- if repl := replace(existing); repl == nil {
- // delete
+ return pn.inode.Child(name, func(existing inode) (inode, error) {
+ if repl, err := replace(existing); err != nil {
+ return existing, err
+ } else if repl == nil {
+ if existing == nil {
+ return nil, nil
+ }
+ // rmdir
// (TODO)
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
+ } else if existing != nil {
+ // clobber
+ return existing, ErrInvalidArgument
} else if repl.FileInfo().IsDir() {
// mkdir
// TODO: repl.SetParent(pn, name), etc.
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
} else {
// create file
- // TODO: repl.SetParent(pn, name), etc.
- return pn.Child(name, nil) // not implemented
+ return existing, ErrInvalidArgument
}
})
}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index c8d7360..701711e 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -37,7 +37,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.inode.Child("by_id", func(inode) inode {
+ root.inode.Child("by_id", func(inode) (inode, error) {
return &vdirnode{
inode: &treenode{
fs: fs,
@@ -50,10 +50,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: fs.mountCollection,
- }
+ }, nil
})
- root.inode.Child("home", func(inode) inode {
- return fs.newProjectNode(fs.root, "home", "")
+ root.inode.Child("home", func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, "home", ""), nil
})
return fs
}
@@ -104,18 +104,23 @@ type vdirnode struct {
create func(parent inode, name string) inode
}
-func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
- return vn.inode.Child(name, func(existing inode) inode {
- if existing != nil {
- return existing
- } else if vn.create == nil {
- return nil
+func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return vn.inode.Child(name, func(existing inode) (inode, error) {
+ if existing == nil && vn.create != nil {
+ existing = vn.create(vn, name)
+ if existing != nil {
+ existing.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ }
}
- n := vn.create(vn, name)
- if n != nil {
- n.SetParent(vn, name)
- vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ if replace == nil {
+ return existing, nil
+ } else if tryRepl, err := replace(existing); err != nil {
+ return existing, err
+ } else if tryRepl != existing {
+ return existing, ErrInvalidArgument
+ } else {
+ return existing, nil
}
- return n
})
}
commit 14fadeaed3dd4f01778eae8342ca0b5e190e3429
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 16:29:40 2018 -0500
13111: Propagate errors in Readdir().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 6402a8e..4d22c1e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -96,7 +96,7 @@ type inode interface {
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
IsDir() bool
- Readdir() []os.FileInfo
+ Readdir() ([]os.FileInfo, error)
Size() int64
FileInfo() os.FileInfo
@@ -198,8 +198,8 @@ func (*nullnode) IsDir() bool {
return false
}
-func (*nullnode) Readdir() []os.FileInfo {
- return nil
+func (*nullnode) Readdir() ([]os.FileInfo, error) {
+ return nil, ErrInvalidOperation
}
func (*nullnode) Child(name string, replace func(inode) inode) inode {
@@ -263,7 +263,7 @@ func (n *treenode) FileInfo() os.FileInfo {
return n.fileinfo
}
-func (n *treenode) Readdir() (fi []os.FileInfo) {
+func (n *treenode) Readdir() (fi []os.FileInfo, err error) {
n.RLock()
defer n.RUnlock()
fi = make([]os.FileInfo, 0, len(n.inodes))
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
index e638838..97fe68b 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs_deferred.go
@@ -89,15 +89,15 @@ func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
return dn.realinode().Child(name, replace)
}
-func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
-func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
-func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
-func (dn *deferrednode) Readdir() []os.FileInfo { return dn.realinode().Readdir() }
-func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
-func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
-func (dn *deferrednode) Lock() { dn.realinode().Lock() }
-func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
-func (dn *deferrednode) RLock() { dn.realinode().RLock() }
-func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
-func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
-func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() ([]os.FileInfo, error) { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index d586531..127bee8 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -74,10 +74,14 @@ func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
return nil, ErrInvalidOperation
}
if count <= 0 {
- return f.inode.Readdir(), nil
+ return f.inode.Readdir()
}
if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
+ var err error
+ f.unreaddirs, err = f.inode.Readdir()
+ if err != nil {
+ return nil, err
+ }
}
if len(f.unreaddirs) == 0 {
return nil, io.EOF
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 4dd8699..f9cb799 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -79,8 +79,11 @@ func (pn *projectnode) setup() {
}
}
-func (pn *projectnode) Readdir() []os.FileInfo {
+func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
pn.setupOnce.Do(pn.setup)
+ if pn.err != nil {
+ return nil, pn.err
+ }
return pn.inode.Readdir()
}
commit 3c230a24be3e99f89805199311ec8b36665c2b74
Merge: f67ede4 acb392d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 8 14:41:05 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit f67ede4c778f5ac6f3bd0e04d1f47993603b3375
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 1 09:45:47 2018 -0500
13111: Add spying client for testing deferred loading.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 92d8672..1de0714 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -5,11 +5,38 @@
package arvados
import (
+ "bytes"
+ "encoding/json"
+ "io"
"os"
check "gopkg.in/check.v1"
)
+type spiedRequest struct {
+ method string
+ path string
+ params map[string]interface{}
+}
+
+type spyingClient struct {
+ *Client
+ calls []spiedRequest
+}
+
+func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
+ var paramsCopy map[string]interface{}
+ var buf bytes.Buffer
+ json.NewEncoder(&buf).Encode(params)
+ json.NewDecoder(&buf).Decode(¶msCopy)
+ sc.calls = append(sc.calls, spiedRequest{
+ method: method,
+ path: path,
+ params: paramsCopy,
+ })
+ return sc.Client.RequestAndDecode(dst, method, path, body, params)
+}
+
func (s *SiteFSSuite) TestHomeProject(c *check.C) {
f, err := s.fs.Open("/home")
c.Assert(err, check.IsNil)
commit e1bdb7bb963f3754f3aebced8810b062ee4fecf8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Feb 1 09:37:31 2018 -0500
13111: Add /home test case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
new file mode 100644
index 0000000..92d8672
--- /dev/null
+++ b/sdk/go/arvados/fs_project_test.go
@@ -0,0 +1,51 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+func (s *SiteFSSuite) TestHomeProject(c *check.C) {
+ f, err := s.fs.Open("/home")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Not(check.Equals), 0)
+
+ ok := false
+ for _, fi := range fis {
+ c.Check(fi.Name(), check.Not(check.Equals), "")
+ if fi.Name() == "Unrestricted public data" {
+ ok = true
+ }
+ }
+ c.Check(ok, check.Equals, true)
+
+ f, err = s.fs.Open("/home/Unrestricted public data/..")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Check(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+ c.Check(fi.Name(), check.Equals, "home")
+
+ f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+ c.Check(err, check.IsNil)
+ fi, err = f.Stat()
+ c.Check(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+
+ for _, nx := range []string{
+ "/home/A Project",
+ "/home/A Project/does not exist",
+ "/home/Unrestricted public data/does not exist",
+ } {
+ c.Log(nx)
+ f, err = s.fs.Open(nx)
+ c.Check(err, check.NotNil)
+ c.Check(os.IsNotExist(err), check.Equals, true)
+ }
+}
commit 0ff4662fab347279897b285e3ef3320497ced8c9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 16:57:30 2018 -0500
13111: Add projects hierarchy under "home".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
new file mode 100644
index 0000000..e638838
--- /dev/null
+++ b/sdk/go/arvados/fs_deferred.go
@@ -0,0 +1,103 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+ "time"
+)
+
+func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+ var modTime time.Time
+ if coll.ModifiedAt != nil {
+ modTime = *coll.ModifiedAt
+ } else {
+ modTime = time.Now()
+ }
+ placeholder := &treenode{
+ fs: fs,
+ parent: parent,
+ inodes: nil,
+ fileinfo: fileinfo{
+ name: coll.Name,
+ modTime: modTime,
+ mode: 0755 | os.ModeDir,
+ },
+ }
+ return &deferrednode{wrapped: placeholder, create: func() inode {
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ cfs, err := coll.FileSystem(fs, fs)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ root := cfs.rootnode()
+ root.SetParent(parent, coll.Name)
+ return root
+ }}
+}
+
+// A deferrednode wraps an inode that's expensive to build. Initially,
+// it responds to basic directory functions by proxying to the given
+// placeholder. If a caller uses a read/write/lock operation,
+// deferrednode calls the create() func to create the real inode, and
+// proxies to the real inode from then on.
+//
+// In practice, this means a deferrednode's parent's directory listing
+// can be generated using only the placeholder, instead of waiting for
+// create().
+type deferrednode struct {
+ wrapped inode
+ create func() inode
+ mtx sync.Mutex
+ created bool
+}
+
+func (dn *deferrednode) realinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ if !dn.created {
+ dn.wrapped = dn.create()
+ dn.created = true
+ }
+ return dn.wrapped
+}
+
+func (dn *deferrednode) currentinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ return dn.wrapped
+}
+
+func (dn *deferrednode) Read(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Read(p, pos)
+}
+
+func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Write(p, pos)
+}
+
+func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
+ return dn.realinode().Child(name, replace)
+}
+
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() []os.FileInfo { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
new file mode 100644
index 0000000..4dd8699
--- /dev/null
+++ b/sdk/go/arvados/fs_project.go
@@ -0,0 +1,113 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+)
+
+// projectnode exposes an Arvados project as a filesystem directory.
+type projectnode struct {
+ inode
+ uuid string
+ setupOnce sync.Once
+ err error
+}
+
+func (pn *projectnode) setup() {
+ fs := pn.FS().(*siteFileSystem)
+ if pn.uuid == "" {
+ var resp User
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if pn.err != nil {
+ return
+ }
+ pn.uuid = resp.UUID
+ }
+ filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ params := ResourceListParams{
+ Filters: filters,
+ Order: "uuid",
+ }
+ for {
+ var resp CollectionList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, i := range resp.Items {
+ coll := i
+ if coll.Name == "" {
+ continue
+ }
+ pn.inode.Child(coll.Name, func(inode) inode {
+ return deferredCollectionFS(fs, pn, coll)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+
+ filters = append(filters, Filter{"group_class", "=", "project"})
+ params.Filters = filters
+ for {
+ var resp GroupList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, group := range resp.Items {
+ if group.Name == "" || group.Name == "." || group.Name == ".." {
+ continue
+ }
+ pn.inode.Child(group.Name, func(inode) inode {
+ return fs.newProjectNode(pn, group.Name, group.UUID)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+}
+
+func (pn *projectnode) Readdir() []os.FileInfo {
+ pn.setupOnce.Do(pn.setup)
+ return pn.inode.Readdir()
+}
+
+func (pn *projectnode) Child(name string, replace func(inode) inode) inode {
+ pn.setupOnce.Do(pn.setup)
+ if pn.err != nil {
+ log.Printf("BUG: not propagating error setting up %T %v: %s", pn, pn, pn.err)
+ // TODO: propagate error, instead of just being empty
+ return nil
+ }
+ if replace == nil {
+ // lookup
+ return pn.inode.Child(name, nil)
+ }
+ return pn.inode.Child(name, func(existing inode) inode {
+ if repl := replace(existing); repl == nil {
+ // delete
+ // (TODO)
+ return pn.Child(name, nil) // not implemented
+ } else if repl.FileInfo().IsDir() {
+ // mkdir
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ } else {
+ // create file
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ }
+ })
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 37ec8a3..c8d7360 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -38,8 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
inodes: make(map[string]inode),
}
root.inode.Child("by_id", func(inode) inode {
- var vn inode
- vn = &vdirnode{
+ return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -52,7 +51,9 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
create: fs.mountCollection,
}
- return vn
+ })
+ root.inode.Child("home", func(inode) inode {
+ return fs.newProjectNode(fs.root, "home", "")
})
return fs
}
@@ -76,6 +77,22 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
+func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
// vdirnode wraps an inode by ignoring any requests to add/replace
// children, and calling a create() func when a non-existing child is
// looked up.
commit 649c27a98f398984b6aa1f806992cfdd9831c904
Merge: c2e623c cd3bab6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 13:13:36 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit c2e623cbcf93e996b4fb1d29fcf99d954f7f2ad3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Jan 16 16:40:20 2018 -0500
13111: Fix nil pointer dereference at sitefs root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 9f3dbce..37ec8a3 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -91,13 +91,14 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
return vn.inode.Child(name, func(existing inode) inode {
if existing != nil {
return existing
- } else {
- n := vn.create(vn, name)
- if n != nil {
- n.SetParent(vn, name)
- vn.inode.(*treenode).fileinfo.modTime = time.Now()
- }
- return n
+ } else if vn.create == nil {
+ return nil
}
+ n := vn.create(vn, name)
+ if n != nil {
+ n.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ }
+ return n
})
}
commit 183cf741009a42fa3dd2af911c416755e4c5cb0e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Jan 13 15:09:40 2018 -0500
13111: Improve deadlock prevention. Prevent unsupported renames.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fd8c18b..6402a8e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -53,6 +53,10 @@ type FileSystem interface {
rootnode() inode
+ // filesystem-wide lock: used by Rename() to prevent deadlock
+ // while locking multiple inodes.
+ locker() sync.Locker
+
// create a new node with nil parent.
newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
@@ -272,12 +276,17 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
type fileSystem struct {
root inode
fsBackend
+ mutex sync.Mutex
}
func (fs *fileSystem) rootnode() inode {
return fs.root
}
+func (fs *fileSystem) locker() sync.Locker {
+ return &fs.mutex
+}
+
// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.openFile(name, flag, perm)
@@ -424,12 +433,31 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
defer newdirf.Close()
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from
- // filesystem root to newdir, then locking the path from
- // filesystem root to olddir, skipping any already-locked
- // nodes.
+ // TODO: If the nearest common ancestor ("nca") of olddirf and
+ // newdirf is on a different filesystem than fs, we should
+ // call nca.FS().Rename() instead of proceeding. Until then
+ // it's awkward for filesystems to implement their own Rename
+ // methods effectively: the only one that runs is the one on
+ // the root filesystem exposed to the caller (webdav, fuse,
+ // etc).
+
+ // When acquiring locks on multiple inodes, avoid deadlock by
+ // locking the entire containing filesystem first.
+ cfs := olddirf.inode.FS()
+ cfs.locker().Lock()
+ defer cfs.locker().Unlock()
+
+ if cfs != newdirf.inode.FS() {
+ // Moving inodes across filesystems is not (yet)
+ // supported. Locking inodes from different
+ // filesystems could deadlock, so we must error out
+ // now.
+ return ErrInvalidArgument
+ }
+
+ // To ensure we can test reliably whether we're about to move
+ // a directory into itself, lock all potential common
+ // ancestors of olddir and newdir.
needLock := []sync.Locker{}
for _, node := range []inode{olddirf.inode, newdirf.inode} {
needLock = append(needLock, node)
@@ -447,8 +475,11 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- err = nil
+ // Return ErrInvalidOperation if olddirf.inode doesn't even
+ // bother calling our "remove oldname entry" replacer func.
+ err = ErrInvalidArgument
olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ err = nil
if oldinode == nil {
err = os.ErrNotExist
return nil
@@ -458,6 +489,12 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = ErrInvalidArgument
return oldinode
}
+ if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
+ // moving a mount point to a different parent
+ // is not (yet) supported.
+ err = ErrInvalidArgument
+ return oldinode
+ }
accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index d3ca510..9f3dbce 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -20,13 +20,16 @@ type siteFileSystem struct {
// there are significant known bugs and shortcomings. For example,
// writes are not persisted until Sync() is called.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ root := &vdirnode{}
fs := &siteFileSystem{
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ root: root,
},
}
- root := &treenode{
- fs: fs,
+ root.inode = &treenode{
+ fs: fs,
+ parent: root,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -34,13 +37,12 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.parent = root
- root.Child("by_id", func(inode) inode {
+ root.inode.Child("by_id", func(inode) inode {
var vn inode
vn = &vdirnode{
inode: &treenode{
fs: fs,
- parent: root,
+ parent: fs.root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
name: "by_id",
@@ -52,7 +54,6 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
return vn
})
- fs.root = root
return fs
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index a8c369f..26a2212 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -6,6 +6,7 @@ package arvados
import (
"net/http"
+ "os"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -47,6 +48,12 @@ func (s *SiteFSSuite) TestByUUID(c *check.C) {
c.Check(err, check.IsNil)
c.Check(len(fis), check.Equals, 0)
+ err = s.fs.Mkdir("/by_id/"+arvadostest.FooCollection, 0755)
+ c.Check(err, check.Equals, os.ErrExist)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.NonexistentCollection)
+ c.Assert(err, check.Equals, os.ErrNotExist)
+
f, err = s.fs.Open("/by_id/" + arvadostest.FooCollection)
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
@@ -55,4 +62,18 @@ func (s *SiteFSSuite) TestByUUID(c *check.C) {
names = append(names, fi.Name())
}
c.Check(names, check.DeepEquals, []string{"foo"})
+
+ _, err = s.fs.OpenFile("/by_id/"+arvadostest.NonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
+ c.Check(err, check.Equals, ErrInvalidOperation)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection, "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ _, err = s.fs.Stat("/by_id/beep")
+ c.Check(err, check.Equals, os.ErrNotExist)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/"+arvadostest.FooCollection+"/bar")
+ c.Check(err, check.IsNil)
+
+ err = s.fs.Rename("/by_id", "/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
}
commit 4668cf881aef0b21245a419b661b010720035f3c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Jan 13 13:39:19 2018 -0500
13111: Add cgofuse to vendor file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/vendor/vendor.json b/vendor/vendor.json
index aeac93e..1bff878 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -48,6 +48,12 @@
"revisionTime": "2016-12-14T20:08:43Z"
},
{
+ "checksumSHA1": "gMBls0ytB5wHvZizUQE8Eivv9WQ=",
+ "path": "github.com/curoverse/cgofuse/fuse",
+ "revision": "d08d9e36b4ca1364eb7a4eb9db0b7fa76c9250a2",
+ "revisionTime": "2017-12-17T05:18:50Z"
+ },
+ {
"checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
"path": "github.com/docker/distribution/digestset",
"revision": "277ed486c948042cab91ad367c379524f3b25e18",
commit 5d5af52a33ec8b10a9af6afd50141db3923441ec
Merge: c02ceff 707e31d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Jan 13 13:37:48 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit c02ceff00fce94ec5794b53fe890f681acf31121
Merge: 699039c cd9b660
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Jan 7 19:58:07 2018 -0500
13111: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 699039c46b728bf27013adeb19d8fe92eab76071
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 3 03:27:36 2018 -0500
13111: Move SetParent() responsibility from Child() to caller.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 419be7f..fd8c18b 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -103,6 +103,14 @@ type inode interface {
// the child inode is replaced with the one returned by
// replace().
//
+ // If replace(x) returns an inode (besides x or nil) that is
+ // subsequently returned by Child(), then Child()'s caller
+ // must ensure the new child's name and parent are set/updated
+ // to Child()'s name argument and its receiver respectively.
+ // This is not necessarily done before replace(x) returns, but
+ // it must be done before Child()'s caller releases the
+ // parent's lock.
+ //
// Nil represents "no child". replace(nil) signifies that no
// child with this name exists yet. If replace() returns nil,
// the existing child should be deleted if possible.
@@ -232,7 +240,6 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
if newchild == nil {
delete(n.inodes, name)
} else if newchild != child {
- newchild.SetParent(n, name)
n.inodes[name] = newchild
n.fileinfo.modTime = time.Now()
child = newchild
@@ -323,6 +330,7 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
var err error
n = parent.Child(name, func(inode) inode {
n, err = parent.FS().newNode(name, perm|0755, time.Now())
+ n.SetParent(parent, name)
return n
})
if err != nil {
@@ -371,6 +379,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
}
child := n.Child(name, func(inode) (child inode) {
child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
+ child.SetParent(n, name)
return
})
if err != nil {
@@ -464,6 +473,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// Leave oldinode in olddir.
return oldinode
}
+ accepted.SetParent(newdirf.inode, newname)
return nil
})
return err
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 27ea904..fbd9775 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -532,19 +532,14 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
}
data, err := json.Marshal(&coll)
if err == nil {
- data = append(data, 10)
+ data = append(data, '\n')
}
return data, err
}}
gn.SetParent(dn, name)
return gn
}
- oldchild := dn.treenode.Child(name, nil)
- child := dn.treenode.Child(name, replace)
- if child != nil && child != oldchild {
- child.SetParent(dn, name)
- }
- return child
+ return dn.treenode.Child(name, replace)
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -844,8 +839,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
}
node.Child(name, func(child inode) inode {
if child == nil {
- node, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
- child = node
+ child, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child.SetParent(node, name)
+ node = child
} else if !child.IsDir() {
err = ErrFileExists
} else {
@@ -861,6 +857,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
switch child := child.(type) {
case nil:
child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+ child.SetParent(node, basename)
fn = child.(*filenode)
return child
case *filenode:
commit 5571e3a20a88a1d9b4dd61df7a5fa9be674b2562
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 3 01:20:47 2018 -0500
13111: Port "rename a to a/b/c" fix from 11377b3b.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index a3df168..419be7f 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -444,6 +444,11 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = os.ErrNotExist
return nil
}
+ if locked[oldinode] {
+ // oldinode cannot become a descendant of itself.
+ err = ErrInvalidArgument
+ return oldinode
+ }
accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e7e6f19..27ea904 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -539,7 +539,12 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
gn.SetParent(dn, name)
return gn
}
- return dn.treenode.Child(name, replace)
+ oldchild := dn.treenode.Child(name, nil)
+ child := dn.treenode.Child(name, replace)
+ if child != nil && child != oldchild {
+ child.SetParent(dn, name)
+ }
+ return child
}
// sync flushes in-memory data (for all files in the tree rooted at
commit a7f0a9c188fd9c2bfe52b421fd30691631e63875
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:23:32 2017 -0500
13111: Update parent dir modtime when adding/removing/renaming.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 9978c43..a3df168 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -232,8 +232,9 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
if newchild == nil {
delete(n.inodes, name)
} else if newchild != child {
- n.inodes[name] = newchild
newchild.SetParent(n, name)
+ n.inodes[name] = newchild
+ n.fileinfo.modTime = time.Now()
child = newchild
}
}
@@ -458,8 +459,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// Leave oldinode in olddir.
return oldinode
}
- //TODO: olddirf.setModTime(time.Now())
- //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 0121d2d..e7e6f19 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -67,10 +67,23 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ backdateTree(root, modTime)
fs.root = root
return fs, nil
}
+func backdateTree(n inode, modTime time.Time) {
+ switch n := n.(type) {
+ case *filenode:
+ n.fileinfo.modTime = modTime
+ case *dirnode:
+ n.fileinfo.modTime = modTime
+ for _, n := range n.inodes {
+ backdateTree(n, modTime)
+ }
+ }
+}
+
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 8a54b44..d3ca510 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -94,6 +94,7 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
n := vn.create(vn, name)
if n != nil {
n.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
}
return n
}
commit 840e18117654b70c8cdb28888f227810369f82c1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:07:22 2017 -0500
13111: Sync name when changing parents.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index ebf2ad6..9978c43 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -85,7 +85,7 @@ type FileSystem interface {
}
type inode interface {
- SetParent(inode)
+ SetParent(parent inode, name string)
Parent() inode
FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
@@ -207,10 +207,11 @@ func (n *treenode) FS() FileSystem {
return n.fs
}
-func (n *treenode) SetParent(p inode) {
- n.RLock()
- defer n.RUnlock()
+func (n *treenode) SetParent(p inode, name string) {
+ n.Lock()
+ defer n.Unlock()
n.parent = p
+ n.fileinfo.name = name
}
func (n *treenode) Parent() inode {
@@ -232,7 +233,7 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
delete(n.inodes, name)
} else if newchild != child {
n.inodes[name] = newchild
- newchild.SetParent(n)
+ newchild.SetParent(n, name)
child = newchild
}
}
@@ -442,28 +443,21 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
err = os.ErrNotExist
return nil
}
- newdirf.inode.Child(newname, func(existing inode) inode {
+ accepted := newdirf.inode.Child(newname, func(existing inode) inode {
if existing != nil && existing.IsDir() {
err = ErrIsDirectory
return existing
}
return oldinode
})
- if err != nil {
+ if accepted != oldinode {
+ if err == nil {
+ // newdirf didn't accept oldinode.
+ err = ErrInvalidArgument
+ }
+ // Leave oldinode in olddir.
return oldinode
}
- oldinode.Lock()
- defer oldinode.Unlock()
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdirf.inode
- n.fileinfo.name = newname
- case *filenode:
- n.parent = newdirf.inode
- n.fileinfo.name = newname
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
//TODO: olddirf.setModTime(time.Now())
//TODO: newdirf.setModTime(time.Now())
return nil
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 96977cb..0121d2d 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -63,7 +63,7 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
inodes: make(map[string]inode),
},
}
- root.SetParent(root)
+ root.SetParent(root, ".")
if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -219,10 +219,11 @@ func (fn *filenode) appendSegment(e segment) {
fn.fileinfo.size += int64(e.Len())
}
-func (fn *filenode) SetParent(p inode) {
- fn.RLock()
- defer fn.RUnlock()
+func (fn *filenode) SetParent(p inode, name string) {
+ fn.Lock()
+ defer fn.Unlock()
fn.parent = p
+ fn.fileinfo.name = name
}
func (fn *filenode) Parent() inode {
@@ -522,7 +523,7 @@ func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
}
return data, err
}}
- gn.SetParent(dn)
+ gn.SetParent(dn, name)
return gn
}
return dn.treenode.Child(name, replace)
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 974dad8..8a54b44 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -71,8 +71,7 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return nil
}
root := cfs.rootnode()
- root.SetParent(parent)
- root.(*dirnode).fileinfo.name = id
+ root.SetParent(parent, id)
return root
}
@@ -94,7 +93,7 @@ func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
} else {
n := vn.create(vn, name)
if n != nil {
- n.SetParent(vn)
+ n.SetParent(vn, name)
}
return n
}
commit af83c3a047bebc48db9490f3523208c4c3f87b6f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 21 01:01:44 2017 -0500
13111: Fix collectionfs crash.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index af0068f..96977cb 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -480,7 +480,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
+ locator, _, err := fn.FS().PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -489,7 +489,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.FS(),
+ kc: fn.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
commit 504e0eb8c75d79e5664897709935512f2c59ff95
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:31 2017 -0500
13111: Tidy up, add comments.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs_backend.go
new file mode 100644
index 0000000..301f0b4
--- /dev/null
+++ b/sdk/go/arvados/fs_backend.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "io"
+
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
+type keepClient interface {
+ ReadAt(locator string, p []byte, off int) (int, error)
+ PutB(p []byte) (string, int, error)
+}
+
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index a64ce8c..af0068f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -21,28 +21,6 @@ import (
var maxBlockSize = 1 << 26
-type fsBackend interface {
- keepClient
- apiClient
-}
-
-// Ideally *Client would do everything; meanwhile keepBackend
-// implements fsBackend by merging the two kinds of arvados client.
-type keepBackend struct {
- keepClient
- apiClient
-}
-
-type keepClient interface {
- ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
-}
-
-type apiClient interface {
- RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -55,6 +33,11 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
+type collectionFileSystem struct {
+ fileSystem
+ uuid string
+}
+
// FileSystem returns a CollectionFileSystem for the collection.
func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
@@ -88,11 +71,6 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
return fs, nil
}
-type collectionFileSystem struct {
- fileSystem
- uuid string
-}
-
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
@@ -142,27 +120,6 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
- if dn == dn.fs.rootnode() && name == ".arvados#collection" {
- gn := &getternode{Getter: func() ([]byte, error) {
- var coll Collection
- var err error
- coll.ManifestText, err = dn.fs.MarshalManifest(".")
- if err != nil {
- return nil, err
- }
- data, err := json.Marshal(&coll)
- if err == nil {
- data = append(data, 10)
- }
- return data, err
- }}
- gn.SetParent(dn)
- return gn
- }
- return dn.treenode.Child(name, replace)
-}
-
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
@@ -550,6 +507,27 @@ func (dn *dirnode) FS() FileSystem {
return dn.fs
}
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ gn.SetParent(dn)
+ return gn
+ }
+ return dn.treenode.Child(name, replace)
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index fd842f0..974dad8 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -56,11 +56,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- return nil, ErrInvalidOperation
-}
-
-func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
commit 0f2575bc8348637a02bc0165456886ce7f3b57bf
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 22:01:09 2017 -0500
13111: Combine newDirnode and newFIlenode to just newNode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index fe185a4..ebf2ad6 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -52,8 +52,9 @@ type FileSystem interface {
fsBackend
rootnode() inode
- newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
- newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+
+ // create a new node with nil parent.
+ newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -319,11 +320,7 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- if perm.IsDir() {
- n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
- } else {
- n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
- }
+ n, err = parent.FS().newNode(name, perm|0755, time.Now())
return n
})
if err != nil {
@@ -371,7 +368,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = n.FS().newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
return
})
if err != nil {
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 36a92af..a64ce8c 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -93,37 +93,32 @@ type collectionFileSystem struct {
uuid string
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
if name == "" || name == "." || name == ".." {
return nil, ErrInvalidArgument
}
- return &dirnode{
- fs: fs,
- treenode: treenode{
+ if perm.IsDir() {
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+ } else {
+ return &filenode{
+ fs: fs,
fileinfo: fileinfo{
name: name,
- mode: perm | os.ModeDir,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
+ }, nil
}
- return &filenode{
- fs: fs,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
}
func (fs *collectionFileSystem) Sync() error {
@@ -852,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
}
node.Child(name, func(child inode) inode {
if child == nil {
- node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ node, err = node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
} else if !child.IsDir() {
err = ErrFileExists
@@ -868,7 +863,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
fn = child.(*filenode)
return child
case *filenode:
commit a0a02d2f4180f908a39293ef4adb00f927b7ad6f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 20 17:11:07 2017 -0500
13111: Clear up some type assertions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 8d987d4..fe185a4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -51,6 +51,7 @@ type FileSystem interface {
http.FileSystem
fsBackend
+ rootnode() inode
newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
@@ -83,6 +84,7 @@ type FileSystem interface {
}
type inode interface {
+ SetParent(inode)
Parent() inode
FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
@@ -204,6 +206,12 @@ func (n *treenode) FS() FileSystem {
return n.fs
}
+func (n *treenode) SetParent(p inode) {
+ n.RLock()
+ defer n.RUnlock()
+ n.parent = p
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -218,11 +226,13 @@ func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
// TODO: special treatment for "", ".", ".."
child = n.inodes[name]
if replace != nil {
- child = replace(child)
- if child == nil {
+ newchild := replace(child)
+ if newchild == nil {
delete(n.inodes, name)
- } else {
- n.inodes[name] = child
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ newchild.SetParent(n)
+ child = newchild
}
}
return
@@ -254,6 +264,10 @@ type fileSystem struct {
fsBackend
}
+func (fs *fileSystem) rootnode() inode {
+ return fs.root
+}
+
// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.openFile(name, flag, perm)
@@ -306,9 +320,9 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
var err error
n = parent.Child(name, func(inode) inode {
if perm.IsDir() {
- n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
+ n, err = parent.FS().newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -357,7 +371,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
return os.ErrExist
}
child := n.Child(name, func(inode) (child inode) {
- child, err = fs.newDirnode(n, name, perm, time.Now())
+ child, err = n.FS().newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -404,12 +418,12 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
// When acquiring locks on multiple nodes, all common
// ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
+ // deadlock. This is assured by locking the path from
+ // filesystem root to newdir, then locking the path from
+ // filesystem root to olddir, skipping any already-locked
+ // nodes.
needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
+ for _, node := range []inode{olddirf.inode, newdirf.inode} {
needLock = append(needLock, node)
for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
@@ -499,41 +513,6 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
return err
}
-// Caller must have parent lock, and must have already ensured
-// parent.Child(name,nil) is nil.
-func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- treenode: treenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- fs: parent.FS(),
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
func (fs *fileSystem) Sync() error {
log.Printf("TODO: sync fileSystem")
return ErrInvalidOperation
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index fc00335..36a92af 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -64,14 +64,14 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
modTime = *c.ModifiedAt
}
fs := &collectionFileSystem{
+ uuid: c.UUID,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: client, keepClient: kc},
},
- uuid: c.UUID,
}
- dn := &dirnode{
+ root := &dirnode{
+ fs: fs,
treenode: treenode{
- fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
@@ -80,11 +80,11 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
inodes: make(map[string]inode),
},
}
- dn.parent = dn
- fs.fileSystem.root = dn
- if err := dn.loadManifest(c.ManifestText); err != nil {
+ root.SetParent(root)
+ if err := root.loadManifest(c.ManifestText); err != nil {
return nil, err
}
+ fs.root = root
return fs, nil
}
@@ -93,6 +93,39 @@ type collectionFileSystem struct {
uuid string
}
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *collectionFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *collectionFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: fs,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
func (fs *collectionFileSystem) Sync() error {
log.Printf("cfs.Sync()")
if fs.uuid == "" {
@@ -114,12 +147,12 @@ func (fs *collectionFileSystem) Sync() error {
return err
}
-func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
- if name == ".arvados#collection" {
- return &getternode{Getter: func() ([]byte, error) {
+func (dn *dirnode) Child(name string, replace func(inode) inode) inode {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
var coll Collection
var err error
- coll.ManifestText, err = fs.MarshalManifest(".")
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
if err != nil {
return nil, err
}
@@ -129,8 +162,10 @@ func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) in
}
return data, err
}}
+ gn.SetParent(dn)
+ return gn
}
- return fs.fileSystem.root.Child(name, replace)
+ return dn.treenode.Child(name, replace)
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
@@ -232,6 +267,12 @@ func (fn *filenode) appendSegment(e segment) {
fn.fileinfo.size += int64(e.Len())
}
+func (fn *filenode) SetParent(p inode) {
+ fn.RLock()
+ defer fn.RUnlock()
+ fn.parent = p
+}
+
func (fn *filenode) Parent() inode {
fn.RLock()
defer fn.RUnlock()
@@ -496,7 +537,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.(fsBackend),
+ kc: fn.parent.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -506,9 +547,14 @@ func (fn *filenode) pruneMemSegments() {
}
type dirnode struct {
+ fs *collectionFileSystem
treenode
}
+func (dn *dirnode) FS() FileSystem {
+ return dn.fs
+}
+
// sync flushes in-memory data (for all files in the tree rooted at
// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
@@ -801,7 +847,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
// can't be sure parent will be a *dirnode
return nil, ErrInvalidArgument
}
- node = node.Parent().(*dirnode)
+ node = node.Parent()
continue
}
node.Child(name, func(child inode) inode {
@@ -832,7 +878,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
err = ErrIsDirectory
return child
default:
- err = ErrInvalidOperation
+ err = ErrInvalidArgument
return child
}
})
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 5326131..fd842f0 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,17 +9,21 @@ import (
"time"
)
+type siteFileSystem struct {
+ fileSystem
+}
+
// SiteFileSystem returns a FileSystem that maps collections and other
// Arvados objects onto a filesystem layout.
//
// This is experimental: the filesystem layout is not stable, and
// there are significant known bugs and shortcomings. For example,
-// although the FileSystem allows files to be added and modified in
-// collections, these changes are not persistent or visible to other
-// Arvados clients.
+// writes are not persisted until Sync() is called.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
- fs := &fileSystem{
- fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ fs := &siteFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ },
}
root := &treenode{
fs: fs,
@@ -34,7 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root.Child("by_id", func(inode) inode {
var vn inode
vn = &vdirnode{
- treenode: treenode{
+ inode: &treenode{
fs: fs,
parent: root,
inodes: make(map[string]inode),
@@ -44,9 +48,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
mode: 0755 | os.ModeDir,
},
},
- create: func(name string) inode {
- return newEntByID(vn, name)
- },
+ create: fs.mountCollection,
}
return vn
})
@@ -54,33 +56,51 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
-func newEntByID(parent inode, id string) inode {
+func (fs *siteFileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
- err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(parent.FS(), parent.FS())
+ cfs, err := coll.FileSystem(fs, fs)
if err != nil {
return nil
}
- root := fs.(*collectionFileSystem).root.(*dirnode)
- root.fileinfo.name = id
- root.parent = parent
+ root := cfs.rootnode()
+ root.SetParent(parent)
+ root.(*dirnode).fileinfo.name = id
return root
}
+// vdirnode wraps an inode by ignoring any requests to add/replace
+// children, and calling a create() func when a non-existing child is
+// looked up.
+//
+// create() can return either a new node, which will be added to the
+// treenode, or nil for ENOENT.
type vdirnode struct {
- treenode
- create func(string) inode
+ inode
+ create func(parent inode, name string) inode
}
func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
- return vn.treenode.Child(name, func(existing inode) inode {
+ return vn.inode.Child(name, func(existing inode) inode {
if existing != nil {
return existing
} else {
- return vn.create(name)
+ n := vn.create(vn, name)
+ if n != nil {
+ n.SetParent(vn)
+ }
+ return n
}
})
}
commit 18c8fa2337a2db04ee6060184885731f4b5c7d7b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 19 09:30:56 2017 -0500
13111: Save on sync().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 3be3f5e..8d987d4 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "log"
"net/http"
"os"
"path"
@@ -40,6 +41,7 @@ type File interface {
Readdir(int) ([]os.FileInfo, error)
Stat() (os.FileInfo, error)
Truncate(int64) error
+ Sync() error
}
// A FileSystem is an http.Filesystem plus Stat() and support for
@@ -47,8 +49,10 @@ type File interface {
// goroutines.
type FileSystem interface {
http.FileSystem
+ fsBackend
- inode
+ newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+ newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error)
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -75,10 +79,12 @@ type FileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+ Sync() error
}
type inode interface {
Parent() inode
+ FS() FileSystem
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
@@ -186,6 +192,7 @@ func (*nullnode) Child(name string, replace func(inode) inode) inode {
}
type treenode struct {
+ fs FileSystem
parent inode
inodes map[string]inode
fileinfo fileinfo
@@ -193,6 +200,10 @@ type treenode struct {
nullnode
}
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
func (n *treenode) Parent() inode {
n.RLock()
defer n.RUnlock()
@@ -239,7 +250,8 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
type fileSystem struct {
- inode
+ root inode
+ fsBackend
}
// OpenFile is analogous to os.OpenFile().
@@ -248,12 +260,11 @@ func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, e
}
func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
if flag&os.O_SYNC != 0 {
return nil, ErrSyncNotSupported
}
dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
+ parent := rlookup(fs.root, dirname)
if parent == nil {
return nil, os.ErrNotExist
}
@@ -294,20 +305,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
}
var err error
n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ n, err = fs.newDirnode(parent, name, perm|0755, time.Now())
} else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ n, err = fs.newFilenode(parent, name, perm|0755, time.Now())
}
return n
})
@@ -322,10 +323,10 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
} else if flag&os.O_TRUNC != 0 {
if !writable {
return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
+ } else if n.IsDir() {
return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
}
}
return &filehandle{
@@ -346,7 +347,7 @@ func (fs *fileSystem) Create(name string) (File, error) {
func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
+ n := rlookup(fs.root, dirname)
if n == nil {
return os.ErrNotExist
}
@@ -355,12 +356,8 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
if n.Child(name, nil) != nil {
return os.ErrExist
}
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
+ child, err = fs.newDirnode(n, name, perm, time.Now())
return
})
if err != nil {
@@ -372,7 +369,7 @@ func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
+ node := rlookup(fs.root, name)
if node == nil {
err = os.ErrNotExist
} else {
@@ -414,7 +411,7 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
for _, f := range []*filehandle{olddirf, newdirf} {
node := f.inode
needLock = append(needLock, node)
- for node.Parent() != node {
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
node = node.Parent()
needLock = append(needLock, node)
}
@@ -428,10 +425,6 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
}
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
err = nil
olddirf.inode.Child(oldname, func(oldinode inode) inode {
if oldinode == nil {
@@ -450,20 +443,18 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
}
oldinode.Lock()
defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
+ n.fileinfo.name = newname
case *filenode:
- n.parent = newdn
+ n.parent = newdirf.inode
n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
+ //TODO: olddirf.setModTime(time.Now())
+ //TODO: newdirf.setModTime(time.Now())
return nil
})
return err
@@ -488,7 +479,7 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
if name == "" || name == "." || name == ".." {
return ErrInvalidArgument
}
- dir := rlookup(fs, dirname)
+ dir := rlookup(fs.root, dirname)
if dir == nil {
return os.ErrNotExist
}
@@ -507,3 +498,70 @@ func (fs *fileSystem) remove(name string, recursive bool) (err error) {
})
return err
}
+
+// Caller must have parent lock, and must have already ensured
+// parent.Child(name,nil) is nil.
+func (fs *fileSystem) newDirnode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &dirnode{
+ treenode: treenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
+}
+
+func (fs *fileSystem) newFilenode(parent inode, name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ fs: parent.FS(),
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
+ }
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ }
+ return
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index e451189..fc00335 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"os"
"path"
"regexp"
@@ -20,11 +21,28 @@ import (
var maxBlockSize = 1 << 26
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
+
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -38,28 +56,32 @@ type CollectionFileSystem interface {
}
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
} else {
modTime = *c.ModifiedAt
}
+ fs := &collectionFileSystem{
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ uuid: c.UUID,
+ }
dn := &dirnode{
- client: client,
- kc: kc,
treenode: treenode{
+ fs: fs,
fileinfo: fileinfo{
name: ".",
mode: os.ModeDir | 0755,
modTime: modTime,
},
- parent: nil,
inodes: make(map[string]inode),
},
}
dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ fs.fileSystem.root = dn
if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
@@ -68,9 +90,31 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
type collectionFileSystem struct {
fileSystem
+ uuid string
}
-func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
+}
+
+func (fs *collectionFileSystem) Child(name string, replace func(inode) inode) inode {
if name == ".arvados#collection" {
return &getternode{Getter: func() ([]byte, error) {
var coll Collection
@@ -86,13 +130,13 @@ func (fs collectionFileSystem) Child(name string, replace func(inode) inode) ino
return data, err
}}
}
- return fs.fileSystem.Child(name, replace)
+ return fs.fileSystem.root.Child(name, replace)
}
-func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
- fs.fileSystem.inode.Lock()
- defer fs.fileSystem.inode.Unlock()
- return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -170,8 +214,9 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
// filenode implements inode.
type filenode struct {
+ parent inode
+ fs FileSystem
fileinfo fileinfo
- parent *dirnode
segments []segment
// number of times `segments` has changed in a
// way that might invalidate a filenodePtr
@@ -193,6 +238,10 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -438,7 +487,7 @@ func (fn *filenode) pruneMemSegments() {
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.parent.(fsBackend).PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -447,7 +496,7 @@ func (fn *filenode) pruneMemSegments() {
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.parent.(fsBackend),
locator: locator,
size: seg.Len(),
offset: 0,
@@ -458,8 +507,6 @@ func (fn *filenode) pruneMemSegments() {
type dirnode struct {
treenode
- client *Client
- kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -480,7 +527,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
@@ -488,7 +535,7 @@ func (dn *dirnode) sync() error {
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
@@ -709,7 +756,7 @@ func (dn *dirnode) loadManifest(txt string) error {
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
@@ -738,7 +785,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
- node := dn
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -758,16 +805,13 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
continue
}
node.Child(name, func(child inode) inode {
- switch child.(type) {
- case nil:
- node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if child == nil {
+ node, err = node.FS().newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
child = node
- case *dirnode:
- node = child.(*dirnode)
- case *filenode:
+ } else if !child.IsDir() {
err = ErrFileExists
- default:
- err = ErrInvalidOperation
+ } else {
+ node = child
}
return child
})
@@ -778,8 +822,9 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node.Child(basename, func(child inode) inode {
switch child := child.(type) {
case nil:
- fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
- return fn
+ child, err = node.FS().newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ fn = child.(*filenode)
+ return child
case *filenode:
fn = child
return child
@@ -794,68 +839,6 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return
}
-// rlookup (recursive lookup) returns the inode for the file/directory
-// with the given name (which may contain "/" separators). If no such
-// file/directory exists, the returned node is nil.
-func rlookup(start inode, path string) (node inode) {
- node = start
- for _, name := range strings.Split(path, "/") {
- if node == nil {
- break
- }
- if node.IsDir() {
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- }
- node = func() inode {
- node.RLock()
- defer node.RUnlock()
- return node.Child(name, nil)
- }()
- }
- return
-}
-
-// Caller must have lock, and must have already ensured
-// Children(name,nil) is nil.
-func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &dirnode{
- client: dn.client,
- kc: dn.kc,
- treenode: treenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm | os.ModeDir,
- modTime: modTime,
- },
- inodes: make(map[string]inode),
- },
- }, nil
-}
-
-func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
- if name == "" || name == "." || name == ".." {
- return nil, ErrInvalidArgument
- }
- return &filenode{
- parent: parent,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
@@ -920,7 +903,7 @@ func (me *memSegment) ReadAt(p []byte, off int64) (n int, err error) {
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index 56963b6..d586531 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -97,3 +97,8 @@ func (f *filehandle) Stat() (os.FileInfo, error) {
func (f *filehandle) Close() error {
return nil
}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 66856b7..5326131 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -18,7 +18,11 @@ import (
// collections, these changes are not persistent or visible to other
// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ fs := &fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ }
root := &treenode{
+ fs: fs,
fileinfo: fileinfo{
name: "/",
mode: os.ModeDir | 0755,
@@ -28,8 +32,10 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
}
root.parent = root
root.Child("by_id", func(inode) inode {
- return &vdirnode{
+ var vn inode
+ vn = &vdirnode{
treenode: treenode{
+ fs: fs,
parent: root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
@@ -39,25 +45,29 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
},
create: func(name string) inode {
- return newEntByID(c, kc, name)
+ return newEntByID(vn, name)
},
}
+ return vn
})
- return &fileSystem{inode: root}
+ fs.root = root
+ return fs
}
-func newEntByID(c *Client, kc keepClient, id string) inode {
+func newEntByID(parent inode, id string) inode {
var coll Collection
- err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ err := parent.FS().RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- fs, err := coll.FileSystem(c, kc)
- fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ fs, err := coll.FileSystem(parent.FS(), parent.FS())
if err != nil {
return nil
}
- return fs
+ root := fs.(*collectionFileSystem).root.(*dirnode)
+ root.fileinfo.name = id
+ root.parent = parent
+ return root
}
type vdirnode struct {
commit 67bd03fc4c2b3fbb25613c76a9470b53ebaa832a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:54:36 2017 -0500
13111: Move code from fs_collection to fs_base and fs_filehandle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 4de1bf8..3be3f5e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -5,10 +5,152 @@
package arvados
import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
"os"
+ "path"
+ "strings"
"sync"
+ "time"
)
+var (
+ ErrReadOnlyFile = errors.New("read-only file")
+ ErrNegativeOffset = errors.New("cannot seek to negative offset")
+ ErrFileExists = errors.New("file exists")
+ ErrInvalidOperation = errors.New("invalid operation")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
+ ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrPermission = os.ErrPermission
+)
+
+// A File is an *os.File-like interface for reading and writing files
+// in a FileSystem.
+type File interface {
+ io.Reader
+ io.Writer
+ io.Closer
+ io.Seeker
+ Size() int64
+ Readdir(int) ([]os.FileInfo, error)
+ Stat() (os.FileInfo, error)
+ Truncate(int64) error
+}
+
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
+ http.FileSystem
+
+ inode
+
+ // analogous to os.Stat()
+ Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
+ Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
+ OpenFile(name string, flag int, perm os.FileMode) (File, error)
+
+ Mkdir(name string, perm os.FileMode) error
+ Remove(name string) error
+ RemoveAll(name string) error
+ Rename(oldname, newname string) error
+}
+
+type inode interface {
+ Parent() inode
+ Read([]byte, filenodePtr) (int, filenodePtr, error)
+ Write([]byte, filenodePtr) (int, filenodePtr, error)
+ Truncate(int64) error
+ IsDir() bool
+ Readdir() []os.FileInfo
+ Size() int64
+ FileInfo() os.FileInfo
+
+ // Child() performs lookups and updates of named child nodes.
+ //
+ // If replace is non-nil, Child calls replace(x) where x is
+ // the current child inode with the given name. If possible,
+ // the child inode is replaced with the one returned by
+ // replace().
+ //
+ // Nil represents "no child". replace(nil) signifies that no
+ // child with this name exists yet. If replace() returns nil,
+ // the existing child should be deleted if possible.
+ //
+ // An implementation of Child() is permitted to ignore
+ // replace() or its return value. For example, a regular file
+ // inode does not have children, so Child() always returns
+ // nil.
+ //
+ // Child() returns the child, if any, with the given name: if
+ // a child was added or changed, the new child is returned.
+ //
+ // Caller must have lock (or rlock if replace is nil).
+ Child(name string, replace func(inode) inode) inode
+
+ sync.Locker
+ RLock()
+ RUnlock()
+}
+
+type fileinfo struct {
+ name string
+ mode os.FileMode
+ size int64
+ modTime time.Time
+}
+
+// Name implements os.FileInfo.
+func (fi fileinfo) Name() string {
+ return fi.name
+}
+
+// ModTime implements os.FileInfo.
+func (fi fileinfo) ModTime() time.Time {
+ return fi.modTime
+}
+
+// Mode implements os.FileInfo.
+func (fi fileinfo) Mode() os.FileMode {
+ return fi.mode
+}
+
+// IsDir implements os.FileInfo.
+func (fi fileinfo) IsDir() bool {
+ return fi.mode&os.ModeDir != 0
+}
+
+// Size implements os.FileInfo.
+func (fi fileinfo) Size() int64 {
+ return fi.size
+}
+
+// Sys implements os.FileInfo.
+func (fi fileinfo) Sys() interface{} {
+ return nil
+}
+
type nullnode struct{}
func (*nullnode) Mkdir(string, os.FileMode) error {
@@ -95,3 +237,273 @@ func (n *treenode) Readdir() (fi []os.FileInfo) {
}
return
}
+
+type fileSystem struct {
+ inode
+}
+
+// OpenFile is analogous to os.OpenFile().
+func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
+}
+
+func (fs *fileSystem) Open(name string) (http.File, error) {
+ return fs.OpenFile(name, os.O_RDONLY, 0)
+}
+
+func (fs *fileSystem) Create(name string) (File, error) {
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
+}
+
+func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
+ node := rlookup(fs.inode, name)
+ if node == nil {
+ err = os.ErrNotExist
+ } else {
+ fi = node.FileInfo()
+ }
+ return
+}
+
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ oldinode.Lock()
+ defer oldinode.Unlock()
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ n.treenode.fileinfo.name = newname
+ case *filenode:
+ n.parent = newdn
+ n.fileinfo.name = newname
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ olddn.treenode.fileinfo.modTime = time.Now()
+ newdn.treenode.fileinfo.modTime = time.Now()
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 72ef49f..e451189 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -6,10 +6,8 @@ package arvados
import (
"encoding/json"
- "errors"
"fmt"
"io"
- "net/http"
"os"
"path"
"regexp"
@@ -20,111 +18,13 @@ import (
"time"
)
-var (
- ErrReadOnlyFile = errors.New("read-only file")
- ErrNegativeOffset = errors.New("cannot seek to negative offset")
- ErrFileExists = errors.New("file exists")
- ErrInvalidOperation = errors.New("invalid operation")
- ErrInvalidArgument = errors.New("invalid argument")
- ErrDirectoryNotEmpty = errors.New("directory not empty")
- ErrWriteOnlyMode = errors.New("file is O_WRONLY")
- ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
- ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
- ErrPermission = os.ErrPermission
-
- maxBlockSize = 1 << 26
-)
-
-// A File is an *os.File-like interface for reading and writing files
-// in a CollectionFileSystem.
-type File interface {
- io.Reader
- io.Writer
- io.Closer
- io.Seeker
- Size() int64
- Readdir(int) ([]os.FileInfo, error)
- Stat() (os.FileInfo, error)
- Truncate(int64) error
-}
+var maxBlockSize = 1 << 26
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
PutB(p []byte) (string, int, error)
}
-type fileinfo struct {
- name string
- mode os.FileMode
- size int64
- modTime time.Time
-}
-
-// Name implements os.FileInfo.
-func (fi fileinfo) Name() string {
- return fi.name
-}
-
-// ModTime implements os.FileInfo.
-func (fi fileinfo) ModTime() time.Time {
- return fi.modTime
-}
-
-// Mode implements os.FileInfo.
-func (fi fileinfo) Mode() os.FileMode {
- return fi.mode
-}
-
-// IsDir implements os.FileInfo.
-func (fi fileinfo) IsDir() bool {
- return fi.mode&os.ModeDir != 0
-}
-
-// Size implements os.FileInfo.
-func (fi fileinfo) Size() int64 {
- return fi.size
-}
-
-// Sys implements os.FileInfo.
-func (fi fileinfo) Sys() interface{} {
- return nil
-}
-
-// A FileSystem is an http.Filesystem plus Stat() and support for
-// opening writable files. All methods are safe to call from multiple
-// goroutines.
-type FileSystem interface {
- http.FileSystem
-
- inode
-
- // analogous to os.Stat()
- Stat(name string) (os.FileInfo, error)
-
- // analogous to os.Create(): create/truncate a file and open it O_RDWR.
- Create(name string) (File, error)
-
- // Like os.OpenFile(): create or open a file or directory.
- //
- // If flag&os.O_EXCL==0, it opens an existing file or
- // directory if one exists. If flag&os.O_CREATE!=0, it creates
- // a new empty file or directory if one does not already
- // exist.
- //
- // When creating a new item, perm&os.ModeDir determines
- // whether it is a file or a directory.
- //
- // A file can be opened multiple times and used concurrently
- // from multiple goroutines. However, each File object should
- // be used by only one goroutine at a time.
- OpenFile(name string, flag int, perm os.FileMode) (File, error)
-
- Mkdir(name string, perm os.FileMode) error
- Remove(name string) error
- RemoveAll(name string) error
- Rename(oldname, newname string) error
-}
-
// A CollectionFileSystem is a FileSystem that can be serialized as a
// manifest and stored as a collection.
type CollectionFileSystem interface {
@@ -137,8 +37,33 @@ type CollectionFileSystem interface {
MarshalManifest(prefix string) (string, error)
}
-type fileSystem struct {
- inode
+// FileSystem returns a CollectionFileSystem for the collection.
+func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+ var modTime time.Time
+ if c.ModifiedAt == nil {
+ modTime = time.Now()
+ } else {
+ modTime = *c.ModifiedAt
+ }
+ dn := &dirnode{
+ client: client,
+ kc: kc,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
+ },
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
+ return nil, err
+ }
+ return fs, nil
}
type collectionFileSystem struct {
@@ -170,301 +95,6 @@ func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
}
-// OpenFile is analogous to os.OpenFile().
-func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.openFile(name, flag, perm)
-}
-
-func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- var dn inode = fs.inode
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- parent := rlookup(dn, dirname)
- if parent == nil {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable && parent.IsDir() {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: parent}, nil
- case "..":
- return &filehandle{inode: parent.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- parent.Lock()
- defer parent.Unlock()
- } else {
- parent.RLock()
- defer parent.RUnlock()
- }
- n := parent.Child(name, nil)
- if n == nil {
- if !createMode {
- return nil, os.ErrNotExist
- }
- var err error
- n = parent.Child(name, func(inode) inode {
- var dn *dirnode
- switch parent := parent.(type) {
- case *dirnode:
- dn = parent
- case *collectionFileSystem:
- dn = parent.inode.(*dirnode)
- default:
- err = ErrInvalidArgument
- return nil
- }
- if perm.IsDir() {
- n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
- } else {
- n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
- }
- return n
- })
- if err != nil {
- return nil, err
- } else if n == nil {
- // parent rejected new child
- return nil, ErrInvalidOperation
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
- }, nil
-}
-
-func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.OpenFile(name, os.O_RDONLY, 0)
-}
-
-func (fs *fileSystem) Create(name string) (File, error) {
- return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
-}
-
-func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
- dirname, name := path.Split(name)
- n := rlookup(fs.inode, dirname)
- if n == nil {
- return os.ErrNotExist
- }
- n.Lock()
- defer n.Unlock()
- if n.Child(name, nil) != nil {
- return os.ErrExist
- }
- dn, ok := n.(*dirnode)
- if !ok {
- return ErrInvalidArgument
- }
- child := n.Child(name, func(inode) (child inode) {
- child, err = dn.newDirnode(dn, name, perm, time.Now())
- return
- })
- if err != nil {
- return err
- } else if child == nil {
- return ErrInvalidArgument
- }
- return nil
-}
-
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := rlookup(fs.inode, name)
- if node == nil {
- err = os.ErrNotExist
- } else {
- fi = node.FileInfo()
- }
- return
-}
-
-func (fs *fileSystem) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
-
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
- }
- }
-
- if _, ok := newdirf.inode.(*dirnode); !ok {
- return ErrInvalidOperation
- }
-
- err = nil
- olddirf.inode.Child(oldname, func(oldinode inode) inode {
- if oldinode == nil {
- err = os.ErrNotExist
- return nil
- }
- newdirf.inode.Child(newname, func(existing inode) inode {
- if existing != nil && existing.IsDir() {
- err = ErrIsDirectory
- return existing
- }
- return oldinode
- })
- if err != nil {
- return oldinode
- }
- oldinode.Lock()
- defer oldinode.Unlock()
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdirf.inode
- n.treenode.fileinfo.name = newname
- case *filenode:
- n.parent = newdn
- n.fileinfo.name = newname
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- olddn.treenode.fileinfo.modTime = time.Now()
- newdn.treenode.fileinfo.modTime = time.Now()
- return nil
- })
- return err
-}
-
-func (fs *fileSystem) Remove(name string) error {
- return fs.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (fs *fileSystem) RemoveAll(name string) error {
- err := fs.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (fs *fileSystem) remove(name string, recursive bool) (err error) {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dir := rlookup(fs, dirname)
- if dir == nil {
- return os.ErrNotExist
- }
- dir.Lock()
- defer dir.Unlock()
- dir.Child(name, func(node inode) inode {
- if node == nil {
- err = os.ErrNotExist
- return nil
- }
- if !recursive && node.IsDir() && node.Size() > 0 {
- err = ErrDirectoryNotEmpty
- return node
- }
- return nil
- })
- return err
-}
-
-type inode interface {
- Parent() inode
- Read([]byte, filenodePtr) (int, filenodePtr, error)
- Write([]byte, filenodePtr) (int, filenodePtr, error)
- Truncate(int64) error
- IsDir() bool
- Readdir() []os.FileInfo
- Size() int64
- FileInfo() os.FileInfo
- // Caller must have lock (or rlock if func is nil)
- Child(string, func(inode) inode) inode
- sync.Locker
- RLock()
- RUnlock()
-}
-
-// filenode implements inode.
-type filenode struct {
- fileinfo fileinfo
- parent *dirnode
- segments []segment
- // number of times `segments` has changed in a
- // way that might invalidate a filenodePtr
- repacked int64
- memsize int64 // bytes in memSegments
- sync.RWMutex
- nullnode
-}
-
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
@@ -538,6 +168,19 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
return
}
+// filenode implements inode.
+type filenode struct {
+ fileinfo fileinfo
+ parent *dirnode
+ segments []segment
+ // number of times `segments` has changed in a
+ // way that might invalidate a filenodePtr
+ repacked int64
+ memsize int64 // bytes in memSegments
+ sync.RWMutex
+ nullnode
+}
+
// caller must have lock
func (fn *filenode) appendSegment(e segment) {
fn.segments = append(fn.segments, e)
@@ -813,128 +456,6 @@ func (fn *filenode) pruneMemSegments() {
}
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
- var modTime time.Time
- if c.ModifiedAt == nil {
- modTime = time.Now()
- } else {
- modTime = *c.ModifiedAt
- }
- dn := &dirnode{
- client: client,
- kc: kc,
- treenode: treenode{
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
- },
- parent: nil,
- inodes: make(map[string]inode),
- },
- }
- dn.parent = dn
- fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
- if err := dn.loadManifest(c.ManifestText); err != nil {
- return nil, err
- }
- return fs, nil
-}
-
-type filehandle struct {
- inode
- ptr filenodePtr
- append bool
- readable bool
- writable bool
- unreaddirs []os.FileInfo
-}
-
-func (f *filehandle) Read(p []byte) (n int, err error) {
- if !f.readable {
- return 0, ErrWriteOnlyMode
- }
- f.inode.RLock()
- defer f.inode.RUnlock()
- n, f.ptr, err = f.inode.Read(p, f.ptr)
- return
-}
-
-func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
- size := f.inode.Size()
- ptr := f.ptr
- switch whence {
- case io.SeekStart:
- ptr.off = off
- case io.SeekCurrent:
- ptr.off += off
- case io.SeekEnd:
- ptr.off = size + off
- }
- if ptr.off < 0 {
- return f.ptr.off, ErrNegativeOffset
- }
- if ptr.off != f.ptr.off {
- f.ptr = ptr
- // force filenode to recompute f.ptr fields on next
- // use
- f.ptr.repacked = -1
- }
- return f.ptr.off, nil
-}
-
-func (f *filehandle) Truncate(size int64) error {
- return f.inode.Truncate(size)
-}
-
-func (f *filehandle) Write(p []byte) (n int, err error) {
- if !f.writable {
- return 0, ErrReadOnlyFile
- }
- f.inode.Lock()
- defer f.inode.Unlock()
- if fn, ok := f.inode.(*filenode); ok && f.append {
- f.ptr = filenodePtr{
- off: fn.fileinfo.size,
- segmentIdx: len(fn.segments),
- segmentOff: 0,
- repacked: fn.repacked,
- }
- }
- n, f.ptr, err = f.inode.Write(p, f.ptr)
- return
-}
-
-func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.IsDir() {
- return nil, ErrInvalidOperation
- }
- if count <= 0 {
- return f.inode.Readdir(), nil
- }
- if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
- }
- if len(f.unreaddirs) == 0 {
- return nil, io.EOF
- }
- if count > len(f.unreaddirs) {
- count = len(f.unreaddirs)
- }
- ret := f.unreaddirs[:count]
- f.unreaddirs = f.unreaddirs[count:]
- return ret, nil
-}
-
-func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.FileInfo(), nil
-}
-
-func (f *filehandle) Close() error {
- return nil
-}
-
type dirnode struct {
treenode
client *Client
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
new file mode 100644
index 0000000..56963b6
--- /dev/null
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -0,0 +1,99 @@
+package arvados
+
+import (
+ "io"
+ "os"
+)
+
+type filehandle struct {
+ inode
+ ptr filenodePtr
+ append bool
+ readable bool
+ writable bool
+ unreaddirs []os.FileInfo
+}
+
+func (f *filehandle) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
+ n, f.ptr, err = f.inode.Read(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
+ size := f.inode.Size()
+ ptr := f.ptr
+ switch whence {
+ case io.SeekStart:
+ ptr.off = off
+ case io.SeekCurrent:
+ ptr.off += off
+ case io.SeekEnd:
+ ptr.off = size + off
+ }
+ if ptr.off < 0 {
+ return f.ptr.off, ErrNegativeOffset
+ }
+ if ptr.off != f.ptr.off {
+ f.ptr = ptr
+ // force filenode to recompute f.ptr fields on next
+ // use
+ f.ptr.repacked = -1
+ }
+ return f.ptr.off, nil
+}
+
+func (f *filehandle) Truncate(size int64) error {
+ return f.inode.Truncate(size)
+}
+
+func (f *filehandle) Write(p []byte) (n int, err error) {
+ if !f.writable {
+ return 0, ErrReadOnlyFile
+ }
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ segmentIdx: len(fn.segments),
+ segmentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
+ n, f.ptr, err = f.inode.Write(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
+ if !f.inode.IsDir() {
+ return nil, ErrInvalidOperation
+ }
+ if count <= 0 {
+ return f.inode.Readdir(), nil
+ }
+ if f.unreaddirs == nil {
+ f.unreaddirs = f.inode.Readdir()
+ }
+ if len(f.unreaddirs) == 0 {
+ return nil, io.EOF
+ }
+ if count > len(f.unreaddirs) {
+ count = len(f.unreaddirs)
+ }
+ ret := f.unreaddirs[:count]
+ f.unreaddirs = f.unreaddirs[count:]
+ return ret, nil
+}
+
+func (f *filehandle) Stat() (os.FileInfo, error) {
+ return f.inode.FileInfo(), nil
+}
+
+func (f *filehandle) Close() error {
+ return nil
+}
commit 2eb576727b1c9551141083b82e1165f9571e2d2f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 02:18:01 2017 -0500
13111: Rearrange source files.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/fs_base.go
similarity index 100%
rename from sdk/go/arvados/base_fs.go
rename to sdk/go/arvados/fs_base.go
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/fs_collection.go
similarity index 100%
rename from sdk/go/arvados/collection_fs.go
rename to sdk/go/arvados/fs_collection.go
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/fs_collection_test.go
similarity index 100%
rename from sdk/go/arvados/collection_fs_test.go
rename to sdk/go/arvados/fs_collection_test.go
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/fs_site.go
similarity index 77%
rename from sdk/go/arvados/site_fs.go
rename to sdk/go/arvados/fs_site.go
index 853bd32..66856b7 100644
--- a/sdk/go/arvados/site_fs.go
+++ b/sdk/go/arvados/fs_site.go
@@ -9,6 +9,14 @@ import (
"time"
)
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// although the FileSystem allows files to be added and modified in
+// collections, these changes are not persistent or visible to other
+// Arvados clients.
func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
root := &treenode{
fileinfo: fileinfo{
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/fs_site_test.go
similarity index 100%
rename from sdk/go/arvados/site_fs_test.go
rename to sdk/go/arvados/fs_site_test.go
commit 10d551cbba521857f3967968348fd107f32651df
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Dec 18 00:49:36 2017 -0500
13111: .arvados#collection special file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 4647055..72ef49f 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -5,6 +5,7 @@
package arvados
import (
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -144,6 +145,25 @@ type collectionFileSystem struct {
fileSystem
}
+func (fs collectionFileSystem) Child(name string, replace func(inode) inode) inode {
+ if name == ".arvados#collection" {
+ return &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, 10)
+ }
+ return data, err
+ }}
+ }
+ return fs.fileSystem.Child(name, replace)
+}
+
func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
fs.fileSystem.inode.Lock()
defer fs.fileSystem.inode.Unlock()
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs_getternode.go
new file mode 100644
index 0000000..c9ffb38
--- /dev/null
+++ b/sdk/go/arvados/fs_getternode.go
@@ -0,0 +1,66 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "os"
+ "time"
+)
+
+// A getternode is a read-only character device that returns whatever
+// data is returned by the supplied function.
+type getternode struct {
+ Getter func() ([]byte, error)
+
+ treenode
+ data *bytes.Reader
+}
+
+func (*getternode) IsDir() bool {
+ return false
+}
+
+func (*getternode) Child(string, func(inode) inode) inode {
+ return nil
+}
+
+func (gn *getternode) get() error {
+ if gn.data != nil {
+ return nil
+ }
+ data, err := gn.Getter()
+ if err != nil {
+ return err
+ }
+ gn.data = bytes.NewReader(data)
+ return nil
+}
+
+func (gn *getternode) Size() int64 {
+ return gn.FileInfo().Size()
+}
+
+func (gn *getternode) FileInfo() os.FileInfo {
+ gn.Lock()
+ defer gn.Unlock()
+ var size int64
+ if gn.get() == nil {
+ size = gn.data.Size()
+ }
+ return fileinfo{
+ modTime: time.Now(),
+ mode: 0444,
+ size: size,
+ }
+}
+
+func (gn *getternode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
+ if err := gn.get(); err != nil {
+ return 0, ptr, err
+ }
+ n, err := gn.data.ReadAt(p, ptr.off)
+ return n, filenodePtr{off: ptr.off + int64(n)}, err
+}
commit dcba8d1ddddc578d97b210f32eb6879a4652039f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sun Dec 17 17:46:53 2017 -0500
13111: Fix inode not updated during rename().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index dcfd098..4647055 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -356,14 +356,22 @@ func (fs *fileSystem) Rename(oldname, newname string) error {
if err != nil {
return oldinode
}
+ oldinode.Lock()
+ defer oldinode.Unlock()
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
switch n := oldinode.(type) {
case *dirnode:
n.parent = newdirf.inode
+ n.treenode.fileinfo.name = newname
case *filenode:
- n.parent = newdirf.inode.(*dirnode)
+ n.parent = newdn
+ n.fileinfo.name = newname
default:
panic(fmt.Sprintf("bad inode type %T", n))
}
+ olddn.treenode.fileinfo.modTime = time.Now()
+ newdn.treenode.fileinfo.modTime = time.Now()
return nil
})
return err
commit 5aeb3af0666da32adce3fbc0c9cc38d9c67de8ec
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 09:22:03 2017 -0500
13111: Mount collections on demand in mnt/by_id/.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/base_fs.go b/sdk/go/arvados/base_fs.go
new file mode 100644
index 0000000..4de1bf8
--- /dev/null
+++ b/sdk/go/arvados/base_fs.go
@@ -0,0 +1,97 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+)
+
+type nullnode struct{}
+
+func (*nullnode) Mkdir(string, os.FileMode) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) Read([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Write([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Truncate(int64) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) FileInfo() os.FileInfo {
+ return fileinfo{}
+}
+
+func (*nullnode) IsDir() bool {
+ return false
+}
+
+func (*nullnode) Readdir() []os.FileInfo {
+ return nil
+}
+
+func (*nullnode) Child(name string, replace func(inode) inode) inode {
+ return nil
+}
+
+type treenode struct {
+ parent inode
+ inodes map[string]inode
+ fileinfo fileinfo
+ sync.RWMutex
+ nullnode
+}
+
+func (n *treenode) Parent() inode {
+ n.RLock()
+ defer n.RUnlock()
+ return n.parent
+}
+
+func (n *treenode) IsDir() bool {
+ return true
+}
+
+func (n *treenode) Child(name string, replace func(inode) inode) (child inode) {
+ // TODO: special treatment for "", ".", ".."
+ child = n.inodes[name]
+ if replace != nil {
+ child = replace(child)
+ if child == nil {
+ delete(n.inodes, name)
+ } else {
+ n.inodes[name] = child
+ }
+ }
+ return
+}
+
+func (n *treenode) Size() int64 {
+ return n.FileInfo().Size()
+}
+
+func (n *treenode) FileInfo() os.FileInfo {
+ n.Lock()
+ defer n.Unlock()
+ n.fileinfo.size = int64(len(n.inodes))
+ return n.fileinfo
+}
+
+func (n *treenode) Readdir() (fi []os.FileInfo) {
+ n.RLock()
+ defer n.RUnlock()
+ fi = make([]os.FileInfo, 0, len(n.inodes))
+ for _, inode := range n.inodes {
+ fi = append(fi, inode.FileInfo())
+ }
+ return
+}
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index d8ee2a2..dcfd098 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -89,12 +89,14 @@ func (fi fileinfo) Sys() interface{} {
return nil
}
-// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files. All methods are safe to call
-// from multiple goroutines.
-type CollectionFileSystem interface {
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
http.FileSystem
+ inode
+
// analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
@@ -120,6 +122,12 @@ type CollectionFileSystem interface {
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
+}
+
+// A CollectionFileSystem is a FileSystem that can be serialized as a
+// manifest and stored as a collection.
+type CollectionFileSystem interface {
+ FileSystem
// Flush all file data to Keep and return a snapshot of the
// filesystem suitable for saving as (Collection)ManifestText.
@@ -129,39 +137,288 @@ type CollectionFileSystem interface {
}
type fileSystem struct {
- dirnode
+ inode
+}
+
+type collectionFileSystem struct {
+ fileSystem
}
+func (fs collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.inode.Lock()
+ defer fs.fileSystem.inode.Unlock()
+ return fs.fileSystem.inode.(*dirnode).marshalManifest(prefix)
+}
+
+// OpenFile is analogous to os.OpenFile().
func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.dirnode.OpenFile(name, flag, perm)
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ var dn inode = fs.inode
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent := rlookup(dn, dirname)
+ if parent == nil {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n := parent.Child(name, nil)
+ if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ var err error
+ n = parent.Child(name, func(inode) inode {
+ var dn *dirnode
+ switch parent := parent.(type) {
+ case *dirnode:
+ dn = parent
+ case *collectionFileSystem:
+ dn = parent.inode.(*dirnode)
+ default:
+ err = ErrInvalidArgument
+ return nil
+ }
+ if perm.IsDir() {
+ n, err = dn.newDirnode(dn, name, perm|0755, time.Now())
+ } else {
+ n, err = dn.newFilenode(dn, name, perm|0755, time.Now())
+ }
+ return n
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // parent rejected new child
+ return nil, ErrInvalidOperation
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
}
func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.dirnode.OpenFile(name, os.O_RDONLY, 0)
+ return fs.OpenFile(name, os.O_RDONLY, 0)
}
func (fs *fileSystem) Create(name string) (File, error) {
- return fs.dirnode.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) (err error) {
+ dirname, name := path.Split(name)
+ n := rlookup(fs.inode, dirname)
+ if n == nil {
+ return os.ErrNotExist
+ }
+ n.Lock()
+ defer n.Unlock()
+ if n.Child(name, nil) != nil {
+ return os.ErrExist
+ }
+ dn, ok := n.(*dirnode)
+ if !ok {
+ return ErrInvalidArgument
+ }
+ child := n.Child(name, func(inode) (child inode) {
+ child, err = dn.newDirnode(dn, name, perm, time.Now())
+ return
+ })
+ if err != nil {
+ return err
+ } else if child == nil {
+ return ErrInvalidArgument
+ }
+ return nil
}
func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := fs.dirnode.lookupPath(name)
+ node := rlookup(fs.inode, name)
if node == nil {
err = os.ErrNotExist
} else {
- fi = node.Stat()
+ fi = node.FileInfo()
}
return
}
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*filehandle{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ if _, ok := newdirf.inode.(*dirnode); !ok {
+ return ErrInvalidOperation
+ }
+
+ err = nil
+ olddirf.inode.Child(oldname, func(oldinode inode) inode {
+ if oldinode == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ newdirf.inode.Child(newname, func(existing inode) inode {
+ if existing != nil && existing.IsDir() {
+ err = ErrIsDirectory
+ return existing
+ }
+ return oldinode
+ })
+ if err != nil {
+ return oldinode
+ }
+ switch n := oldinode.(type) {
+ case *dirnode:
+ n.parent = newdirf.inode
+ case *filenode:
+ n.parent = newdirf.inode.(*dirnode)
+ default:
+ panic(fmt.Sprintf("bad inode type %T", n))
+ }
+ return nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) (err error) {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir := rlookup(fs, dirname)
+ if dir == nil {
+ return os.ErrNotExist
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ dir.Child(name, func(node inode) inode {
+ if node == nil {
+ err = os.ErrNotExist
+ return nil
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ err = ErrDirectoryNotEmpty
+ return node
+ }
+ return nil
+ })
+ return err
+}
+
type inode interface {
Parent() inode
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
+ IsDir() bool
Readdir() []os.FileInfo
Size() int64
- Stat() os.FileInfo
+ FileInfo() os.FileInfo
+ // Caller must have lock (or rlock if func is nil)
+ Child(string, func(inode) inode) inode
sync.Locker
RLock()
RUnlock()
@@ -177,6 +434,7 @@ type filenode struct {
repacked int64
memsize int64 // bytes in memSegments
sync.RWMutex
+ nullnode
}
// filenodePtr is an offset into a file that is (usually) efficient to
@@ -264,10 +522,6 @@ func (fn *filenode) Parent() inode {
return fn.parent
}
-func (fn *filenode) Readdir() []os.FileInfo {
- return nil
-}
-
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -302,7 +556,7 @@ func (fn *filenode) Size() int64 {
return fn.fileinfo.Size()
}
-func (fn *filenode) Stat() os.FileInfo {
+func (fn *filenode) FileInfo() os.FileInfo {
fn.RLock()
defer fn.RUnlock()
return fn.fileinfo
@@ -539,19 +793,22 @@ func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSy
} else {
modTime = *c.ModifiedAt
}
- fs := &fileSystem{dirnode: dirnode{
+ dn := &dirnode{
client: client,
kc: kc,
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
},
- parent: nil,
- inodes: make(map[string]inode),
- }}
- fs.dirnode.parent = &fs.dirnode
- if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
+ }
+ dn.parent = dn
+ fs := &collectionFileSystem{fileSystem: fileSystem{inode: dn}}
+ if err := dn.loadManifest(c.ManifestText); err != nil {
return nil, err
}
return fs, nil
@@ -622,7 +879,7 @@ func (f *filehandle) Write(p []byte) (n int, err error) {
}
func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.Stat().IsDir() {
+ if !f.inode.IsDir() {
return nil, ErrInvalidOperation
}
if count <= 0 {
@@ -643,7 +900,7 @@ func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
}
func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.Stat(), nil
+ return f.inode.FileInfo(), nil
}
func (f *filehandle) Close() error {
@@ -651,12 +908,9 @@ func (f *filehandle) Close() error {
}
type dirnode struct {
- fileinfo fileinfo
- parent *dirnode
- client *Client
- kc keepClient
- inodes map[string]inode
- sync.RWMutex
+ treenode
+ client *Client
+ kc keepClient
}
// sync flushes in-memory data (for all files in the tree rooted at
@@ -735,12 +989,6 @@ func (dn *dirnode) sync() error {
return flush(pending)
}
-func (dn *dirnode) MarshalManifest(prefix string) (string, error) {
- dn.Lock()
- defer dn.Unlock()
- return dn.marshalManifest(prefix)
-}
-
// caller must have read lock.
func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var streamLen int64
@@ -941,6 +1189,7 @@ func (dn *dirnode) loadManifest(txt string) error {
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
+ node := dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
if basename == "" || basename == "." || basename == ".." {
@@ -950,336 +1199,111 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
for _, name := range names[:len(names)-1] {
switch name {
case "", ".":
+ continue
case "..":
- dn = dn.parent
- default:
- switch node := dn.inodes[name].(type) {
+ if node == dn {
+ // can't be sure parent will be a *dirnode
+ return nil, ErrInvalidArgument
+ }
+ node = node.Parent().(*dirnode)
+ continue
+ }
+ node.Child(name, func(child inode) inode {
+ switch child.(type) {
case nil:
- dn = dn.newDirnode(name, 0755, dn.fileinfo.modTime)
+ node, err = dn.newDirnode(node, name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ child = node
case *dirnode:
- dn = node
+ node = child.(*dirnode)
case *filenode:
err = ErrFileExists
- return
+ default:
+ err = ErrInvalidOperation
}
+ return child
+ })
+ if err != nil {
+ return
}
}
- switch node := dn.inodes[basename].(type) {
- case nil:
- fn = dn.newFilenode(basename, 0755, dn.fileinfo.modTime)
- case *filenode:
- fn = node
- case *dirnode:
- err = ErrIsDirectory
- }
- return
-}
-
-func (dn *dirnode) mkdir(name string) (*filehandle, error) {
- return dn.OpenFile(name, os.O_CREATE|os.O_EXCL, os.ModeDir|0755)
-}
-
-func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
- f, err := dn.mkdir(name)
- if err == nil {
- err = f.Close()
- }
- return err
-}
-
-func (dn *dirnode) Remove(name string) error {
- return dn.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (dn *dirnode) RemoveAll(name string) error {
- err := dn.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (dn *dirnode) remove(name string, recursive bool) error {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return os.ErrNotExist
- }
- dn.Lock()
- defer dn.Unlock()
- switch node := dn.inodes[name].(type) {
- case nil:
- return os.ErrNotExist
- case *dirnode:
- node.RLock()
- defer node.RUnlock()
- if !recursive && len(node.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ node.Child(basename, func(child inode) inode {
+ switch child := child.(type) {
+ case nil:
+ fn, err = dn.newFilenode(node, basename, 0755, node.FileInfo().ModTime())
+ return fn
+ case *filenode:
+ fn = child
+ return child
+ case *dirnode:
+ err = ErrIsDirectory
+ return child
+ default:
+ err = ErrInvalidOperation
+ return child
}
- }
- delete(dn.inodes, name)
- return nil
+ })
+ return
}
-func (dn *dirnode) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := dn.OpenFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := dn.OpenFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
- node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node == nil {
+ break
}
- }
-
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- oldinode, ok := olddn.inodes[oldname]
- if !ok {
- return os.ErrNotExist
- }
- if locked[oldinode] {
- // oldinode cannot become a descendant of itself.
- return ErrInvalidArgument
- }
- if existing, ok := newdn.inodes[newname]; ok {
- // overwriting an existing file or dir
- if dn, ok := existing.(*dirnode); ok {
- if !oldinode.Stat().IsDir() {
- return ErrIsDirectory
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
}
- dn.RLock()
- defer dn.RUnlock()
- if len(dn.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ if name == ".." {
+ node = node.Parent()
+ continue
}
}
- } else {
- if newdn.inodes == nil {
- newdn.inodes = make(map[string]inode)
- }
- newdn.fileinfo.size++
- }
- newdn.inodes[newname] = oldinode
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdn
- case *filenode:
- n.parent = newdn
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- delete(olddn.inodes, oldname)
- olddn.fileinfo.size--
- return nil
-}
-
-func (dn *dirnode) Parent() inode {
- dn.RLock()
- defer dn.RUnlock()
- return dn.parent
-}
-
-func (dn *dirnode) Readdir() (fi []os.FileInfo) {
- dn.RLock()
- defer dn.RUnlock()
- fi = make([]os.FileInfo, 0, len(dn.inodes))
- for _, inode := range dn.inodes {
- fi = append(fi, inode.Stat())
+ node = func() inode {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
}
return
}
-func (dn *dirnode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Write(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Size() int64 {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo.Size()
-}
-
-func (dn *dirnode) Stat() os.FileInfo {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo
-}
-
-func (dn *dirnode) Truncate(int64) error {
- return ErrInvalidOperation
-}
-
-// lookupPath returns the inode for the file/directory with the given
-// name (which may contain "/" separators), along with its parent
-// node. If no such file/directory exists, the returned node is nil.
-func (dn *dirnode) lookupPath(path string) (node inode) {
- node = dn
- for _, name := range strings.Split(path, "/") {
- dn, ok := node.(*dirnode)
- if !ok {
- return nil
- }
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
- }
- dn.RLock()
- node = dn.inodes[name]
- dn.RUnlock()
+// Caller must have lock, and must have already ensured
+// Children(name,nil) is nil.
+func (dn *dirnode) newDirnode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *dirnode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
}
- return
-}
-
-func (dn *dirnode) newDirnode(name string, perm os.FileMode, modTime time.Time) *dirnode {
- child := &dirnode{
- parent: dn,
+ return &dirnode{
client: dn.client,
kc: dn.kc,
- fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | perm,
- modTime: modTime,
+ treenode: treenode{
+ parent: parent,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
+ }, nil
}
-func (dn *dirnode) newFilenode(name string, perm os.FileMode, modTime time.Time) *filenode {
- child := &filenode{
- parent: dn,
+func (dn *dirnode) newFilenode(parent *dirnode, name string, perm os.FileMode, modTime time.Time) (node *filenode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ return &filenode{
+ parent: parent,
fileinfo: fileinfo{
name: name,
- mode: perm,
+ mode: perm & ^os.ModeDir,
modTime: modTime,
},
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
-}
-
-// OpenFile is analogous to os.OpenFile().
-func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: dn}, nil
- case "..":
- return &filehandle{inode: dn.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- dn.Lock()
- defer dn.Unlock()
- } else {
- dn.RLock()
- defer dn.RUnlock()
- }
- n, ok := dn.inodes[name]
- if !ok {
- if !createMode {
- return nil, os.ErrNotExist
- }
- if perm.IsDir() {
- n = dn.newDirnode(name, 0755, time.Now())
- } else {
- n = dn.newFilenode(name, 0755, time.Now())
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
}, nil
}
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index bd5d08b..023226f 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -465,6 +465,10 @@ func (s *CollectionFSSuite) TestMkdir(c *check.C) {
}
func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
maxBlockSize = 8
defer func() { maxBlockSize = 2 << 26 }()
@@ -1014,6 +1018,10 @@ var _ = check.Suite(&CollectionFSUnitSuite{})
// expect ~2 seconds to load a manifest with 256K files
func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
const (
dirCount = 512
fileCount = 512
diff --git a/sdk/go/arvados/site_fs.go b/sdk/go/arvados/site_fs.go
new file mode 100644
index 0000000..853bd32
--- /dev/null
+++ b/sdk/go/arvados/site_fs.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "time"
+)
+
+func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+ root := &treenode{
+ fileinfo: fileinfo{
+ name: "/",
+ mode: os.ModeDir | 0755,
+ modTime: time.Now(),
+ },
+ inodes: make(map[string]inode),
+ }
+ root.parent = root
+ root.Child("by_id", func(inode) inode {
+ return &vdirnode{
+ treenode: treenode{
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: "by_id",
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ create: func(name string) inode {
+ return newEntByID(c, kc, name)
+ },
+ }
+ })
+ return &fileSystem{inode: root}
+}
+
+func newEntByID(c *Client, kc keepClient, id string) inode {
+ var coll Collection
+ err := c.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ if err != nil {
+ return nil
+ }
+ fs, err := coll.FileSystem(c, kc)
+ fs.(*collectionFileSystem).inode.(*dirnode).fileinfo.name = id
+ if err != nil {
+ return nil
+ }
+ return fs
+}
+
+type vdirnode struct {
+ treenode
+ create func(string) inode
+}
+
+func (vn *vdirnode) Child(name string, _ func(inode) inode) inode {
+ return vn.treenode.Child(name, func(existing inode) inode {
+ if existing != nil {
+ return existing
+ } else {
+ return vn.create(name)
+ }
+ })
+}
diff --git a/sdk/go/arvados/site_fs_test.go b/sdk/go/arvados/site_fs_test.go
new file mode 100644
index 0000000..a8c369f
--- /dev/null
+++ b/sdk/go/arvados/site_fs_test.go
@@ -0,0 +1,58 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&SiteFSSuite{})
+
+type SiteFSSuite struct {
+ client *Client
+ fs FileSystem
+ kc keepClient
+}
+
+func (s *SiteFSSuite) SetUpTest(c *check.C) {
+ s.client = NewClientFromEnv()
+ s.kc = &keepClientStub{
+ blocks: map[string][]byte{
+ "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
+ }}
+ s.fs = s.client.SiteFileSystem(s.kc)
+}
+
+func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
+ _, ok := s.fs.(http.FileSystem)
+ c.Check(ok, check.Equals, true)
+}
+
+func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Equals, 0)
+}
+
+func (s *SiteFSSuite) TestByUUID(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 0)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.FooCollection)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(-1)
+ var names []string
+ for _, fi := range fis {
+ names = append(names, fi.Name())
+ }
+ c.Check(names, check.DeepEquals, []string{"foo"})
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list