[ARVADOS] updated: 1.1.2-315-g2c3b9be
Git user
git at public.curoverse.com
Tue Feb 13 17:32:27 EST 2018
Summary of changes:
sdk/go/arvados/collection.go | 1 +
sdk/go/arvados/fs_project.go | 26 +++++++++++-----
sdk/go/arvados/fs_project_test.go | 62 +++++++++++++++++++++++++++++++++++----
sdk/go/arvados/fs_site.go | 19 ++++++++++++
sdk/go/arvados/fs_site_test.go | 6 +++-
sdk/go/arvadostest/fixtures.go | 3 ++
6 files changed, 103 insertions(+), 14 deletions(-)
via 2c3b9beb3345ac31722bcd04cb41ed318469bb79 (commit)
from 9451dec1ace2370015b4c75367076cb5d4901f7e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2c3b9beb3345ac31722bcd04cb41ed318469bb79
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Feb 13 17:29:38 2018 -0500
12308: Reload project dir if fsync(2) was called since last load.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 999b4e9..aea0cc0 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -16,6 +16,7 @@ import (
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
TrashAt *time.Time `json:"trash_at,omitempty"`
ManifestText string `json:"manifest_text,omitempty"`
UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index a5e4710..776c8a3 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -7,18 +7,29 @@ package arvados
import (
"os"
"sync"
+ "time"
)
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
- uuid string
- setupOnce sync.Once
- err error
+ uuid string
+ err error
+
+ loadLock sync.Mutex
+ loadStart time.Time
}
-func (pn *projectnode) setup() {
+func (pn *projectnode) load() {
fs := pn.FS().(*siteFileSystem)
+
+ pn.loadLock.Lock()
+ defer pn.loadLock.Unlock()
+ if !fs.Stale(pn.loadStart) {
+ return
+ }
+ pn.loadStart = time.Now()
+
if pn.uuid == "" {
var resp User
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
@@ -36,7 +47,6 @@ func (pn *projectnode) setup() {
var resp CollectionList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -60,7 +70,6 @@ func (pn *projectnode) setup() {
var resp GroupList
pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
if pn.err != nil {
- // TODO: retry on next access, instead of returning the same error forever
return
}
if len(resp.Items) == 0 {
@@ -76,10 +85,11 @@ func (pn *projectnode) setup() {
}
params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
+ pn.err = nil
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
@@ -87,7 +97,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.setupOnce.Do(pn.setup)
+ pn.load()
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 1de0714..b7dc08e 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -46,29 +47,29 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
ok := false
for _, fi := range fis {
c.Check(fi.Name(), check.Not(check.Equals), "")
- if fi.Name() == "Unrestricted public data" {
+ if fi.Name() == "A Project" {
ok = true
}
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/Unrestricted public data/..")
+ f, err = s.fs.Open("/home/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
c.Check(fi.Name(), check.Equals, "home")
- f, err = s.fs.Open("/home/Unrestricted public data/Subproject in anonymous accessible project")
+ f, err = s.fs.Open("/home/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/A Project",
- "/home/A Project/does not exist",
+ "/home/Unrestricted public data",
"/home/Unrestricted public data/does not exist",
+ "/home/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -76,3 +77,54 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
c.Check(os.IsNotExist(err), check.Equals, true)
}
}
+
+func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ project, err := s.fs.OpenFile("/home/A Project", 0, 0)
+ c.Check(err, check.IsNil)
+
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+
+ oob := Collection{
+ Name: "oob",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ f, err := s.fs.Open("/home/A Project/oob")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Check(fi.IsDir(), check.Equals, true)
+ f.Close()
+
+ wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
+ c.Assert(err, check.IsNil)
+ _, err = wf.Write([]byte("hello oob\n"))
+ c.Check(err, check.IsNil)
+ err = wf.Close()
+ c.Check(err, check.IsNil)
+
+ // Delete test.txt behind s.fs's back by updating the
+ // collection record with the old (empty) ManifestText.
+ err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob/test.txt")
+ c.Check(err, check.NotNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.IsNil)
+
+ err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 701711e..cdcf40e 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -6,11 +6,15 @@ package arvados
import (
"os"
+ "sync"
"time"
)
type siteFileSystem struct {
fileSystem
+
+ staleThreshold time.Time
+ staleLock sync.Mutex
}
// SiteFileSystem returns a FileSystem that maps collections and other
@@ -58,6 +62,21 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
return fs
}
+func (fs *siteFileSystem) Sync() error {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ fs.staleThreshold = time.Now()
+ return nil
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *siteFileSystem) Stale(t time.Time) bool {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ return !fs.staleThreshold.Before(t)
+}
+
func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 26a2212..a3a9712 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -21,7 +21,11 @@ type SiteFSSuite struct {
}
func (s *SiteFSSuite) SetUpTest(c *check.C) {
- s.client = NewClientFromEnv()
+ s.client = &Client{
+ APIHost: os.Getenv("ARVADOS_API_HOST"),
+ AuthToken: arvadostest.ActiveToken,
+ Insecure: true,
+ }
s.kc = &keepClientStub{
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index d057c09..5fccfb3 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -25,6 +25,9 @@ const (
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
+ AProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
+
FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list