[ARVADOS] updated: 1.2.0-41-g12a2757d6
Git user
git at public.curoverse.com
Thu Sep 13 14:05:27 EDT 2018
Summary of changes:
build/run-tests.sh | 2 ++
sdk/go/arvados/client.go | 10 ++++--
sdk/go/arvados/{ => fs}/fs_backend.go | 10 ++++--
sdk/go/arvados/{ => fs}/fs_base.go | 2 +-
sdk/go/arvados/{ => fs}/fs_collection.go | 13 +++++---
sdk/go/arvados/{ => fs}/fs_collection_test.go | 47 ++++++++++++++-------------
sdk/go/arvados/{ => fs}/fs_deferred.go | 8 +++--
sdk/go/arvados/{ => fs}/fs_filehandle.go | 2 +-
sdk/go/arvados/{ => fs}/fs_getternode.go | 2 +-
sdk/go/arvados/{ => fs}/fs_lookup.go | 2 +-
sdk/go/arvados/{ => fs}/fs_project.go | 26 ++++++++-------
sdk/go/arvados/{ => fs}/fs_project_test.go | 11 ++++---
sdk/go/arvados/{ => fs}/fs_site.go | 16 +++++----
sdk/go/arvados/{ => fs}/fs_site_test.go | 9 ++---
sdk/go/arvados/{ => fs}/fs_users.go | 16 +++++----
sdk/go/keepclient/collectionreader.go | 9 ++---
services/keep-web/cache.go | 3 +-
services/keep-web/handler.go | 19 ++++++-----
services/keep-web/handler_test.go | 3 +-
services/keep-web/webdav.go | 5 ++-
20 files changed, 121 insertions(+), 94 deletions(-)
rename sdk/go/arvados/{ => fs}/fs_backend.go (82%)
rename sdk/go/arvados/{ => fs}/fs_base.go (99%)
rename sdk/go/arvados/{ => fs}/fs_collection.go (98%)
rename sdk/go/arvados/{ => fs}/fs_collection_test.go (95%)
rename sdk/go/arvados/{ => fs}/fs_deferred.go (94%)
rename sdk/go/arvados/{ => fs}/fs_filehandle.go (99%)
rename sdk/go/arvados/{ => fs}/fs_getternode.go (98%)
rename sdk/go/arvados/{ => fs}/fs_lookup.go (99%)
rename sdk/go/arvados/{ => fs}/fs_project.go (80%)
rename sdk/go/arvados/{ => fs}/fs_project_test.go (97%)
rename sdk/go/arvados/{ => fs}/fs_site.go (92%)
rename sdk/go/arvados/{ => fs}/fs_site_test.go (94%)
rename sdk/go/arvados/{ => fs}/fs_users.go (74%)
via 12a2757d65bbbca161e30d2e7c25f87d6537e5c7 (commit)
from 55f6178b9a9a0d165e952eeec9a04d0234299397 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 12a2757d65bbbca161e30d2e7c25f87d6537e5c7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Sep 13 14:04:47 2018 -0400
13994: Extract filesystem to its own package.
Fixes import cycle in tests (arvados -> arvadostest -> arvados).
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4ddbf89c1..81aa5dc98 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -100,6 +100,7 @@ sdk/python
sdk/python:py3
sdk/ruby
sdk/go/arvados
+sdk/go/arvados/fs
sdk/go/arvadosclient
sdk/go/dispatch
sdk/go/keepclient
@@ -924,6 +925,7 @@ gostuff=(
lib/crunchstat
lib/dispatchcloud
sdk/go/arvados
+ sdk/go/arvados/fs
sdk/go/arvadosclient
sdk/go/blockdigest
sdk/go/dispatch
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index cca9f9bf1..fb2a33e8c 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -227,13 +227,13 @@ func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.
return c.DoAndDecode(dst, req)
}
-type resource interface {
+type NamedResource interface {
resourceName() string
}
// UpdateBody returns an io.Reader suitable for use as an http.Request
// Body for a create or update API call.
-func (c *Client) UpdateBody(rsc resource) io.Reader {
+func (c *Client) UpdateBody(rsc NamedResource) io.Reader {
j, err := json.Marshal(rsc)
if err != nil {
// Return a reader that returns errors.
@@ -323,8 +323,12 @@ func (c *Client) DiscoveryDocument() (*DiscoveryDocument, error) {
var pdhRegexp = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`)
+func IsPDH(s string) bool {
+ return pdhRegexp.MatchString(s)
+}
+
func (c *Client) modelForUUID(dd *DiscoveryDocument, uuid string) (string, error) {
- if pdhRegexp.MatchString(uuid) {
+ if IsPDH(uuid) {
return "Collection", nil
}
if len(uuid) != 27 {
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs/fs_backend.go
similarity index 82%
rename from sdk/go/arvados/fs_backend.go
rename to sdk/go/arvados/fs/fs_backend.go
index 301f0b48b..69d0e653f 100644
--- a/sdk/go/arvados/fs_backend.go
+++ b/sdk/go/arvados/fs/fs_backend.go
@@ -2,9 +2,13 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
-import "io"
+import (
+ "io"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
type fsBackend interface {
keepClient
@@ -25,5 +29,5 @@ type keepClient interface {
type apiClient interface {
RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
+ UpdateBody(rsc arvados.NamedResource) io.Reader
}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs/fs_base.go
similarity index 99%
rename from sdk/go/arvados/fs_base.go
rename to sdk/go/arvados/fs/fs_base.go
index 3058a7609..f0992754c 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs/fs_base.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"errors"
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs/fs_collection.go
similarity index 98%
rename from sdk/go/arvados/fs_collection.go
rename to sdk/go/arvados/fs/fs_collection.go
index 7ce37aa24..e68dce034 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs/fs_collection.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"encoding/json"
@@ -17,6 +17,8 @@ import (
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
var maxBlockSize = 1 << 26
@@ -38,8 +40,9 @@ type collectionFileSystem struct {
uuid string
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
+// NewFileSystem returns a new CollectionFileSystem with initial state
+// based on the given collection.
+func NewFileSystem(c arvados.Collection, client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
@@ -122,7 +125,7 @@ func (fs *collectionFileSystem) Sync() error {
log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
return err
}
- coll := &Collection{
+ coll := &arvados.Collection{
UUID: fs.uuid,
ManifestText: txt,
}
@@ -524,7 +527,7 @@ func (dn *dirnode) FS() FileSystem {
func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
- var coll Collection
+ var coll arvados.Collection
var err error
coll.ManifestText, err = dn.fs.MarshalManifest(".")
if err != nil {
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs/fs_collection_test.go
similarity index 95%
rename from sdk/go/arvados/fs_collection_test.go
rename to sdk/go/arvados/fs/fs_collection_test.go
index d2f55d0e3..08a9745b4 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs/fs_collection_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
@@ -20,6 +20,7 @@ import (
"testing"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -54,21 +55,21 @@ func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
}
type CollectionFSSuite struct {
- client *Client
- coll Collection
+ client *arvados.Client
+ coll arvados.Collection
fs CollectionFileSystem
kc keepClient
}
func (s *CollectionFSSuite) SetUpTest(c *check.C) {
- s.client = NewClientFromEnv()
+ s.client = arvados.NewClientFromEnv()
err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
c.Assert(err, check.IsNil)
s.kc = &keepClientStub{
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
}}
- s.fs, err = s.coll.FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(s.coll, s.client, s.kc)
c.Assert(err, check.IsNil)
}
@@ -78,9 +79,9 @@ func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
}
func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
- fs, err := (&Collection{
+ fs, err := NewFileSystem(arvados.Collection{
ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
- }).FileSystem(s.client, s.kc)
+ }, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.Open("/foo:foo")
@@ -356,7 +357,7 @@ func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
}
func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
c.Assert(err, check.IsNil)
@@ -403,7 +404,7 @@ func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
for _, name := range []string{"foo", "bar", "baz"} {
f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
@@ -520,7 +521,7 @@ func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
const nfiles = 256
@@ -580,7 +581,7 @@ func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
}
func (s *CollectionFSSuite) TestRemove(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = fs.Mkdir("dir0", 0755)
c.Assert(err, check.IsNil)
@@ -613,7 +614,7 @@ func (s *CollectionFSSuite) TestRemove(c *check.C) {
}
func (s *CollectionFSSuite) TestRenameError(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = fs.Mkdir("first", 0755)
c.Assert(err, check.IsNil)
@@ -637,7 +638,7 @@ func (s *CollectionFSSuite) TestRenameError(c *check.C) {
}
func (s *CollectionFSSuite) TestRename(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
const (
outer = 16
@@ -724,7 +725,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = s.fs.Mkdir("d:r", 0755)
c.Assert(err, check.IsNil)
@@ -765,7 +766,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
c.Check(err, check.IsNil)
c.Check(len(fi), check.Equals, 4)
- persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
+ persisted, err := NewFileSystem(arvados.Collection{ManifestText: m}, s.client, s.kc)
c.Assert(err, check.IsNil)
root, err = persisted.Open("/")
@@ -788,7 +789,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
err = s.fs.Mkdir(name, 0755)
@@ -819,7 +820,7 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
c.Check(err, check.IsNil)
c.Logf("%q", m)
- persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
+ persisted, err := NewFileSystem(arvados.Collection{ManifestText: m}, s.client, s.kc)
c.Assert(err, check.IsNil)
for name, data := range expect {
@@ -839,7 +840,7 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
}
func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
@@ -940,7 +941,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
c.Assert(err, check.IsNil)
@@ -991,7 +992,7 @@ func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
"./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
} {
c.Logf("<-%q", txt)
- fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{ManifestText: txt}, s.client, s.kc)
c.Check(fs, check.IsNil)
c.Logf("-> %s", err)
c.Check(err, check.NotNil)
@@ -1007,7 +1008,7 @@ func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
} {
c.Logf("<-%q", txt)
- fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{ManifestText: txt}, s.client, s.kc)
c.Check(err, check.IsNil)
c.Check(fs, check.NotNil)
}
@@ -1050,14 +1051,14 @@ func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
}
mb.Write([]byte{'\n'})
}
- coll := Collection{ManifestText: mb.String()}
+ coll := arvados.Collection{ManifestText: mb.String()}
c.Logf("%s built", time.Now())
var memstats runtime.MemStats
runtime.ReadMemStats(&memstats)
c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
- f, err := coll.FileSystem(nil, nil)
+ f, err := NewFileSystem(coll, nil, nil)
c.Check(err, check.IsNil)
c.Logf("%s loaded", time.Now())
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs/fs_deferred.go
similarity index 94%
rename from sdk/go/arvados/fs_deferred.go
rename to sdk/go/arvados/fs/fs_deferred.go
index a84f64fe7..ecca3cd3c 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs/fs_deferred.go
@@ -2,16 +2,18 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"log"
"os"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
-func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+func deferredCollectionFS(fs FileSystem, parent inode, coll arvados.Collection) inode {
var modTime time.Time
if coll.ModifiedAt != nil {
modTime = *coll.ModifiedAt
@@ -34,7 +36,7 @@ func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
log.Printf("BUG: unhandled error: %s", err)
return placeholder
}
- cfs, err := coll.FileSystem(fs, fs)
+ cfs, err := NewFileSystem(coll, fs, fs)
if err != nil {
log.Printf("BUG: unhandled error: %s", err)
return placeholder
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs/fs_filehandle.go
similarity index 99%
rename from sdk/go/arvados/fs_filehandle.go
rename to sdk/go/arvados/fs/fs_filehandle.go
index 9af8d0ad4..09f65ce80 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs/fs_filehandle.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"io"
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs/fs_getternode.go
similarity index 98%
rename from sdk/go/arvados/fs_getternode.go
rename to sdk/go/arvados/fs/fs_getternode.go
index 966fe9d5c..b98dde35b 100644
--- a/sdk/go/arvados/fs_getternode.go
+++ b/sdk/go/arvados/fs/fs_getternode.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs/fs_lookup.go
similarity index 99%
rename from sdk/go/arvados/fs_lookup.go
rename to sdk/go/arvados/fs/fs_lookup.go
index 42322a14a..28a022cd5 100644
--- a/sdk/go/arvados/fs_lookup.go
+++ b/sdk/go/arvados/fs/fs_lookup.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs/fs_project.go
similarity index 80%
rename from sdk/go/arvados/fs_project.go
rename to sdk/go/arvados/fs/fs_project.go
index 92995510c..7cbf7dbd1 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs/fs_project.go
@@ -2,19 +2,21 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"log"
"os"
"strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
if uuid != "" {
return uuid, nil
}
- var resp User
+ var resp arvados.User
err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return "", err
@@ -29,10 +31,10 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in
return nil, err
}
- var contents CollectionList
- err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ var contents arvados.CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, arvados.ResourceListParams{
Count: "none",
- Filters: []Filter{
+ Filters: []arvados.Filter{
{"name", "=", name},
{"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
{"groups.group_class", "=", "project"},
@@ -69,14 +71,14 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", uuid}}
- params := ResourceListParams{
+ filters := []arvados.Filter{{"owner_uuid", "=", uuid}}
+ params := arvados.ResourceListParams{
Count: "none",
Filters: filters,
Order: "uuid",
}
for {
- var resp CollectionList
+ var resp arvados.CollectionList
err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
if err != nil {
return nil, err
@@ -91,13 +93,13 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
}
inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
- params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ params.Filters = append(filters, arvados.Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- filters = append(filters, Filter{"group_class", "=", "project"})
+ filters = append(filters, arvados.Filter{"group_class", "=", "project"})
params.Filters = filters
for {
- var resp GroupList
+ var resp arvados.GroupList
err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
if err != nil {
return nil, err
@@ -111,7 +113,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
}
inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
- params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ params.Filters = append(filters, arvados.Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs/fs_project_test.go
similarity index 97%
rename from sdk/go/arvados/fs_project_test.go
rename to sdk/go/arvados/fs/fs_project_test.go
index 1a06ce146..3d9c944e4 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs/fs_project_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
@@ -12,6 +12,7 @@ import (
"path/filepath"
"strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -23,7 +24,7 @@ type spiedRequest struct {
}
type spyingClient struct {
- *Client
+ *arvados.Client
calls []spiedRequest
}
@@ -119,7 +120,7 @@ func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
}
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
- badCollection := Collection{
+ badCollection := arvados.Collection{
Name: "bad/collection",
OwnerUUID: arvadostest.AProjectUUID,
}
@@ -127,7 +128,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
c.Assert(err, check.IsNil)
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
- badProject := Group{
+ badProject := arvados.Group{
Name: "bad/project",
GroupClass: "project",
OwnerUUID: arvadostest.AProjectUUID,
@@ -155,7 +156,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
- oob := Collection{
+ oob := arvados.Collection{
Name: "oob",
OwnerUUID: arvadostest.AProjectUUID,
}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs/fs_site.go
similarity index 92%
rename from sdk/go/arvados/fs_site.go
rename to sdk/go/arvados/fs/fs_site.go
index 82114e2ea..ab02c6610 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs/fs_site.go
@@ -2,13 +2,15 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
type CustomFileSystem interface {
@@ -26,7 +28,7 @@ type customFileSystem struct {
staleLock sync.Mutex
}
-func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
+func NewCustomFileSystem(c apiClient, kc keepClient) CustomFileSystem {
root := &vdirnode{}
fs := &customFileSystem{
root: root,
@@ -98,8 +100,8 @@ func (fs *customFileSystem) MountUsers(mount string) {
// This is experimental: the filesystem layout is not stable, and
// there are significant known bugs and shortcomings. For example,
// writes are not persisted until Sync() is called.
-func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
- fs := c.CustomFileSystem(kc)
+func NewSiteFileSystem(c apiClient, kc keepClient) CustomFileSystem {
+ fs := NewCustomFileSystem(c, kc)
fs.MountByID("by_id")
fs.MountUsers("users")
return fs
@@ -125,7 +127,7 @@ func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.
}
func (fs *customFileSystem) mountByID(parent inode, id string) inode {
- if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
+ if strings.Contains(id, "-4zz18-") || arvados.IsPDH(id) {
return fs.mountCollection(parent, id)
} else if strings.Contains(id, "-j7d0g-") {
return fs.newProjectNode(fs.root, id, id)
@@ -135,12 +137,12 @@ func (fs *customFileSystem) mountByID(parent inode, id string) inode {
}
func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
- var coll Collection
+ var coll arvados.Collection
err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- cfs, err := coll.FileSystem(fs, fs)
+ cfs, err := NewFileSystem(coll, fs, fs)
if err != nil {
return nil
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs/fs_site_test.go
similarity index 94%
rename from sdk/go/arvados/fs_site_test.go
rename to sdk/go/arvados/fs/fs_site_test.go
index 80028dc59..5d2a92256 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs/fs_site_test.go
@@ -2,12 +2,13 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"net/http"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -15,13 +16,13 @@ import (
var _ = check.Suite(&SiteFSSuite{})
type SiteFSSuite struct {
- client *Client
+ client *arvados.Client
fs CustomFileSystem
kc keepClient
}
func (s *SiteFSSuite) SetUpTest(c *check.C) {
- s.client = &Client{
+ s.client = &arvados.Client{
APIHost: os.Getenv("ARVADOS_API_HOST"),
AuthToken: arvadostest.ActiveToken,
Insecure: true,
@@ -30,7 +31,7 @@ func (s *SiteFSSuite) SetUpTest(c *check.C) {
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
}}
- s.fs = s.client.SiteFileSystem(s.kc)
+ s.fs = NewSiteFileSystem(s.client, s.kc)
}
func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs/fs_users.go
similarity index 74%
rename from sdk/go/arvados/fs_users.go
rename to sdk/go/arvados/fs/fs_users.go
index 00f703696..a4792dfea 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs/fs_users.go
@@ -2,17 +2,19 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
- var resp UserList
- err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ var resp arvados.UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, arvados.ResourceListParams{
Count: "none",
- Filters: []Filter{{"username", "=", name}},
+ Filters: []arvados.Filter{{"username", "=", name}},
})
if err != nil {
return nil, err
@@ -24,13 +26,13 @@ func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, erro
}
func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
- params := ResourceListParams{
+ params := arvados.ResourceListParams{
Count: "none",
Order: "uuid",
}
var inodes []inode
for {
- var resp UserList
+ var resp arvados.UserList
err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
if err != nil {
return nil, err
@@ -43,6 +45,6 @@ func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
}
inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
- params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ params.Filters = []arvados.Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
}
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index fa309f655..c05ba7faf 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -9,6 +9,7 @@ import (
"os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
@@ -20,20 +21,20 @@ var ErrNoManifest = errors.New("Collection has no manifest")
// CollectionFileReader returns a Reader that reads content from a single file
// in the collection. The filename must be relative to the root of the
// collection. A leading prefix of "/" or "./" in the filename is ignored.
-func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (arvados.File, error) {
+func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (fs.File, error) {
mText, ok := collection["manifest_text"].(string)
if !ok {
return nil, ErrNoManifest
}
- fs, err := (&arvados.Collection{ManifestText: mText}).FileSystem(nil, kc)
+ fs, err := fs.NewFileSystem(arvados.Collection{ManifestText: mText}, nil, kc)
if err != nil {
return nil, err
}
return fs.OpenFile(filename, os.O_RDONLY, 0)
}
-func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
- fs, err := (&arvados.Collection{ManifestText: m.Text}).FileSystem(nil, kc)
+func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (fs.File, error) {
+ fs, err := fs.NewFileSystem(arvados.Collection{ManifestText: m.Text}, nil, kc)
if err != nil {
return nil, err
}
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 8336b78f9..711dd700b 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -9,6 +9,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
"github.com/prometheus/client_golang/prometheus"
@@ -147,7 +148,7 @@ var selectPDH = map[string]interface{}{
// Update saves a modified version (fs) to an existing collection
// (coll) and, if successful, updates the relevant cache entries so
// subsequent calls to Get() reflect the modifications.
-func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error {
+func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs fs.CollectionFileSystem) error {
c.setupOnce.Do(c.setup)
if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText {
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 912398fa6..4ef86be99 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -20,6 +20,7 @@ import (
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/health"
@@ -438,13 +439,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
- fs, err := collection.FileSystem(client, kc)
+ collfs, err := fs.NewFileSystem(*collection, client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ writefs, writeOK := collfs.(fs.CollectionFileSystem)
targetIsPDH := arvadosclient.PDHMatch(collectionID)
if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
@@ -466,7 +467,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h := webdav.Handler{
Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
FileSystem: &webdavFS{
- collfs: fs,
+ collfs: collfs,
writing: writeMethod[r.Method],
alwaysReadEOF: r.Method == "PROPFIND",
},
@@ -482,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
openPath := "/" + strings.Join(targetPath, "/")
- if f, err := fs.Open(openPath); os.IsNotExist(err) {
+ if f, err := collfs.Open(openPath); os.IsNotExist(err) {
// Requested non-existent path
statusCode = http.StatusNotFound
} else if err != nil {
@@ -498,7 +499,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, true)
+ h.serveDirectory(w, r, collection.Name, collfs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -543,8 +544,8 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
- fs := client.SiteFileSystem(kc)
- f, err := fs.Open(r.URL.Path)
+ collfs := fs.NewSiteFileSystem(client, kc)
+ f, err := collfs.Open(r.URL.Path)
if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
@@ -557,7 +558,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
if !strings.HasSuffix(r.URL.Path, "/") {
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else {
- h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ h.serveDirectory(w, r, fi.Name(), collfs, r.URL.Path, false)
}
return
}
@@ -568,7 +569,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
wh := webdav.Handler{
Prefix: "/",
FileSystem: &webdavFS{
- collfs: fs,
+ collfs: collfs,
writing: writeMethod[r.Method],
alwaysReadEOF: r.Method == "PROPFIND",
},
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index bced67ed2..7cba782b0 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -18,6 +18,7 @@ import (
"strings"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/auth"
check "gopkg.in/check.v1"
@@ -436,7 +437,7 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
client := s.testServer.Config.Client
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := fs.NewFileSystem(arvados.Collection{}, &client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 5b23c9c5f..006676b08 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -16,8 +16,7 @@ import (
"sync/atomic"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
-
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"golang.org/x/net/context"
"golang.org/x/net/webdav"
)
@@ -36,7 +35,7 @@ var (
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.FileSystem
+ collfs fs.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list