[ARVADOS] created: 1.2.0-41-g84bc63b96
Git user
git at public.curoverse.com
Thu Sep 13 14:34:30 EDT 2018
at 84bc63b965253ce010598b9591f2666bc2831ddb (commit)
commit 84bc63b965253ce010598b9591f2666bc2831ddb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Sep 13 14:30:53 2018 -0400
13994: Extract filesystem to its own package.
Fixes import cycle in tests (arvados -> arvadostest -> arvados).
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4ddbf89c1..81aa5dc98 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -100,6 +100,7 @@ sdk/python
sdk/python:py3
sdk/ruby
sdk/go/arvados
+sdk/go/arvados/fs
sdk/go/arvadosclient
sdk/go/dispatch
sdk/go/keepclient
@@ -924,6 +925,7 @@ gostuff=(
lib/crunchstat
lib/dispatchcloud
sdk/go/arvados
+ sdk/go/arvados/fs
sdk/go/arvadosclient
sdk/go/blockdigest
sdk/go/dispatch
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index cca9f9bf1..fb2a33e8c 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -227,13 +227,13 @@ func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.
return c.DoAndDecode(dst, req)
}
-type resource interface {
+type NamedResource interface {
resourceName() string
}
// UpdateBody returns an io.Reader suitable for use as an http.Request
// Body for a create or update API call.
-func (c *Client) UpdateBody(rsc resource) io.Reader {
+func (c *Client) UpdateBody(rsc NamedResource) io.Reader {
j, err := json.Marshal(rsc)
if err != nil {
// Return a reader that returns errors.
@@ -323,8 +323,12 @@ func (c *Client) DiscoveryDocument() (*DiscoveryDocument, error) {
var pdhRegexp = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`)
+func IsPDH(s string) bool {
+ return pdhRegexp.MatchString(s)
+}
+
func (c *Client) modelForUUID(dd *DiscoveryDocument, uuid string) (string, error) {
- if pdhRegexp.MatchString(uuid) {
+ if IsPDH(uuid) {
return "Collection", nil
}
if len(uuid) != 27 {
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs/fs_backend.go
similarity index 82%
rename from sdk/go/arvados/fs_backend.go
rename to sdk/go/arvados/fs/fs_backend.go
index 301f0b48b..69d0e653f 100644
--- a/sdk/go/arvados/fs_backend.go
+++ b/sdk/go/arvados/fs/fs_backend.go
@@ -2,9 +2,13 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
-import "io"
+import (
+ "io"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
type fsBackend interface {
keepClient
@@ -25,5 +29,5 @@ type keepClient interface {
type apiClient interface {
RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
- UpdateBody(rsc resource) io.Reader
+ UpdateBody(rsc arvados.NamedResource) io.Reader
}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs/fs_base.go
similarity index 99%
rename from sdk/go/arvados/fs_base.go
rename to sdk/go/arvados/fs/fs_base.go
index 3058a7609..f0992754c 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs/fs_base.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"errors"
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs/fs_collection.go
similarity index 98%
rename from sdk/go/arvados/fs_collection.go
rename to sdk/go/arvados/fs/fs_collection.go
index 7ce37aa24..e68dce034 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs/fs_collection.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"encoding/json"
@@ -17,6 +17,8 @@ import (
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
var maxBlockSize = 1 << 26
@@ -38,8 +40,9 @@ type collectionFileSystem struct {
uuid string
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
+// NewFileSystem returns a new CollectionFileSystem with initial state
+// based on the given collection.
+func NewFileSystem(c arvados.Collection, client apiClient, kc keepClient) (CollectionFileSystem, error) {
var modTime time.Time
if c.ModifiedAt == nil {
modTime = time.Now()
@@ -122,7 +125,7 @@ func (fs *collectionFileSystem) Sync() error {
log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
return err
}
- coll := &Collection{
+ coll := &arvados.Collection{
UUID: fs.uuid,
ManifestText: txt,
}
@@ -524,7 +527,7 @@ func (dn *dirnode) FS() FileSystem {
func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
if dn == dn.fs.rootnode() && name == ".arvados#collection" {
gn := &getternode{Getter: func() ([]byte, error) {
- var coll Collection
+ var coll arvados.Collection
var err error
coll.ManifestText, err = dn.fs.MarshalManifest(".")
if err != nil {
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs/fs_collection_test.go
similarity index 95%
rename from sdk/go/arvados/fs_collection_test.go
rename to sdk/go/arvados/fs/fs_collection_test.go
index d2f55d0e3..08a9745b4 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs/fs_collection_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
@@ -20,6 +20,7 @@ import (
"testing"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -54,21 +55,21 @@ func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
}
type CollectionFSSuite struct {
- client *Client
- coll Collection
+ client *arvados.Client
+ coll arvados.Collection
fs CollectionFileSystem
kc keepClient
}
func (s *CollectionFSSuite) SetUpTest(c *check.C) {
- s.client = NewClientFromEnv()
+ s.client = arvados.NewClientFromEnv()
err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
c.Assert(err, check.IsNil)
s.kc = &keepClientStub{
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
}}
- s.fs, err = s.coll.FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(s.coll, s.client, s.kc)
c.Assert(err, check.IsNil)
}
@@ -78,9 +79,9 @@ func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
}
func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
- fs, err := (&Collection{
+ fs, err := NewFileSystem(arvados.Collection{
ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
- }).FileSystem(s.client, s.kc)
+ }, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.Open("/foo:foo")
@@ -356,7 +357,7 @@ func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
}
func (s *CollectionFSSuite) TestSeekSparse(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
c.Assert(err, check.IsNil)
@@ -403,7 +404,7 @@ func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
for _, name := range []string{"foo", "bar", "baz"} {
f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
@@ -520,7 +521,7 @@ func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
const nfiles = 256
@@ -580,7 +581,7 @@ func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
}
func (s *CollectionFSSuite) TestRemove(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = fs.Mkdir("dir0", 0755)
c.Assert(err, check.IsNil)
@@ -613,7 +614,7 @@ func (s *CollectionFSSuite) TestRemove(c *check.C) {
}
func (s *CollectionFSSuite) TestRenameError(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = fs.Mkdir("first", 0755)
c.Assert(err, check.IsNil)
@@ -637,7 +638,7 @@ func (s *CollectionFSSuite) TestRenameError(c *check.C) {
}
func (s *CollectionFSSuite) TestRename(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
const (
outer = 16
@@ -724,7 +725,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
defer func() { maxBlockSize = 2 << 26 }()
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
err = s.fs.Mkdir("d:r", 0755)
c.Assert(err, check.IsNil)
@@ -765,7 +766,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
c.Check(err, check.IsNil)
c.Check(len(fi), check.Equals, 4)
- persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
+ persisted, err := NewFileSystem(arvados.Collection{ManifestText: m}, s.client, s.kc)
c.Assert(err, check.IsNil)
root, err = persisted.Open("/")
@@ -788,7 +789,7 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
var err error
- s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ s.fs, err = NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
err = s.fs.Mkdir(name, 0755)
@@ -819,7 +820,7 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
c.Check(err, check.IsNil)
c.Logf("%q", m)
- persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
+ persisted, err := NewFileSystem(arvados.Collection{ManifestText: m}, s.client, s.kc)
c.Assert(err, check.IsNil)
for name, data := range expect {
@@ -839,7 +840,7 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
}
func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("missing", os.O_WRONLY, 0)
@@ -940,7 +941,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
- fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{}, s.client, s.kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("50K", os.O_WRONLY|os.O_CREATE, 0)
c.Assert(err, check.IsNil)
@@ -991,7 +992,7 @@ func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
"./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
} {
c.Logf("<-%q", txt)
- fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{ManifestText: txt}, s.client, s.kc)
c.Check(fs, check.IsNil)
c.Logf("-> %s", err)
c.Check(err, check.NotNil)
@@ -1007,7 +1008,7 @@ func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
} {
c.Logf("<-%q", txt)
- fs, err := (&Collection{ManifestText: txt}).FileSystem(s.client, s.kc)
+ fs, err := NewFileSystem(arvados.Collection{ManifestText: txt}, s.client, s.kc)
c.Check(err, check.IsNil)
c.Check(fs, check.NotNil)
}
@@ -1050,14 +1051,14 @@ func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
}
mb.Write([]byte{'\n'})
}
- coll := Collection{ManifestText: mb.String()}
+ coll := arvados.Collection{ManifestText: mb.String()}
c.Logf("%s built", time.Now())
var memstats runtime.MemStats
runtime.ReadMemStats(&memstats)
c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
- f, err := coll.FileSystem(nil, nil)
+ f, err := NewFileSystem(coll, nil, nil)
c.Check(err, check.IsNil)
c.Logf("%s loaded", time.Now())
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs/fs_deferred.go
similarity index 94%
rename from sdk/go/arvados/fs_deferred.go
rename to sdk/go/arvados/fs/fs_deferred.go
index a84f64fe7..ecca3cd3c 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs/fs_deferred.go
@@ -2,16 +2,18 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"log"
"os"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
-func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+func deferredCollectionFS(fs FileSystem, parent inode, coll arvados.Collection) inode {
var modTime time.Time
if coll.ModifiedAt != nil {
modTime = *coll.ModifiedAt
@@ -34,7 +36,7 @@ func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
log.Printf("BUG: unhandled error: %s", err)
return placeholder
}
- cfs, err := coll.FileSystem(fs, fs)
+ cfs, err := NewFileSystem(coll, fs, fs)
if err != nil {
log.Printf("BUG: unhandled error: %s", err)
return placeholder
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs/fs_filehandle.go
similarity index 99%
rename from sdk/go/arvados/fs_filehandle.go
rename to sdk/go/arvados/fs/fs_filehandle.go
index 9af8d0ad4..09f65ce80 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs/fs_filehandle.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"io"
diff --git a/sdk/go/arvados/fs_getternode.go b/sdk/go/arvados/fs/fs_getternode.go
similarity index 98%
rename from sdk/go/arvados/fs_getternode.go
rename to sdk/go/arvados/fs/fs_getternode.go
index 966fe9d5c..b98dde35b 100644
--- a/sdk/go/arvados/fs_getternode.go
+++ b/sdk/go/arvados/fs/fs_getternode.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
diff --git a/sdk/go/arvados/fs_lookup.go b/sdk/go/arvados/fs/fs_lookup.go
similarity index 99%
rename from sdk/go/arvados/fs_lookup.go
rename to sdk/go/arvados/fs/fs_lookup.go
index 42322a14a..28a022cd5 100644
--- a/sdk/go/arvados/fs_lookup.go
+++ b/sdk/go/arvados/fs/fs_lookup.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs/fs_project.go
similarity index 80%
rename from sdk/go/arvados/fs_project.go
rename to sdk/go/arvados/fs/fs_project.go
index 92995510c..7cbf7dbd1 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs/fs_project.go
@@ -2,19 +2,21 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"log"
"os"
"strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
if uuid != "" {
return uuid, nil
}
- var resp User
+ var resp arvados.User
err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return "", err
@@ -29,10 +31,10 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in
return nil, err
}
- var contents CollectionList
- err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ var contents arvados.CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, arvados.ResourceListParams{
Count: "none",
- Filters: []Filter{
+ Filters: []arvados.Filter{
{"name", "=", name},
{"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
{"groups.group_class", "=", "project"},
@@ -69,14 +71,14 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
// Note: the "filters" slice's backing array might be reused
// by append(filters,...) below. This isn't goroutine safe,
// but all accesses are in the same goroutine, so it's OK.
- filters := []Filter{{"owner_uuid", "=", uuid}}
- params := ResourceListParams{
+ filters := []arvados.Filter{{"owner_uuid", "=", uuid}}
+ params := arvados.ResourceListParams{
Count: "none",
Filters: filters,
Order: "uuid",
}
for {
- var resp CollectionList
+ var resp arvados.CollectionList
err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
if err != nil {
return nil, err
@@ -91,13 +93,13 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
}
inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
}
- params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ params.Filters = append(filters, arvados.Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
- filters = append(filters, Filter{"group_class", "=", "project"})
+ filters = append(filters, arvados.Filter{"group_class", "=", "project"})
params.Filters = filters
for {
- var resp GroupList
+ var resp arvados.GroupList
err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
if err != nil {
return nil, err
@@ -111,7 +113,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
}
inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
}
- params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ params.Filters = append(filters, arvados.Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
}
return inodes, nil
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs/fs_project_test.go
similarity index 97%
rename from sdk/go/arvados/fs_project_test.go
rename to sdk/go/arvados/fs/fs_project_test.go
index 1a06ce146..3d9c944e4 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs/fs_project_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"bytes"
@@ -12,6 +12,7 @@ import (
"path/filepath"
"strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -23,7 +24,7 @@ type spiedRequest struct {
}
type spyingClient struct {
- *Client
+ *arvados.Client
calls []spiedRequest
}
@@ -119,7 +120,7 @@ func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
}
func (s *SiteFSSuite) TestSlashInName(c *check.C) {
- badCollection := Collection{
+ badCollection := arvados.Collection{
Name: "bad/collection",
OwnerUUID: arvadostest.AProjectUUID,
}
@@ -127,7 +128,7 @@ func (s *SiteFSSuite) TestSlashInName(c *check.C) {
c.Assert(err, check.IsNil)
defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
- badProject := Group{
+ badProject := arvados.Group{
Name: "bad/project",
GroupClass: "project",
OwnerUUID: arvadostest.AProjectUUID,
@@ -155,7 +156,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
_, err = s.fs.Open("/home/A Project/oob")
c.Check(err, check.NotNil)
- oob := Collection{
+ oob := arvados.Collection{
Name: "oob",
OwnerUUID: arvadostest.AProjectUUID,
}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs/fs_site.go
similarity index 92%
rename from sdk/go/arvados/fs_site.go
rename to sdk/go/arvados/fs/fs_site.go
index 82114e2ea..ab02c6610 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs/fs_site.go
@@ -2,13 +2,15 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
type CustomFileSystem interface {
@@ -26,7 +28,7 @@ type customFileSystem struct {
staleLock sync.Mutex
}
-func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
+func NewCustomFileSystem(c apiClient, kc keepClient) CustomFileSystem {
root := &vdirnode{}
fs := &customFileSystem{
root: root,
@@ -98,8 +100,8 @@ func (fs *customFileSystem) MountUsers(mount string) {
// This is experimental: the filesystem layout is not stable, and
// there are significant known bugs and shortcomings. For example,
// writes are not persisted until Sync() is called.
-func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
- fs := c.CustomFileSystem(kc)
+func NewSiteFileSystem(c apiClient, kc keepClient) CustomFileSystem {
+ fs := NewCustomFileSystem(c, kc)
fs.MountByID("by_id")
fs.MountUsers("users")
return fs
@@ -125,7 +127,7 @@ func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.
}
func (fs *customFileSystem) mountByID(parent inode, id string) inode {
- if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
+ if strings.Contains(id, "-4zz18-") || arvados.IsPDH(id) {
return fs.mountCollection(parent, id)
} else if strings.Contains(id, "-j7d0g-") {
return fs.newProjectNode(fs.root, id, id)
@@ -135,12 +137,12 @@ func (fs *customFileSystem) mountByID(parent inode, id string) inode {
}
func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
- var coll Collection
+ var coll arvados.Collection
err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
return nil
}
- cfs, err := coll.FileSystem(fs, fs)
+ cfs, err := NewFileSystem(coll, fs, fs)
if err != nil {
return nil
}
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs/fs_site_test.go
similarity index 94%
rename from sdk/go/arvados/fs_site_test.go
rename to sdk/go/arvados/fs/fs_site_test.go
index 80028dc59..5d2a92256 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs/fs_site_test.go
@@ -2,12 +2,13 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"net/http"
"os"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -15,13 +16,13 @@ import (
var _ = check.Suite(&SiteFSSuite{})
type SiteFSSuite struct {
- client *Client
+ client *arvados.Client
fs CustomFileSystem
kc keepClient
}
func (s *SiteFSSuite) SetUpTest(c *check.C) {
- s.client = &Client{
+ s.client = &arvados.Client{
APIHost: os.Getenv("ARVADOS_API_HOST"),
AuthToken: arvadostest.ActiveToken,
Insecure: true,
@@ -30,7 +31,7 @@ func (s *SiteFSSuite) SetUpTest(c *check.C) {
blocks: map[string][]byte{
"3858f62230ac3c915f300c664312c63f": []byte("foobar"),
}}
- s.fs = s.client.SiteFileSystem(s.kc)
+ s.fs = NewSiteFileSystem(s.client, s.kc)
}
func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs/fs_users.go
similarity index 74%
rename from sdk/go/arvados/fs_users.go
rename to sdk/go/arvados/fs/fs_users.go
index 00f703696..a4792dfea 100644
--- a/sdk/go/arvados/fs_users.go
+++ b/sdk/go/arvados/fs/fs_users.go
@@ -2,17 +2,19 @@
//
// SPDX-License-Identifier: Apache-2.0
-package arvados
+package fs
import (
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
- var resp UserList
- err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ var resp arvados.UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, arvados.ResourceListParams{
Count: "none",
- Filters: []Filter{{"username", "=", name}},
+ Filters: []arvados.Filter{{"username", "=", name}},
})
if err != nil {
return nil, err
@@ -24,13 +26,13 @@ func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, erro
}
func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
- params := ResourceListParams{
+ params := arvados.ResourceListParams{
Count: "none",
Order: "uuid",
}
var inodes []inode
for {
- var resp UserList
+ var resp arvados.UserList
err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
if err != nil {
return nil, err
@@ -43,6 +45,6 @@ func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
}
inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
}
- params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ params.Filters = []arvados.Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
}
}
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index fa309f655..c05ba7faf 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -9,6 +9,7 @@ import (
"os"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
@@ -20,20 +21,20 @@ var ErrNoManifest = errors.New("Collection has no manifest")
// CollectionFileReader returns a Reader that reads content from a single file
// in the collection. The filename must be relative to the root of the
// collection. A leading prefix of "/" or "./" in the filename is ignored.
-func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (arvados.File, error) {
+func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (fs.File, error) {
mText, ok := collection["manifest_text"].(string)
if !ok {
return nil, ErrNoManifest
}
- fs, err := (&arvados.Collection{ManifestText: mText}).FileSystem(nil, kc)
+ fs, err := fs.NewFileSystem(arvados.Collection{ManifestText: mText}, nil, kc)
if err != nil {
return nil, err
}
return fs.OpenFile(filename, os.O_RDONLY, 0)
}
-func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
- fs, err := (&arvados.Collection{ManifestText: m.Text}).FileSystem(nil, kc)
+func (kc *KeepClient) ManifestFileReader(m manifest.Manifest, filename string) (fs.File, error) {
+ fs, err := fs.NewFileSystem(arvados.Collection{ManifestText: m.Text}, nil, kc)
if err != nil {
return nil, err
}
diff --git a/services/crunch-run/copier.go b/services/crunch-run/copier.go
index 4c45f6acb..5dc362779 100644
--- a/services/crunch-run/copier.go
+++ b/services/crunch-run/copier.go
@@ -15,6 +15,7 @@ import (
"strings"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/manifest"
)
@@ -72,7 +73,7 @@ func (cp *copier) Copy() (string, error) {
if err != nil {
return "", err
}
- fs, err := (&arvados.Collection{ManifestText: cp.manifest}).FileSystem(cp.client, cp.keepClient)
+ fs, err := fs.NewFileSystem(arvados.Collection{ManifestText: cp.manifest}, cp.client, cp.keepClient)
if err != nil {
return "", err
}
@@ -91,7 +92,7 @@ func (cp *copier) Copy() (string, error) {
return fs.MarshalManifest(".")
}
-func (cp *copier) copyFile(fs arvados.CollectionFileSystem, f filetodo) error {
+func (cp *copier) copyFile(fs fs.CollectionFileSystem, f filetodo) error {
cp.logger.Printf("copying %q (%d bytes)", f.dst, f.size)
dst, err := fs.OpenFile(f.dst, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 0a980b9ce..4b5d3dc0c 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -29,6 +29,7 @@ import (
"git.curoverse.com/arvados.git/lib/crunchstat"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
@@ -60,7 +61,7 @@ var ErrCancelled = errors.New("Cancelled")
type IKeepClient interface {
PutB(buf []byte) (string, int, error)
ReadAt(locator string, p []byte, off int) (int, error)
- ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
+ ManifestFileReader(m manifest.Manifest, filename string) (fs.File, error)
ClearBlockCache()
}
@@ -106,7 +107,7 @@ type ContainerRunner struct {
CrunchLog *ThrottledLogger
Stdout io.WriteCloser
Stderr io.WriteCloser
- LogCollection arvados.CollectionFileSystem
+ LogCollection fs.CollectionFileSystem
LogsPDH *string
RunArvMount
MkTempDir
@@ -887,7 +888,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
runner.CrunchLog.Print("Attaching container streams")
// If stdin mount is provided, attach it to the docker container
- var stdinRdr arvados.File
+ var stdinRdr fs.File
var stdinJson []byte
if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok {
if stdinMnt.Kind == "collection" {
@@ -1617,7 +1618,7 @@ func NewContainerRunner(client *arvados.Client, api IArvadosClient, kc IKeepClie
return cl, nil
}
var err error
- cr.LogCollection, err = (&arvados.Collection{}).FileSystem(cr.client, cr.Kc)
+ cr.LogCollection, err = fs.NewFileSystem(arvados.Collection{}, cr.client, cr.Kc)
if err != nil {
return nil, err
}
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 8336b78f9..711dd700b 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -9,6 +9,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
"github.com/prometheus/client_golang/prometheus"
@@ -147,7 +148,7 @@ var selectPDH = map[string]interface{}{
// Update saves a modified version (fs) to an existing collection
// (coll) and, if successful, updates the relevant cache entries so
// subsequent calls to Get() reflect the modifications.
-func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error {
+func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs fs.CollectionFileSystem) error {
c.setupOnce.Do(c.setup)
if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText {
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 912398fa6..4ef86be99 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -20,6 +20,7 @@ import (
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/health"
@@ -438,13 +439,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
- fs, err := collection.FileSystem(client, kc)
+ collfs, err := fs.NewFileSystem(*collection, client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ writefs, writeOK := collfs.(fs.CollectionFileSystem)
targetIsPDH := arvadosclient.PDHMatch(collectionID)
if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
@@ -466,7 +467,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h := webdav.Handler{
Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
FileSystem: &webdavFS{
- collfs: fs,
+ collfs: collfs,
writing: writeMethod[r.Method],
alwaysReadEOF: r.Method == "PROPFIND",
},
@@ -482,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
openPath := "/" + strings.Join(targetPath, "/")
- if f, err := fs.Open(openPath); os.IsNotExist(err) {
+ if f, err := collfs.Open(openPath); os.IsNotExist(err) {
// Requested non-existent path
statusCode = http.StatusNotFound
} else if err != nil {
@@ -498,7 +499,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, true)
+ h.serveDirectory(w, r, collection.Name, collfs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -543,8 +544,8 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
- fs := client.SiteFileSystem(kc)
- f, err := fs.Open(r.URL.Path)
+ collfs := fs.NewSiteFileSystem(client, kc)
+ f, err := collfs.Open(r.URL.Path)
if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
@@ -557,7 +558,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
if !strings.HasSuffix(r.URL.Path, "/") {
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else {
- h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ h.serveDirectory(w, r, fi.Name(), collfs, r.URL.Path, false)
}
return
}
@@ -568,7 +569,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
wh := webdav.Handler{
Prefix: "/",
FileSystem: &webdavFS{
- collfs: fs,
+ collfs: collfs,
writing: writeMethod[r.Method],
alwaysReadEOF: r.Method == "PROPFIND",
},
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index bced67ed2..7cba782b0 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -18,6 +18,7 @@ import (
"strings"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/auth"
check "gopkg.in/check.v1"
@@ -436,7 +437,7 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
client := s.testServer.Config.Client
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := fs.NewFileSystem(arvados.Collection{}, &client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 5b23c9c5f..006676b08 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -16,8 +16,7 @@ import (
"sync/atomic"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
-
+ "git.curoverse.com/arvados.git/sdk/go/arvados/fs"
"golang.org/x/net/context"
"golang.org/x/net/webdav"
)
@@ -36,7 +35,7 @@ var (
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.FileSystem
+ collfs fs.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
commit 55f6178b9a9a0d165e952eeec9a04d0234299397
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 11 12:55:05 2018 -0400
13994: Fix deadlock in keepstore tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 2140dceab..6778b39fb 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -23,7 +23,10 @@ func RefreshServiceDiscovery() {
svcListCacheMtx.Lock()
defer svcListCacheMtx.Unlock()
for _, ent := range svcListCache {
- ent.clear <- struct{}{}
+ select {
+ case ent.clear <- struct{}{}:
+ default:
+ }
}
}
@@ -136,7 +139,7 @@ func (kc *KeepClient) discoverServices() error {
arv := *kc.Arvados
cacheEnt = cachedSvcList{
latest: make(chan svcList),
- clear: make(chan struct{}),
+ clear: make(chan struct{}, 1),
arv: &arv,
}
go cacheEnt.poll()
commit d6993a413a1290f110c52ed7339b09caf8b87b15
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 11 12:54:41 2018 -0400
13994: Proxy to remote cluster if +R hint given.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 6a4b6232a..e6984601f 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -8,6 +8,7 @@ package arvadostest
const (
SpectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
ActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ ActiveTokenV2 = "v2/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
AdminToken = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h"
AnonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
DataManagerToken = "320mkve8qkswstz7ff61glpk3mhgghmg67wmic7elw4z41pke1"
diff --git a/sdk/go/arvadostest/integration_test_cluster.go b/sdk/go/arvadostest/integration_test_cluster.go
new file mode 100644
index 000000000..ac08ce184
--- /dev/null
+++ b/sdk/go/arvadostest/integration_test_cluster.go
@@ -0,0 +1,21 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvadostest
+
+import (
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ check "gopkg.in/check.v1"
+)
+
+func IntegrationTestCluster(c *check.C) *arvados.Cluster {
+ config, err := arvados.GetConfig(filepath.Join(os.Getenv("WORKSPACE"), "tmp", "arvados.yml"))
+ c.Assert(err, check.IsNil)
+ cluster, err := config.GetCluster("")
+ c.Assert(err, check.IsNil)
+ return cluster
+}
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index f012ea390..c37a4d112 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -30,6 +30,10 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
)
+var testCluster = &arvados.Cluster{
+ ClusterID: "zzzzz",
+}
+
// A RequestTester represents the parameters for an HTTP request to
// be issued on behalf of a unit test.
type RequestTester struct {
@@ -823,7 +827,7 @@ func IssueRequest(rt *RequestTester) *httptest.ResponseRecorder {
if rt.apiToken != "" {
req.Header.Set("Authorization", "OAuth2 "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter()
+ loggingRouter := MakeRESTRouter(testCluster)
loggingRouter.ServeHTTP(response, req)
return response
}
@@ -835,7 +839,7 @@ func IssueHealthCheckRequest(rt *RequestTester) *httptest.ResponseRecorder {
if rt.apiToken != "" {
req.Header.Set("Authorization", "Bearer "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter()
+ loggingRouter := MakeRESTRouter(testCluster)
loggingRouter.ServeHTTP(response, req)
return response
}
@@ -975,7 +979,7 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
ok := make(chan struct{})
go func() {
req, _ := http.NewRequest("GET", fmt.Sprintf("/%s+%d", TestHash, len(TestBlock)), nil)
- MakeRESTRouter().ServeHTTP(resp, req)
+ MakeRESTRouter(testCluster).ServeHTTP(resp, req)
ok <- struct{}{}
}()
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index c31ab9c2e..2426c9cbd 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -4,13 +4,6 @@
package main
-// REST handlers for Keep are implemented here.
-//
-// GetBlockHandler (GET /locator)
-// PutBlockHandler (PUT /locator)
-// IndexHandler (GET /index, GET /index/prefix)
-// StatusHandler (GET /status.json)
-
import (
"container/list"
"context"
@@ -29,27 +22,33 @@ import (
"github.com/gorilla/mux"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
type router struct {
*mux.Router
- limiter httpserver.RequestCounter
+ limiter httpserver.RequestCounter
+ cluster *arvados.Cluster
+ remoteProxy remoteProxy
}
// MakeRESTRouter returns a new router that forwards all Keep requests
// to the appropriate handlers.
-func MakeRESTRouter() http.Handler {
- rtr := &router{Router: mux.NewRouter()}
+func MakeRESTRouter(cluster *arvados.Cluster) http.Handler {
+ rtr := &router{
+ Router: mux.NewRouter(),
+ cluster: cluster,
+ }
rtr.HandleFunc(
- `/{hash:[0-9a-f]{32}}`, GetBlockHandler).Methods("GET", "HEAD")
+ `/{hash:[0-9a-f]{32}}`, rtr.handleGET).Methods("GET", "HEAD")
rtr.HandleFunc(
`/{hash:[0-9a-f]{32}}+{hints}`,
- GetBlockHandler).Methods("GET", "HEAD")
+ rtr.handleGET).Methods("GET", "HEAD")
- rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, PutBlockHandler).Methods("PUT")
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handlePUT).Methods("PUT")
rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
// List all blocks stored here. Privileged client only.
rtr.HandleFunc(`/index`, rtr.IndexHandler).Methods("GET", "HEAD")
@@ -98,11 +97,16 @@ func BadRequestHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, BadRequestError.Error(), BadRequestError.HTTPCode)
}
-// GetBlockHandler is a HandleFunc to address Get block requests.
-func GetBlockHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
ctx, cancel := contextForResponse(context.TODO(), resp)
defer cancel()
+ locator := req.URL.Path[1:]
+ if strings.Contains(locator, "+R") && !strings.Contains(locator, "+A") {
+ rtr.remoteProxy.Get(resp, req, rtr.cluster)
+ return
+ }
+
if theConfig.RequireSignatures {
locator := req.URL.Path[1:] // strip leading slash
if err := VerifySignature(locator, GetAPIToken(req)); err != nil {
@@ -177,8 +181,7 @@ func getBufferWithContext(ctx context.Context, bufs *bufferPool, bufSize int) ([
}
}
-// PutBlockHandler is a HandleFunc to address Put block requests.
-func PutBlockHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
ctx, cancel := contextForResponse(context.TODO(), resp)
defer cancel()
@@ -826,7 +829,7 @@ func IsValidLocator(loc string) bool {
return validLocatorRe.MatchString(loc)
}
-var authRe = regexp.MustCompile(`^OAuth2\s+(.*)`)
+var authRe = regexp.MustCompile(`^(OAuth2|Bearer)\s+(.*)`)
// GetAPIToken returns the OAuth2 token from the Authorization
// header of a HTTP request, or an empty string if no matching
@@ -834,7 +837,7 @@ var authRe = regexp.MustCompile(`^OAuth2\s+(.*)`)
func GetAPIToken(req *http.Request) string {
if auth, ok := req.Header["Authorization"]; ok {
if match := authRe.FindStringSubmatch(auth[0]); match != nil {
- return match[1]
+ return match[2]
}
}
return ""
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 79e3017d5..6ae414bf9 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -13,6 +13,7 @@ import (
"syscall"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/config"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
@@ -149,6 +150,22 @@ func main() {
}
}
+ var cluster *arvados.Cluster
+ cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
+ if err != nil && os.IsNotExist(err) {
+ log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
+ cluster = &arvados.Cluster{
+ ClusterID: "xxxxx",
+ }
+ } else if err != nil {
+ log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
+ } else {
+ cluster, err = cfg.GetCluster("")
+ if err != nil {
+ log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
+ }
+ }
+
log.Println("keepstore starting, pid", os.Getpid())
defer log.Println("keepstore exiting, pid", os.Getpid())
@@ -156,7 +173,7 @@ func main() {
KeepVM = MakeRRVolumeManager(theConfig.Volumes)
// Middleware/handler stack
- router := MakeRESTRouter()
+ router := MakeRESTRouter(cluster)
// Set up a TCP listener.
listener, err := net.Listen("tcp", theConfig.Listen)
diff --git a/services/keepstore/mounts_test.go b/services/keepstore/mounts_test.go
index 0f7b6e973..9fa0090aa 100644
--- a/services/keepstore/mounts_test.go
+++ b/services/keepstore/mounts_test.go
@@ -28,7 +28,7 @@ func (s *MountsSuite) SetUpTest(c *check.C) {
theConfig = DefaultConfig()
theConfig.systemAuthToken = arvadostest.DataManagerToken
theConfig.Start()
- s.rtr = MakeRESTRouter()
+ s.rtr = MakeRESTRouter(testCluster)
}
func (s *MountsSuite) TearDownTest(c *check.C) {
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
new file mode 100644
index 000000000..2e3d66351
--- /dev/null
+++ b/services/keepstore/proxy_remote.go
@@ -0,0 +1,113 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+)
+
+type remoteProxy struct {
+ clients map[string]*keepclient.KeepClient
+ mtx sync.Mutex
+}
+
+func (rp *remoteProxy) Get(w http.ResponseWriter, r *http.Request, cluster *arvados.Cluster) {
+ var remoteClient *keepclient.KeepClient
+ var parts []string
+ for i, part := range strings.Split(r.URL.Path[1:], "+") {
+ switch {
+ case i == 0:
+ // don't try to parse hash part as hint
+ case strings.HasPrefix(part, "A"):
+ // drop local permission hint
+ continue
+ case len(part) > 7 && part[0] == 'R' && part[6] == '-':
+ remoteID := part[1:6]
+ remote, ok := cluster.RemoteClusters[remoteID]
+ if !ok {
+ http.Error(w, "remote cluster not configured", http.StatusBadGateway)
+ return
+ }
+ token := GetAPIToken(r)
+ if token == "" {
+ http.Error(w, "no token provided in Authorization header", http.StatusUnauthorized)
+ return
+ }
+ kc, err := rp.remoteClient(remoteID, remote, token)
+ if err == auth.ErrObsoleteToken {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ remoteClient = kc
+ part = "A" + part[7:]
+ }
+ parts = append(parts, part)
+ }
+ if remoteClient == nil {
+ http.Error(w, "bad request", http.StatusBadRequest)
+ return
+ }
+ locator := strings.Join(parts, "+")
+ rdr, _, _, err := remoteClient.Get(locator)
+ switch err.(type) {
+ case nil:
+ defer rdr.Close()
+ io.Copy(w, rdr)
+ case *keepclient.ErrNotFound:
+ http.Error(w, err.Error(), http.StatusNotFound)
+ default:
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ }
+}
+
+func (rp *remoteProxy) remoteClient(remoteID string, remoteCluster arvados.RemoteCluster, token string) (*keepclient.KeepClient, error) {
+ rp.mtx.Lock()
+ kc, ok := rp.clients[remoteID]
+ rp.mtx.Unlock()
+ if !ok {
+ c := &arvados.Client{
+ APIHost: remoteCluster.Host,
+ AuthToken: "xxx",
+ Insecure: remoteCluster.Insecure,
+ }
+ ac, err := arvadosclient.New(c)
+ if err != nil {
+ return nil, err
+ }
+ kc, err = keepclient.MakeKeepClient(ac)
+ if err != nil {
+ return nil, err
+ }
+
+ rp.mtx.Lock()
+ if rp.clients == nil {
+ rp.clients = map[string]*keepclient.KeepClient{remoteID: kc}
+ } else {
+ rp.clients[remoteID] = kc
+ }
+ rp.mtx.Unlock()
+ }
+ accopy := *kc.Arvados
+ accopy.ApiToken = token
+ kccopy := *kc
+ kccopy.Arvados = &accopy
+ token, err := auth.SaltToken(token, remoteID)
+ if err != nil {
+ return nil, err
+ }
+ kccopy.Arvados.ApiToken = token
+ return &kccopy, nil
+}
diff --git a/services/keepstore/proxy_remote_test.go b/services/keepstore/proxy_remote_test.go
new file mode 100644
index 000000000..84c84d653
--- /dev/null
+++ b/services/keepstore/proxy_remote_test.go
@@ -0,0 +1,149 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "crypto/md5"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ProxyRemoteSuite{})
+
+type ProxyRemoteSuite struct {
+ cluster *arvados.Cluster
+ vm VolumeManager
+ rtr http.Handler
+
+ remoteClusterID string
+ remoteBlobSigningKey []byte
+ remoteKeepLocator string
+ remoteKeepData []byte
+ remoteKeepproxy *httptest.Server
+ remoteKeepRequests int64
+ remoteAPI *httptest.Server
+}
+
+func (s *ProxyRemoteSuite) remoteKeepproxyHandler(w http.ResponseWriter, r *http.Request) {
+ expectToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, s.remoteClusterID)
+ if err != nil {
+ panic(err)
+ }
+ atomic.AddInt64(&s.remoteKeepRequests, 1)
+ var token string
+ if auth := strings.Split(r.Header.Get("Authorization"), " "); len(auth) == 2 && (auth[0] == "OAuth2" || auth[0] == "Bearer") {
+ token = auth[1]
+ }
+ if r.Method == "GET" && r.URL.Path == "/"+s.remoteKeepLocator && token == expectToken {
+ w.Write(s.remoteKeepData)
+ return
+ }
+ http.Error(w, "404", 404)
+}
+
+func (s *ProxyRemoteSuite) remoteAPIHandler(w http.ResponseWriter, r *http.Request) {
+ host, port, _ := net.SplitHostPort(strings.Split(s.remoteKeepproxy.URL, "//")[1])
+ portnum, _ := strconv.Atoi(port)
+ if r.URL.Path == "/arvados/v1/discovery/v1/rest" {
+ json.NewEncoder(w).Encode(arvados.DiscoveryDocument{})
+ return
+ }
+ if r.URL.Path == "/arvados/v1/keep_services/accessible" {
+ json.NewEncoder(w).Encode(arvados.KeepServiceList{
+ Items: []arvados.KeepService{
+ {
+ UUID: s.remoteClusterID + "-bi6l4-proxyproxyproxy",
+ ServiceType: "proxy",
+ ServiceHost: host,
+ ServicePort: portnum,
+ ServiceSSLFlag: false,
+ },
+ },
+ })
+ return
+ }
+ http.Error(w, "404", 404)
+}
+
+func (s *ProxyRemoteSuite) SetUpTest(c *check.C) {
+ s.remoteClusterID = "z0000"
+ s.remoteBlobSigningKey = []byte("3b6df6fb6518afe12922a5bc8e67bf180a358bc8")
+ s.remoteKeepproxy = httptest.NewServer(http.HandlerFunc(s.remoteKeepproxyHandler))
+ s.remoteAPI = httptest.NewUnstartedServer(http.HandlerFunc(s.remoteAPIHandler))
+ s.remoteAPI.StartTLS()
+ s.cluster = arvadostest.IntegrationTestCluster(c)
+ s.cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+ s.remoteClusterID: arvados.RemoteCluster{
+ Host: strings.Split(s.remoteAPI.URL, "//")[1],
+ Proxy: true,
+ Scheme: "http",
+ Insecure: true,
+ },
+ }
+ s.vm = MakeTestVolumeManager(2)
+ KeepVM = s.vm
+ theConfig = DefaultConfig()
+ theConfig.systemAuthToken = arvadostest.DataManagerToken
+ theConfig.Start()
+ s.rtr = MakeRESTRouter(s.cluster)
+}
+
+func (s *ProxyRemoteSuite) TearDownTest(c *check.C) {
+ s.vm.Close()
+ KeepVM = nil
+ theConfig = DefaultConfig()
+ theConfig.Start()
+ s.remoteAPI.Close()
+ s.remoteKeepproxy.Close()
+}
+
+func (s *ProxyRemoteSuite) TestProxyRemote(c *check.C) {
+ data := []byte("foo bar")
+ s.remoteKeepData = data
+ locator := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
+ s.remoteKeepLocator = keepclient.SignLocator(locator, arvadostest.ActiveTokenV2, time.Now().Add(time.Minute), time.Minute, s.remoteBlobSigningKey)
+
+ path := "/" + strings.Replace(s.remoteKeepLocator, "+A", "+R"+s.remoteClusterID+"-", 1)
+
+ var req *http.Request
+ var resp *httptest.ResponseRecorder
+ tryWithToken := func(token string) {
+ req = httptest.NewRequest("GET", path, nil)
+ req.Header.Set("Authorization", "Bearer "+token)
+ resp = httptest.NewRecorder()
+ s.rtr.ServeHTTP(resp, req)
+ }
+
+ // Happy path
+ tryWithToken(arvadostest.ActiveTokenV2)
+ c.Check(s.remoteKeepRequests, check.Equals, int64(1))
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, string(data))
+
+ // Obsolete token
+ tryWithToken(arvadostest.ActiveToken)
+ c.Check(s.remoteKeepRequests, check.Equals, int64(1))
+ c.Check(resp.Code, check.Equals, http.StatusBadRequest)
+ c.Check(resp.Body.String(), check.Not(check.Equals), string(data))
+
+ // Bad token
+ tryWithToken(arvadostest.ActiveTokenV2[:len(arvadostest.ActiveTokenV2)-3] + "xxx")
+ c.Check(s.remoteKeepRequests, check.Equals, int64(2))
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Body.String(), check.Not(check.Equals), string(data))
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list