[ARVADOS] updated: 1.1.0-150-g9cee268
Git user
git at public.curoverse.com
Wed Nov 15 13:36:12 EST 2017
Summary of changes:
sdk/go/arvados/client.go | 23 +++++++++
sdk/go/arvados/collection.go | 4 ++
sdk/go/arvados/collection_fs.go | 15 ++++--
sdk/go/arvados/collection_fs_test.go | 52 ++++++++++++++++++++
services/keep-web/cache.go | 23 +++++++++
services/keep-web/cadaver_test.go | 79 +++++++++++++++++++++++++++---
services/keep-web/handler.go | 21 ++++++--
services/keep-web/handler_test.go | 2 +-
services/keep-web/webdav.go | 94 ++++++++----------------------------
services/keep-web/webdav_test.go | 5 ++
10 files changed, 229 insertions(+), 89 deletions(-)
create mode 100644 services/keep-web/webdav_test.go
via 9cee2680c3b1f74d2a43d9a0e2b572168e3b488d (commit)
via b33889df9191d3725212220c1304fdeb9c9798a9 (commit)
via 794d493c18ec19b7c2437eb898b5e13645e6ca2a (commit)
from 056a3fa0fcace48ea0053ed5b89cd6e0d1ca792e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9cee2680c3b1f74d2a43d9a0e2b572168e3b488d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Nov 15 13:24:16 2017 -0500
12483: Support file create/write via webdav.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 47a953a..a38d95c 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -5,6 +5,7 @@
package arvados
import (
+ "bytes"
"crypto/tls"
"encoding/json"
"fmt"
@@ -180,6 +181,10 @@ func anythingToValues(params interface{}) (url.Values, error) {
//
// path must not contain a query string.
func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
+ if body, ok := body.(io.Closer); ok {
+ // Ensure body is closed even if we error out early
+ defer body.Close()
+ }
urlString := c.apiURL(path)
urlValues, err := anythingToValues(params)
if err != nil {
@@ -202,6 +207,24 @@ func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.
return c.DoAndDecode(dst, req)
}
+type resource interface {
+ resourceName() string
+}
+
+// UpdateBody returns an io.Reader suitable for use as an http.Request
+// Body for a create or update API call.
+func (c *Client) UpdateBody(rsc resource) io.Reader {
+ j, err := json.Marshal(rsc)
+ if err != nil {
+ // Return a reader that returns errors.
+ r, w := io.Pipe()
+ w.CloseWithError(err)
+ return r
+ }
+ v := url.Values{rsc.resourceName(): {string(j)}}
+ return bytes.NewBufferString(v.Encode())
+}
+
func (c *Client) httpClient() *http.Client {
switch {
case c.Client != nil:
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 61bcd7f..999b4e9 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -30,6 +30,10 @@ type Collection struct {
IsTrashed bool `json:"is_trashed,omitempty"`
}
+func (c Collection) resourceName() string {
+ return "collection"
+}
+
// SizedDigests returns the hash+size part of each data block
// referenced by the collection.
func (c *Collection) SizedDigests() ([]SizedDigest, error) {
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index ce1168a..9ee9990 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -86,6 +86,29 @@ func (c *cache) Stats() cacheStats {
}
}
+// Update saves a modified version (fs) to an existing collection
+// (coll) and, if successful, updates the relevant cache entries so
+// subsequent calls to Get() reflect the modifications.
+func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error {
+ c.setupOnce.Do(c.setup)
+
+ if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText {
+ return err
+ } else {
+ coll.ManifestText = m
+ }
+ var updated arvados.Collection
+ defer c.pdhs.Remove(coll.UUID)
+ err := client.RequestAndDecode(&updated, "PATCH", "/arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
+ if err == nil {
+ c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
+ expire: time.Now().Add(time.Duration(c.TTL)),
+ collection: &updated,
+ })
+ }
+ return err
+}
+
func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
c.setupOnce.Do(c.setup)
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 87a712f..f5fdbec 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -7,42 +7,99 @@ package main
import (
"bytes"
"io"
+ "io/ioutil"
+ "net/url"
+ "os"
"os/exec"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
- basePath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
+ testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
+
+ localfile, err := ioutil.TempFile("", "localfile")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(localfile.Name())
+ localfile.Write(testdata)
+
+ emptyfile, err := ioutil.TempFile("", "emptyfile")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(emptyfile.Name())
+
+ checkfile, err := ioutil.TempFile("", "checkfile")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(checkfile.Name())
+
+ var newCollection arvados.Collection
+ arv := arvados.NewClientFromEnv()
+ arv.AuthToken = arvadostest.ActiveToken
+ err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
+ c.Assert(err, check.IsNil)
+ writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
+
+ readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
type testcase struct {
path string
cmd string
match string
+ data []byte
}
for _, trial := range []testcase{
{
- path: basePath,
+ path: readPath,
cmd: "ls\n",
match: `(?ms).*dir1 *0 .*`,
},
{
- path: basePath,
+ path: readPath,
cmd: "ls dir1\n",
match: `(?ms).*bar *3.*foo *3 .*`,
},
{
- path: basePath + "_/dir1",
+ path: readPath + "_/dir1",
cmd: "ls\n",
match: `(?ms).*bar *3.*foo *3 .*`,
},
{
- path: basePath + "dir1/",
+ path: readPath + "dir1/",
cmd: "ls\n",
match: `(?ms).*bar *3.*foo *3 .*`,
},
+ {
+ path: writePath,
+ cmd: "get emptyfile '" + checkfile.Name() + "'\n",
+ match: `(?ms).*Not Found.*`,
+ },
+ {
+ path: writePath,
+ cmd: "put '" + emptyfile.Name() + "' emptyfile\n",
+ match: `(?ms).*Uploading .* succeeded.*`,
+ },
+ {
+ path: writePath,
+ cmd: "get emptyfile '" + checkfile.Name() + "'\n",
+ match: `(?ms).*great success.*`,
+ data: []byte{},
+ },
+ {
+ path: writePath,
+ cmd: "put '" + localfile.Name() + "' testfile\n",
+ match: `(?ms).*Uploading .* succeeded.*`,
+ },
+ {
+ path: writePath,
+ cmd: "get testfile '" + checkfile.Name() + "'\n",
+ match: `(?ms).*succeeded.*`,
+ data: testdata,
+ },
} {
- c.Logf("%s %#v", "http://"+s.testServer.Addr, trial)
+ c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+
+ os.Remove(checkfile.Name())
+
cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
cmd.Stdin = bytes.NewBufferString(trial.cmd)
stdout, err := cmd.StdoutPipe()
@@ -56,5 +113,15 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
err = cmd.Wait()
c.Check(err, check.Equals, nil)
c.Check(buf.String(), check.Matches, trial.match)
+
+ if trial.data == nil {
+ continue
+ }
+ checkfile, err = os.Open(checkfile.Name())
+ c.Assert(err, check.IsNil)
+ checkfile.Seek(0, os.SEEK_SET)
+ got, err := ioutil.ReadAll(checkfile)
+ c.Check(got, check.DeepEquals, trial.data)
+ c.Check(err, check.IsNil)
}
}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index b4f3062..42f6c51 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -100,6 +100,7 @@ var (
webdavMethod = map[string]bool{
"OPTIONS": true,
"PROPFIND": true,
+ "PUT": true,
}
browserMethod = map[string]bool{
"GET": true,
@@ -147,7 +148,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Range")
- w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, PROPFIND")
+ w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, PROPFIND, PUT")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Max-Age", "86400")
statusCode = http.StatusOK
@@ -352,19 +353,29 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
applyContentDispositionHdr(w, r, basename, attachment)
- fs, err := collection.FileSystem(&arvados.Client{
+ client := &arvados.Client{
APIHost: arv.ApiServer,
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
- }, kc)
+ }
+ fs, err := collection.FileSystem(client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
if webdavMethod[r.Method] {
+ var update func() error
+ if !arvadosclient.PDHMatch(targetID) {
+ update = func() error {
+ return h.Config.Cache.Update(client, *collection, fs)
+ }
+ }
h := webdav.Handler{
- Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
- FileSystem: &webdavFS{collfs: fs},
+ Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
+ FileSystem: &webdavFS{
+ collfs: fs,
+ update: update,
+ },
LockSystem: h.webdavLS,
Logger: func(_ *http.Request, err error) {
if os.IsNotExist(err) {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 6bd34d7..32174cb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -45,7 +45,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
- c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST, OPTIONS, PROPFIND")
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST, OPTIONS, PROPFIND, PUT")
c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range")
// Check preflight for a disallowed request
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 57f3f53..e763a55 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -8,10 +8,9 @@ import (
"crypto/rand"
"errors"
"fmt"
+ "log"
prand "math/rand"
- "net/http"
"os"
- "sync"
"sync/atomic"
"time"
@@ -27,24 +26,28 @@ var (
errReadOnly = errors.New("read-only filesystem")
)
-// webdavFS implements a read-only webdav.FileSystem by wrapping an
+// webdavFS implements a webdav.FileSystem by wrapping an
// arvados.CollectionFilesystem.
type webdavFS struct {
collfs arvados.CollectionFileSystem
+ update func() error
}
-var _ webdav.FileSystem = &webdavFS{}
-
func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
- return errReadOnly
+ return fs.collfs.Mkdir(name, 0755)
}
-func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
- fi, err := fs.collfs.Stat(name)
- if err != nil {
- return nil, err
+func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
+ writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+ if writing && fs.update == nil {
+ return nil, errReadOnly
+ }
+ f, err = fs.collfs.OpenFile(name, flag, perm)
+ log.Printf("OpenFile(%q,%x,%o) => %v, %v", name, flag, perm, f, err)
+ if writing && err == nil {
+ f = writingFile{File: f, update: fs.update}
}
- return &webdavFile{collfs: fs.collfs, fileInfo: fi, name: name}, nil
+ return
}
func (fs *webdavFS) RemoveAll(ctx context.Context, name string) error {
@@ -59,71 +62,16 @@ func (fs *webdavFS) Stat(ctx context.Context, name string) (os.FileInfo, error)
return fs.collfs.Stat(name)
}
-// webdavFile implements a read-only webdav.File by wrapping
-// http.File.
-//
-// The http.File is opened from an arvados.CollectionFileSystem, but
-// not until Seek, Read, or Readdir is called. This deferred-open
-// strategy makes webdav's OpenFile-Stat-Close cycle fast even though
-// the collfs's Open method is slow. This is relevant because webdav
-// does OpenFile-Stat-Close on each file when preparing directory
-// listings.
-//
-// Writes to a webdavFile always fail.
-type webdavFile struct {
- // fields populated by (*webdavFS).OpenFile()
- collfs http.FileSystem
- fileInfo os.FileInfo
- name string
-
- // internal fields
- file http.File
- loadOnce sync.Once
- err error
-}
-
-func (f *webdavFile) load() {
- f.file, f.err = f.collfs.Open(f.name)
-}
-
-func (f *webdavFile) Write([]byte) (int, error) {
- return 0, errReadOnly
-}
-
-func (f *webdavFile) Seek(offset int64, whence int) (int64, error) {
- f.loadOnce.Do(f.load)
- if f.err != nil {
- return 0, f.err
- }
- return f.file.Seek(offset, whence)
-}
-
-func (f *webdavFile) Read(buf []byte) (int, error) {
- f.loadOnce.Do(f.load)
- if f.err != nil {
- return 0, f.err
- }
- return f.file.Read(buf)
-}
-
-func (f *webdavFile) Close() error {
- if f.file == nil {
- // We never called load(), or load() failed
- return f.err
- }
- return f.file.Close()
+type writingFile struct {
+ webdav.File
+ update func() error
}
-func (f *webdavFile) Readdir(n int) ([]os.FileInfo, error) {
- f.loadOnce.Do(f.load)
- if f.err != nil {
- return nil, f.err
+func (f writingFile) Close() error {
+ if err := f.File.Close(); err != nil || f.update == nil {
+ return err
}
- return f.file.Readdir(n)
-}
-
-func (f *webdavFile) Stat() (os.FileInfo, error) {
- return f.fileInfo, nil
+ return f.update()
}
// noLockSystem implements webdav.LockSystem by returning success for
diff --git a/services/keep-web/webdav_test.go b/services/keep-web/webdav_test.go
new file mode 100644
index 0000000..52db776
--- /dev/null
+++ b/services/keep-web/webdav_test.go
@@ -0,0 +1,5 @@
+package main
+
+import "golang.org/x/net/webdav"
+
+var _ webdav.FileSystem = &webdavFS{}
commit b33889df9191d3725212220c1304fdeb9c9798a9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Nov 15 13:14:13 2017 -0500
12483: Fix makeParentDirs bug when higher levels already exist.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 6019cfb..328d68f 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -794,7 +794,10 @@ func (dn *dirnode) loadManifest(txt string) error {
return fmt.Errorf("line %d: bad file segment %q", lineno, token)
}
name := path.Clean(dirname + "/" + manifestUnescape(toks[2]))
- dn.makeParentDirs(name)
+ err = dn.makeParentDirs(name)
+ if err != nil {
+ return fmt.Errorf("line %d: cannot use path %q: %s", lineno, name, err)
+ }
f, err := dn.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0700)
if err != nil {
return fmt.Errorf("line %d: cannot append to %q: %s", lineno, name, err)
@@ -852,7 +855,7 @@ func (dn *dirnode) loadManifest(txt string) error {
func (dn *dirnode) makeParentDirs(name string) (err error) {
names := strings.Split(name, "/")
for _, name := range names[:len(names)-1] {
- f, err := dn.mkdir(name)
+ f, err := dn.OpenFile(name, os.O_CREATE, os.ModeDir|0755)
if err != nil {
return err
}
@@ -872,8 +875,8 @@ func (dn *dirnode) mkdir(name string) (*file, error) {
func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
f, err := dn.mkdir(name)
- if err != nil {
- f.Close()
+ if err == nil {
+ err = f.Close()
}
return err
}
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 6dc7286..ecb1f89 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -578,19 +578,20 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
var err error
s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
- for _, name := range []string{"dir", "zero", "zero/zero"} {
+ for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
err = s.fs.Mkdir(name, 0755)
c.Assert(err, check.IsNil)
}
expect := map[string][]byte{
- "0": nil,
- "00": []byte{},
- "one": []byte{1},
- "dir/0": nil,
- "dir/two": []byte{1, 2},
- "dir/zero": nil,
- "zero/zero/zero": nil,
+ "0": nil,
+ "00": []byte{},
+ "one": []byte{1},
+ "dir/0": nil,
+ "dir/two": []byte{1, 2},
+ "dir/zero": nil,
+ "dir/zerodir/zero": nil,
+ "zero/zero/zero": nil,
}
for name, data := range expect {
f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
commit 794d493c18ec19b7c2437eb898b5e13645e6ca2a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Nov 15 13:01:29 2017 -0500
12483: Persist empty files.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 7962650..6019cfb 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -699,6 +699,10 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
}
subdirs = subdirs + subdir
case *filenode:
+ if len(node.extents) == 0 {
+ segments = append(segments, m1segment{name: name})
+ break
+ }
for _, e := range node.extents {
switch e := e.(type) {
case storedExtent:
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 4be1a3d..6dc7286 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -574,6 +574,57 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
}
}
+func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
+ var err error
+ s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ for _, name := range []string{"dir", "zero", "zero/zero"} {
+ err = s.fs.Mkdir(name, 0755)
+ c.Assert(err, check.IsNil)
+ }
+
+ expect := map[string][]byte{
+ "0": nil,
+ "00": []byte{},
+ "one": []byte{1},
+ "dir/0": nil,
+ "dir/two": []byte{1, 2},
+ "dir/zero": nil,
+ "zero/zero/zero": nil,
+ }
+ for name, data := range expect {
+ f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
+ c.Assert(err, check.IsNil)
+ if data != nil {
+ _, err := f.Write(data)
+ c.Assert(err, check.IsNil)
+ }
+ f.Close()
+ }
+
+ m, err := s.fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
+ c.Logf("%q", m)
+
+ persisted, err := (&Collection{ManifestText: m}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ for name, data := range expect {
+ f, err := persisted.Open("bogus-" + name)
+ c.Check(err, check.NotNil)
+
+ f, err = persisted.Open(name)
+ c.Assert(err, check.IsNil)
+
+ if data == nil {
+ data = []byte{}
+ }
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(buf, check.DeepEquals, data)
+ }
+}
+
func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
maxBlockSize = 1024
defer func() { maxBlockSize = 2 << 26 }()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list