[ARVADOS] updated: 1.1.0-180-g1f29f37
Git user
git at public.curoverse.com
Tue Nov 28 16:42:25 EST 2017
Summary of changes:
sdk/go/arvados/collection_fs.go | 54 +++++++++++++++++++++++-------------
sdk/go/arvados/collection_fs_test.go | 10 +++----
services/keep-web/cadaver_test.go | 26 +++++++++++++++--
services/keep-web/doc.go | 12 ++++----
services/keep-web/handler.go | 12 ++++++--
5 files changed, 77 insertions(+), 37 deletions(-)
via 1f29f3766ec760168695c8f5bc64aa5ca0c497f1 (commit)
via 9af8e82f25fd6eb6ff3ebe9b7aeb3ef647b98ff0 (commit)
via 1d4d5726abdb9e2899431125d0c44e2a11caf669 (commit)
via 8ab7085c18193e3072481546a8c9f273497c177e (commit)
via 400e4596d20b4b57a6d7bd2dd280f027800a8da6 (commit)
from 65495abc7634b2a7cdccd01f7b6b3188b80bf052 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1f29f3766ec760168695c8f5bc64aa5ca0c497f1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 28 16:24:55 2017 -0500
12483: Update docs to acknowledge read/write webdav support.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 598fabc..b7da3b0 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -2,11 +2,11 @@
//
// SPDX-License-Identifier: AGPL-3.0
-// Keep-web provides read-only HTTP access to files stored in Keep. It
-// serves public data to anonymous and unauthenticated clients, and
-// serves private data to clients that supply Arvados API tokens. It
-// can be installed anywhere with access to Keep services, typically
-// behind a web proxy that supports TLS.
+// Keep-web provides read/write HTTP (WebDAV) access to files stored
+// in Keep. It serves public data to anonymous and unauthenticated
+// clients, and serves private data to clients that supply Arvados API
+// tokens. It can be installed anywhere with access to Keep services,
+// typically behind a web proxy that supports TLS.
//
// See http://doc.arvados.org/install/install-keep-web.html.
//
@@ -40,7 +40,7 @@
//
// Proxy configuration
//
-// Keep-web does not support SSL natively. Typically, it is installed
+// Keep-web does not support TLS natively. Typically, it is installed
// behind a proxy like nginx.
//
// Here is an example nginx configuration.
commit 9af8e82f25fd6eb6ff3ebe9b7aeb3ef647b98ff0
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 28 16:21:10 2017 -0500
12483: Add comments.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index dd8a8e4..28629e3 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -1345,9 +1345,9 @@ func (me *memSegment) ReadAt(p []byte, off int64) (n int, err error) {
type storedSegment struct {
kc keepClient
locator string
- size int
- offset int
- length int
+ size int // size of stored block (also encoded in locator)
+ offset int // position of segment within the stored block
+ length int // bytes in this segment (offset + length <= size)
}
func (se storedSegment) Len() int {
commit 1d4d5726abdb9e2899431125d0c44e2a11caf669
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 28 16:04:55 2017 -0500
12483: Return 405 for all write methods when target is PDH.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 6c6d322..d4a89c8 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -42,6 +42,8 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
c.Assert(err, check.IsNil)
writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
+ pdhPath := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arv.AuthToken + "/"
+
matchToday := time.Now().Format("Jan +2")
readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
@@ -177,10 +179,25 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
},
{
- path: "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arv.AuthToken + "/",
+ path: pdhPath,
cmd: "put '" + localfile.Name() + "' foo\n",
match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
},
+ {
+ path: pdhPath,
+ cmd: "move foo bar\n",
+ match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
+ },
+ {
+ path: pdhPath,
+ cmd: "copy foo bar\n",
+ match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
+ },
+ {
+ path: pdhPath,
+ cmd: "delete foo\n",
+ match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
+ },
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 131214d..4222e38 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -417,9 +417,15 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
+
+ targetIsPDH := arvadosclient.PDHMatch(targetID)
+ if targetIsPDH && writeMethod[r.Method] {
+ statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
+ return
+ }
+
if webdavMethod[r.Method] {
- writing := !arvadosclient.PDHMatch(targetID) && writeMethod[r.Method]
- if writing {
+ if writeMethod[r.Method] {
// Save the collection only if/when all
// webdav->filesystem operations succeed --
// and send a 500 error if the modified
@@ -434,7 +440,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
FileSystem: &webdavFS{
collfs: fs,
- writing: writing,
+ writing: writeMethod[r.Method],
},
LockSystem: h.webdavLS,
Logger: func(_ *http.Request, err error) {
commit 8ab7085c18193e3072481546a8c9f273497c177e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 28 16:02:42 2017 -0500
12483: Don't escape "-", "_", etc. in filenames.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 429fabf..dd8a8e4 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -1389,7 +1389,7 @@ func canonicalName(name string) string {
return name
}
-var manifestEscapeSeq = regexp.MustCompile(`\\([0-9]{3}|\\)`)
+var manifestEscapeSeq = regexp.MustCompile(`\\([0-7]{3}|\\)`)
func manifestUnescapeFunc(seq string) string {
if seq == `\\` {
@@ -1407,7 +1407,7 @@ func manifestUnescape(s string) string {
return manifestEscapeSeq.ReplaceAllStringFunc(s, manifestUnescapeFunc)
}
-var manifestEscapedChar = regexp.MustCompile(`[^\.\w/]`)
+var manifestEscapedChar = regexp.MustCompile(`[\000-\040:\s\\]`)
func manifestEscapeFunc(seq string) string {
return fmt.Sprintf("\\%03o", byte(seq[0]))
diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index fac0b61..f1a3475 100644
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -173,7 +173,7 @@ func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
}
func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
- f, err := s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE, 0)
+ f, err := s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE, 0)
c.Assert(err, check.IsNil)
st, err := f.Stat()
c.Assert(err, check.IsNil)
@@ -185,11 +185,11 @@ func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
c.Check(f.Close(), check.IsNil)
- f, err = s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
+ f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
c.Check(f, check.IsNil)
c.Assert(err, check.NotNil)
- f, err = s.fs.OpenFile("/newfile", os.O_RDWR, 0)
+ f, err = s.fs.OpenFile("/new-file 1", os.O_RDWR, 0)
c.Assert(err, check.IsNil)
st, err = f.Stat()
c.Assert(err, check.IsNil)
@@ -197,8 +197,8 @@ func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
c.Check(f.Close(), check.IsNil)
- // TODO: serialize to Collection, confirm manifest contents,
- // make new FileSystem, confirm file contents.
+ m, err := s.fs.MarshalManifest(".")
+ c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
}
func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
commit 400e4596d20b4b57a6d7bd2dd280f027800a8da6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 28 16:01:17 2017 -0500
12483: Use real modTimes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 56717fd..429fabf 100644
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -496,6 +496,8 @@ func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePt
ptr.segmentOff = 0
ptr.segmentIdx++
}
+
+ fn.fileinfo.modTime = time.Now()
}
return
}
@@ -531,12 +533,22 @@ func (fn *filenode) pruneMemSegments() {
// FileSystem returns a CollectionFileSystem for the collection.
func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
+ var modTime time.Time
+ if c.ModifiedAt == nil {
+ modTime = time.Now()
+ } else {
+ modTime = *c.ModifiedAt
+ }
fs := &fileSystem{dirnode: dirnode{
- client: client,
- kc: kc,
- fileinfo: fileinfo{name: ".", mode: os.ModeDir | 0755},
- parent: nil,
- inodes: make(map[string]inode),
+ client: client,
+ kc: kc,
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ parent: nil,
+ inodes: make(map[string]inode),
}}
fs.dirnode.parent = &fs.dirnode
if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
@@ -943,7 +955,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
default:
switch node := dn.inodes[name].(type) {
case nil:
- dn = dn.newDirnode(name, 0755)
+ dn = dn.newDirnode(name, 0755, dn.fileinfo.modTime)
case *dirnode:
dn = node
case *filenode:
@@ -954,7 +966,7 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
}
switch node := dn.inodes[basename].(type) {
case nil:
- fn = dn.newFilenode(basename, 0755)
+ fn = dn.newFilenode(basename, 0755, dn.fileinfo.modTime)
case *filenode:
fn = node
case *dirnode:
@@ -1162,14 +1174,15 @@ func (dn *dirnode) lookupPath(path string) (node inode) {
return
}
-func (dn *dirnode) newDirnode(name string, perm os.FileMode) *dirnode {
+func (dn *dirnode) newDirnode(name string, perm os.FileMode, modTime time.Time) *dirnode {
child := &dirnode{
parent: dn,
client: dn.client,
kc: dn.kc,
fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | perm,
+ name: name,
+ mode: os.ModeDir | perm,
+ modTime: modTime,
},
}
if dn.inodes == nil {
@@ -1180,12 +1193,13 @@ func (dn *dirnode) newDirnode(name string, perm os.FileMode) *dirnode {
return child
}
-func (dn *dirnode) newFilenode(name string, perm os.FileMode) *filenode {
+func (dn *dirnode) newFilenode(name string, perm os.FileMode, modTime time.Time) *filenode {
child := &filenode{
parent: dn,
fileinfo: fileinfo{
- name: name,
- mode: perm,
+ name: name,
+ mode: perm,
+ modTime: modTime,
},
}
if dn.inodes == nil {
@@ -1242,9 +1256,9 @@ func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*filehandl
return nil, os.ErrNotExist
}
if perm.IsDir() {
- n = dn.newDirnode(name, 0755)
+ n = dn.newDirnode(name, 0755, time.Now())
} else {
- n = dn.newFilenode(name, 0755)
+ n = dn.newFilenode(name, 0755, time.Now())
}
} else if flag&os.O_EXCL != 0 {
return nil, ErrFileExists
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index e80facc..6c6d322 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -12,6 +12,7 @@ import (
"os"
"os/exec"
"strings"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -41,6 +42,8 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
c.Assert(err, check.IsNil)
writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
+ matchToday := time.Now().Format("Jan +2")
+
readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
type testcase struct {
path string
@@ -67,7 +70,7 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
{
path: readPath + "dir1/",
cmd: "ls\n",
- match: `(?ms).*bar *3.*foo *3 .*`,
+ match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
},
{
path: writePath,
@@ -109,7 +112,7 @@ func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
{
path: writePath,
cmd: "ls\n",
- match: `(?ms).*newdir0.* 0 .*`,
+ match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
},
{
path: writePath,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list