[ARVADOS] updated: 1.1.0-18-ga23fa06
Git user
git at public.curoverse.com
Thu Oct 12 15:05:31 EDT 2017
Summary of changes:
build/run-tests.sh | 7 +++--
services/keep-web/cadaver_test.go | 60 +++++++++++++++++++++++++++++++++++
services/keep-web/handler.go | 16 +++++-----
services/keep-web/handler_test.go | 42 +++++++++++++++++++++++--
services/keep-web/webdav.go | 66 ++++++++++++++++++++++++++++++++++++++-
5 files changed, 178 insertions(+), 13 deletions(-)
create mode 100644 services/keep-web/cadaver_test.go
via a23fa06e9849f2ab76fa271624e22a245c2abc47 (commit)
via 165993fac96251ec0ec2e881fab40a5db113a282 (commit)
via ec4c5a76c2050f7bf91c00ae0f6da9ec0aab9f63 (commit)
from 654ee9154fe85832a0862c27fd7b982831a75a0d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a23fa06e9849f2ab76fa271624e22a245c2abc47
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Oct 12 14:42:34 2017 -0400
12216: Test webdav directory listings.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 81c6612..496ba0c 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -205,8 +205,11 @@ sanity_checks() {
which gitolite \
|| fatal "No gitolite. Try: apt-get install gitolite3"
echo -n 'npm: '
- which npm \
- || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/"
+ npm --version \
+ || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/"
+ echo -n 'cadaver: '
+ cadaver --version | grep -w cadaver \
+ || fatal "No cadaver. Try: apt-get install cadaver"
}
rotate_logfile() {
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
new file mode 100644
index 0000000..87a712f
--- /dev/null
+++ b/services/keep-web/cadaver_test.go
@@ -0,0 +1,60 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "io"
+ "os/exec"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
+ basePath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
+ type testcase struct {
+ path string
+ cmd string
+ match string
+ }
+ for _, trial := range []testcase{
+ {
+ path: basePath,
+ cmd: "ls\n",
+ match: `(?ms).*dir1 *0 .*`,
+ },
+ {
+ path: basePath,
+ cmd: "ls dir1\n",
+ match: `(?ms).*bar *3.*foo *3 .*`,
+ },
+ {
+ path: basePath + "_/dir1",
+ cmd: "ls\n",
+ match: `(?ms).*bar *3.*foo *3 .*`,
+ },
+ {
+ path: basePath + "dir1/",
+ cmd: "ls\n",
+ match: `(?ms).*bar *3.*foo *3 .*`,
+ },
+ } {
+ c.Logf("%s %#v", "http://"+s.testServer.Addr, trial)
+ cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
+ cmd.Stdin = bytes.NewBufferString(trial.cmd)
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ c.Check(buf.String(), check.Matches, trial.match)
+ }
+}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 8822767..1798d2c 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -101,7 +101,7 @@ var (
"OPTIONS": true,
"PROPFIND": true,
}
- fsMethod = map[string]bool{
+ browserMethod = map[string]bool{
"GET": true,
"HEAD": true,
"POST": true,
@@ -142,7 +142,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" {
- if !fsMethod[method] && !webdavMethod[method] {
+ if !browserMethod[method] && !webdavMethod[method] {
statusCode = http.StatusMethodNotAllowed
return
}
@@ -154,7 +154,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
}
- if !fsMethod[r.Method] && !webdavMethod[r.Method] {
+ if !browserMethod[r.Method] && !webdavMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, r.Method
return
}
@@ -239,7 +239,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// * The token isn't embedded in the URL, so we don't
// need to worry about bookmarks and copy/paste.
tokens = append(tokens, formToken)
- } else if formToken != "" {
+ } else if formToken != "" && browserMethod[r.Method] {
// The client provided an explicit token in the query
// string, or a form in POST body. We must put the
// token in an HttpOnly cookie, and redirect to the
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 3bbb4ed..d877a5f 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -5,12 +5,14 @@
package main
import (
+ "bytes"
"fmt"
"html"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
+ "path/filepath"
"regexp"
"strings"
@@ -550,7 +552,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
expect: nil,
},
} {
- c.Logf("%q => %q", trial.uri, trial.expect)
+ c.Logf("HTML: %q => %q", trial.uri, trial.expect)
resp := httptest.NewRecorder()
u := mustParseURL("//" + trial.uri)
req := &http.Request{
@@ -587,6 +589,42 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
}
c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
}
+
+ c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
+ req = &http.Request{
+ Method: "OPTIONS",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: trial.header,
+ Body: ioutil.NopCloser(&bytes.Buffer{}),
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ }
+
+ req = &http.Request{
+ Method: "PROPFIND",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: trial.header,
+ Body: ioutil.NopCloser(&bytes.Buffer{}),
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+ for _, e := range trial.expect {
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
+ }
+ }
}
}
commit 165993fac96251ec0ec2e881fab40a5db113a282
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Oct 12 01:20:19 2017 -0400
12216: Disable lock operations.
Lock support in a webdav server is optional, and none of the
operations that would be affected by locks (i.e., writes) are
supported here anyway.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 4c12b8e..8822767 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -82,6 +82,8 @@ func (h *handler) setup() {
Prefix: "/_health/",
}
+ // Even though we don't accept LOCK requests, every webdav
+ // handler must have a non-nil LockSystem.
h.webdavLS = &noLockSystem{}
}
@@ -98,8 +100,6 @@ var (
webdavMethod = map[string]bool{
"OPTIONS": true,
"PROPFIND": true,
- "LOCK": true,
- "UNLOCK": true,
}
fsMethod = map[string]bool{
"GET": true,
@@ -147,7 +147,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set("Access-Control-Allow-Headers", "Range")
- w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, PROPFIND, LOCK, UNLOCK")
+ w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, PROPFIND")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Max-Age", "86400")
statusCode = http.StatusOK
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index d0e92ee..3bbb4ed 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -43,7 +43,7 @@ func (s *UnitSuite) TestCORSPreflight(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
- c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST, OPTIONS, PROPFIND, LOCK, UNLOCK")
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST, OPTIONS, PROPFIND")
c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Range")
// Check preflight for a disallowed request
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 109bfa7..0a7b782 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -87,6 +87,10 @@ func (f *webdavFile) Write([]byte) (int, error) {
// However, it does also permit impossible operations, like acquiring
// conflicting locks and releasing non-existent locks. This might
// confuse some clients if they try to probe for correctness.
+//
+// Currently this is a moot point: the LOCK and UNLOCK methods are not
+// accepted by keep-web, so it suffices to implement the
+// webdav.LockSystem interface.
type noLockSystem struct{}
func (*noLockSystem) Confirm(time.Time, string, string, ...webdav.Condition) (func(), error) {
commit ec4c5a76c2050f7bf91c00ae0f6da9ec0aab9f63
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Oct 12 00:59:02 2017 -0400
12216: Use no-op locking system.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 432fc21..4c12b8e 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -82,7 +82,7 @@ func (h *handler) setup() {
Prefix: "/_health/",
}
- h.webdavLS = webdav.NewMemLS()
+ h.webdavLS = &noLockSystem{}
}
func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 1b5811d..109bfa7 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -5,15 +5,24 @@
package main
import (
+ "crypto/rand"
"errors"
+ "fmt"
+ prand "math/rand"
"net/http"
"os"
+ "sync/atomic"
+ "time"
"golang.org/x/net/context"
"golang.org/x/net/webdav"
)
-var errReadOnly = errors.New("read-only filesystem")
+var (
+ lockPrefix string = uuid()
+ nextLockSuffix int64 = prand.Int63()
+ errReadOnly = errors.New("read-only filesystem")
+)
// webdavFS implements a read-only webdav.FileSystem by wrapping
// http.Filesystem.
@@ -60,3 +69,54 @@ type webdavFile struct {
func (f *webdavFile) Write([]byte) (int, error) {
return 0, errReadOnly
}
+
+// noLockSystem implements webdav.LockSystem by returning success for
+// every possible locking operation, even though it has no side
+// effects such as actually locking anything. This works for a
+// read-only webdav filesystem because webdav locks only apply to
+// writes.
+//
+// This is more suitable than webdav.NewMemLS() for two reasons:
+// First, it allows keep-web to use one locker for all collections
+// even though coll1.vhost/foo and coll2.vhost/foo have the same path
+// but represent different resources. Additionally, it returns valid
+// tokens (rfc2518 specifies that tokens are represented as URIs and
+// are unique across all resources for all time), which might improve
+// client compatibility.
+//
+// However, it does also permit impossible operations, like acquiring
+// conflicting locks and releasing non-existent locks. This might
+// confuse some clients if they try to probe for correctness.
+type noLockSystem struct{}
+
+func (*noLockSystem) Confirm(time.Time, string, string, ...webdav.Condition) (func(), error) {
+ return noop, nil
+}
+
+func (*noLockSystem) Create(now time.Time, details webdav.LockDetails) (token string, err error) {
+ return fmt.Sprintf("opaquelocktoken:%s-%x", lockPrefix, atomic.AddInt64(&nextLockSuffix, 1)), nil
+}
+
+func (*noLockSystem) Refresh(now time.Time, token string, duration time.Duration) (webdav.LockDetails, error) {
+ return webdav.LockDetails{}, nil
+}
+
+func (*noLockSystem) Unlock(now time.Time, token string) error {
+ return nil
+}
+
+func noop() {}
+
+// Return a version 1 variant 4 UUID, meaning all bits are random
+// except the ones indicating the version and variant.
+func uuid() string {
+ var data [16]byte
+ if _, err := rand.Read(data[:]); err != nil {
+ panic(err)
+ }
+ // variant 1: N=10xx
+ data[8] = data[8]&0x3f | 0x80
+ // version 4: M=0100
+ data[6] = data[6]&0x0f | 0x40
+ return fmt.Sprintf("%x-%x-%x-%x-%x", data[0:4], data[4:6], data[6:8], data[8:10], data[10:])
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list