[ARVADOS] updated: 1.1.3-266-g127c858
Git user
git at public.curoverse.com
Mon Mar 26 15:59:50 EDT 2018
Summary of changes:
sdk/go/arvados/fs_project_test.go | 29 +++--
sdk/go/arvados/fs_site.go | 8 +-
sdk/go/arvados/fs_site_test.go | 2 +-
.../app/controllers/arvados/v1/users_controller.rb | 2 +-
services/keep-balance/balance_run_test.go | 1 -
services/keep-web/cadaver_test.go | 20 +++-
services/keep-web/handler.go | 132 ++++++++++++++++-----
services/keep-web/handler_test.go | 31 +++--
services/keep-web/webdav.go | 19 ++-
vendor/vendor.json | 12 +-
10 files changed, 192 insertions(+), 64 deletions(-)
discards 7633b9438d0c75693eb167c146b26764dd7161ed (commit)
discards 161de9f7fe71e328837df5c053f3663bc59245f3 (commit)
via 127c8584fa1a2336fd6569d9d6013f3f2b166088 (commit)
via c3cb135e3ff0617956608fbeadf64de0528dc9dd (commit)
via 8c7d6a0a98c0bb197ff5384d5bb3da1f3d4d8007 (commit)
via ef0356b856c06ba07b357dcd52c6cb723f63cf19 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (7633b9438d0c75693eb167c146b26764dd7161ed)
\
N -- N -- N (127c8584fa1a2336fd6569d9d6013f3f2b166088)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 127c8584fa1a2336fd6569d9d6013f3f2b166088
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Mar 23 17:25:27 2018 -0400
13111: Expose read-only /users/ tree.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 2d6dd2b..6a7ce2a 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -5,7 +5,6 @@
package main
import (
- _ "encoding/json"
"fmt"
"io"
"io/ioutil"
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 5f5f69a..843a8e8 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -27,7 +27,7 @@ func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
@@ -36,19 +36,23 @@ func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
- })
+ }, nil)
}
func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ rpath := "/users/active/foo_file_in_dir/"
s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
- r := "/users/active/foo_file_in_dir/"
- w := "/users/active/" + newCollection.Name
+ wpath := "/users/active/" + newCollection.Name
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
- return r, w, pdh
+ return rpath, wpath, pdh
+ }, func(path string) bool {
+ // Skip tests that rely on writes, because /users/
+ // tree is read-only.
+ return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -239,6 +243,10 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
},
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ if skip != nil && skip(trial.path) {
+ c.Log("(skip)")
+ continue
+ }
os.Remove(checkfile.Name())
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5ab4f70..00af0f4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -226,13 +226,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
}
- arv := h.clientPool.Get()
- if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
- return
- }
- defer h.clientPool.Put(arv)
-
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
@@ -241,6 +234,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
var attachment bool
+ var useSiteFS bool
credentialsOK := h.Config.TrustAllContent
if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
@@ -256,6 +250,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
+ } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+ useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
collectionID = parseCollectionIDFromURL(pathParts[0][2:])
@@ -275,6 +271,16 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+ if collectionID == "" && !useSiteFS {
+ statusCode = http.StatusNotFound
+ return
+ }
+
+ forceReload := false
+ if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
+ forceReload = true
+ }
+
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -322,6 +328,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokens = append(reqTokens, h.Config.AnonymousTokens...)
}
+ if useSiteFS {
+ h.serveSiteFS(w, r, tokens)
+ return
+ }
+
if len(targetPath) > 0 && targetPath[0] == "_" {
// If a collection has a directory called "t=foo" or
// "_", it can be served at
@@ -333,10 +344,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
stripParts++
}
- forceReload := false
- if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
- forceReload = true
+ arv := h.clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
+ return
}
+ defer h.clientPool.Put(arv)
var collection *arvados.Collection
tokenResult := make(map[string]int)
@@ -410,12 +423,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Insecure: arv.ApiInsecure,
}
- var fs arvados.FileSystem
- if collectionID == "" {
- fs = client.SiteFileSystem(kc)
- } else {
- fs, err = collection.FileSystem(client, kc)
- }
+ fs, err := collection.FileSystem(client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
@@ -475,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
+ h.serveDirectory(w, r, collection.Name, fs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -491,10 +499,69 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+ if len(tokens) == 0 {
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+ http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ return
+ }
+ if writeMethod[r.Method] {
+ http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
+ return
+ }
+ arv := h.clientPool.Get()
+ if arv == nil {
+ http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ return
+ }
+ defer h.clientPool.Put(arv)
+ arv.ApiToken = tokens[0]
+
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ client := &arvados.Client{
+ APIHost: arv.ApiServer,
+ AuthToken: arv.ApiToken,
+ Insecure: arv.ApiInsecure,
+ }
+ fs := client.SiteFileSystem(kc)
+ if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ return
+ } else {
+ f.Close()
+ }
+ wh := webdav.Handler{
+ Prefix: "/",
+ FileSystem: &webdavFS{
+ collfs: fs,
+ writing: writeMethod[r.Method],
+ alwaysReadEOF: r.Method == "PROPFIND",
+ },
+ LockSystem: h.webdavLS,
+ Logger: func(_ *http.Request, err error) {
+ if err != nil {
+ log.Printf("error from webdav handler: %q", err)
+ }
+ },
+ }
+ wh.ServeHTTP(w, r)
+}
+
var dirListingTemplate = `<!DOCTYPE HTML>
<HTML><HEAD>
<META name="robots" content="NOINDEX">
- <TITLE>{{ .Collection.Name }}</TITLE>
+ <TITLE>{{ .CollectionName }}</TITLE>
<STYLE type="text/css">
body {
margin: 1.5em;
@@ -518,19 +585,26 @@ var dirListingTemplate = `<!DOCTYPE HTML>
</STYLE>
</HEAD>
<BODY>
+
<H1>{{ .CollectionName }}</H1>
<P>This collection of data files is being shared with you through
Arvados. You can download individual files listed below. To download
-the entire collection with wget, try:</P>
+the entire directory tree with wget, try:</P>
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
<H2>File Listing</H2>
{{if .Files}}
<UL>
-{{range .Files}} <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
+{{range .Files}}
+{{if .IsDir }}
+ <LI>{{" " | printf "%15s " | nbsp}}<A href="{{.Name}}/">{{.Name}}/</A></LI>
+{{else}}
+ <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>
+{{end}}
+{{end}}
</UL>
{{else}}
<P>(No files; this collection is empty.)</P>
@@ -550,11 +624,12 @@ the entire collection with wget, try:</P>
`
type fileListEnt struct {
- Name string
- Size int64
+ Name string
+ Size int64
+ IsDir bool
}
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
var files []fileListEnt
var walk func(string) error
if !strings.HasSuffix(base, "/") {
@@ -574,15 +649,16 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
return err
}
for _, ent := range ents {
- if ent.IsDir() {
+ if recurse && ent.IsDir() {
err = walk(path + ent.Name() + "/")
if err != nil {
return err
}
} else {
files = append(files, fileListEnt{
- Name: path + ent.Name(),
- Size: ent.Size(),
+ Name: path + ent.Name(),
+ Size: ent.Size(),
+ IsDir: ent.IsDir(),
})
}
}
@@ -611,7 +687,7 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
"CollectionName": collectionName,
"Files": files,
"Request": r,
- "StripParts": stripParts,
+ "StripParts": strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
})
}
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 3e7ae5f..7fed6fb 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -508,7 +508,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
@@ -517,12 +517,30 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
- uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ uri: "download.example.com/users/active/foo_file_in_dir/",
header: authHeader,
- expect: []string{"dir1/foo", "dir1/bar"},
+ expect: []string{"dir1/"},
cutDirs: 3,
},
{
+ uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
+ header: authHeader,
+ expect: []string{"bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/users/",
+ header: authHeader,
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/active/",
+ header: authHeader,
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
@@ -550,19 +568,19 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 1,
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 2,
+ cutDirs: 3,
},
{
uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 941090a..3e62b19 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -47,6 +47,9 @@ type webdavFS struct {
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
dir, name := path.Split(name)
if dir == "" || dir == "/" {
return
@@ -66,7 +69,7 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
}
func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
- writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+ writing := flag&(os.O_WRONLY|os.O_RDWR|os.O_TRUNC) != 0
if writing {
fs.makeparents(name)
}
@@ -75,8 +78,13 @@ func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os
// webdav module returns 404 on all OpenFile errors,
// but returns 405 Method Not Allowed if OpenFile()
// succeeds but Write() or Close() fails. We'd rather
- // have 405.
- f = writeFailer{File: f, err: errReadOnly}
+ // have 405. writeFailer ensures Close() fails if the
+ // file is opened for writing *or* Write() is called.
+ var err error
+ if writing {
+ err = errReadOnly
+ }
+ f = writeFailer{File: f, err: err}
}
if fs.alwaysReadEOF {
f = readEOF{File: f}
@@ -109,10 +117,15 @@ type writeFailer struct {
}
func (wf writeFailer) Write([]byte) (int, error) {
+ wf.err = errReadOnly
return 0, wf.err
}
func (wf writeFailer) Close() error {
+ err := wf.File.Close()
+ if err != nil {
+ wf.err = err
+ }
return wf.err
}
commit c3cb135e3ff0617956608fbeadf64de0528dc9dd
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 16:56:16 2018 -0400
13111: Add /users virtual dir to siteFS.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index d269779..1633296 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -33,7 +33,7 @@ type keepFS struct {
Uid int
Gid int
- root arvados.FileSystem
+ root arvados.CustomFileSystem
open map[uint64]*sharedFile
lastFH uint64
sync.Mutex
@@ -70,6 +70,7 @@ func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
func (fs *keepFS) Init() {
defer fs.debugPanics()
fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
+ fs.root.MountProject("home", "")
if fs.ready != nil {
close(fs.ready)
}
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 776c8a3..7f34a42 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -10,25 +10,31 @@ import (
"time"
)
+type staleChecker struct {
+ mtx sync.Mutex
+ last time.Time
+}
+
+func (sc *staleChecker) DoIfStale(fn func(), staleFunc func(time.Time) bool) {
+ sc.mtx.Lock()
+ defer sc.mtx.Unlock()
+ if !staleFunc(sc.last) {
+ return
+ }
+ sc.last = time.Now()
+ fn()
+}
+
// projectnode exposes an Arvados project as a filesystem directory.
type projectnode struct {
inode
+ staleChecker
uuid string
err error
-
- loadLock sync.Mutex
- loadStart time.Time
}
func (pn *projectnode) load() {
- fs := pn.FS().(*siteFileSystem)
-
- pn.loadLock.Lock()
- defer pn.loadLock.Unlock()
- if !fs.Stale(pn.loadStart) {
- return
- }
- pn.loadStart = time.Now()
+ fs := pn.FS().(*customFileSystem)
if pn.uuid == "" {
var resp User
@@ -89,7 +95,7 @@ func (pn *projectnode) load() {
}
func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
@@ -97,7 +103,7 @@ func (pn *projectnode) Readdir() ([]os.FileInfo, error) {
}
func (pn *projectnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
- pn.load()
+ pn.staleChecker.DoIfStale(pn.load, pn.FS().(*customFileSystem).Stale)
if pn.err != nil {
return nil, pn.err
}
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index b7dc08e..69bae89 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"io"
"os"
+ "path/filepath"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -38,8 +39,17 @@ func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, b
return sc.Client.RequestAndDecode(dst, method, path, body, params)
}
-func (s *SiteFSSuite) TestHomeProject(c *check.C) {
- f, err := s.fs.Open("/home")
+func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
+ s.fs.MountProject("home", "")
+ s.testHomeProject(c, "/home")
+}
+
+func (s *SiteFSSuite) TestUsersDir(c *check.C) {
+ s.testHomeProject(c, "/users/active")
+}
+
+func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
+ f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
c.Check(len(fis), check.Not(check.Equals), 0)
@@ -53,23 +63,24 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
c.Check(ok, check.Equals, true)
- f, err = s.fs.Open("/home/A Project/..")
+ f, err = s.fs.Open(path + "/A Project/..")
c.Assert(err, check.IsNil)
fi, err := f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
- c.Check(fi.Name(), check.Equals, "home")
+ _, basename := filepath.Split(path)
+ c.Check(fi.Name(), check.Equals, basename)
- f, err = s.fs.Open("/home/A Project/A Subproject")
+ f, err = s.fs.Open(path + "/A Project/A Subproject")
c.Check(err, check.IsNil)
fi, err = f.Stat()
c.Check(err, check.IsNil)
c.Check(fi.IsDir(), check.Equals, true)
for _, nx := range []string{
- "/home/Unrestricted public data",
- "/home/Unrestricted public data/does not exist",
- "/home/A Project/does not exist",
+ path + "/Unrestricted public data",
+ path + "/Unrestricted public data/does not exist",
+ path + "/A Project/does not exist",
} {
c.Log(nx)
f, err = s.fs.Open(nx)
@@ -79,6 +90,8 @@ func (s *SiteFSSuite) TestHomeProject(c *check.C) {
}
func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ s.fs.MountProject("home", "")
+
project, err := s.fs.OpenFile("/home/A Project", 0, 0)
c.Check(err, check.IsNil)
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index cdcf40e..e9c8387 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -10,22 +10,25 @@ import (
"time"
)
-type siteFileSystem struct {
+type CustomFileSystem interface {
+ FileSystem
+ MountByID(mount string)
+ MountProject(mount, uuid string)
+ MountUsers(mount string)
+}
+
+type customFileSystem struct {
fileSystem
+ root *vdirnode
staleThreshold time.Time
staleLock sync.Mutex
}
-// SiteFileSystem returns a FileSystem that maps collections and other
-// Arvados objects onto a filesystem layout.
-//
-// This is experimental: the filesystem layout is not stable, and
-// there are significant known bugs and shortcomings. For example,
-// writes are not persisted until Sync() is called.
-func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
+func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
root := &vdirnode{}
- fs := &siteFileSystem{
+ fs := &customFileSystem{
+ root: root,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: c, keepClient: kc},
root: root,
@@ -41,14 +44,18 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
inodes: make(map[string]inode),
}
- root.inode.Child("by_id", func(inode) (inode, error) {
+ return fs
+}
+
+func (fs *customFileSystem) MountByID(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
inodes: make(map[string]inode),
fileinfo: fileinfo{
- name: "by_id",
+ name: mount,
modTime: time.Now(),
mode: 0755 | os.ModeDir,
},
@@ -56,13 +63,45 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
create: fs.mountCollection,
}, nil
})
- root.inode.Child("home", func(inode) (inode, error) {
- return fs.newProjectNode(fs.root, "home", ""), nil
+}
+
+func (fs *customFileSystem) MountProject(mount, uuid string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, mount, uuid), nil
})
+}
+
+func (fs *customFileSystem) MountUsers(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return &usersnode{
+ inode: &treenode{
+ fs: fs,
+ parent: fs.root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: mount,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }, nil
+ })
+}
+
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// writes are not persisted until Sync() is called.
+func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
+ fs := c.CustomFileSystem(kc)
+ fs.MountByID("by_id")
+ fs.MountUsers("users")
return fs
}
-func (fs *siteFileSystem) Sync() error {
+func (fs *customFileSystem) Sync() error {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
fs.staleThreshold = time.Now()
@@ -71,17 +110,17 @@ func (fs *siteFileSystem) Sync() error {
// Stale returns true if information obtained at time t should be
// considered stale.
-func (fs *siteFileSystem) Stale(t time.Time) bool {
+func (fs *customFileSystem) Stale(t time.Time) bool {
fs.staleLock.Lock()
defer fs.staleLock.Unlock()
return !fs.staleThreshold.Before(t)
}
-func (fs *siteFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
return nil, ErrInvalidOperation
}
-func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
+func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
var coll Collection
err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
if err != nil {
@@ -96,7 +135,23 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
-func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
+func (fs *customFileSystem) newUserNode(root inode, name, uuid string) inode {
return &projectnode{
uuid: uuid,
inode: &treenode{
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index a3a9712..e35ae48 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -16,7 +16,7 @@ var _ = check.Suite(&SiteFSSuite{})
type SiteFSSuite struct {
client *Client
- fs FileSystem
+ fs CustomFileSystem
kc keepClient
}
diff --git a/sdk/go/arvados/fs_users.go b/sdk/go/arvados/fs_users.go
new file mode 100644
index 0000000..ccfe2c5
--- /dev/null
+++ b/sdk/go/arvados/fs_users.go
@@ -0,0 +1,62 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+)
+
+// usersnode is a virtual directory with an entry for each visible
+// Arvados username, each showing the respective user's "home
+// projects".
+type usersnode struct {
+ inode
+ staleChecker
+ err error
+}
+
+func (un *usersnode) load() {
+ fs := un.FS().(*customFileSystem)
+
+ params := ResourceListParams{
+ Order: "uuid",
+ }
+ for {
+ var resp UserList
+ un.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if un.err != nil {
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, user := range resp.Items {
+ if user.Username == "" {
+ continue
+ }
+ un.inode.Child(user.Username, func(inode) (inode, error) {
+ return fs.newProjectNode(un, user.Username, user.UUID), nil
+ })
+ }
+ params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ }
+ un.err = nil
+}
+
+func (un *usersnode) Readdir() ([]os.FileInfo, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Readdir()
+}
+
+func (un *usersnode) Child(name string, _ func(inode) (inode, error)) (inode, error) {
+ un.staleChecker.DoIfStale(un.load, un.FS().(*customFileSystem).Stale)
+ if un.err != nil {
+ return nil, un.err
+ }
+ return un.inode.Child(name, nil)
+}
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index dc7e62f..587f4db 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -159,7 +159,7 @@ class Arvados::V1::UsersController < ApplicationController
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name"]
+ safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username"]
if @select
@select = @select & safe_attrs
else
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 19a2040..5ab4f70 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -236,7 +236,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
- var targetID string
+ var collectionID string
var tokens []string
var reqTokens []string
var pathToken bool
@@ -250,7 +250,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
attachment = true
}
- if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
+ if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
@@ -258,28 +258,23 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
return
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
- targetID = parseCollectionIDFromURL(pathParts[0][2:])
+ collectionID = parseCollectionIDFromURL(pathParts[0][2:])
stripParts = 1
} else if len(pathParts) >= 2 && pathParts[0] == "collections" {
if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetID = parseCollectionIDFromURL(pathParts[2])
+ collectionID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
stripParts = 4
pathToken = true
} else {
// /collections/ID/PATH...
- targetID = parseCollectionIDFromURL(pathParts[1])
+ collectionID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
stripParts = 2
}
}
- if targetID == "" {
- statusCode = http.StatusNotFound
- return
- }
-
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
@@ -347,7 +342,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
+ collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
@@ -414,14 +409,21 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
}
- fs, err := collection.FileSystem(client, kc)
+
+ var fs arvados.FileSystem
+ if collectionID == "" {
+ fs = client.SiteFileSystem(kc)
+ } else {
+ fs, err = collection.FileSystem(client, kc)
+ }
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- targetIsPDH := arvadosclient.PDHMatch(targetID)
- if targetIsPDH && writeMethod[r.Method] {
+ writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ targetIsPDH := arvadosclient.PDHMatch(collectionID)
+ if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
return
}
@@ -435,7 +437,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w = &updateOnSuccess{
ResponseWriter: w,
update: func() error {
- return h.Config.Cache.Update(client, *collection, fs)
+ return h.Config.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index af83681..941090a 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -36,7 +36,7 @@ var (
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.CollectionFileSystem
+ collfs arvados.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
commit 8c7d6a0a98c0bb197ff5384d5bb3da1f3d4d8007
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Mar 20 10:52:09 2018 -0400
13111: Add tests for /users/ paths.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index b7ec1de..5f5f69a 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -22,24 +22,33 @@ import (
)
func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
- s.testCadaver(c, arvadostest.ActiveToken, func(newUUID string) (string, string, string) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
- w := "/c=" + newUUID + "/"
+ w := "/c=" + newCollection.UUID + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
return r, w, pdh
})
}
func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
- s.testCadaver(c, "", func(newUUID string) (string, string, string) {
+ s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
- w := "/c=" + newUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
return r, w, pdh
})
}
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(string) (string, string, string)) {
+func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
+ r := "/users/active/foo_file_in_dir/"
+ w := "/users/active/" + newCollection.Name
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -62,7 +71,7 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- readPath, writePath, pdhPath := pathFunc(newCollection.UUID)
+ readPath, writePath, pdhPath := pathFunc(newCollection)
matchToday := time.Now().Format("Jan +2")
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 21e47c8..3e7ae5f 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -517,6 +517,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
cutDirs: 2,
},
{
+ uri: "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 3,
+ },
+ {
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
expect: []string{"dir1/foo", "dir1/bar"},
commit ef0356b856c06ba07b357dcd52c6cb723f63cf19
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Mar 26 15:23:15 2018 -0400
13111: Sort vendor.json
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 1189ca8..7af81ba 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -84,18 +84,18 @@
"revisionTime": "2018-01-08T08:51:32Z"
},
{
- "checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
- "path": "github.com/dgrijalva/jwt-go",
- "revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
- "revisionTime": "2017-10-19T21:57:19Z"
- },
- {
"checksumSHA1": "gMBls0ytB5wHvZizUQE8Eivv9WQ=",
"path": "github.com/curoverse/cgofuse/fuse",
"revision": "d08d9e36b4ca1364eb7a4eb9db0b7fa76c9250a2",
"revisionTime": "2017-12-17T05:18:50Z"
},
{
+ "checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
+ "path": "github.com/dgrijalva/jwt-go",
+ "revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
+ "revisionTime": "2017-10-19T21:57:19Z"
+ },
+ {
"checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
"path": "github.com/docker/distribution/digestset",
"revision": "277ed486c948042cab91ad367c379524f3b25e18",
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list