[ARVADOS] updated: 2.1.0-892-gfb343cbbf
Git user
git at public.arvados.org
Fri Jun 11 21:20:03 UTC 2021
Summary of changes:
sdk/go/arvados/fs_collection.go | 1 +
services/keep-web/cache.go | 2 +-
services/keep-web/handler.go | 45 +++++++++++++++++++++++++++++++++------
services/keep-web/handler_test.go | 21 ++++++++++++++++++
services/keep-web/s3.go | 4 ++--
5 files changed, 64 insertions(+), 9 deletions(-)
via fb343cbbf8157315ec1a34a547beb0f2ddd5314c (commit)
via 9860df8c037d1230d543a06043d7af3c85ae2d27 (commit)
from fa956f9afdf53aafaa5b8fab1bb923af03d6120c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fb343cbbf8157315ec1a34a547beb0f2ddd5314c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jun 11 17:19:36 2021 -0400
17464: Fix shadowed "sess" variable
Testing WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 350e95ce8..a52af8048 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -225,7 +225,7 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, *cachedSessi
expired := false
if sess == nil {
c.metrics.sessionMisses.Inc()
- sess := &cachedSession{
+ sess = &cachedSession{
expire: now.Add(c.config.TTL.Duration()),
}
var err error
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 446d591bf..94fbb71ff 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -1174,6 +1174,27 @@ func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
checkWithID(colls[0].UUID, http.StatusOK)
}
+// func (s *IntegrationSuite) TestUploadDownloadLogging(c *check.C) {
+// u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
+// req := &http.Request{
+// Method: "GET",
+// Host: u.Host,
+// URL: u,
+// RequestURI: u.RequestURI(),
+// Header: http.Header{
+// "Authorization": {"Bearer " + arvadostest.ActiveToken},
+// },
+// }
+
+// var logbuf bytes.Buffer
+// logger := logrus.New()
+// logger.Out = &logbuf
+// req = req.WithContext(ctxlog.Context(context.Background(), logger))
+// s.doReq(req)
+
+// c.Check(logbuf.String(), check.Matches, `Download file*`)
+// }
+
func copyHeader(h http.Header) http.Header {
hc := http.Header{}
for k, v := range h {
commit 9860df8c037d1230d543a06043d7af3c85ae2d27
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jun 11 15:26:05 2021 -0400
17464: Log collection uuid for FileSystem requests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 0233826a7..4dbd4b774 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -674,6 +674,7 @@ func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode
if err != nil {
return nil, err
}
+ coll.UUID = dn.fs.uuid
data, err := json.Marshal(&coll)
if err == nil {
data = append(data, '\n')
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index ba9114664..43ce57904 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -491,7 +491,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
http.Error(w, "Not permitted", http.StatusForbidden)
return
}
- h.LogUploadOrDownload(r, sess.arvadosclient, collection, tokenUser)
+ h.LogUploadOrDownload(r, sess.arvadosclient, nil, strings.Join(targetPath, "/"), collection, tokenUser)
if webdavMethod[r.Method] {
if writeMethod[r.Method] {
@@ -623,7 +623,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
http.Error(w, "Not permitted", http.StatusForbidden)
return
}
- h.LogUploadOrDownload(r, sess.arvadosclient, nil, tokenUser)
+ h.LogUploadOrDownload(r, sess.arvadosclient, fs, r.URL.Path, nil, tokenUser)
if r.Method == "GET" {
_, basename := filepath.Split(r.URL.Path)
@@ -882,17 +882,29 @@ func (h *handler) UserPermittedToUploadOrDownload(method string, tokenUser *arva
return true
}
-func (h *handler) LogUploadOrDownload(r *http.Request, client *arvadosclient.ArvadosClient, collection *arvados.Collection, user *arvados.User) {
+func (h *handler) LogUploadOrDownload(
+ r *http.Request,
+ client *arvadosclient.ArvadosClient,
+ fs arvados.CustomFileSystem,
+ filepath string,
+ collection *arvados.Collection,
+ user *arvados.User) {
+
log := ctxlog.FromContext(r.Context())
props := make(map[string]string)
props["reqPath"] = r.URL.Path
if user != nil {
log = log.WithField("user_uuid", user.UUID).
- WithField("full_name", user.FullName)
+ WithField("user_full_name", user.FullName)
+ }
+ if collection == nil && fs != nil {
+ collection, filepath = h.DetermineCollection(fs, filepath)
}
if collection != nil {
- log = log.WithField("collection_uuid", collection.UUID)
+ log = log.WithField("collection_uuid", collection.UUID).
+ WithField("collection_file_path", filepath)
props["collection_uuid"] = collection.UUID
+ props["collection_file_path"] = filepath
}
if r.Method == "PUT" || r.Method == "POST" {
log.Info("File upload")
@@ -904,7 +916,7 @@ func (h *handler) LogUploadOrDownload(r *http.Request, client *arvadosclient.Arv
client.Create("logs", lr, nil)
}()
} else if r.Method == "GET" {
- if collection != nil {
+ if collection != nil && collection.PortableDataHash != "" {
log = log.WithField("portable_data_hash", collection.PortableDataHash)
props["portable_data_hash"] = collection.PortableDataHash
}
@@ -918,3 +930,24 @@ func (h *handler) LogUploadOrDownload(r *http.Request, client *arvadosclient.Arv
}()
}
}
+
+func (h *handler) DetermineCollection(fs arvados.CustomFileSystem, path string) (*arvados.Collection, string) {
+ segments := strings.Split(path, "/")
+ var i int
+ for i = len(segments) - 1; i >= 0; i-- {
+ dir := append([]string{}, segments[0:i]...)
+ dir = append(dir, ".arvados#collection")
+ f, err := fs.OpenFile(strings.Join(dir, "/"), os.O_RDONLY, 0)
+ if err == nil {
+ decoder := json.NewDecoder(f)
+ var collection arvados.Collection
+ err = decoder.Decode(&collection)
+ if err != nil {
+ return nil, ""
+ }
+ return &collection, strings.Join(segments[i:], "/")
+ }
+ f.Close()
+ }
+ return nil, ""
+}
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index db693741b..447c37cbc 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -386,7 +386,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
}
- h.LogUploadOrDownload(r, arvclient, nil, tokenUser)
+ h.LogUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser)
// shallow copy r, and change URL path
r := *r
@@ -477,7 +477,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
}
- h.LogUploadOrDownload(r, arvclient, nil, tokenUser)
+ h.LogUploadOrDownload(r, arvclient, fs, fspath, nil, tokenUser)
_, err = io.Copy(f, r.Body)
if err != nil {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list