[ARVADOS] updated: 7a093f23bd68f9034701cdfdaa49543c50759094
Git user
git at public.curoverse.com
Mon Jul 4 15:50:05 EDT 2016
Summary of changes:
discards c47c3d3b01cbb5e966a963513a02f20e5d840cc1 (commit)
via 7a093f23bd68f9034701cdfdaa49543c50759094 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (c47c3d3b01cbb5e966a963513a02f20e5d840cc1)
\
N -- N -- N (7a093f23bd68f9034701cdfdaa49543c50759094)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7a093f23bd68f9034701cdfdaa49543c50759094
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 4 15:49:50 2016 -0400
9541: Serve HEAD requests without using the buffer pool.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 48cb026..60565e6 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -259,6 +259,18 @@ func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
return actualSize, nil
}
+func (v *AzureBlobVolume) GetReader(loc string) (io.ReadCloser, error) {
+ trashed, _, err := v.checkTrashed(loc)
+ if err != nil {
+ return nil, err
+ }
+ if trashed {
+ return nil, os.ErrNotExist
+ }
+ rdr, err := v.bsClient.GetBlob(v.containerName, loc)
+ return rdr, v.translateError(err)
+}
+
// Compare the given data with existing stored data.
func (v *AzureBlobVolume) Compare(loc string, expect []byte) error {
trashed, _, err := v.checkTrashed(loc)
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index f698982..363ca70 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -31,16 +31,19 @@ import (
func MakeRESTRouter() *mux.Router {
rest := mux.NewRouter()
- rest.HandleFunc(
- `/{hash:[0-9a-f]{32}}`, GetBlockHandler).Methods("GET", "HEAD")
- rest.HandleFunc(
- `/{hash:[0-9a-f]{32}}+{hints}`,
- GetBlockHandler).Methods("GET", "HEAD")
+ rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, GetBlockHandler).Methods("GET")
+ rest.HandleFunc(`/{hash:[0-9a-f]{32}}+{hints}`, GetBlockHandler).Methods("GET")
+
+ rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, Head).Methods("HEAD")
+ rest.HandleFunc(`/{hash:[0-9a-f]{32}}+{hints}`, Head).Methods("HEAD")
rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, PutBlockHandler).Methods("PUT")
+
rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
+
// List all blocks stored here. Privileged client only.
rest.HandleFunc(`/index`, IndexHandler).Methods("GET", "HEAD")
+
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler).Methods("GET", "HEAD")
@@ -64,19 +67,66 @@ func MakeRESTRouter() *mux.Router {
return rest
}
+func permissionOK(resp http.ResponseWriter, req *http.Request) bool {
+ if !enforcePermissions {
+ return true
+ }
+ locator := req.URL.Path[1:] // strip leading slash
+ if err := VerifySignature(locator, GetApiToken(req)); err == nil {
+ return true
+ } else {
+ http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
+ return false
+ }
+}
+
// BadRequestHandler is a HandleFunc to address bad requests.
func BadRequestHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, BadRequestError.Error(), BadRequestError.HTTPCode)
}
+// Head computes the checksum of stored data and returns 200 if it
+// finds a copy on any volume with a valid checksum.
+func Head(resp http.ResponseWriter, req *http.Request) {
+ if !permissionOK(resp, req) {
+ return
+ }
+ locator := mux.Vars(req)["hash"]
+ for _, vol := range KeepVM.AllReadable() {
+ rdr, err := vol.GetReader(locator)
+ if err != nil {
+ continue
+ }
+ h := md5.New()
+ size, err := io.Copy(h, rdr)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ log.Printf("%s: read %s: %s", vol, locator, err)
+ }
+ continue
+ }
+ if rdr, ok := rdr.(io.Closer); ok {
+ err = rdr.Close()
+ if err != nil {
+ log.Printf("%s: read %s: close: %s", vol, locator, err)
+ }
+ continue
+ }
+ if h := fmt.Sprintf("%x", h); h != locator {
+ log.Printf("%s: verify %s: checksum mismatch: %s", vol, locator, h)
+ continue
+ }
+ resp.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+ resp.Header().Set("Content-Type", "application/octet-stream")
+ return
+ }
+ http.Error(resp, "Not found", http.StatusNotFound)
+}
+
// GetBlockHandler is a HandleFunc to address Get block requests.
func GetBlockHandler(resp http.ResponseWriter, req *http.Request) {
- if enforcePermissions {
- locator := req.URL.Path[1:] // strip leading slash
- if err := VerifySignature(locator, GetApiToken(req)); err != nil {
- http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
- return
- }
+ if !permissionOK(resp, req) {
+ return
}
// TODO: Probe volumes to check whether the block _might_
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 98e1203..4628329 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -169,6 +169,14 @@ func (v *S3Volume) Get(loc string, buf []byte) (int, error) {
}
}
+func (v *S3Volume) GetReader(loc string) (io.ReadCloser, error) {
+ rdr, err := v.Bucket.GetReader(loc)
+ if err != nil {
+ return nil, v.translateError(err)
+ }
+ return rdr, err
+}
+
func (v *S3Volume) Compare(loc string, expect []byte) error {
rdr, err := v.Bucket.GetReader(loc)
if err != nil {
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 8ae6660..cd475bc 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -47,6 +47,13 @@ type Volume interface {
// was encountered opening/reading the stored data.
Compare(loc string, data []byte) error
+ // Return an io.ReadCloser that reads data for the given
+ // locator.
+ //
+ // As with Get, if the block does not exist, the returned
+ // error should satisfy os.IsNotExist(err).
+ GetReader(loc string) (io.ReadCloser, error)
+
// Put writes a block to an underlying storage device.
//
// loc is as described in Get.
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 5671b8d..686b8e8 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
+ "io/ioutil"
"os"
"strings"
"sync"
@@ -125,6 +126,17 @@ func (v *MockVolume) Get(loc string, buf []byte) (int, error) {
return 0, os.ErrNotExist
}
+func (v *MockVolume) GetReader(loc string) (io.ReadCloser, error) {
+ v.gotCall("GetReader")
+ <-v.Gate
+ if v.Bad {
+ return nil, errors.New("Bad volume")
+ } else if block, ok := v.Store[loc]; ok {
+ return ioutil.NopCloser(bytes.NewReader(block)), nil
+ }
+ return nil, os.ErrNotExist
+}
+
func (v *MockVolume) Put(loc string, block []byte) error {
v.gotCall("Put")
<-v.Gate
diff --git a/services/keepstore/volume_unix.go b/services/keepstore/volume_unix.go
index 90189dc..a39e762 100644
--- a/services/keepstore/volume_unix.go
+++ b/services/keepstore/volume_unix.go
@@ -180,6 +180,25 @@ func (v *UnixVolume) stat(path string) (os.FileInfo, error) {
return stat, err
}
+// GetReader returns a ReadCloser that respects the serialize lock.
+func (v *UnixVolume) GetReader(loc string) (io.ReadCloser, error) {
+ unlock := nop
+ if v.locker != nil {
+ v.locker.Lock()
+ unlock = v.locker.Unlock
+ }
+ path := v.blockPath(loc)
+ f, err := os.Open(path)
+ if err != nil {
+ unlock()
+ return nil, err
+ }
+ return &unlockingReadCloser{
+ ReadCloser: f,
+ unlock: unlock,
+ }, nil
+}
+
// Get retrieves a block, copies it to the given slice, and returns
// the number of bytes copied.
func (v *UnixVolume) Get(loc string, buf []byte) (int, error) {
@@ -583,3 +602,15 @@ func (v *UnixVolume) EmptyTrash() {
log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
}
+
+type unlockingReadCloser struct {
+ io.ReadCloser
+ unlock func()
+}
+
+func (urc *unlockingReadCloser) Close() error {
+ defer urc.unlock()
+ return urc.ReadCloser.Close()
+}
+
+func nop() {}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list