[ARVADOS] updated: f75abd5da51c048fd26e7076a8a8fe96f6f5e0a2

[git at public.curoverse.com]

Summary of changes:
 services/keep/keep.go      |   59 ++++++++++++++++++++++++++++++++++++++------
 services/keep/keep_test.go |   53 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+), 8 deletions(-)
 create mode 100644 services/keep/keep_test.go

       via  f75abd5da51c048fd26e7076a8a8fe96f6f5e0a2 (commit)
      from  971452bda26fc20bfff3c07f6203d61fd4ca0df8 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit f75abd5da51c048fd26e7076a8a8fe96f6f5e0a2
Author: Tim Pierce <twp at curoverse.com>
Date:   Mon Mar 24 17:15:43 2014 -0400

    Adding unit tests for GetBlock. (refs #2291)

diff --git a/services/keep/keep.go b/services/keep/keep.go
index cec20be..919bccc 100644
--- a/services/keep/keep.go
+++ b/services/keep/keep.go
@@ -2,9 +2,10 @@ package main
 
 import (
 	"bufio"
+	"crypto/md5"
+	"errors"
 	"fmt"
 	"github.com/gorilla/mux"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -12,6 +13,7 @@ import (
 )
 
 const DEFAULT_PORT = 25107
+const BLOCKSIZE = 64 * 1024 * 1024
 
 var KeepVolumes []string
 
@@ -27,26 +29,67 @@ func main() {
 
 	// Set up REST handlers.
 	rest := mux.NewRouter()
-	rest.HandleFunc("/{hash}", GetBlock).Methods("GET")
+	rest.HandleFunc("/{hash:[0-9a-f]{32}}", GetBlockHandler).Methods("GET")
 	http.Handle("/", rest)
 
 	port := fmt.Sprintf(":%d", DEFAULT_PORT)
 	http.ListenAndServe(port, nil)
 }
 
-func GetBlock(w http.ResponseWriter, req *http.Request) {
+func GetBlockHandler(w http.ResponseWriter, req *http.Request) {
 	hash := mux.Vars(req)["hash"]
 
+	block, err := GetBlock(hash)
+	if err != nil {
+		http.Error(w, err.Error(), 404)
+		return
+	}
+
+	_, err = w.Write(block)
+	if err != nil {
+		log.Printf("GetBlockHandler: writing response: %s", err)
+	}
+
+	return
+}
+
+func GetBlock(hash string) ([]byte, error) {
+	var buf = make([]byte, BLOCKSIZE)
+
 	// Attempt to read the requested hash from a keep volume.
 	for _, vol := range KeepVolumes {
+		var f *os.File
+		var err error
+		var nread int
+
 		path := fmt.Sprintf("%s/%s/%s", vol, hash[0:3], hash)
-		if f, err := os.Open(path); err == nil {
-			io.Copy(w, f)
-			break
-		} else {
-			log.Printf("%s: reading block %s: %s\n", vol, hash, err)
+
+		f, err = os.Open(path)
+		if err != nil {
+			log.Printf("%s: opening %s: %s\n", vol, path, err)
+			continue
+		}
+
+		nread, err = f.Read(buf)
+		if err != nil {
+			log.Printf("%s: reading %s: %s\n", vol, path, err)
+			continue
 		}
+
+		// Double check the file checksum.
+		filehash := fmt.Sprintf("%x", md5.Sum(buf[:nread]))
+		if filehash != hash {
+			log.Printf("%s: checksum mismatch: %s (actual hash %s)\n",
+				vol, path, filehash)
+			continue
+		}
+
+		// Success!
+		return buf[:nread], nil
 	}
+
+	log.Printf("%s: all keep volumes failed, giving up\n", hash)
+	return buf, errors.New("not found: " + hash)
 }
 
 // FindKeepVolumes
diff --git a/services/keep/keep_test.go b/services/keep/keep_test.go
new file mode 100644
index 0000000..6210eae
--- /dev/null
+++ b/services/keep/keep_test.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+var TEST_BLOCK = []byte("The quick brown fox jumps over the lazy dog.")
+var TEST_HASH = "e4d909c290d0fb1ca068ffaddf22cbd0"
+
+func TestGetBlockOK(t *testing.T) {
+	var err error
+
+	// Manually populate keep1 and keep2 with a block.
+	KeepVolumes = make([]string, 2)
+	for i := range KeepVolumes {
+		if dir, err := ioutil.TempDir(os.TempDir(), "keeptest"); err == nil {
+			KeepVolumes[i] = dir + "/keep"
+		} else {
+			t.Fatal(err)
+		}
+
+		blockdir := fmt.Sprintf("%s/%s", KeepVolumes[i], TEST_HASH[:3])
+		if err := os.MkdirAll(blockdir, 0755); err != nil {
+			t.Fatal(err)
+		}
+
+		blockpath := fmt.Sprintf("%s/%s", blockdir, TEST_HASH)
+		if f, err := os.Create(blockpath); err == nil {
+			f.Write(TEST_BLOCK)
+			f.Close()
+		} else {
+			t.Fatal(err)
+		}
+
+	}
+
+	// Check that GetBlock returns success.
+	result, err := GetBlock(TEST_HASH)
+	if err != nil {
+		t.Errorf("GetBlock error: %s", err)
+	}
+	if fmt.Sprint(result) != fmt.Sprint(TEST_BLOCK) {
+		t.Errorf("expected %s, got %s", TEST_BLOCK, result)
+	}
+
+	for _, vol := range KeepVolumes {
+		os.RemoveAll(path.Dir(vol))
+	}
+}

-----------------------------------------------------------------------


hooks/post-receive
--