[ARVADOS] updated: 2b8857f631f58df2baa93077185fb7a5a29c6aad
git at public.curoverse.com
git at public.curoverse.com
Tue May 6 17:41:20 EDT 2014
Summary of changes:
services/keep/src/keep/keep.go | 106 +++++++++++++++++++++++++++++------
services/keep/src/keep/keep_test.go | 103 +++++++++++++++++++++++++++++-----
services/keep/src/keep/perms.go | 10 ++-
3 files changed, 184 insertions(+), 35 deletions(-)
via 2b8857f631f58df2baa93077185fb7a5a29c6aad (commit)
from bdc9139d17c184a58e5088270f2ce6ba361fb8a7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2b8857f631f58df2baa93077185fb7a5a29c6aad
Author: Tim Pierce <twp at curoverse.com>
Date: Tue May 6 17:40:28 2014 -0400
Added permission flags and unit tests.
New flags:
--enforce-permissions enables permission checking for GET requests.
--permission-ttl sets the expiration time on signed locators returned
by PUT.
--data-manager-token defines a privileged token for the Data Manager
to issue DELETE and "GET /index" requests.
PUT now responds with a signed locator if a permission key has been
set.
Unit test TestGetHandler tests the GetBlockHandler both when permission
checking is off, and tests signed, unsigned and expired requests when
permission checking is enabled.
Refs #2328
diff --git a/services/keep/src/keep/keep.go b/services/keep/src/keep/keep.go
index 6619a80..278023b 100644
--- a/services/keep/src/keep/keep.go
+++ b/services/keep/src/keep/keep.go
@@ -15,8 +15,10 @@ import (
"net/http"
"os"
"regexp"
+ "strconv"
"strings"
"syscall"
+ "time"
)
// ======================
@@ -40,6 +42,19 @@ var PROC_MOUNTS = "/proc/mounts"
// The Keep VolumeManager maintains a list of available volumes.
var KeepVM VolumeManager
+// enforce_permissions controls whether permission signatures
+// should be enforced (affecting GET and DELETE requests)
+var enforce_permissions bool
+
+// permission_ttl is the time duration (in seconds) for which
+// new permission signatures (returned by PUT requests) will be
+// valid.
+var permission_ttl int
+
+// data_manager_token represents the API token used by the
+// Data Manager, and is required on certain privileged operations.
+var data_manager_token string
+
// ==========
// Error types.
//
@@ -88,9 +103,19 @@ func main() {
// by looking at currently mounted filesystems for /keep top-level
// directories.
- var listen, permission_key, volumearg string
+ var data_manager_token, listen, permission_key, volumearg string
var serialize_io bool
flag.StringVar(
+ &data_manager_token,
+ "data-manager-token",
+ "",
+ "API token used by the Data Manager. All DELETE requests or unqualified GET /index requests must carry this token.")
+ flag.BoolVar(
+ &enforce_permissions,
+ "enforce-permissions",
+ false,
+ "Enforce permission signatures on requests.")
+ flag.StringVar(
&listen,
"listen",
DEFAULT_ADDR,
@@ -100,6 +125,11 @@ func main() {
"permission-key",
"",
"Secret key to use for generating and verifying permission signatures.")
+ flag.IntVar(
+ &permission_ttl,
+ "permission-ttl",
+ 300,
+ "Expiration time (in seconds) for newly generated permission signatures.")
flag.BoolVar(
&serialize_io,
"serialize",
@@ -144,27 +174,35 @@ func main() {
PermissionSecret = []byte(permission_key)
}
+ // If --enforce-permissions is true, we must have a permission key to continue.
+ if enforce_permissions && PermissionSecret == nil {
+ log.Fatal("--enforce-permissions requires a permission key")
+ }
+
// Start a round-robin VolumeManager with the volumes we have found.
KeepVM = MakeRRVolumeManager(goodvols)
- // Set up REST handlers.
- //
- // Start with a router that will route each URL path to an
- // appropriate handler.
- //
+ // Tell the built-in HTTP server to direct all requests to the REST
+ // router.
+ http.Handle("/", NewRESTRouter())
+
+ // Start listening for requests.
+ http.ListenAndServe(listen, nil)
+}
+
+// NewRESTRouter
+// Returns a mux.Router that passes GET and PUT requests to the
+// appropriate handlers.
+//
+func NewRESTRouter() *mux.Router {
rest := mux.NewRouter()
rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, GetBlockHandler).Methods("GET", "HEAD")
+ rest.HandleFunc(`/{hash:[0-9a-f]{32}}+A{signature:[0-9a-f]+}@{timestamp:[0-9a-f]+}`, GetBlockHandler).Methods("GET", "HEAD")
rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, PutBlockHandler).Methods("PUT")
rest.HandleFunc(`/index`, IndexHandler).Methods("GET", "HEAD")
rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler).Methods("GET", "HEAD")
rest.HandleFunc(`/status.json`, StatusHandler).Methods("GET", "HEAD")
-
- // Tell the built-in HTTP server to direct all requests to the REST
- // router.
- http.Handle("/", rest)
-
- // Start listening for requests.
- http.ListenAndServe(listen, nil)
+ return rest
}
// FindKeepVolumes
@@ -199,18 +237,27 @@ func FindKeepVolumes() []string {
func GetBlockHandler(w http.ResponseWriter, req *http.Request) {
hash := mux.Vars(req)["hash"]
+ signature := mux.Vars(req)["signature"]
+ timestamp := mux.Vars(req)["timestamp"]
// If permission checking is in effect, verify this
// request's permission signature.
- if PermissionSecret != nil {
- if !VerifySignature(hash, GetApiToken(req)) {
- http.Error(w, PermissionError.Error(), 401)
+ if enforce_permissions {
+ if signature == "" || timestamp == "" {
+ http.Error(w, PermissionError.Error(), PermissionError.HTTPCode)
+ return
+ } else if IsExpired(timestamp) {
+ http.Error(w, ExpiredError.Error(), ExpiredError.HTTPCode)
+ return
+ } else if signature != MakePermSignature(hash, GetApiToken(req), timestamp) {
+ http.Error(w, PermissionError.Error(), PermissionError.HTTPCode)
+ return
}
}
block, err := GetBlock(hash)
if err != nil {
- http.Error(w, err.Error(), 404)
+ http.Error(w, err.Error(), err.(*KeepError).HTTPCode)
return
}
@@ -237,7 +284,12 @@ func PutBlockHandler(w http.ResponseWriter, req *http.Request) {
//
if buf, err := ReadAtMost(req.Body, BLOCKSIZE); err == nil {
if err := PutBlock(buf, hash); err == nil {
- w.WriteHeader(http.StatusOK)
+ // Success; sign the locator and return it to the client.
+ api_token := GetApiToken(req)
+ expiry := time.Now().Add( // convert permission_ttl to time.Duration
+ time.Duration(permission_ttl) * time.Second)
+ signed_loc := SignLocator(hash, api_token, expiry)
+ w.Write([]byte(signed_loc))
} else {
ke := err.(*KeepError)
http.Error(w, ke.Error(), ke.HTTPCode)
@@ -260,6 +312,13 @@ func PutBlockHandler(w http.ResponseWriter, req *http.Request) {
func IndexHandler(w http.ResponseWriter, req *http.Request) {
prefix := mux.Vars(req)["prefix"]
+ // Only the data manager may issue unqualified "GET /index" requests.
+ if prefix == "" {
+ if data_manager_token != GetApiToken(req) {
+ http.Error(w, PermissionError.Error(), PermissionError.HTTPCode)
+ return
+ }
+ }
var index string
for _, vol := range KeepVM.Volumes() {
index = index + vol.Index(prefix)
@@ -500,3 +559,14 @@ func GetApiToken(req *http.Request) string {
}
return ""
}
+
+// IsExpired returns true if the given Unix timestamp (expressed as a
+// hexadecimal string) is in the past.
+func IsExpired(timestamp_hex string) bool {
+ ts, err := strconv.ParseInt(timestamp_hex, 16, 0)
+ if err != nil {
+ log.Printf("IsExpired: %s\n", err)
+ return true
+ }
+ return time.Unix(ts, 0).Before(time.Now())
+}
diff --git a/services/keep/src/keep/keep_test.go b/services/keep/src/keep/keep_test.go
index cfbb62e..ed6f0be 100644
--- a/services/keep/src/keep/keep_test.go
+++ b/services/keep/src/keep/keep_test.go
@@ -4,10 +4,13 @@ import (
"bytes"
"fmt"
"io/ioutil"
+ "net/http"
+ "net/http/httptest"
"os"
"path"
"regexp"
"testing"
+ "time"
)
var TEST_BLOCK = []byte("The quick brown fox jumps over the lazy dog.")
@@ -104,20 +107,6 @@ func TestGetBlockCorrupt(t *testing.T) {
}
}
-/*
-// TestGetBlockPermissionOK
-// When enforce_permissions is set, GetBlock correctly
-// handles a request with a valid permission signature.
-func TestGetBlockPermissionOK(t *testing.T) {
- defer teardown()
-
- enforce_permissions = true
- PermissionSecret =
- // Create two test Keep volumes and store a block.
-
-}
-*/
-
// ========================================
// PutBlock tests
// ========================================
@@ -407,6 +396,92 @@ func TestNodeStatus(t *testing.T) {
}
// ========================================
+// Tests for HTTP handlers
+// ========================================
+
+func TestGetHandler(t *testing.T) {
+ defer teardown()
+
+ // Prepare two test Keep volumes. Our block is stored on the second volume.
+ KeepVM = MakeTestVolumeManager(2)
+ defer func() { KeepVM.Quit() }()
+
+ vols := KeepVM.Volumes()
+ if err := vols[0].Put(TEST_HASH, TEST_BLOCK); err != nil {
+ t.Error(err)
+ }
+
+ // Set up a REST router for testing the handlers.
+ rest := NewRESTRouter()
+
+ // Test an unsigned GET request.
+ test_url := "http://localhost:25107/" + TEST_HASH
+ req, _ := http.NewRequest("GET", test_url, nil)
+ resp := httptest.NewRecorder()
+ rest.ServeHTTP(resp, req)
+
+ if resp.Code != 200 {
+ t.Errorf("bad response code: %v", resp)
+ }
+ if bytes.Compare(resp.Body.Bytes(), TEST_BLOCK) != 0 {
+ t.Errorf("bad response body: %v", resp)
+ }
+
+ // Enable permissions.
+ enforce_permissions = true
+ PermissionSecret = []byte(known_key)
+ permission_ttl = 300
+ expiry := time.Now().Add(time.Duration(permission_ttl) * time.Second)
+
+ // Test GET with a signed locator.
+ test_url = "http://localhost:25107/" + SignLocator(TEST_HASH, known_token, expiry)
+ resp = httptest.NewRecorder()
+ req, _ = http.NewRequest("GET", test_url, nil)
+ req.Header.Set("Authorization", "OAuth "+known_token)
+ rest.ServeHTTP(resp, req)
+
+ if resp.Code != 200 {
+ t.Errorf("signed request: bad response code: %v", resp)
+ }
+ if bytes.Compare(resp.Body.Bytes(), TEST_BLOCK) != 0 {
+ t.Errorf("signed request: bad response body: %v", resp)
+ }
+
+ // Test GET with an unsigned locator.
+ test_url = "http://localhost:25107/" + TEST_HASH
+ resp = httptest.NewRecorder()
+ req, _ = http.NewRequest("GET", test_url, nil)
+ req.Header.Set("Authorization", "OAuth "+known_token)
+ rest.ServeHTTP(resp, req)
+
+ if resp.Code != PermissionError.HTTPCode {
+ t.Errorf("unsigned request: bad response code: %v", resp)
+ }
+
+ // Test GET with a signed locator and an unauthenticated request.
+ test_url = "http://localhost:25107/" + SignLocator(TEST_HASH, known_token, expiry)
+ resp = httptest.NewRecorder()
+ req, _ = http.NewRequest("GET", test_url, nil)
+ rest.ServeHTTP(resp, req)
+
+ if resp.Code != PermissionError.HTTPCode {
+ t.Errorf("signed locator, unauthenticated request: bad response code: %v", resp)
+ }
+
+ // Test GET with an expired, signed locator.
+ expired_ts := time.Now().Add(-time.Hour)
+ test_url = "http://localhost:25107/" + SignLocator(TEST_HASH, known_token, expired_ts)
+ resp = httptest.NewRecorder()
+ req, _ = http.NewRequest("GET", test_url, nil)
+ req.Header.Set("Authorization", "OAuth "+known_token)
+ rest.ServeHTTP(resp, req)
+
+ if resp.Code != ExpiredError.HTTPCode {
+ t.Errorf("expired signature: bad response code: %v", resp)
+ }
+}
+
+// ========================================
// Helper functions for unit tests.
// ========================================
diff --git a/services/keep/src/keep/perms.go b/services/keep/src/keep/perms.go
index 183bc2f..3ad7bb0 100644
--- a/services/keep/src/keep/perms.go
+++ b/services/keep/src/keep/perms.go
@@ -50,9 +50,9 @@ import (
// key.
var PermissionSecret []byte
-// makePermSignature returns a string representing the signed permission
+// MakePermSignature returns a string representing the signed permission
// hint for the blob identified by blob_hash, api_token and expiration timestamp.
-func makePermSignature(blob_hash string, api_token string, expiry string) string {
+func MakePermSignature(blob_hash string, api_token string, expiry string) string {
hmac := hmac.New(sha1.New, PermissionSecret)
hmac.Write([]byte(blob_hash))
hmac.Write([]byte("@"))
@@ -66,12 +66,16 @@ func makePermSignature(blob_hash string, api_token string, expiry string) string
// SignLocator takes a blob_locator, an api_token and an expiry time, and
// returns a signed locator string.
func SignLocator(blob_locator string, api_token string, expiry time.Time) string {
+ // If the permission secret has not been set, return an unsigned locator.
+ if PermissionSecret == nil {
+ return blob_locator
+ }
// Extract the hash from the blob locator, omitting any size hint that may be present.
blob_hash := strings.Split(blob_locator, "+")[0]
// Return the signed locator string.
timestamp_hex := fmt.Sprintf("%08x", expiry.Unix())
return blob_locator +
- "+A" + makePermSignature(blob_hash, api_token, timestamp_hex) +
+ "+A" + MakePermSignature(blob_hash, api_token, timestamp_hex) +
"@" + timestamp_hex
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list