[ARVADOS] updated: 46ccec84f9af75a15fc84cd9b060625b710279c1
git at public.curoverse.com
git at public.curoverse.com
Mon Sep 28 15:30:09 EDT 2015
Summary of changes:
apps/workbench/config/application.default.yml | 47 ++++++++-
sdk/cli/bin/crunch-job | 17 ++--
sdk/go/keepclient/keepclient.go | 77 +++++++++++++--
sdk/go/keepclient/keepclient_test.go | 136 ++++++++++++++++++++++++++
sdk/go/keepclient/support.go | 44 +++++----
sdk/python/arvados/keep.py | 15 ++-
services/api/config/application.default.yml | 43 +++++++-
services/api/script/crunch-dispatch.rb | 5 +
services/datamanager/keep/keep.go | 19 +++-
services/keepproxy/keepproxy.go | 91 +++++++++++++++++
services/keepproxy/keepproxy_test.go | 62 ++++++++++++
services/keepstore/handler_test.go | 30 ++++++
12 files changed, 536 insertions(+), 50 deletions(-)
via 46ccec84f9af75a15fc84cd9b060625b710279c1 (commit)
via ffa7f6dbbf52ab063610796378e90a054f900581 (commit)
via b4112f2e4220312c71c5c3d288f888133a5ba41a (commit)
via e31d1ec3c19d88042b47516fad9ed952d6561fd6 (commit)
via c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd (commit)
via 72748d17a195be1b65f847b078d1924f4648d45a (commit)
via fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c (commit)
via 57694f580feff10b274d6b13e9b4b013bfdad937 (commit)
via b8ad249763a75f3eb7bea758b88cdc87389639f1 (commit)
via 0ba145115071473ba46a3fac30c2dc98b74f49ad (commit)
via 0fdfc23c640c831cb2c0ee1e04fb639eb032de6f (commit)
via 73ce0cf7675e060d33e75488edfa4f533c177f82 (commit)
via 32a29ee7d171dccdb424990ff9c73e4b893dc3e4 (commit)
via 5a55113805af906145449be18ec86b1a95a5017b (commit)
via 285c0092f8ae4d757e31754c91314baa4db5af44 (commit)
via 559dc0ac07ea3d6820f5e220b9f6faf850ae63bf (commit)
via ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)
via 743022a885b563e47bc5d31e144b5165ab4688c9 (commit)
via 5a974b0568f7645a7e4377394da04940fa1ec137 (commit)
via 2b20a0ec9a32abfd85c09474e044cfd4c09f4963 (commit)
via 99349abd0ee7347b5bac3d4a9638853c6d4b97ab (commit)
via acd1241cec9260d54c2dca55785e309644334c41 (commit)
via c21c6330cc8a3c8cc96d83ff9a315fe10dff7c8f (commit)
via e45aa811c8489cae3ac83836aceca7f2ce6ba398 (commit)
via 9db403a87830f7afec86dae5bb279cf87df8fd0e (commit)
via 20ad8adb02808a85ac38f865fd870b00fb400b74 (commit)
via af901a9a0434c9b5ed8b9349835a1749f45f95d3 (commit)
via 67f730557e062608f842fa3abc612fb717fbce2f (commit)
via b2ed333572a4db15e82ba23a162b0f065d24c2b7 (commit)
via 6d352c4bfe2d0e9a405a52863c28b00de47f4be5 (commit)
via 7fdfbc3eae8de4fe7f6912c160e408ef57a89487 (commit)
via 0333c360cdd24b8fdd28a88d6d99bebe7e7a9bed (commit)
via c7b4e4802762eada81652224878a74b538a1d704 (commit)
via 0219fab6b0eb8b4710af2ae03c5231bf4d932a28 (commit)
via 6cca3e129b6de3fc4b34a1664f494fcbbbd940d1 (commit)
via bb663993cc851971d9444d834acfc416363931b2 (commit)
from 33596fce13c3bd5c368f5efeb89d625f684a622d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 46ccec84f9af75a15fc84cd9b060625b710279c1
Author: radhika <radhika at curoverse.com>
Date: Mon Sep 28 15:28:24 2015 -0400
7200: The incomplete response when no such prefix exists will be "\n". Update keepclient and keepproxy to expect this.
Also, added some more comments and golint checks for the newly added code.
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index a7522eb..c7a543c 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -28,9 +28,15 @@ var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.It
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
var InvalidLocatorError = errors.New("Invalid locator")
-var NoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
-var GetIndexError = errors.New("Error getting index")
-var IncompleteIndexError = errors.New("Got incomplete index")
+
+// ErrorGetIndex is the generic GetIndex error
+var ErrorGetIndex = errors.New("Error getting index")
+
+// ErrorNoSuchKeepServer is returned when GetIndex is invoked with a UUID with no matching keep server
+var ErrorNoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
+
+// ErrorIncompleteIndex is returned when the Index response does not end with a new empty line
+var ErrorIncompleteIndex = errors.New("Got incomplete index")
const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
@@ -194,7 +200,7 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
url := kc.LocalRoots()[keepServiceUUID]
if url == "" {
log.Printf("No such keep server found: %v", keepServiceUUID)
- return nil, NoSuchKeepServer
+ return nil, ErrorNoSuchKeepServer
}
url += "/index"
@@ -205,33 +211,35 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("GET index error: %v", err)
- return nil, GetIndexError
+ return nil, ErrorGetIndex
}
req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
resp, err := kc.Client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
log.Printf("GET index error: %v; status code: %v", err, resp.StatusCode)
- return nil, GetIndexError
+ return nil, ErrorGetIndex
}
- var respbody []byte
+ var respBody []byte
if resp.Body != nil {
- respbody, err = ioutil.ReadAll(resp.Body)
+ respBody, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("GET index error: %v", err)
- return nil, GetIndexError
+ return nil, ErrorGetIndex
}
// Got index; verify that it is complete
- if !strings.HasSuffix(string(respbody), "\n\n") {
+ // The response should be "\n" if no locators matched the prefix
+ // Else, it should be a list of locators followed by a blank line
+ if (!strings.HasSuffix(string(respBody), "\n\n")) && (string(respBody) != "\n") {
log.Printf("Got incomplete index for %v", url)
- return nil, IncompleteIndexError
+ return nil, ErrorIncompleteIndex
}
}
- // Got complete index or "" if no locators matching prefix
- return strings.NewReader(string(respbody)), nil
+ // Got complete index
+ return strings.NewReader(string(respBody)), nil
}
// LocalRoots() returns the map of local (i.e., disk and proxy) Keep
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index f83bee1..5880373 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1064,10 +1064,10 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
st := StubGetIndexHandler{
c,
- "/index/xyz",
+ "/index/abcd",
"abc123",
http.StatusOK,
- []byte("")}
+ []byte("\n")}
ks := RunFakeKeepServer(st)
defer ks.listener.Close()
@@ -1077,6 +1077,10 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
- _, err = kc.GetIndex("x", "xyz")
- c.Check((err != nil), Equals, true)
+ r, err := kc.GetIndex("x", "abcd")
+ c.Check(err, Equals, nil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body)
}
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 788cbfc..865212d 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -273,7 +273,7 @@ func MakeRESTRouter(
rest.Handle(`/index`, IndexHandler{kc, t}).Methods("GET")
// List blocks whose hash has the given prefix
- rest.Handle(`/index/{prefix}`, IndexHandler{kc, t}).Methods("GET")
+ rest.Handle(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler{kc, t}).Methods("GET")
}
if enable_put {
@@ -494,7 +494,14 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
}
}
-func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+// ServeHTTP implemenation for IndexHandler
+// Supports only GET requests for /index/{prefix:[0-9a-f]{0,32}}
+// For each keep server found in LocalRoots:
+// Invokes GetIndex using keepclient
+// Expects "complete" response (terminating with blank new line)
+// Aborts on any errors
+// Concatenates responses from all those keep servers and returns
+func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
SetCorsHeaders(resp)
prefix := mux.Vars(req)["prefix"]
@@ -507,11 +514,11 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
}
}()
- kc := *this.KeepClient
+ kc := *handler.KeepClient
var pass bool
var tok string
- if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+ if pass, tok = CheckAuthorizationHeader(kc, handler.ApiTokenCache, req); !pass {
status, err = http.StatusForbidden, BadAuthorizationHeader
return
}
@@ -526,7 +533,7 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
switch req.Method {
case "GET":
- for uuid, _ := range kc.LocalRoots() {
+ for uuid := range kc.LocalRoots() {
reader, err = kc.GetIndex(uuid, prefix)
if err != nil {
break
@@ -539,12 +546,17 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
}
// Got index; verify that it is complete
- if !strings.HasSuffix(string(readBytes), "\n\n") {
+ // The response should be "\n" if no locators matched the prefix
+ // Else, it should be a list of locators followed by a blank line
+ if (!strings.HasSuffix(string(readBytes), "\n\n")) && (string(readBytes) != "\n") {
err = errors.New("Got incomplete index")
}
+ // Trim the extra empty new line found in response from each server
indexResp = append(indexResp, (readBytes[0 : len(readBytes)-1])...)
}
+
+ // Append empty line at the end of concatenation of all server responses
indexResp = append(indexResp, ([]byte("\n"))...)
default:
status, err = http.StatusNotImplemented, MethodNotSupported
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index a08d1b6..1afe373 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -407,6 +407,12 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
}
+// Test GetIndex
+// Put one block, with 2 replicas
+// With no prefix (expect the block locator, twice)
+// With an existing prefix (expect the block locator, twice)
+// With a valid but non-existing prefix (expect "\n")
+// With an invalid prefix (expect error)
func (s *ServerRequiredSuite) TestGetIndex(c *C) {
kc := runProxy(c, []string{"keepproxy"}, 28852, false)
waitForListener()
@@ -434,7 +440,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
count := 0
for _, locator := range locators {
if strings.HasPrefix(locator, hash) {
- count += 1
+ count++
}
}
c.Assert(2, Equals, count)
@@ -447,12 +453,18 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
count = 0
for _, locator := range locators {
if strings.HasPrefix(locator, hash) {
- count += 1
+ count++
}
}
c.Assert(2, Equals, count)
- // GetIndex with no such prefix
+ // GetIndex with valid but no such prefix
+ indexReader, err = kc.GetIndex("proxy", "abcd")
+ c.Assert(err, Equals, nil)
+ indexResp, err = ioutil.ReadAll(indexReader)
+ c.Assert(string(indexResp), Equals, "\n")
+
+ // GetIndex with invalid prefix
indexReader, err = kc.GetIndex("proxy", "xyz")
c.Assert((err != nil), Equals, true)
}
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index dd83666..ba923ca 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -321,6 +321,16 @@ func TestIndexHandler(t *testing.T) {
uri: "/index/" + TestHash[0:3],
apiToken: dataManagerToken,
}
+ superuserNoSuchPrefixReq := &RequestTester{
+ method: "GET",
+ uri: "/index/abcd",
+ apiToken: dataManagerToken,
+ }
+ superuserInvalidPrefixReq := &RequestTester{
+ method: "GET",
+ uri: "/index/xyz",
+ apiToken: dataManagerToken,
+ }
// -------------------------------------------------------------
// Only the superuser should be allowed to issue /index requests.
@@ -407,6 +417,26 @@ func TestIndexHandler(t *testing.T) {
"permissions on, superuser /index/prefix request: expected %s, got:\n%s",
expected, response.Body.String())
}
+
+ // superuser /index/{no-such-prefix} request
+ // => OK
+ response = IssueRequest(superuserNoSuchPrefixReq)
+ ExpectStatusCode(t,
+ "permissions on, superuser request",
+ http.StatusOK,
+ response)
+
+ if "\n" != response.Body.String() {
+ t.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
+ }
+
+ // superuser /index/{invalid-prefix} request
+ // => StatusBadRequest
+ response = IssueRequest(superuserInvalidPrefixReq)
+ ExpectStatusCode(t,
+ "permissions on, superuser request",
+ http.StatusBadRequest,
+ response)
}
// TestDeleteHandler
commit ffa7f6dbbf52ab063610796378e90a054f900581
Author: radhika <radhika at curoverse.com>
Date: Mon Sep 28 10:10:48 2015 -0400
7200: add GetIndex function to keepclient; add IndexHandler to keepproxy.
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 70aa374..a7522eb 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -28,6 +28,9 @@ var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.It
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
var InvalidLocatorError = errors.New("Invalid locator")
+var NoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
+var GetIndexError = errors.New("Error getting index")
+var IncompleteIndexError = errors.New("Got incomplete index")
const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
@@ -182,6 +185,55 @@ func (kc *KeepClient) Ask(locator string) (int64, string, error) {
return 0, "", BlockNotFound
}
+// GetIndex retrieves a list of blocks stored on the given server whose hashes
+// begin with the given prefix. The returned reader will return an error (other
+// than EOF) if the complete index cannot be retrieved. This should only be
+// expected to return useful results if the client is using a "data manager token"
+// recognized by the Keep services.
+func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error) {
+ url := kc.LocalRoots()[keepServiceUUID]
+ if url == "" {
+ log.Printf("No such keep server found: %v", keepServiceUUID)
+ return nil, NoSuchKeepServer
+ }
+
+ url += "/index"
+ if prefix != "" {
+ url += "/" + prefix
+ }
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Printf("GET index error: %v", err)
+ return nil, GetIndexError
+ }
+
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+ resp, err := kc.Client.Do(req)
+ if err != nil || resp.StatusCode != http.StatusOK {
+ log.Printf("GET index error: %v; status code: %v", err, resp.StatusCode)
+ return nil, GetIndexError
+ }
+
+ var respbody []byte
+ if resp.Body != nil {
+ respbody, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ log.Printf("GET index error: %v", err)
+ return nil, GetIndexError
+ }
+
+ // Got index; verify that it is complete
+ if !strings.HasSuffix(string(respbody), "\n\n") {
+ log.Printf("Got incomplete index for %v", url)
+ return nil, IncompleteIndexError
+ }
+ }
+
+ // Got complete index or "" if no locators matching prefix
+ return strings.NewReader(string(respbody)), nil
+}
+
// LocalRoots() returns the map of local (i.e., disk and proxy) Keep
// services: uuid -> baseURI.
func (kc *KeepClient) LocalRoots() map[string]string {
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index e4e459e..f83bee1 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -948,3 +948,135 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
c.Check(err, Equals, InsufficientReplicasError)
c.Check(replicas, Equals, 0)
}
+
+type StubGetIndexHandler struct {
+ c *C
+ expectPath string
+ expectApiToken string
+ httpStatus int
+ body []byte
+}
+
+func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ h.c.Check(req.URL.Path, Equals, h.expectPath)
+ h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectApiToken))
+ resp.WriteHeader(h.httpStatus)
+ resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
+ resp.Write(h.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index",
+ "abc123",
+ http.StatusOK,
+ []byte(string(hash) + "\n\n")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+ r, err := kc.GetIndex("x", "")
+ c.Check(err, Equals, nil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(string(hash) + "\n\n")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+ r, err := kc.GetIndex("x", hash[0:3])
+ c.Check(err, Equals, nil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(string(hash))}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+ _, err = kc.GetIndex("x", hash[0:3])
+ c.Check(err, Equals, IncompleteIndexError)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(string(hash))}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+ _, err = kc.GetIndex("y", hash[0:3])
+ c.Check(err, Equals, NoSuchKeepServer)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
+ st := StubGetIndexHandler{
+ c,
+ "/index/xyz",
+ "abc123",
+ http.StatusOK,
+ []byte("")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+ _, err = kc.GetIndex("x", "xyz")
+ c.Check((err != nil), Equals, true)
+}
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index d0af4a5..788cbfc 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -16,6 +16,7 @@ import (
"os/signal"
"reflect"
"regexp"
+ "strings"
"sync"
"syscall"
"time"
@@ -241,6 +242,11 @@ type PutBlockHandler struct {
*ApiTokenCache
}
+type IndexHandler struct {
+ *keepclient.KeepClient
+ *ApiTokenCache
+}
+
type InvalidPathHandler struct{}
type OptionsHandler struct{}
@@ -262,6 +268,12 @@ func MakeRESTRouter(
rest.Handle(`/{locator:[0-9a-f]{32}\+.*}`,
GetBlockHandler{kc, t}).Methods("GET", "HEAD")
rest.Handle(`/{locator:[0-9a-f]{32}}`, GetBlockHandler{kc, t}).Methods("GET", "HEAD")
+
+ // List all blocks
+ rest.Handle(`/index`, IndexHandler{kc, t}).Methods("GET")
+
+ // List blocks whose hash has the given prefix
+ rest.Handle(`/index/{prefix}`, IndexHandler{kc, t}).Methods("GET")
}
if enable_put {
@@ -481,3 +493,70 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
status = http.StatusBadGateway
}
}
+
+func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ SetCorsHeaders(resp)
+
+ prefix := mux.Vars(req)["prefix"]
+ var err error
+ var status int
+
+ defer func() {
+ if status != http.StatusOK {
+ http.Error(resp, err.Error(), status)
+ }
+ }()
+
+ kc := *this.KeepClient
+
+ var pass bool
+ var tok string
+ if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+ status, err = http.StatusForbidden, BadAuthorizationHeader
+ return
+ }
+
+ // Copy ArvadosClient struct and use the client's API token
+ arvclient := *kc.Arvados
+ arvclient.ApiToken = tok
+ kc.Arvados = &arvclient
+
+ var indexResp []byte
+ var reader io.Reader
+
+ switch req.Method {
+ case "GET":
+ for uuid, _ := range kc.LocalRoots() {
+ reader, err = kc.GetIndex(uuid, prefix)
+ if err != nil {
+ break
+ }
+
+ var readBytes []byte
+ readBytes, err = ioutil.ReadAll(reader)
+ if err != nil {
+ break
+ }
+
+ // Got index; verify that it is complete
+ if !strings.HasSuffix(string(readBytes), "\n\n") {
+ err = errors.New("Got incomplete index")
+ }
+
+ indexResp = append(indexResp, (readBytes[0 : len(readBytes)-1])...)
+ }
+ indexResp = append(indexResp, ([]byte("\n"))...)
+ default:
+ status, err = http.StatusNotImplemented, MethodNotSupported
+ return
+ }
+
+ switch err {
+ case nil:
+ status = http.StatusOK
+ resp.Header().Set("Content-Length", fmt.Sprint(len(indexResp)))
+ _, err = resp.Write(indexResp)
+ default:
+ status = http.StatusBadGateway
+ }
+}
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 22cc72e..a08d1b6 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -406,3 +406,53 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
}
+
+func (s *ServerRequiredSuite) TestGetIndex(c *C) {
+ kc := runProxy(c, []string{"keepproxy"}, 28852, false)
+ waitForListener()
+ defer closeListener()
+
+ data := []byte("index-data")
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kc.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kc.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(10))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ // GetIndex with no prefix
+ indexReader, err := kc.GetIndex("proxy", "")
+ c.Assert(err, Equals, nil)
+ indexResp, err := ioutil.ReadAll(indexReader)
+ locators := strings.Split(string(indexResp), "\n")
+ count := 0
+ for _, locator := range locators {
+ if strings.HasPrefix(locator, hash) {
+ count += 1
+ }
+ }
+ c.Assert(2, Equals, count)
+
+ // GetIndex with prefix
+ indexReader, err = kc.GetIndex("proxy", hash[0:3])
+ c.Assert(err, Equals, nil)
+ indexResp, err = ioutil.ReadAll(indexReader)
+ locators = strings.Split(string(indexResp), "\n")
+ count = 0
+ for _, locator := range locators {
+ if strings.HasPrefix(locator, hash) {
+ count += 1
+ }
+ }
+ c.Assert(2, Equals, count)
+
+ // GetIndex with no such prefix
+ indexReader, err = kc.GetIndex("proxy", "xyz")
+ c.Assert((err != nil), Equals, true)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list