[ARVADOS] updated: 46ccec84f9af75a15fc84cd9b060625b710279c1

git at public.curoverse.com git at public.curoverse.com
Mon Sep 28 15:30:09 EDT 2015


Summary of changes:
 apps/workbench/config/application.default.yml |  47 ++++++++-
 sdk/cli/bin/crunch-job                        |  17 ++--
 sdk/go/keepclient/keepclient.go               |  77 +++++++++++++--
 sdk/go/keepclient/keepclient_test.go          | 136 ++++++++++++++++++++++++++
 sdk/go/keepclient/support.go                  |  44 +++++----
 sdk/python/arvados/keep.py                    |  15 ++-
 services/api/config/application.default.yml   |  43 +++++++-
 services/api/script/crunch-dispatch.rb        |   5 +
 services/datamanager/keep/keep.go             |  19 +++-
 services/keepproxy/keepproxy.go               |  91 +++++++++++++++++
 services/keepproxy/keepproxy_test.go          |  62 ++++++++++++
 services/keepstore/handler_test.go            |  30 ++++++
 12 files changed, 536 insertions(+), 50 deletions(-)

       via  46ccec84f9af75a15fc84cd9b060625b710279c1 (commit)
       via  ffa7f6dbbf52ab063610796378e90a054f900581 (commit)
       via  b4112f2e4220312c71c5c3d288f888133a5ba41a (commit)
       via  e31d1ec3c19d88042b47516fad9ed952d6561fd6 (commit)
       via  c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd (commit)
       via  72748d17a195be1b65f847b078d1924f4648d45a (commit)
       via  fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c (commit)
       via  57694f580feff10b274d6b13e9b4b013bfdad937 (commit)
       via  b8ad249763a75f3eb7bea758b88cdc87389639f1 (commit)
       via  0ba145115071473ba46a3fac30c2dc98b74f49ad (commit)
       via  0fdfc23c640c831cb2c0ee1e04fb639eb032de6f (commit)
       via  73ce0cf7675e060d33e75488edfa4f533c177f82 (commit)
       via  32a29ee7d171dccdb424990ff9c73e4b893dc3e4 (commit)
       via  5a55113805af906145449be18ec86b1a95a5017b (commit)
       via  285c0092f8ae4d757e31754c91314baa4db5af44 (commit)
       via  559dc0ac07ea3d6820f5e220b9f6faf850ae63bf (commit)
       via  ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)
       via  743022a885b563e47bc5d31e144b5165ab4688c9 (commit)
       via  5a974b0568f7645a7e4377394da04940fa1ec137 (commit)
       via  2b20a0ec9a32abfd85c09474e044cfd4c09f4963 (commit)
       via  99349abd0ee7347b5bac3d4a9638853c6d4b97ab (commit)
       via  acd1241cec9260d54c2dca55785e309644334c41 (commit)
       via  c21c6330cc8a3c8cc96d83ff9a315fe10dff7c8f (commit)
       via  e45aa811c8489cae3ac83836aceca7f2ce6ba398 (commit)
       via  9db403a87830f7afec86dae5bb279cf87df8fd0e (commit)
       via  20ad8adb02808a85ac38f865fd870b00fb400b74 (commit)
       via  af901a9a0434c9b5ed8b9349835a1749f45f95d3 (commit)
       via  67f730557e062608f842fa3abc612fb717fbce2f (commit)
       via  b2ed333572a4db15e82ba23a162b0f065d24c2b7 (commit)
       via  6d352c4bfe2d0e9a405a52863c28b00de47f4be5 (commit)
       via  7fdfbc3eae8de4fe7f6912c160e408ef57a89487 (commit)
       via  0333c360cdd24b8fdd28a88d6d99bebe7e7a9bed (commit)
       via  c7b4e4802762eada81652224878a74b538a1d704 (commit)
       via  0219fab6b0eb8b4710af2ae03c5231bf4d932a28 (commit)
       via  6cca3e129b6de3fc4b34a1664f494fcbbbd940d1 (commit)
       via  bb663993cc851971d9444d834acfc416363931b2 (commit)
      from  33596fce13c3bd5c368f5efeb89d625f684a622d (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 46ccec84f9af75a15fc84cd9b060625b710279c1
Author: radhika <radhika at curoverse.com>
Date:   Mon Sep 28 15:28:24 2015 -0400

    7200: The incomplete response when no such prefix exists will be "\n". Update keepclient and keepproxy to expect this.
    Also, added some more comments and golint checks for the newly added code.

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index a7522eb..c7a543c 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -28,9 +28,15 @@ var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.It
 var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
 var InvalidLocatorError = errors.New("Invalid locator")
-var NoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
-var GetIndexError = errors.New("Error getting index")
-var IncompleteIndexError = errors.New("Got incomplete index")
+
+// ErrorGetIndex is the generic GetIndex error
+var ErrorGetIndex = errors.New("Error getting index")
+
+// ErrorNoSuchKeepServer is returned when GetIndex is invoked with a UUID with no matching keep server
+var ErrorNoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
+
+// ErrorIncompleteIndex is returned when the Index response does not end with a new empty line
+var ErrorIncompleteIndex = errors.New("Got incomplete index")
 
 const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
 const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
@@ -194,7 +200,7 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
 	url := kc.LocalRoots()[keepServiceUUID]
 	if url == "" {
 		log.Printf("No such keep server found: %v", keepServiceUUID)
-		return nil, NoSuchKeepServer
+		return nil, ErrorNoSuchKeepServer
 	}
 
 	url += "/index"
@@ -205,33 +211,35 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		log.Printf("GET index error: %v", err)
-		return nil, GetIndexError
+		return nil, ErrorGetIndex
 	}
 
 	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
 	resp, err := kc.Client.Do(req)
 	if err != nil || resp.StatusCode != http.StatusOK {
 		log.Printf("GET index error: %v; status code: %v", err, resp.StatusCode)
-		return nil, GetIndexError
+		return nil, ErrorGetIndex
 	}
 
-	var respbody []byte
+	var respBody []byte
 	if resp.Body != nil {
-		respbody, err = ioutil.ReadAll(resp.Body)
+		respBody, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
 			log.Printf("GET index error: %v", err)
-			return nil, GetIndexError
+			return nil, ErrorGetIndex
 		}
 
 		// Got index; verify that it is complete
-		if !strings.HasSuffix(string(respbody), "\n\n") {
+		// The response should be "\n" if no locators matched the prefix
+		// Else, it should be a list of locators followed by a blank line
+		if (!strings.HasSuffix(string(respBody), "\n\n")) && (string(respBody) != "\n") {
 			log.Printf("Got incomplete index for %v", url)
-			return nil, IncompleteIndexError
+			return nil, ErrorIncompleteIndex
 		}
 	}
 
-	// Got complete index or "" if no locators matching prefix
-	return strings.NewReader(string(respbody)), nil
+	// Got complete index
+	return strings.NewReader(string(respBody)), nil
 }
 
 // LocalRoots() returns the map of local (i.e., disk and proxy) Keep
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index f83bee1..5880373 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1064,10 +1064,10 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 	st := StubGetIndexHandler{
 		c,
-		"/index/xyz",
+		"/index/abcd",
 		"abc123",
 		http.StatusOK,
-		[]byte("")}
+		[]byte("\n")}
 
 	ks := RunFakeKeepServer(st)
 	defer ks.listener.Close()
@@ -1077,6 +1077,10 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 	arv.ApiToken = "abc123"
 	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
 
-	_, err = kc.GetIndex("x", "xyz")
-	c.Check((err != nil), Equals, true)
+	r, err := kc.GetIndex("x", "abcd")
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body)
 }
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 788cbfc..865212d 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -273,7 +273,7 @@ func MakeRESTRouter(
 		rest.Handle(`/index`, IndexHandler{kc, t}).Methods("GET")
 
 		// List blocks whose hash has the given prefix
-		rest.Handle(`/index/{prefix}`, IndexHandler{kc, t}).Methods("GET")
+		rest.Handle(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler{kc, t}).Methods("GET")
 	}
 
 	if enable_put {
@@ -494,7 +494,14 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 	}
 }
 
-func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+// ServeHTTP implemenation for IndexHandler
+// Supports only GET requests for /index/{prefix:[0-9a-f]{0,32}}
+// For each keep server found in LocalRoots:
+//   Invokes GetIndex using keepclient
+//   Expects "complete" response (terminating with blank new line)
+//   Aborts on any errors
+// Concatenates responses from all those keep servers and returns
+func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	SetCorsHeaders(resp)
 
 	prefix := mux.Vars(req)["prefix"]
@@ -507,11 +514,11 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 		}
 	}()
 
-	kc := *this.KeepClient
+	kc := *handler.KeepClient
 
 	var pass bool
 	var tok string
-	if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+	if pass, tok = CheckAuthorizationHeader(kc, handler.ApiTokenCache, req); !pass {
 		status, err = http.StatusForbidden, BadAuthorizationHeader
 		return
 	}
@@ -526,7 +533,7 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 
 	switch req.Method {
 	case "GET":
-		for uuid, _ := range kc.LocalRoots() {
+		for uuid := range kc.LocalRoots() {
 			reader, err = kc.GetIndex(uuid, prefix)
 			if err != nil {
 				break
@@ -539,12 +546,17 @@ func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 			}
 
 			// Got index; verify that it is complete
-			if !strings.HasSuffix(string(readBytes), "\n\n") {
+			// The response should be "\n" if no locators matched the prefix
+			// Else, it should be a list of locators followed by a blank line
+			if (!strings.HasSuffix(string(readBytes), "\n\n")) && (string(readBytes) != "\n") {
 				err = errors.New("Got incomplete index")
 			}
 
+			// Trim the extra empty new line found in response from each server
 			indexResp = append(indexResp, (readBytes[0 : len(readBytes)-1])...)
 		}
+
+		// Append empty line at the end of concatenation of all server responses
 		indexResp = append(indexResp, ([]byte("\n"))...)
 	default:
 		status, err = http.StatusNotImplemented, MethodNotSupported
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index a08d1b6..1afe373 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -407,6 +407,12 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
 
 }
 
+// Test GetIndex
+//   Put one block, with 2 replicas
+//   With no prefix (expect the block locator, twice)
+//   With an existing prefix (expect the block locator, twice)
+//   With a valid but non-existing prefix (expect "\n")
+//   With an invalid prefix (expect error)
 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 	kc := runProxy(c, []string{"keepproxy"}, 28852, false)
 	waitForListener()
@@ -434,7 +440,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 	count := 0
 	for _, locator := range locators {
 		if strings.HasPrefix(locator, hash) {
-			count += 1
+			count++
 		}
 	}
 	c.Assert(2, Equals, count)
@@ -447,12 +453,18 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 	count = 0
 	for _, locator := range locators {
 		if strings.HasPrefix(locator, hash) {
-			count += 1
+			count++
 		}
 	}
 	c.Assert(2, Equals, count)
 
-	// GetIndex with no such prefix
+	// GetIndex with valid but no such prefix
+	indexReader, err = kc.GetIndex("proxy", "abcd")
+	c.Assert(err, Equals, nil)
+	indexResp, err = ioutil.ReadAll(indexReader)
+	c.Assert(string(indexResp), Equals, "\n")
+
+	// GetIndex with invalid prefix
 	indexReader, err = kc.GetIndex("proxy", "xyz")
 	c.Assert((err != nil), Equals, true)
 }
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index dd83666..ba923ca 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -321,6 +321,16 @@ func TestIndexHandler(t *testing.T) {
 		uri:      "/index/" + TestHash[0:3],
 		apiToken: dataManagerToken,
 	}
+	superuserNoSuchPrefixReq := &RequestTester{
+		method:   "GET",
+		uri:      "/index/abcd",
+		apiToken: dataManagerToken,
+	}
+	superuserInvalidPrefixReq := &RequestTester{
+		method:   "GET",
+		uri:      "/index/xyz",
+		apiToken: dataManagerToken,
+	}
 
 	// -------------------------------------------------------------
 	// Only the superuser should be allowed to issue /index requests.
@@ -407,6 +417,26 @@ func TestIndexHandler(t *testing.T) {
 			"permissions on, superuser /index/prefix request: expected %s, got:\n%s",
 			expected, response.Body.String())
 	}
+
+	// superuser /index/{no-such-prefix} request
+	// => OK
+	response = IssueRequest(superuserNoSuchPrefixReq)
+	ExpectStatusCode(t,
+		"permissions on, superuser request",
+		http.StatusOK,
+		response)
+
+	if "\n" != response.Body.String() {
+		t.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
+	}
+
+	// superuser /index/{invalid-prefix} request
+	// => StatusBadRequest
+	response = IssueRequest(superuserInvalidPrefixReq)
+	ExpectStatusCode(t,
+		"permissions on, superuser request",
+		http.StatusBadRequest,
+		response)
 }
 
 // TestDeleteHandler

commit ffa7f6dbbf52ab063610796378e90a054f900581
Author: radhika <radhika at curoverse.com>
Date:   Mon Sep 28 10:10:48 2015 -0400

    7200: add GetIndex function to keepclient; add IndexHandler to keepproxy.

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 70aa374..a7522eb 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -28,6 +28,9 @@ var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.It
 var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
 var InvalidLocatorError = errors.New("Invalid locator")
+var NoSuchKeepServer = errors.New("No keep server matching the given UUID is found")
+var GetIndexError = errors.New("Error getting index")
+var IncompleteIndexError = errors.New("Got incomplete index")
 
 const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
 const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
@@ -182,6 +185,55 @@ func (kc *KeepClient) Ask(locator string) (int64, string, error) {
 	return 0, "", BlockNotFound
 }
 
+// GetIndex retrieves a list of blocks stored on the given server whose hashes
+// begin with the given prefix. The returned reader will return an error (other
+// than EOF) if the complete index cannot be retrieved. This should only be
+// expected to return useful results if the client is using a "data manager token"
+// recognized by the Keep services.
+func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error) {
+	url := kc.LocalRoots()[keepServiceUUID]
+	if url == "" {
+		log.Printf("No such keep server found: %v", keepServiceUUID)
+		return nil, NoSuchKeepServer
+	}
+
+	url += "/index"
+	if prefix != "" {
+		url += "/" + prefix
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		log.Printf("GET index error: %v", err)
+		return nil, GetIndexError
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+	resp, err := kc.Client.Do(req)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		log.Printf("GET index error: %v; status code: %v", err, resp.StatusCode)
+		return nil, GetIndexError
+	}
+
+	var respbody []byte
+	if resp.Body != nil {
+		respbody, err = ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Printf("GET index error: %v", err)
+			return nil, GetIndexError
+		}
+
+		// Got index; verify that it is complete
+		if !strings.HasSuffix(string(respbody), "\n\n") {
+			log.Printf("Got incomplete index for %v", url)
+			return nil, IncompleteIndexError
+		}
+	}
+
+	// Got complete index or "" if no locators matching prefix
+	return strings.NewReader(string(respbody)), nil
+}
+
 // LocalRoots() returns the map of local (i.e., disk and proxy) Keep
 // services: uuid -> baseURI.
 func (kc *KeepClient) LocalRoots() map[string]string {
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index e4e459e..f83bee1 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -948,3 +948,135 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
 	c.Check(err, Equals, InsufficientReplicasError)
 	c.Check(replicas, Equals, 0)
 }
+
+type StubGetIndexHandler struct {
+	c              *C
+	expectPath     string
+	expectApiToken string
+	httpStatus     int
+	body           []byte
+}
+
+func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	h.c.Check(req.URL.Path, Equals, h.expectPath)
+	h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectApiToken))
+	resp.WriteHeader(h.httpStatus)
+	resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
+	resp.Write(h.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index",
+		"abc123",
+		http.StatusOK,
+		[]byte(string(hash) + "\n\n")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+	r, err := kc.GetIndex("x", "")
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(string(hash) + "\n\n")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+	r, err := kc.GetIndex("x", hash[0:3])
+	c.Check(err, Equals, nil)
+
+	content, err2 := ioutil.ReadAll(r)
+	c.Check(err2, Equals, nil)
+	c.Check(content, DeepEquals, st.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(string(hash))}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+	_, err = kc.GetIndex("x", hash[0:3])
+	c.Check(err, Equals, IncompleteIndexError)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	st := StubGetIndexHandler{
+		c,
+		"/index/" + hash[0:3],
+		"abc123",
+		http.StatusOK,
+		[]byte(string(hash))}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+	_, err = kc.GetIndex("y", hash[0:3])
+	c.Check(err, Equals, NoSuchKeepServer)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
+	st := StubGetIndexHandler{
+		c,
+		"/index/xyz",
+		"abc123",
+		http.StatusOK,
+		[]byte("")}
+
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(map[string]string{"x": ks.url}, map[string]string{ks.url: ""}, nil)
+
+	_, err = kc.GetIndex("x", "xyz")
+	c.Check((err != nil), Equals, true)
+}
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index d0af4a5..788cbfc 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -16,6 +16,7 @@ import (
 	"os/signal"
 	"reflect"
 	"regexp"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -241,6 +242,11 @@ type PutBlockHandler struct {
 	*ApiTokenCache
 }
 
+type IndexHandler struct {
+	*keepclient.KeepClient
+	*ApiTokenCache
+}
+
 type InvalidPathHandler struct{}
 
 type OptionsHandler struct{}
@@ -262,6 +268,12 @@ func MakeRESTRouter(
 		rest.Handle(`/{locator:[0-9a-f]{32}\+.*}`,
 			GetBlockHandler{kc, t}).Methods("GET", "HEAD")
 		rest.Handle(`/{locator:[0-9a-f]{32}}`, GetBlockHandler{kc, t}).Methods("GET", "HEAD")
+
+		// List all blocks
+		rest.Handle(`/index`, IndexHandler{kc, t}).Methods("GET")
+
+		// List blocks whose hash has the given prefix
+		rest.Handle(`/index/{prefix}`, IndexHandler{kc, t}).Methods("GET")
 	}
 
 	if enable_put {
@@ -481,3 +493,70 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 		status = http.StatusBadGateway
 	}
 }
+
+func (this IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	SetCorsHeaders(resp)
+
+	prefix := mux.Vars(req)["prefix"]
+	var err error
+	var status int
+
+	defer func() {
+		if status != http.StatusOK {
+			http.Error(resp, err.Error(), status)
+		}
+	}()
+
+	kc := *this.KeepClient
+
+	var pass bool
+	var tok string
+	if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
+		status, err = http.StatusForbidden, BadAuthorizationHeader
+		return
+	}
+
+	// Copy ArvadosClient struct and use the client's API token
+	arvclient := *kc.Arvados
+	arvclient.ApiToken = tok
+	kc.Arvados = &arvclient
+
+	var indexResp []byte
+	var reader io.Reader
+
+	switch req.Method {
+	case "GET":
+		for uuid, _ := range kc.LocalRoots() {
+			reader, err = kc.GetIndex(uuid, prefix)
+			if err != nil {
+				break
+			}
+
+			var readBytes []byte
+			readBytes, err = ioutil.ReadAll(reader)
+			if err != nil {
+				break
+			}
+
+			// Got index; verify that it is complete
+			if !strings.HasSuffix(string(readBytes), "\n\n") {
+				err = errors.New("Got incomplete index")
+			}
+
+			indexResp = append(indexResp, (readBytes[0 : len(readBytes)-1])...)
+		}
+		indexResp = append(indexResp, ([]byte("\n"))...)
+	default:
+		status, err = http.StatusNotImplemented, MethodNotSupported
+		return
+	}
+
+	switch err {
+	case nil:
+		status = http.StatusOK
+		resp.Header().Set("Content-Length", fmt.Sprint(len(indexResp)))
+		_, err = resp.Write(indexResp)
+	default:
+		status = http.StatusBadGateway
+	}
+}
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 22cc72e..a08d1b6 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -406,3 +406,53 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
 		"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
 
 }
+
+func (s *ServerRequiredSuite) TestGetIndex(c *C) {
+	kc := runProxy(c, []string{"keepproxy"}, 28852, false)
+	waitForListener()
+	defer closeListener()
+
+	data := []byte("index-data")
+	hash := fmt.Sprintf("%x", md5.Sum(data))
+
+	hash2, rep, err := kc.PutB(data)
+	c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
+	c.Check(rep, Equals, 2)
+	c.Check(err, Equals, nil)
+
+	reader, blocklen, _, err := kc.Get(hash)
+	c.Assert(err, Equals, nil)
+	c.Check(blocklen, Equals, int64(10))
+	all, err := ioutil.ReadAll(reader)
+	c.Check(all, DeepEquals, data)
+
+	// GetIndex with no prefix
+	indexReader, err := kc.GetIndex("proxy", "")
+	c.Assert(err, Equals, nil)
+	indexResp, err := ioutil.ReadAll(indexReader)
+	locators := strings.Split(string(indexResp), "\n")
+	count := 0
+	for _, locator := range locators {
+		if strings.HasPrefix(locator, hash) {
+			count += 1
+		}
+	}
+	c.Assert(2, Equals, count)
+
+	// GetIndex with prefix
+	indexReader, err = kc.GetIndex("proxy", hash[0:3])
+	c.Assert(err, Equals, nil)
+	indexResp, err = ioutil.ReadAll(indexReader)
+	locators = strings.Split(string(indexResp), "\n")
+	count = 0
+	for _, locator := range locators {
+		if strings.HasPrefix(locator, hash) {
+			count += 1
+		}
+	}
+	c.Assert(2, Equals, count)
+
+	// GetIndex with no such prefix
+	indexReader, err = kc.GetIndex("proxy", "xyz")
+	c.Assert((err != nil), Equals, true)
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list