[ARVADOS] updated: 86dffe03ec74387d14da9ceb17934bbdab1239b9
git at public.curoverse.com
git at public.curoverse.com
Thu May 22 11:17:36 EDT 2014
Summary of changes:
sdk/go/go.sh | 6 +-
sdk/go/src/arvados.org/keepclient/keepclient.go | 5 +-
.../src/arvados.org/keepclient/keepclient_test.go | 6 +-
sdk/go/src/arvados.org/keepclient/support.go | 9 +-
.../arvados/v1/keep_services_controller.rb | 2 +-
services/api/test/integration/keep_proxy_test.rb | 2 +-
services/keep/go.sh | 4 +-
.../keep/src/arvados.org/keepproxy/keepproxy.go | 66 +++++++----
.../src/arvados.org/keepproxy/keepproxy_test.go | 121 ++++++++++++++++++++-
9 files changed, 184 insertions(+), 37 deletions(-)
via 86dffe03ec74387d14da9ceb17934bbdab1239b9 (commit)
via fff404a0e2ebe2582f616526c486ad0dbecce3c8 (commit)
via 66f1be77149feb033e3adc17c37a09a391eec6e9 (commit)
via c2e70e05178c9569e9de2ac5b3ef47440c49efc5 (commit)
from 77504f6d369bd7bd323748d5347d6d99ed9c75ca (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 86dffe03ec74387d14da9ceb17934bbdab1239b9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu May 22 11:17:32 2014 -0400
1885: Full-stack integration test (api+keep+keepproxy+keepclient) works!
diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go
index 89c5e4b..ef4a8e1 100644
--- a/sdk/go/src/arvados.org/keepclient/support.go
+++ b/sdk/go/src/arvados.org/keepclient/support.go
@@ -186,6 +186,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
upload_status <- uploadStatus{err, url, 0, 0}
+ body.Close()
return
}
@@ -231,7 +232,6 @@ func (this KeepClient) putReplicas(
next_server += 1
active += 1
} else {
- fmt.Print(active)
if active == 0 {
return (this.Want_replicas - remaining_replicas), InsufficientReplicasError
} else {
@@ -251,7 +251,7 @@ func (this KeepClient) putReplicas(
status.url, status.err)
}
active -= 1
- log.Printf("Upload status %v %v %v", status.statusCode, remaining_replicas, active)
+ log.Printf("Upload status code: %v remaining replicas: %v active: %v", status.statusCode, remaining_replicas, active)
}
return this.Want_replicas, nil
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index 4213ce0..8b8ff18 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -240,25 +240,27 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
var reader io.ReadCloser
var err error
+ var blocklen int64
if req.Method == "GET" {
- reader, _, _, err = this.KeepClient.AuthorizedGet(hash, signature, timestamp)
+ reader, blocklen, _, err = this.KeepClient.AuthorizedGet(hash, signature, timestamp)
+ defer reader.Close()
} else if req.Method == "HEAD" {
- _, _, err = this.KeepClient.AuthorizedAsk(hash, signature, timestamp)
+ blocklen, _, err = this.KeepClient.AuthorizedAsk(hash, signature, timestamp)
}
+ resp.Header().Set("Content-Length", fmt.Sprint(blocklen))
+
switch err {
case nil:
- io.Copy(resp, reader)
+ if reader != nil {
+ io.Copy(resp, reader)
+ }
case keepclient.BlockNotFound:
http.Error(resp, "Not found", http.StatusNotFound)
default:
http.Error(resp, err.Error(), http.StatusBadGateway)
}
-
- if reader != nil {
- reader.Close()
- }
}
func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
@@ -294,6 +296,8 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
// Now try to put the block through
replicas, err := this.KeepClient.PutHR(hash, req.Body, contentLength)
+ log.Printf("Replicas stored: %v err: %v", replicas, err)
+
// Tell the client how many successful PUTs we accomplished
resp.Header().Set("X-Keep-Replicas-Stored", fmt.Sprintf("%d", replicas))
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index 256f7f4..9871281 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -2,6 +2,7 @@ package main
import (
"arvados.org/keepclient"
+ "crypto/md5"
"crypto/tls"
"fmt"
. "gopkg.in/check.v1"
@@ -14,6 +15,7 @@ import (
"os/exec"
"strings"
"testing"
+ "time"
)
// Gocheck boilerplate
@@ -94,7 +96,7 @@ func SetupProxyService() {
}
}
-func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
+func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
log.Print("TestPutAndGet start")
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
@@ -111,22 +113,43 @@ func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
os.Args = []string{"keepproxy", "-listen=:29950"}
go main()
- log.Print("keepproxy main started")
-
- hash, rep, err2 := kc.PutB([]byte("foo"))
-
- log.Print("PutB")
+ time.Sleep(100 * time.Millisecond)
- c.Check(rep, Equals, 2)
- c.Check(err2, Equals, nil)
+ log.Print("keepproxy main started")
- reader, blocklen, _, err3 := kc.Get(hash)
- all, err := ioutil.ReadAll(reader)
- c.Check(all, DeepEquals, []byte("foo"))
- c.Check(blocklen, Equals, int64(3))
- c.Check(err3, Equals, nil)
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ // Uncomment this when actual keep server supports HEAD
+ /*{
+ _, _, err := kc.Ask(hash)
+ c.Check(err, Equals, keepclient.BlockNotFound)
+ log.Print("Ask 1")
+ }*/
+
+ {
+ hash2, rep, err := kc.PutB([]byte("foo"))
+ c.Check(hash2, Equals, hash)
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+ log.Print("PutB")
+ }
- log.Print("Get")
+ // Uncomment this when actual keep server supports HEAD
+ /*{
+ blocklen, _, err := kc.Ask(hash)
+ c.Check(blocklen, Equals, int64(3))
+ c.Check(err, Equals, nil)
+ log.Print("Ask 2")
+ }*/
+
+ {
+ reader, blocklen, _, err := kc.Get(hash)
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, []byte("foo"))
+ c.Check(blocklen, Equals, int64(3))
+ c.Check(err, Equals, nil)
+ log.Print("Get")
+ }
// Close internal listener socket.
listener.Close()
commit fff404a0e2ebe2582f616526c486ad0dbecce3c8
Merge: 66f1be7 c2e70e0
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu May 22 10:45:33 2014 -0400
Merge remote-tracking branch 'origin/master' into 1885-keep-proxy
commit 66f1be77149feb033e3adc17c37a09a391eec6e9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu May 22 10:45:18 2014 -0400
1885: Integration test of proxy work in progress.
diff --git a/sdk/go/go.sh b/sdk/go/go.sh
index 0203a1c..89f81fb 100755
--- a/sdk/go/go.sh
+++ b/sdk/go/go.sh
@@ -1,11 +1,11 @@
#! /bin/sh
rootdir=$(dirname $0)
-GOPATH=$rootdir:$rootdir/../../sdk/go:$GOPATH
+GOPATH=$rootdir:$GOPATH
export GOPATH
-mkdir -p pkg
-mkdir -p bin
+mkdir -p $rootdir/pkg
+mkdir -p $rootdir/bin
go get gopkg.in/check.v1
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient.go b/sdk/go/src/arvados.org/keepclient/keepclient.go
index e16c853..91989bd 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient.go
@@ -29,6 +29,7 @@ type KeepClient struct {
Want_replicas int
Client *http.Client
Using_proxy bool
+ External bool
}
// Create a new KeepClient, initialized with standard Arvados environment
@@ -37,6 +38,7 @@ type KeepClient struct {
// Keep servers.
func MakeKeepClient() (kc KeepClient, err error) {
insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true")
+ external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true")
kc = KeepClient{
ApiServer: os.Getenv("ARVADOS_API_HOST"),
@@ -45,7 +47,8 @@ func MakeKeepClient() (kc KeepClient, err error) {
Want_replicas: 2,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
- Using_proxy: false}
+ Using_proxy: false,
+ External: external}
err = (&kc).discoverKeepServers()
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
index 1ef5fd6..c67e09f 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
@@ -111,17 +111,15 @@ func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request
}
func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) {
- server := http.Server{Handler: st}
-
var err error
listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port})
if err != nil {
panic(fmt.Sprintf("Could not listen on tcp port %v", port))
}
- url = fmt.Sprintf("http://localhost:%d", listener.Addr().(*net.TCPAddr).Port)
+ url = fmt.Sprintf("http://localhost:%d", port)
- go server.Serve(listener)
+ go http.Serve(listener, st)
return listener, url
}
diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go
index d0ea967..89c5e4b 100644
--- a/sdk/go/src/arvados.org/keepclient/support.go
+++ b/sdk/go/src/arvados.org/keepclient/support.go
@@ -33,12 +33,15 @@ func (this *KeepClient) discoverKeepServers() error {
var req *http.Request
var err error
- if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible", this.ApiServer), nil); err != nil {
+ if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible?format=json", this.ApiServer), nil); err != nil {
return err
}
// Add api token header
req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+ if this.External {
+ req.Header.Add("X-External-Client", "1")
+ }
// Make the request
var resp *http.Response
diff --git a/services/keep/go.sh b/services/keep/go.sh
index fa6b5f6..177b27d 100755
--- a/services/keep/go.sh
+++ b/services/keep/go.sh
@@ -4,8 +4,8 @@ rootdir=$(dirname $0)
GOPATH=$rootdir:$rootdir/../../sdk/go:$GOPATH
export GOPATH
-mkdir -p pkg
-mkdir -p bin
+mkdir -p $rootdir/pkg
+mkdir -p $rootdir/bin
go get github.com/gorilla/mux
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index c118f22..4213ce0 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -7,15 +7,19 @@ import (
"github.com/gorilla/mux"
"io"
"log"
+ "net"
"net/http"
+ "os"
"sync"
"time"
)
// Default TCP address on which to listen for requests.
-// Initialized by the --listen flag.
+// Initialized by the -listen flag.
const DEFAULT_ADDR = ":25107"
+var listener net.Listener
+
func main() {
var (
listen string
@@ -23,6 +27,7 @@ func main() {
no_put bool
no_head bool
default_replicas int
+ pidfile string
)
flag.StringVar(
@@ -32,23 +37,24 @@ func main() {
"Interface on which to listen for requests, in the format "+
"ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port "+
"to listen on all network interfaces.")
+
flag.BoolVar(
&no_get,
"no-get",
true,
- "If true, disable GET operations")
+ "If set, disable GET operations")
flag.BoolVar(
&no_get,
"no-put",
false,
- "If true, disable PUT operations")
+ "If set, disable PUT operations")
flag.BoolVar(
&no_head,
"no-head",
- false,
- "If true, disable HEAD operations")
+ true,
+ "If set, disable HEAD operations")
flag.IntVar(
&default_replicas,
@@ -56,12 +62,18 @@ func main() {
2,
"Default number of replicas to write if not specified by the client.")
+ flag.StringVar(
+ &pidfile,
+ "pid",
+ "",
+ "Path to write pid file")
+
flag.Parse()
- if no_get == false {
- log.Print("Must specify --no-get")
+ /*if no_get == false || no_head == false {
+ log.Print("Must specify -no-get and -no-head")
return
- }
+ }*/
kc, err := keepclient.MakeKeepClient()
if err != nil {
@@ -69,14 +81,26 @@ func main() {
return
}
+ if pidfile != "" {
+ f, err := os.Create(pidfile)
+ if err == nil {
+ fmt.Fprint(f, os.Getpid())
+ f.Close()
+ } else {
+ log.Printf("Error writing pid file (%s): %s", pidfile, err.Error())
+ }
+ }
+
kc.Want_replicas = default_replicas
- // Tell the built-in HTTP server to direct all requests to the REST
- // router.
- http.Handle("/", MakeRESTRouter(!no_get, !no_put, !no_head, kc))
+ listener, err = net.Listen("tcp", listen)
+ if err != nil {
+ log.Printf("Could not listen on %v", listen)
+ return
+ }
// Start listening for requests.
- http.ListenAndServe(listen, nil)
+ http.Serve(listener, MakeRESTRouter(!no_get, !no_put, !no_head, kc))
}
type ApiTokenCache struct {
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index d4f3ef4..256f7f4 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -2,8 +2,14 @@ package main
import (
"arvados.org/keepclient"
+ "crypto/tls"
"fmt"
. "gopkg.in/check.v1"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "net/url"
"os"
"os/exec"
"strings"
@@ -23,21 +29,107 @@ type ServerRequiredSuite struct{}
func pythonDir() string {
gopath := os.Getenv("GOPATH")
- return fmt.Sprintf("%s/../python", strings.Split(gopath, ":")[0])
+ return fmt.Sprintf("%s/../../sdk/python", strings.Split(gopath, ":")[0])
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+
os.Chdir(pythonDir())
exec.Command("python", "run_test_server.py", "start").Run()
exec.Command("python", "run_test_server.py", "start_keep").Run()
+
+ os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+ os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+
+ SetupProxyService()
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+
os.Chdir(pythonDir())
exec.Command("python", "run_test_server.py", "stop_keep").Run()
exec.Command("python", "run_test_server.py", "stop").Run()
}
+func SetupProxyService() {
+
+ client := &http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+
+ var req *http.Request
+ var err error
+ if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
+ panic(err.Error())
+ }
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
+
+ reader, writer := io.Pipe()
+
+ req.Body = reader
+
+ go func() {
+ data := url.Values{}
+ data.Set("keep_service", `{
+ "service_host": "localhost",
+ "service_port": 29950,
+ "service_ssl_flag": false,
+ "service_type": "proxy"
+}`)
+
+ writer.Write([]byte(data.Encode()))
+ writer.Close()
+ }()
+
+ var resp *http.Response
+ if resp, err = client.Do(req); err != nil {
+ panic(err.Error())
+ }
+ if resp.StatusCode != 200 {
+ panic(resp.Status)
+ }
+}
+
func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
- kc := keepclient.KeepClient{"localhost", "", true, 29950, nil, 2, nil, true}
+ log.Print("TestPutAndGet start")
+
+ os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
+ kc, err := keepclient.MakeKeepClient()
+ c.Check(kc.External, Equals, true)
+ c.Check(kc.Using_proxy, Equals, true)
+ c.Check(len(kc.Service_roots), Equals, 1)
+ c.Check(kc.Service_roots[0], Equals, "http://localhost:29950")
+ c.Check(err, Equals, nil)
+ os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+
+ log.Print("keepclient created")
+
+ os.Args = []string{"keepproxy", "-listen=:29950"}
+ go main()
+
+ log.Print("keepproxy main started")
+
+ hash, rep, err2 := kc.PutB([]byte("foo"))
+
+ log.Print("PutB")
+
+ c.Check(rep, Equals, 2)
+ c.Check(err2, Equals, nil)
+
+ reader, blocklen, _, err3 := kc.Get(hash)
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, []byte("foo"))
+ c.Check(blocklen, Equals, int64(3))
+ c.Check(err3, Equals, nil)
+
+ log.Print("Get")
+
+ // Close internal listener socket.
+ listener.Close()
+
+ log.Print("TestPutAndGet done")
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list