[ARVADOS] updated: 86dffe03ec74387d14da9ceb17934bbdab1239b9

git at public.curoverse.com git at public.curoverse.com
Thu May 22 11:17:36 EDT 2014


Summary of changes:
 sdk/go/go.sh                                       |   6 +-
 sdk/go/src/arvados.org/keepclient/keepclient.go    |   5 +-
 .../src/arvados.org/keepclient/keepclient_test.go  |   6 +-
 sdk/go/src/arvados.org/keepclient/support.go       |   9 +-
 .../arvados/v1/keep_services_controller.rb         |   2 +-
 services/api/test/integration/keep_proxy_test.rb   |   2 +-
 services/keep/go.sh                                |   4 +-
 .../keep/src/arvados.org/keepproxy/keepproxy.go    |  66 +++++++----
 .../src/arvados.org/keepproxy/keepproxy_test.go    | 121 ++++++++++++++++++++-
 9 files changed, 184 insertions(+), 37 deletions(-)

       via  86dffe03ec74387d14da9ceb17934bbdab1239b9 (commit)
       via  fff404a0e2ebe2582f616526c486ad0dbecce3c8 (commit)
       via  66f1be77149feb033e3adc17c37a09a391eec6e9 (commit)
       via  c2e70e05178c9569e9de2ac5b3ef47440c49efc5 (commit)
      from  77504f6d369bd7bd323748d5347d6d99ed9c75ca (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 86dffe03ec74387d14da9ceb17934bbdab1239b9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu May 22 11:17:32 2014 -0400

    1885: Full-stack integration test (api+keep+keepproxy+keepclient) works!

diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go
index 89c5e4b..ef4a8e1 100644
--- a/sdk/go/src/arvados.org/keepclient/support.go
+++ b/sdk/go/src/arvados.org/keepclient/support.go
@@ -186,6 +186,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
 	var resp *http.Response
 	if resp, err = this.Client.Do(req); err != nil {
 		upload_status <- uploadStatus{err, url, 0, 0}
+		body.Close()
 		return
 	}
 
@@ -231,7 +232,6 @@ func (this KeepClient) putReplicas(
 				next_server += 1
 				active += 1
 			} else {
-				fmt.Print(active)
 				if active == 0 {
 					return (this.Want_replicas - remaining_replicas), InsufficientReplicasError
 				} else {
@@ -251,7 +251,7 @@ func (this KeepClient) putReplicas(
 				status.url, status.err)
 		}
 		active -= 1
-		log.Printf("Upload status %v %v %v", status.statusCode, remaining_replicas, active)
+		log.Printf("Upload status code: %v remaining replicas: %v active: %v", status.statusCode, remaining_replicas, active)
 	}
 
 	return this.Want_replicas, nil
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index 4213ce0..8b8ff18 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -240,25 +240,27 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 
 	var reader io.ReadCloser
 	var err error
+	var blocklen int64
 
 	if req.Method == "GET" {
-		reader, _, _, err = this.KeepClient.AuthorizedGet(hash, signature, timestamp)
+		reader, blocklen, _, err = this.KeepClient.AuthorizedGet(hash, signature, timestamp)
+		defer reader.Close()
 	} else if req.Method == "HEAD" {
-		_, _, err = this.KeepClient.AuthorizedAsk(hash, signature, timestamp)
+		blocklen, _, err = this.KeepClient.AuthorizedAsk(hash, signature, timestamp)
 	}
 
+	resp.Header().Set("Content-Length", fmt.Sprint(blocklen))
+
 	switch err {
 	case nil:
-		io.Copy(resp, reader)
+		if reader != nil {
+			io.Copy(resp, reader)
+		}
 	case keepclient.BlockNotFound:
 		http.Error(resp, "Not found", http.StatusNotFound)
 	default:
 		http.Error(resp, err.Error(), http.StatusBadGateway)
 	}
-
-	if reader != nil {
-		reader.Close()
-	}
 }
 
 func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
@@ -294,6 +296,8 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 	// Now try to put the block through
 	replicas, err := this.KeepClient.PutHR(hash, req.Body, contentLength)
 
+	log.Printf("Replicas stored: %v err: %v", replicas, err)
+
 	// Tell the client how many successful PUTs we accomplished
 	resp.Header().Set("X-Keep-Replicas-Stored", fmt.Sprintf("%d", replicas))
 
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index 256f7f4..9871281 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"arvados.org/keepclient"
+	"crypto/md5"
 	"crypto/tls"
 	"fmt"
 	. "gopkg.in/check.v1"
@@ -14,6 +15,7 @@ import (
 	"os/exec"
 	"strings"
 	"testing"
+	"time"
 )
 
 // Gocheck boilerplate
@@ -94,7 +96,7 @@ func SetupProxyService() {
 	}
 }
 
-func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
+func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 	log.Print("TestPutAndGet start")
 
 	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
@@ -111,22 +113,43 @@ func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
 	os.Args = []string{"keepproxy", "-listen=:29950"}
 	go main()
 
-	log.Print("keepproxy main started")
-
-	hash, rep, err2 := kc.PutB([]byte("foo"))
-
-	log.Print("PutB")
+	time.Sleep(100 * time.Millisecond)
 
-	c.Check(rep, Equals, 2)
-	c.Check(err2, Equals, nil)
+	log.Print("keepproxy main started")
 
-	reader, blocklen, _, err3 := kc.Get(hash)
-	all, err := ioutil.ReadAll(reader)
-	c.Check(all, DeepEquals, []byte("foo"))
-	c.Check(blocklen, Equals, int64(3))
-	c.Check(err3, Equals, nil)
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	// Uncomment this when actual keep server supports HEAD
+	/*{
+		_, _, err := kc.Ask(hash)
+		c.Check(err, Equals, keepclient.BlockNotFound)
+		log.Print("Ask 1")
+	}*/
+
+	{
+		hash2, rep, err := kc.PutB([]byte("foo"))
+		c.Check(hash2, Equals, hash)
+		c.Check(rep, Equals, 2)
+		c.Check(err, Equals, nil)
+		log.Print("PutB")
+	}
 
-	log.Print("Get")
+	// Uncomment this when actual keep server supports HEAD
+	/*{
+		blocklen, _, err := kc.Ask(hash)
+		c.Check(blocklen, Equals, int64(3))
+		c.Check(err, Equals, nil)
+		log.Print("Ask 2")
+	}*/
+
+	{
+		reader, blocklen, _, err := kc.Get(hash)
+		all, err := ioutil.ReadAll(reader)
+		c.Check(all, DeepEquals, []byte("foo"))
+		c.Check(blocklen, Equals, int64(3))
+		c.Check(err, Equals, nil)
+		log.Print("Get")
+	}
 
 	// Close internal listener socket.
 	listener.Close()

commit fff404a0e2ebe2582f616526c486ad0dbecce3c8
Merge: 66f1be7 c2e70e0
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu May 22 10:45:33 2014 -0400

    Merge remote-tracking branch 'origin/master' into 1885-keep-proxy


commit 66f1be77149feb033e3adc17c37a09a391eec6e9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu May 22 10:45:18 2014 -0400

    1885: Integration test of proxy work in progress.

diff --git a/sdk/go/go.sh b/sdk/go/go.sh
index 0203a1c..89f81fb 100755
--- a/sdk/go/go.sh
+++ b/sdk/go/go.sh
@@ -1,11 +1,11 @@
 #! /bin/sh
 
 rootdir=$(dirname $0)
-GOPATH=$rootdir:$rootdir/../../sdk/go:$GOPATH
+GOPATH=$rootdir:$GOPATH
 export GOPATH
 
-mkdir -p pkg
-mkdir -p bin
+mkdir -p $rootdir/pkg
+mkdir -p $rootdir/bin
 
 go get gopkg.in/check.v1
 
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient.go b/sdk/go/src/arvados.org/keepclient/keepclient.go
index e16c853..91989bd 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient.go
@@ -29,6 +29,7 @@ type KeepClient struct {
 	Want_replicas int
 	Client        *http.Client
 	Using_proxy   bool
+	External      bool
 }
 
 // Create a new KeepClient, initialized with standard Arvados environment
@@ -37,6 +38,7 @@ type KeepClient struct {
 // Keep servers.
 func MakeKeepClient() (kc KeepClient, err error) {
 	insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true")
+	external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true")
 
 	kc = KeepClient{
 		ApiServer:     os.Getenv("ARVADOS_API_HOST"),
@@ -45,7 +47,8 @@ func MakeKeepClient() (kc KeepClient, err error) {
 		Want_replicas: 2,
 		Client: &http.Client{Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
-		Using_proxy: false}
+		Using_proxy: false,
+		External:    external}
 
 	err = (&kc).discoverKeepServers()
 
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
index 1ef5fd6..c67e09f 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
@@ -111,17 +111,15 @@ func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request
 }
 
 func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) {
-	server := http.Server{Handler: st}
-
 	var err error
 	listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port})
 	if err != nil {
 		panic(fmt.Sprintf("Could not listen on tcp port %v", port))
 	}
 
-	url = fmt.Sprintf("http://localhost:%d", listener.Addr().(*net.TCPAddr).Port)
+	url = fmt.Sprintf("http://localhost:%d", port)
 
-	go server.Serve(listener)
+	go http.Serve(listener, st)
 	return listener, url
 }
 
diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go
index d0ea967..89c5e4b 100644
--- a/sdk/go/src/arvados.org/keepclient/support.go
+++ b/sdk/go/src/arvados.org/keepclient/support.go
@@ -33,12 +33,15 @@ func (this *KeepClient) discoverKeepServers() error {
 	var req *http.Request
 	var err error
 
-	if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible", this.ApiServer), nil); err != nil {
+	if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible?format=json", this.ApiServer), nil); err != nil {
 		return err
 	}
 
 	// Add api token header
 	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+	if this.External {
+		req.Header.Add("X-External-Client", "1")
+	}
 
 	// Make the request
 	var resp *http.Response
diff --git a/services/keep/go.sh b/services/keep/go.sh
index fa6b5f6..177b27d 100755
--- a/services/keep/go.sh
+++ b/services/keep/go.sh
@@ -4,8 +4,8 @@ rootdir=$(dirname $0)
 GOPATH=$rootdir:$rootdir/../../sdk/go:$GOPATH
 export GOPATH
 
-mkdir -p pkg
-mkdir -p bin
+mkdir -p $rootdir/pkg
+mkdir -p $rootdir/bin
 
 go get github.com/gorilla/mux
 
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index c118f22..4213ce0 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -7,15 +7,19 @@ import (
 	"github.com/gorilla/mux"
 	"io"
 	"log"
+	"net"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 )
 
 // Default TCP address on which to listen for requests.
-// Initialized by the --listen flag.
+// Initialized by the -listen flag.
 const DEFAULT_ADDR = ":25107"
 
+var listener net.Listener
+
 func main() {
 	var (
 		listen           string
@@ -23,6 +27,7 @@ func main() {
 		no_put           bool
 		no_head          bool
 		default_replicas int
+		pidfile          string
 	)
 
 	flag.StringVar(
@@ -32,23 +37,24 @@ func main() {
 		"Interface on which to listen for requests, in the format "+
 			"ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port "+
 			"to listen on all network interfaces.")
+
 	flag.BoolVar(
 		&no_get,
 		"no-get",
 		true,
-		"If true, disable GET operations")
+		"If set, disable GET operations")
 
 	flag.BoolVar(
 		&no_get,
 		"no-put",
 		false,
-		"If true, disable PUT operations")
+		"If set, disable PUT operations")
 
 	flag.BoolVar(
 		&no_head,
 		"no-head",
-		false,
-		"If true, disable HEAD operations")
+		true,
+		"If set, disable HEAD operations")
 
 	flag.IntVar(
 		&default_replicas,
@@ -56,12 +62,18 @@ func main() {
 		2,
 		"Default number of replicas to write if not specified by the client.")
 
+	flag.StringVar(
+		&pidfile,
+		"pid",
+		"",
+		"Path to write pid file")
+
 	flag.Parse()
 
-	if no_get == false {
-		log.Print("Must specify --no-get")
+	/*if no_get == false || no_head == false {
+		log.Print("Must specify -no-get and -no-head")
 		return
-	}
+	}*/
 
 	kc, err := keepclient.MakeKeepClient()
 	if err != nil {
@@ -69,14 +81,26 @@ func main() {
 		return
 	}
 
+	if pidfile != "" {
+		f, err := os.Create(pidfile)
+		if err == nil {
+			fmt.Fprint(f, os.Getpid())
+			f.Close()
+		} else {
+			log.Printf("Error writing pid file (%s): %s", pidfile, err.Error())
+		}
+	}
+
 	kc.Want_replicas = default_replicas
 
-	// Tell the built-in HTTP server to direct all requests to the REST
-	// router.
-	http.Handle("/", MakeRESTRouter(!no_get, !no_put, !no_head, kc))
+	listener, err = net.Listen("tcp", listen)
+	if err != nil {
+		log.Printf("Could not listen on %v", listen)
+		return
+	}
 
 	// Start listening for requests.
-	http.ListenAndServe(listen, nil)
+	http.Serve(listener, MakeRESTRouter(!no_get, !no_put, !no_head, kc))
 }
 
 type ApiTokenCache struct {
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index d4f3ef4..256f7f4 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -2,8 +2,14 @@ package main
 
 import (
 	"arvados.org/keepclient"
+	"crypto/tls"
 	"fmt"
 	. "gopkg.in/check.v1"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"strings"
@@ -23,21 +29,107 @@ type ServerRequiredSuite struct{}
 
 func pythonDir() string {
 	gopath := os.Getenv("GOPATH")
-	return fmt.Sprintf("%s/../python", strings.Split(gopath, ":")[0])
+	return fmt.Sprintf("%s/../../sdk/python", strings.Split(gopath, ":")[0])
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+	cwd, _ := os.Getwd()
+	defer os.Chdir(cwd)
+
 	os.Chdir(pythonDir())
 	exec.Command("python", "run_test_server.py", "start").Run()
 	exec.Command("python", "run_test_server.py", "start_keep").Run()
+
+	os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+
+	SetupProxyService()
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+	cwd, _ := os.Getwd()
+	defer os.Chdir(cwd)
+
 	os.Chdir(pythonDir())
 	exec.Command("python", "run_test_server.py", "stop_keep").Run()
 	exec.Command("python", "run_test_server.py", "stop").Run()
 }
 
+func SetupProxyService() {
+
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+
+	var req *http.Request
+	var err error
+	if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
+		panic(err.Error())
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
+
+	reader, writer := io.Pipe()
+
+	req.Body = reader
+
+	go func() {
+		data := url.Values{}
+		data.Set("keep_service", `{
+  "service_host": "localhost",
+  "service_port": 29950,
+  "service_ssl_flag": false,
+  "service_type": "proxy"
+}`)
+
+		writer.Write([]byte(data.Encode()))
+		writer.Close()
+	}()
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		panic(err.Error())
+	}
+	if resp.StatusCode != 200 {
+		panic(resp.Status)
+	}
+}
+
 func (s *ServerRequiredSuite) TestPutAndGet(c *C) {
-	kc := keepclient.KeepClient{"localhost", "", true, 29950, nil, 2, nil, true}
+	log.Print("TestPutAndGet start")
+
+	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
+	kc, err := keepclient.MakeKeepClient()
+	c.Check(kc.External, Equals, true)
+	c.Check(kc.Using_proxy, Equals, true)
+	c.Check(len(kc.Service_roots), Equals, 1)
+	c.Check(kc.Service_roots[0], Equals, "http://localhost:29950")
+	c.Check(err, Equals, nil)
+	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+
+	log.Print("keepclient created")
+
+	os.Args = []string{"keepproxy", "-listen=:29950"}
+	go main()
+
+	log.Print("keepproxy main started")
+
+	hash, rep, err2 := kc.PutB([]byte("foo"))
+
+	log.Print("PutB")
+
+	c.Check(rep, Equals, 2)
+	c.Check(err2, Equals, nil)
+
+	reader, blocklen, _, err3 := kc.Get(hash)
+	all, err := ioutil.ReadAll(reader)
+	c.Check(all, DeepEquals, []byte("foo"))
+	c.Check(blocklen, Equals, int64(3))
+	c.Check(err3, Equals, nil)
+
+	log.Print("Get")
+
+	// Close internal listener socket.
+	listener.Close()
+
+	log.Print("TestPutAndGet done")
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list