[ARVADOS] created: 0e244ff2826dee32b92fd007c65a84d065740b4c

git at public.curoverse.com git at public.curoverse.com
Wed Jun 17 01:26:04 EDT 2015


        at  0e244ff2826dee32b92fd007c65a84d065740b4c (commit)


commit 0e244ff2826dee32b92fd007c65a84d065740b4c
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 17 01:25:28 2015 -0400

    5824: Add keepdl.

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index bb06126..d2d76e9 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -270,6 +270,21 @@ func (this ArvadosClient) Update(resource string, uuid string, parameters Dict,
 	return this.Call("PUT", resource, uuid, "", parameters, output)
 }
 
+// Get a resource.
+func (c ArvadosClient) Get(resource string, uuid string, parameters Dict, output interface{}) (err error) {
+	if uuid == "" {
+		// There's no endpoint for that because GET /type/ is
+		// the List API. If there were an endpoint, the
+		// response would be 404: no object has uuid == "".
+		return APIServerError{
+			ServerAddress:     c.ApiServer,
+			HttpStatusCode:    404,
+			HttpStatusMessage: "Not Found",
+		}
+	}
+	return c.Call("GET", resource, uuid, "", parameters, output)
+}
+
 // List the instances of a resource
 //
 //   resource - the arvados resource on which to list
diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index 6a9e13b..c2cf83e 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -62,6 +62,13 @@ func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
 	c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2")
 
 	uuid := getback["uuid"].(string)
+
+	getback = make(Dict)
+	err = arv.Get("pipeline_templates", uuid, nil, &getback)
+	c.Assert(err, Equals, nil)
+	c.Assert(getback["name"], Equals, "tmp")
+	c.Assert(getback["components"].(map[string]interface{})["c1"].(map[string]interface{})["script"], Equals, "script1")
+
 	getback = make(Dict)
 	err = arv.Update("pipeline_templates", uuid,
 		Dict{
diff --git a/services/keepdl/.gitignore b/services/keepdl/.gitignore
new file mode 100644
index 0000000..173e306
--- /dev/null
+++ b/services/keepdl/.gitignore
@@ -0,0 +1 @@
+keepdl
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
new file mode 100644
index 0000000..dcb3755
--- /dev/null
+++ b/services/keepdl/handler.go
@@ -0,0 +1,150 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	"git.curoverse.com/arvados.git/sdk/go/auth"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var clientPool = arvadosclient.MakeClientPool()
+
+var anonymousTokens []string
+
+type handler struct{}
+
+func init() {
+	// TODO(TC): Get anonymousTokens from flags
+	anonymousTokens = []string{}
+}
+
+func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+	var statusCode int
+	var statusText string
+
+	w := httpserver.WrapResponseWriter(wOrig)
+	defer func() {
+		if statusCode > 0 {
+			if w.WroteStatus() == 0 {
+				w.WriteHeader(statusCode)
+			} else {
+				httpserver.Log(r.RemoteAddr, "WARNING",
+					fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
+			}
+		}
+		if statusText == "" {
+			statusText = http.StatusText(statusCode)
+		}
+		httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.URL.Path)
+	}()
+
+	arv := clientPool.Get()
+	if arv == nil {
+		statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+		return
+	}
+	defer clientPool.Put(arv)
+
+	pathParts := strings.Split(r.URL.Path[1:], "/")
+
+	if len(pathParts) < 3 || pathParts[0] != "collections" || pathParts[1] == "" || pathParts[2] == "" {
+		statusCode = http.StatusNotFound
+		return
+	}
+
+	var targetId string
+	var targetPath []string
+	var tokens []string
+	var reqTokens []string
+	var pathToken bool
+	if len(pathParts) >= 5 && pathParts[1] == "download" {
+		// "/collections/download/{id}/{token}/path..." form:
+		// Don't use our configured anonymous tokens,
+		// Authorization headers, etc.  Just use the token in
+		// the path.
+		targetId = pathParts[2]
+		tokens = []string{pathParts[3]}
+		targetPath = pathParts[4:]
+		pathToken = true
+	} else {
+		// "/collections/{id}/path..." form
+		targetId = pathParts[1]
+		reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
+		tokens = append(reqTokens, anonymousTokens...)
+		targetPath = pathParts[2:]
+	}
+
+	tokenResult := make(map[string]int)
+	collection := make(map[string]interface{})
+	found := false
+	for _, arv.ApiToken = range tokens {
+		err := arv.Get("collections", targetId, nil, &collection)
+		httpserver.Log(err)
+		if err == nil {
+			// Success
+			found = true
+			break
+		}
+		if srvErr, ok := err.(arvadosclient.APIServerError); ok {
+			switch srvErr.HttpStatusCode {
+			case 404, 401:
+				// Token broken or insufficient to
+				// retrieve collection
+				tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
+				continue
+			}
+		}
+		// Something more serious is wrong
+		statusCode, statusText = http.StatusInternalServerError, err.Error()
+		return
+	}
+	if !found {
+		if pathToken {
+			// The URL is a "secret sharing link", but it
+			// didn't work out. Asking the client for
+			// additional credentials would just be
+			// confusing.
+			statusCode = http.StatusNotFound
+			return
+		}
+		for _, t := range reqTokens {
+			if tokenResult[t] == 404 {
+				// The client provided valid token(s), but the
+				// collection was not found.
+				statusCode = http.StatusNotFound
+				return
+			}
+		}
+		// The client's token was invalid (e.g., expired), or
+		// the client didn't even provide one.  Propagate the
+		// 401 to encourage the client to use a [different]
+		// token.
+		//
+		// TODO(TC): This response would be confusing to
+		// someone trying (anonymously) to download public
+		// data that has been deleted.  Allow a referrer to
+		// provide this context somehow?
+		statusCode = http.StatusUnauthorized
+		w.Header().Add("WWW-Authenticate", "Basic realm=\"dl\"")
+		return
+	}
+
+	filename := strings.Join(targetPath, "/")
+	rdr, err := arvadosclient.CollectionFileReader(collection, filename)
+	if os.IsNotExist(err) {
+		statusCode = http.StatusNotFound
+		return
+	} else if err != nil {
+		statusCode, statusText = http.StatusBadGateway, err.Error()
+		return
+	}
+	_, err = io.Copy(w, rdr)
+	if err != nil {
+		statusCode, statusText = http.StatusBadGateway, err.Error()
+	}
+}
diff --git a/services/keepdl/main.go b/services/keepdl/main.go
new file mode 100644
index 0000000..d780cc3
--- /dev/null
+++ b/services/keepdl/main.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+)
+
+func init() {
+	// MakeArvadosClient returns an error if this env var isn't
+	// available as a default token (even if we explicitly set a
+	// different token before doing anything with the client). We
+	// set this dummy value during init so it doesn't clobber the
+	// one used by "run test servers".
+	os.Setenv("ARVADOS_API_TOKEN", "xxx")
+}
+
+func main() {
+	flag.Parse()
+	srv := &server{}
+	if err := srv.Start(); err != nil {
+		log.Fatal(err)
+	}
+	log.Println("Listening at", srv.Addr)
+	if err := srv.Wait(); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/services/keepdl/server.go b/services/keepdl/server.go
new file mode 100644
index 0000000..01d7ee9
--- /dev/null
+++ b/services/keepdl/server.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var address string
+
+func init() {
+	flag.StringVar(&address, "address", "0.0.0.0:80",
+		"Address to listen on, \"host:port\".")
+}
+
+type server struct {
+	httpserver.Server
+}
+
+func (srv *server) Start() error {
+	mux := http.NewServeMux()
+	mux.Handle("/", &handler{})
+	srv.Handler = mux
+	srv.Addr = address
+	return srv.Server.Start()
+}
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
new file mode 100644
index 0000000..867a829
--- /dev/null
+++ b/services/keepdl/server_test.go
@@ -0,0 +1,166 @@
+package main
+
+import (
+	"crypto/md5"
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+const (
+	spectatorToken  = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
+	activeToken     = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+	anonymousToken  = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
+	fooCollection   = "zzzzz-4zz18-fy296fx3hot09f7"
+	bogusCollection = "zzzzz-4zz18-totallynotexist"
+	hwCollection    = "zzzzz-4zz18-4en62shvi99lxd4"
+)
+
+// IntegrationSuite tests need an API server and an arv-git-httpd server
+type IntegrationSuite struct {
+	testServer *server
+}
+
+func (s *IntegrationSuite) TestNoToken(c *check.C) {
+	for _, token := range []string{
+		"",
+		"bogustoken",
+	} {
+		hdr, body := s.runCurl(c, token, "/collections/"+fooCollection+"/foo")
+		c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
+		c.Assert(body, check.Equals, "")
+
+		if token != "" {
+			hdr, body = s.runCurl(c, token, "/collections/download/"+fooCollection+"/"+token+"/foo")
+			c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+			c.Assert(body, check.Equals, "")
+		}
+
+		hdr, body = s.runCurl(c, token, "/bad-route")
+		c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+		c.Assert(body, check.Equals, "")
+	}
+}
+
+// TODO: Move most cases to functional tests -- at least use Go's own
+// http client instead of forking curl. Just leave enough of an
+// integration test to assure that the documented way of invoking curl
+// really works against the server.
+func (s *IntegrationSuite) Test404(c *check.C) {
+	for _, uri := range []string{
+		// Routing errors
+		"/",
+		"/foo",
+		"/download",
+		"/collections",
+		"/collections/",
+		"/collections/"+fooCollection,
+		"/collections/"+fooCollection+"/",
+		// Non-existent file in collection
+		"/collections/"+fooCollection+"/theperthcountyconspiracy",
+		"/collections/download/"+fooCollection+"/"+activeToken+"/theperthcountyconspiracy",
+		// Non-existent collection
+		"/collections/"+bogusCollection,
+		"/collections/"+bogusCollection+"/",
+		"/collections/"+bogusCollection+"/theperthcountyconspiracy",
+		"/collections/download/"+bogusCollection+"/"+activeToken+"/theperthcountyconspiracy",
+	} {
+		hdr, body := s.runCurl(c, activeToken, uri)
+		c.Assert(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
+		c.Assert(body, check.Equals, "")
+	}
+}
+
+func (s *IntegrationSuite) Test200(c *check.C) {
+	anonymousTokens = []string{anonymousToken}
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, check.Equals, nil)
+	arv.ApiToken = activeToken
+	kc, err := keepclient.MakeKeepClient(&arv)
+	c.Assert(err, check.Equals, nil)
+	kc.PutB([]byte("Hello world\n"))
+	kc.PutB([]byte("foo"))
+	for _, spec := range [][]string{
+		// My collection
+		{activeToken, "/collections/"+fooCollection+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+		{"", "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+		{"tokensobogus", "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+		{activeToken, "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+		{anonymousToken, "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+		// Anonymously accessible user agreement. These should
+		// start working when CollectionFileReader provides
+		// real data instead of fake/stub data.
+		// {"", "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+		// {activeToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+		// {spectatorToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+		// {spectatorToken, "/collections/download/"+hwCollection+"/"+spectatorToken+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+	} {
+		hdr, body := s.runCurl(c, spec[0], spec[1])
+		c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+		c.Assert(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec[2])
+	}
+}
+
+// Return header block and body.
+func (s *IntegrationSuite) runCurl(c *check.C, token, uri string, args ...string) (hdr, body string) {
+	curlArgs := []string{"--silent", "--show-error", "--include"}
+	if token != "" {
+		curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+	}
+	curlArgs = append(curlArgs, args...)
+	curlArgs = append(curlArgs, "http://" + s.testServer.Addr + uri)
+	c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
+	output, err := exec.Command("curl", curlArgs...).CombinedOutput()
+	// Without "-f", curl exits 0 as long as it gets a valid HTTP
+	// response from the server, even if the response status
+	// indicates that the request failed. In our test suite, we
+	// always expect a valid HTTP response, and we parse the
+	// headers ourselves. If curl exits non-zero, our testing
+	// environment is broken.
+	c.Assert(err, check.Equals, nil)
+	hdrsAndBody := strings.SplitN(string(output), "\r\n\r\n", 2)
+	c.Assert(len(hdrsAndBody), check.Equals, 2)
+	hdr = hdrsAndBody[0]
+	body = hdrsAndBody[1]
+	return
+}
+
+func (s *IntegrationSuite) SetUpSuite(c *check.C) {
+	arvadostest.StartAPI()
+	arvadostest.StartKeep()
+}
+
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+	arvadostest.StopKeep()
+	arvadostest.StopAPI()
+}
+
+func (s *IntegrationSuite) SetUpTest(c *check.C) {
+	arvadostest.ResetEnv()
+	s.testServer = &server{}
+	var err error
+	address = "127.0.0.1:0"
+	err = s.testServer.Start()
+	c.Assert(err, check.Equals, nil)
+}
+
+func (s *IntegrationSuite) TearDownTest(c *check.C) {
+	var err error
+	if s.testServer != nil {
+		err = s.testServer.Close()
+	}
+	c.Check(err, check.Equals, nil)
+}
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+	check.TestingT(t)
+}

commit 0c9a309ac1fa0466b6e6bf07b8dcd3b554ae4425
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 17 00:47:12 2015 -0400

    Shut down API server after suite (noticed during 5824, otherwise no issue #)

diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index e5ddc29..318ba8a 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -74,6 +74,10 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 	arvadostest.StartAPI()
 }
 
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+	arvadostest.StopAPI()
+}
+
 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 	arvadostest.ResetEnv()
 	s.testServer = &server{}

commit efbd939ea473a056d71874104bda054bde95cc0e
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Jun 12 02:43:19 2015 -0400

    5824: Move client pool to SDK.

diff --git a/sdk/go/arvadosclient/pool.go b/sdk/go/arvadosclient/pool.go
new file mode 100644
index 0000000..1c5893a
--- /dev/null
+++ b/sdk/go/arvadosclient/pool.go
@@ -0,0 +1,39 @@
+package arvadosclient
+
+import (
+	"sync"
+)
+
+type ClientPool struct {
+	sync.Pool
+	lastErr error
+}
+
+func MakeClientPool() *ClientPool {
+	p := &ClientPool{}
+	p.Pool = sync.Pool{New: func() interface{} {
+		arv, err := MakeArvadosClient()
+		if err != nil {
+			p.lastErr = err
+			return nil
+		}
+		return &arv
+	}}
+	return p
+}
+
+func (p *ClientPool) Err() error {
+	return p.lastErr
+}
+
+func (p *ClientPool) Get() *ArvadosClient {
+	c, ok := p.Pool.Get().(*ArvadosClient)
+	if !ok {
+		return nil
+	}
+	return c
+}
+
+func (p *ClientPool) Put(c *ArvadosClient) {
+	p.Pool.Put(c)
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index f2f5c8c..ddb0706 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -6,7 +6,6 @@ import (
 	"net/http/cgi"
 	"os"
 	"strings"
-	"sync"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -14,16 +13,7 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
-func newArvadosClient() interface{} {
-	arv, err := arvadosclient.MakeArvadosClient()
-	if err != nil {
-		log.Println("MakeArvadosClient:", err)
-		return nil
-	}
-	return &arv
-}
-
-var connectionPool = &sync.Pool{New: newArvadosClient}
+var clientPool = arvadosclient.MakeClientPool()
 
 type authHandler struct {
 	handler *cgi.Handler
@@ -66,12 +56,12 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	repoName = pathParts[0]
 	repoName = strings.TrimRight(repoName, "/")
 
-	arv, ok := connectionPool.Get().(*arvadosclient.ArvadosClient)
-	if !ok || arv == nil {
-		statusCode, statusText = http.StatusInternalServerError, "connection pool failed"
+	arv := clientPool.Get()
+	if arv == nil {
+		statusCode, statusText = http.StatusInternalServerError, "connection pool failed: "+clientPool.Err().Error()
 		return
 	}
-	defer connectionPool.Put(arv)
+	defer clientPool.Put(arv)
 
 	// Ask API server whether the repository is readable using
 	// this token (by trying to read it!)

commit 3c1dc695b209a9b1f4a95e72e39efb67080ce2e8
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Jun 12 01:53:05 2015 -0400

    5824: Move quoted-logging function to SDK.

diff --git a/sdk/go/httpserver/log.go b/sdk/go/httpserver/log.go
new file mode 100644
index 0000000..7bee887
--- /dev/null
+++ b/sdk/go/httpserver/log.go
@@ -0,0 +1,23 @@
+package httpserver
+
+import (
+	"log"
+	"strings"
+)
+
+var escaper = strings.NewReplacer("\"", "\\\"", "\\", "\\\\", "\n", "\\n")
+
+// Log calls log.Println but first transforms strings so they are
+// safer to write in logs (e.g., 'foo"bar' becomes
+// '"foo\"bar"'). Non-string args are left alone.
+func Log(args ...interface{}) {
+	newargs := make([]interface{}, len(args))
+	for i, arg := range args {
+		if s, ok := arg.(string); ok {
+			newargs[i] = "\"" + escaper.Replace(s) + "\""
+		} else {
+			newargs[i] = arg
+		}
+	}
+	log.Println(newargs...)
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 5e63fb7..f2f5c8c 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -44,7 +44,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(statusCode)
 			w.Write([]byte(statusText))
 		}
-		log.Println(quoteStrings(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)...)
+		httpserver.Log(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)
 	}()
 
 	creds := auth.NewCredentialsFromHTTPRequest(r)
@@ -150,16 +150,3 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	handlerCopy.Env = append(handlerCopy.Env, "REMOTE_USER="+r.RemoteAddr) // Should be username
 	handlerCopy.ServeHTTP(&w, r)
 }
-
-var escaper = strings.NewReplacer("\"", "\\\"", "\\", "\\\\", "\n", "\\n")
-
-// Transform strings so they are safer to write in logs (e.g.,
-// 'foo"bar' becomes '"foo\"bar"'). Non-string args are left alone.
-func quoteStrings(args ...interface{}) []interface{} {
-	for i, arg := range args {
-		if s, ok := arg.(string); ok {
-			args[i] = "\"" + escaper.Replace(s) + "\""
-		}
-	}
-	return args
-}

commit 65d773097bc4b67c1a3eb0f4b17f0b1659a4b1c7
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Jun 12 01:39:27 2015 -0400

    5824: Move spying ResponseWriter to SDK.

diff --git a/sdk/go/httpserver/responsewriter.go b/sdk/go/httpserver/responsewriter.go
new file mode 100644
index 0000000..1af4dc8
--- /dev/null
+++ b/sdk/go/httpserver/responsewriter.go
@@ -0,0 +1,43 @@
+package httpserver
+
+import (
+	"net/http"
+)
+
+// ResponseWriter wraps http.ResponseWriter and exposes the status
+// sent, the number of bytes sent to the client, and the last write
+// error.
+type ResponseWriter struct {
+	http.ResponseWriter
+	wroteStatus *int	// Last status given to WriteHeader()
+	wroteBodyBytes *int	// Bytes successfully written
+	err *error		// Last error returned from Write()
+}
+
+func WrapResponseWriter(orig http.ResponseWriter) ResponseWriter {
+	return ResponseWriter{orig, new(int), new(int), new(error)}
+}
+
+func (w ResponseWriter) WriteHeader(s int) {
+	*w.wroteStatus = s
+	w.ResponseWriter.WriteHeader(s)
+}
+
+func (w ResponseWriter) Write(data []byte) (n int, err error) {
+	n, err = w.ResponseWriter.Write(data)
+	*w.wroteBodyBytes += n
+	*w.err = err
+	return
+}
+
+func (w ResponseWriter) WroteStatus() int {
+	return *w.wroteStatus
+}
+
+func (w ResponseWriter) WroteBodyBytes() int {
+	return *w.wroteBodyBytes
+}
+
+func (w ResponseWriter) Err() error {
+	return *w.err
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 0a8f9fd..5e63fb7 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -11,6 +11,7 @@ import (
 
 	"git.curoverse.com/arvados.git/sdk/go/auth"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
 func newArvadosClient() interface{} {
@@ -24,16 +25,6 @@ func newArvadosClient() interface{} {
 
 var connectionPool = &sync.Pool{New: newArvadosClient}
 
-type spyingResponseWriter struct {
-	http.ResponseWriter
-	wroteStatus *int
-}
-
-func (w spyingResponseWriter) WriteHeader(s int) {
-	*w.wroteStatus = s
-	w.ResponseWriter.WriteHeader(s)
-}
-
 type authHandler struct {
 	handler *cgi.Handler
 }
@@ -43,18 +34,17 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	var statusText string
 	var apiToken string
 	var repoName string
-	var wroteStatus int
 
-	w := spyingResponseWriter{wOrig, &wroteStatus}
+	w := httpserver.WrapResponseWriter(wOrig)
 
 	defer func() {
-		if wroteStatus == 0 {
+		if w.WroteStatus() == 0 {
 			// Nobody has called WriteHeader yet: that
 			// must be our job.
 			w.WriteHeader(statusCode)
 			w.Write([]byte(statusText))
 		}
-		log.Println(quoteStrings(r.RemoteAddr, apiToken, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
+		log.Println(quoteStrings(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)...)
 	}()
 
 	creds := auth.NewCredentialsFromHTTPRequest(r)

commit b11f4cdc04da273482b68b1e6e44156a0bb05771
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Jun 12 01:01:52 2015 -0400

    5824: Move HTTP server code to SDK.

diff --git a/services/arv-git-httpd/server.go b/sdk/go/httpserver/httpserver.go
similarity index 56%
copy from services/arv-git-httpd/server.go
copy to sdk/go/httpserver/httpserver.go
index 716b276..396fe42 100644
--- a/services/arv-git-httpd/server.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -1,14 +1,13 @@
-package main
+package httpserver
 
 import (
 	"net"
 	"net/http"
-	"net/http/cgi"
 	"sync"
 	"time"
 )
 
-type server struct {
+type Server struct {
 	http.Server
 	Addr     string // host:port where the server is listening.
 	err      error
@@ -18,28 +17,15 @@ type server struct {
 	wantDown bool
 }
 
-func (srv *server) Start() error {
-	gitHandler := &cgi.Handler{
-		Path: theConfig.GitCommand,
-		Dir:  theConfig.Root,
-		Env: []string{
-			"GIT_PROJECT_ROOT=" + theConfig.Root,
-			"GIT_HTTP_EXPORT_ALL=",
-		},
-		InheritEnv: []string{"PATH"},
-		Args:       []string{"http-backend"},
-	}
-
-	// The rest of the work here is essentially
-	// http.ListenAndServe() with two more features: (1) whoever
-	// called Start() can discover which address:port we end up
-	// listening to -- which makes listening on ":0" useful in
-	// test suites -- and (2) the server can be shut down without
-	// killing the process -- which is useful in test cases, and
-	// makes it possible to shut down gracefully on SIGTERM
-	// without killing active connections.
-
-	addr, err := net.ResolveTCPAddr("tcp", theConfig.Addr)
+// Start is essentially (*http.Server)ListenAndServe() with two more
+// features: (1) by the time Start() returns, Addr is changed to the
+// address:port we ended up listening to -- which makes listening on
+// ":0" useful in test suites -- and (2) the server can be shut down
+// without killing the process -- which is useful in test cases, and
+// makes it possible to shut down gracefully on SIGTERM without
+// killing active connections.
+func (srv *Server) Start() error {
+	addr, err := net.ResolveTCPAddr("tcp", srv.Addr)
 	if err != nil {
 		return err
 	}
@@ -48,9 +34,6 @@ func (srv *server) Start() error {
 		return err
 	}
 	srv.Addr = srv.listener.Addr().String()
-	mux := http.NewServeMux()
-	mux.Handle("/", &authHandler{gitHandler})
-	srv.Handler = mux
 
 	mutex := &sync.RWMutex{}
 	srv.cond = sync.NewCond(mutex.RLocker())
@@ -68,8 +51,15 @@ func (srv *server) Start() error {
 	return nil
 }
 
+// Close shuts down the server and returns when it has stopped.
+func (srv *Server) Close() error {
+	srv.wantDown = true
+	srv.listener.Close()
+	return srv.Wait()
+}
+
 // Wait returns when the server has shut down.
-func (srv *server) Wait() error {
+func (srv *Server) Wait() error {
 	if srv.cond == nil {
 		return nil
 	}
@@ -81,15 +71,7 @@ func (srv *server) Wait() error {
 	return srv.err
 }
 
-// Close shuts down the server and returns when it has stopped.
-func (srv *server) Close() error {
-	srv.wantDown = true
-	srv.listener.Close()
-	return srv.Wait()
-}
-
 // tcpKeepAliveListener is copied from net/http because not exported.
-//
 type tcpKeepAliveListener struct {
 	*net.TCPListener
 }
diff --git a/services/arv-git-httpd/server.go b/services/arv-git-httpd/server.go
index 716b276..9e80481 100644
--- a/services/arv-git-httpd/server.go
+++ b/services/arv-git-httpd/server.go
@@ -1,21 +1,13 @@
 package main
 
 import (
-	"net"
 	"net/http"
 	"net/http/cgi"
-	"sync"
-	"time"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
 type server struct {
-	http.Server
-	Addr     string // host:port where the server is listening.
-	err      error
-	cond     *sync.Cond
-	running  bool
-	listener *net.TCPListener
-	wantDown bool
+	httpserver.Server
 }
 
 func (srv *server) Start() error {
@@ -29,77 +21,9 @@ func (srv *server) Start() error {
 		InheritEnv: []string{"PATH"},
 		Args:       []string{"http-backend"},
 	}
-
-	// The rest of the work here is essentially
-	// http.ListenAndServe() with two more features: (1) whoever
-	// called Start() can discover which address:port we end up
-	// listening to -- which makes listening on ":0" useful in
-	// test suites -- and (2) the server can be shut down without
-	// killing the process -- which is useful in test cases, and
-	// makes it possible to shut down gracefully on SIGTERM
-	// without killing active connections.
-
-	addr, err := net.ResolveTCPAddr("tcp", theConfig.Addr)
-	if err != nil {
-		return err
-	}
-	srv.listener, err = net.ListenTCP("tcp", addr)
-	if err != nil {
-		return err
-	}
-	srv.Addr = srv.listener.Addr().String()
 	mux := http.NewServeMux()
 	mux.Handle("/", &authHandler{gitHandler})
 	srv.Handler = mux
-
-	mutex := &sync.RWMutex{}
-	srv.cond = sync.NewCond(mutex.RLocker())
-	srv.running = true
-	go func() {
-		err = srv.Serve(tcpKeepAliveListener{srv.listener})
-		if !srv.wantDown {
-			srv.err = err
-		}
-		mutex.Lock()
-		srv.running = false
-		srv.cond.Broadcast()
-		mutex.Unlock()
-	}()
-	return nil
-}
-
-// Wait returns when the server has shut down.
-func (srv *server) Wait() error {
-	if srv.cond == nil {
-		return nil
-	}
-	srv.cond.L.Lock()
-	defer srv.cond.L.Unlock()
-	for srv.running {
-		srv.cond.Wait()
-	}
-	return srv.err
-}
-
-// Close shuts down the server and returns when it has stopped.
-func (srv *server) Close() error {
-	srv.wantDown = true
-	srv.listener.Close()
-	return srv.Wait()
-}
-
-// tcpKeepAliveListener is copied from net/http because not exported.
-//
-type tcpKeepAliveListener struct {
-	*net.TCPListener
-}
-
-func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
-	tc, err := ln.AcceptTCP()
-	if err != nil {
-		return
-	}
-	tc.SetKeepAlive(true)
-	tc.SetKeepAlivePeriod(3 * time.Minute)
-	return tc, nil
+	srv.Addr = theConfig.Addr
+	return srv.Server.Start()
 }

commit 489911981539dcbde1e6b37264cc2a952316f114
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu May 21 00:33:56 2015 -0400

    5824: Move request auth code into an SDK package. Support more ways of passing tokens.

diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
new file mode 100644
index 0000000..4a719e9
--- /dev/null
+++ b/sdk/go/auth/auth.go
@@ -0,0 +1,61 @@
+package auth
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type Credentials struct {
+	Tokens []string
+}
+
+func NewCredentials() *Credentials {
+	return &Credentials{Tokens: []string{}}
+}
+
+func NewCredentialsFromHTTPRequest(r *http.Request) *Credentials {
+	c := NewCredentials()
+	c.LoadTokensFromHTTPRequest(r)
+	return c
+}
+
+// LoadTokensFromHttpRequest loads all tokens it can find in the
+// headers and query string of an http query.
+func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
+	// Load plain token from "Authorization: OAuth2 ..." header
+	// (typically used by smart API clients)
+	if toks := strings.SplitN(r.Header.Get("Authorization"), " ", 2); len(toks) == 2 && toks[0] == "OAuth2" {
+		a.Tokens = append(a.Tokens, toks[1])
+	}
+
+	// Load base64-encoded token from "Authorization: Basic ..."
+	// header (typically used by git via credential helper)
+	if _, password, ok := BasicAuth(r); ok {
+		a.Tokens = append(a.Tokens, password)
+	}
+
+	// Load tokens from query string. It's generally not a good
+	// idea to pass tokens around this way, but passing a narrowly
+	// scoped token is a reasonable way to implement "secret link
+	// to an object" in a generic way.
+	//
+	// ParseQuery always returns a non-nil map which might have
+	// valid parameters, even when a decoding error causes it to
+	// return a non-nil err. We ignore err; hopefully the caller
+	// will also need to parse the query string for
+	// application-specific purposes and will therefore
+	// find/report decoding errors in a suitable way.
+	qvalues, _ := url.ParseQuery(r.URL.RawQuery)
+	if val, ok := qvalues["api_token"]; ok {
+		a.Tokens = append(a.Tokens, val...)
+	}
+
+	// TODO: Load token from Rails session cookie (if Rails site
+	// secret is known)
+}
+
+// TODO: LoadTokensFromHttpRequestBody(). We can't assume in
+// LoadTokensFromHttpRequest() that [or how] we should read and parse
+// the request body. This has to be requested explicitly by the
+// application.
diff --git a/services/arv-git-httpd/basic_auth_go13.go b/sdk/go/auth/basic_auth_go13.go
similarity index 97%
rename from services/arv-git-httpd/basic_auth_go13.go
rename to sdk/go/auth/basic_auth_go13.go
index 087f2c8..c0fe5fc 100644
--- a/services/arv-git-httpd/basic_auth_go13.go
+++ b/sdk/go/auth/basic_auth_go13.go
@@ -1,6 +1,6 @@
 // +build !go1.4
 
-package main
+package auth
 
 import (
 	"encoding/base64"
diff --git a/services/arv-git-httpd/basic_auth_go14.go b/sdk/go/auth/basic_auth_go14.go
similarity index 91%
rename from services/arv-git-httpd/basic_auth_go14.go
rename to sdk/go/auth/basic_auth_go14.go
index 6a0079a..aeedb06 100644
--- a/services/arv-git-httpd/basic_auth_go14.go
+++ b/sdk/go/auth/basic_auth_go14.go
@@ -1,6 +1,6 @@
 // +build go1.4
 
-package main
+package auth
 
 import (
 	"net/http"
diff --git a/services/arv-git-httpd/basic_auth_test.go b/sdk/go/auth/basic_auth_test.go
similarity index 98%
rename from services/arv-git-httpd/basic_auth_test.go
rename to sdk/go/auth/basic_auth_test.go
index 2bd84dc..935f696 100644
--- a/services/arv-git-httpd/basic_auth_test.go
+++ b/sdk/go/auth/basic_auth_test.go
@@ -1,4 +1,4 @@
-package main
+package auth
 
 import (
 	"net/http"
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 6313d50..0a8f9fd 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/auth"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 )
 
@@ -40,7 +41,7 @@ type authHandler struct {
 func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	var statusCode int
 	var statusText string
-	var username, password string
+	var apiToken string
 	var repoName string
 	var wroteStatus int
 
@@ -48,21 +49,21 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 
 	defer func() {
 		if wroteStatus == 0 {
-			// Nobody has called WriteHeader yet: that must be our job.
+			// Nobody has called WriteHeader yet: that
+			// must be our job.
 			w.WriteHeader(statusCode)
 			w.Write([]byte(statusText))
 		}
-		log.Println(quoteStrings(r.RemoteAddr, username, password, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
+		log.Println(quoteStrings(r.RemoteAddr, apiToken, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
 	}()
 
-	// HTTP request username is logged, but unused. Password is an
-	// Arvados API token.
-	username, password, ok := BasicAuth(r)
-	if !ok || username == "" || password == "" {
+	creds := auth.NewCredentialsFromHTTPRequest(r)
+	if len(creds.Tokens) == 0 {
 		statusCode, statusText = http.StatusUnauthorized, "no credentials provided"
 		w.Header().Add("WWW-Authenticate", "Basic realm=\"git\"")
 		return
 	}
+	apiToken = creds.Tokens[0]
 
 	// Access to paths "/foo/bar.git/*" and "/foo/bar/.git/*" are
 	// protected by the permissions on the repository named
@@ -84,7 +85,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 
 	// Ask API server whether the repository is readable using
 	// this token (by trying to read it!)
-	arv.ApiToken = password
+	arv.ApiToken = apiToken
 	reposFound := arvadosclient.Dict{}
 	if err := arv.List("repositories", arvadosclient.Dict{
 		"filters": [][]string{{"name", "=", repoName}},

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list