[ARVADOS] created: 0e244ff2826dee32b92fd007c65a84d065740b4c
git at public.curoverse.com
git at public.curoverse.com
Wed Jun 17 01:26:04 EDT 2015
at 0e244ff2826dee32b92fd007c65a84d065740b4c (commit)
commit 0e244ff2826dee32b92fd007c65a84d065740b4c
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 17 01:25:28 2015 -0400
5824: Add keepdl.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index bb06126..d2d76e9 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -270,6 +270,21 @@ func (this ArvadosClient) Update(resource string, uuid string, parameters Dict,
return this.Call("PUT", resource, uuid, "", parameters, output)
}
+// Get a resource.
+func (c ArvadosClient) Get(resource string, uuid string, parameters Dict, output interface{}) (err error) {
+ if uuid == "" {
+ // There's no endpoint for that because GET /type/ is
+ // the List API. If there were an endpoint, the
+ // response would be 404: no object has uuid == "".
+ return APIServerError{
+ ServerAddress: c.ApiServer,
+ HttpStatusCode: 404,
+ HttpStatusMessage: "Not Found",
+ }
+ }
+ return c.Call("GET", resource, uuid, "", parameters, output)
+}
+
// List the instances of a resource
//
// resource - the arvados resource on which to list
diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index 6a9e13b..c2cf83e 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -62,6 +62,13 @@ func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2")
uuid := getback["uuid"].(string)
+
+ getback = make(Dict)
+ err = arv.Get("pipeline_templates", uuid, nil, &getback)
+ c.Assert(err, Equals, nil)
+ c.Assert(getback["name"], Equals, "tmp")
+ c.Assert(getback["components"].(map[string]interface{})["c1"].(map[string]interface{})["script"], Equals, "script1")
+
getback = make(Dict)
err = arv.Update("pipeline_templates", uuid,
Dict{
diff --git a/services/keepdl/.gitignore b/services/keepdl/.gitignore
new file mode 100644
index 0000000..173e306
--- /dev/null
+++ b/services/keepdl/.gitignore
@@ -0,0 +1 @@
+keepdl
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
new file mode 100644
index 0000000..dcb3755
--- /dev/null
+++ b/services/keepdl/handler.go
@@ -0,0 +1,150 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var clientPool = arvadosclient.MakeClientPool()
+
+var anonymousTokens []string
+
+type handler struct{}
+
+func init() {
+ // TODO(TC): Get anonymousTokens from flags
+ anonymousTokens = []string{}
+}
+
+func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+ var statusCode int
+ var statusText string
+
+ w := httpserver.WrapResponseWriter(wOrig)
+ defer func() {
+ if statusCode > 0 {
+ if w.WroteStatus() == 0 {
+ w.WriteHeader(statusCode)
+ } else {
+ httpserver.Log(r.RemoteAddr, "WARNING",
+ fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
+ }
+ }
+ if statusText == "" {
+ statusText = http.StatusText(statusCode)
+ }
+ httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.URL.Path)
+ }()
+
+ arv := clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+ return
+ }
+ defer clientPool.Put(arv)
+
+ pathParts := strings.Split(r.URL.Path[1:], "/")
+
+ if len(pathParts) < 3 || pathParts[0] != "collections" || pathParts[1] == "" || pathParts[2] == "" {
+ statusCode = http.StatusNotFound
+ return
+ }
+
+ var targetId string
+ var targetPath []string
+ var tokens []string
+ var reqTokens []string
+ var pathToken bool
+ if len(pathParts) >= 5 && pathParts[1] == "download" {
+ // "/collections/download/{id}/{token}/path..." form:
+ // Don't use our configured anonymous tokens,
+ // Authorization headers, etc. Just use the token in
+ // the path.
+ targetId = pathParts[2]
+ tokens = []string{pathParts[3]}
+ targetPath = pathParts[4:]
+ pathToken = true
+ } else {
+ // "/collections/{id}/path..." form
+ targetId = pathParts[1]
+ reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
+ tokens = append(reqTokens, anonymousTokens...)
+ targetPath = pathParts[2:]
+ }
+
+ tokenResult := make(map[string]int)
+ collection := make(map[string]interface{})
+ found := false
+ for _, arv.ApiToken = range tokens {
+ err := arv.Get("collections", targetId, nil, &collection)
+ httpserver.Log(err)
+ if err == nil {
+ // Success
+ found = true
+ break
+ }
+ if srvErr, ok := err.(arvadosclient.APIServerError); ok {
+ switch srvErr.HttpStatusCode {
+ case 404, 401:
+ // Token broken or insufficient to
+ // retrieve collection
+ tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
+ continue
+ }
+ }
+ // Something more serious is wrong
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ return
+ }
+ if !found {
+ if pathToken {
+ // The URL is a "secret sharing link", but it
+ // didn't work out. Asking the client for
+ // additional credentials would just be
+ // confusing.
+ statusCode = http.StatusNotFound
+ return
+ }
+ for _, t := range reqTokens {
+ if tokenResult[t] == 404 {
+ // The client provided valid token(s), but the
+ // collection was not found.
+ statusCode = http.StatusNotFound
+ return
+ }
+ }
+ // The client's token was invalid (e.g., expired), or
+ // the client didn't even provide one. Propagate the
+ // 401 to encourage the client to use a [different]
+ // token.
+ //
+ // TODO(TC): This response would be confusing to
+ // someone trying (anonymously) to download public
+ // data that has been deleted. Allow a referrer to
+ // provide this context somehow?
+ statusCode = http.StatusUnauthorized
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"dl\"")
+ return
+ }
+
+ filename := strings.Join(targetPath, "/")
+ rdr, err := arvadosclient.CollectionFileReader(collection, filename)
+ if os.IsNotExist(err) {
+ statusCode = http.StatusNotFound
+ return
+ } else if err != nil {
+ statusCode, statusText = http.StatusBadGateway, err.Error()
+ return
+ }
+ _, err = io.Copy(w, rdr)
+ if err != nil {
+ statusCode, statusText = http.StatusBadGateway, err.Error()
+ }
+}
diff --git a/services/keepdl/main.go b/services/keepdl/main.go
new file mode 100644
index 0000000..d780cc3
--- /dev/null
+++ b/services/keepdl/main.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+ "flag"
+ "log"
+ "os"
+)
+
+func init() {
+ // MakeArvadosClient returns an error if this env var isn't
+ // available as a default token (even if we explicitly set a
+ // different token before doing anything with the client). We
+ // set this dummy value during init so it doesn't clobber the
+ // one used by "run test servers".
+ os.Setenv("ARVADOS_API_TOKEN", "xxx")
+}
+
+func main() {
+ flag.Parse()
+ srv := &server{}
+ if err := srv.Start(); err != nil {
+ log.Fatal(err)
+ }
+ log.Println("Listening at", srv.Addr)
+ if err := srv.Wait(); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/services/keepdl/server.go b/services/keepdl/server.go
new file mode 100644
index 0000000..01d7ee9
--- /dev/null
+++ b/services/keepdl/server.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+ "flag"
+ "net/http"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var address string
+
+func init() {
+ flag.StringVar(&address, "address", "0.0.0.0:80",
+ "Address to listen on, \"host:port\".")
+}
+
+type server struct {
+ httpserver.Server
+}
+
+func (srv *server) Start() error {
+ mux := http.NewServeMux()
+ mux.Handle("/", &handler{})
+ srv.Handler = mux
+ srv.Addr = address
+ return srv.Server.Start()
+}
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
new file mode 100644
index 0000000..867a829
--- /dev/null
+++ b/services/keepdl/server_test.go
@@ -0,0 +1,166 @@
+package main
+
+import (
+ "crypto/md5"
+ "fmt"
+ "os/exec"
+ "strings"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+const (
+ spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
+ activeToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ anonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
+ fooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ bogusCollection = "zzzzz-4zz18-totallynotexist"
+ hwCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+)
+
+// IntegrationSuite tests need an API server and an arv-git-httpd server
+type IntegrationSuite struct {
+ testServer *server
+}
+
+func (s *IntegrationSuite) TestNoToken(c *check.C) {
+ for _, token := range []string{
+ "",
+ "bogustoken",
+ } {
+ hdr, body := s.runCurl(c, token, "/collections/"+fooCollection+"/foo")
+ c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
+ c.Assert(body, check.Equals, "")
+
+ if token != "" {
+ hdr, body = s.runCurl(c, token, "/collections/download/"+fooCollection+"/"+token+"/foo")
+ c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+ c.Assert(body, check.Equals, "")
+ }
+
+ hdr, body = s.runCurl(c, token, "/bad-route")
+ c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+ c.Assert(body, check.Equals, "")
+ }
+}
+
+// TODO: Move most cases to functional tests -- at least use Go's own
+// http client instead of forking curl. Just leave enough of an
+// integration test to assure that the documented way of invoking curl
+// really works against the server.
+func (s *IntegrationSuite) Test404(c *check.C) {
+ for _, uri := range []string{
+ // Routing errors
+ "/",
+ "/foo",
+ "/download",
+ "/collections",
+ "/collections/",
+ "/collections/"+fooCollection,
+ "/collections/"+fooCollection+"/",
+ // Non-existent file in collection
+ "/collections/"+fooCollection+"/theperthcountyconspiracy",
+ "/collections/download/"+fooCollection+"/"+activeToken+"/theperthcountyconspiracy",
+ // Non-existent collection
+ "/collections/"+bogusCollection,
+ "/collections/"+bogusCollection+"/",
+ "/collections/"+bogusCollection+"/theperthcountyconspiracy",
+ "/collections/download/"+bogusCollection+"/"+activeToken+"/theperthcountyconspiracy",
+ } {
+ hdr, body := s.runCurl(c, activeToken, uri)
+ c.Assert(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
+ c.Assert(body, check.Equals, "")
+ }
+}
+
+func (s *IntegrationSuite) Test200(c *check.C) {
+ anonymousTokens = []string{anonymousToken}
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = activeToken
+ kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, check.Equals, nil)
+ kc.PutB([]byte("Hello world\n"))
+ kc.PutB([]byte("foo"))
+ for _, spec := range [][]string{
+ // My collection
+ {activeToken, "/collections/"+fooCollection+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"", "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"tokensobogus", "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {activeToken, "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {anonymousToken, "/collections/download/"+fooCollection+"/"+activeToken+"/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ // Anonymously accessible user agreement. These should
+ // start working when CollectionFileReader provides
+ // real data instead of fake/stub data.
+ // {"", "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ // {activeToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ // {spectatorToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ // {spectatorToken, "/collections/download/"+hwCollection+"/"+spectatorToken+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ } {
+ hdr, body := s.runCurl(c, spec[0], spec[1])
+ c.Assert(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Assert(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec[2])
+ }
+}
+
+// Return header block and body.
+func (s *IntegrationSuite) runCurl(c *check.C, token, uri string, args ...string) (hdr, body string) {
+ curlArgs := []string{"--silent", "--show-error", "--include"}
+ if token != "" {
+ curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ }
+ curlArgs = append(curlArgs, args...)
+ curlArgs = append(curlArgs, "http://" + s.testServer.Addr + uri)
+ c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
+ output, err := exec.Command("curl", curlArgs...).CombinedOutput()
+ // Without "-f", curl exits 0 as long as it gets a valid HTTP
+ // response from the server, even if the response status
+ // indicates that the request failed. In our test suite, we
+ // always expect a valid HTTP response, and we parse the
+ // headers ourselves. If curl exits non-zero, our testing
+ // environment is broken.
+ c.Assert(err, check.Equals, nil)
+ hdrsAndBody := strings.SplitN(string(output), "\r\n\r\n", 2)
+ c.Assert(len(hdrsAndBody), check.Equals, 2)
+ hdr = hdrsAndBody[0]
+ body = hdrsAndBody[1]
+ return
+}
+
+func (s *IntegrationSuite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+}
+
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
+}
+
+func (s *IntegrationSuite) SetUpTest(c *check.C) {
+ arvadostest.ResetEnv()
+ s.testServer = &server{}
+ var err error
+ address = "127.0.0.1:0"
+ err = s.testServer.Start()
+ c.Assert(err, check.Equals, nil)
+}
+
+func (s *IntegrationSuite) TearDownTest(c *check.C) {
+ var err error
+ if s.testServer != nil {
+ err = s.testServer.Close()
+ }
+ c.Check(err, check.Equals, nil)
+}
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
commit 0c9a309ac1fa0466b6e6bf07b8dcd3b554ae4425
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 17 00:47:12 2015 -0400
Shut down API server after suite (noticed during 5824, otherwise no issue #)
diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index e5ddc29..318ba8a 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -74,6 +74,10 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
}
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopAPI()
+}
+
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
s.testServer = &server{}
commit efbd939ea473a056d71874104bda054bde95cc0e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 12 02:43:19 2015 -0400
5824: Move client pool to SDK.
diff --git a/sdk/go/arvadosclient/pool.go b/sdk/go/arvadosclient/pool.go
new file mode 100644
index 0000000..1c5893a
--- /dev/null
+++ b/sdk/go/arvadosclient/pool.go
@@ -0,0 +1,39 @@
+package arvadosclient
+
+import (
+ "sync"
+)
+
+type ClientPool struct {
+ sync.Pool
+ lastErr error
+}
+
+func MakeClientPool() *ClientPool {
+ p := &ClientPool{}
+ p.Pool = sync.Pool{New: func() interface{} {
+ arv, err := MakeArvadosClient()
+ if err != nil {
+ p.lastErr = err
+ return nil
+ }
+ return &arv
+ }}
+ return p
+}
+
+func (p *ClientPool) Err() error {
+ return p.lastErr
+}
+
+func (p *ClientPool) Get() *ArvadosClient {
+ c, ok := p.Pool.Get().(*ArvadosClient)
+ if !ok {
+ return nil
+ }
+ return c
+}
+
+func (p *ClientPool) Put(c *ArvadosClient) {
+ p.Pool.Put(c)
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index f2f5c8c..ddb0706 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -6,7 +6,6 @@ import (
"net/http/cgi"
"os"
"strings"
- "sync"
"time"
"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -14,16 +13,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
-func newArvadosClient() interface{} {
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Println("MakeArvadosClient:", err)
- return nil
- }
- return &arv
-}
-
-var connectionPool = &sync.Pool{New: newArvadosClient}
+var clientPool = arvadosclient.MakeClientPool()
type authHandler struct {
handler *cgi.Handler
@@ -66,12 +56,12 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
repoName = pathParts[0]
repoName = strings.TrimRight(repoName, "/")
- arv, ok := connectionPool.Get().(*arvadosclient.ArvadosClient)
- if !ok || arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "connection pool failed"
+ arv := clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "connection pool failed: "+clientPool.Err().Error()
return
}
- defer connectionPool.Put(arv)
+ defer clientPool.Put(arv)
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
commit 3c1dc695b209a9b1f4a95e72e39efb67080ce2e8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 12 01:53:05 2015 -0400
5824: Move quoted-logging function to SDK.
diff --git a/sdk/go/httpserver/log.go b/sdk/go/httpserver/log.go
new file mode 100644
index 0000000..7bee887
--- /dev/null
+++ b/sdk/go/httpserver/log.go
@@ -0,0 +1,23 @@
+package httpserver
+
+import (
+ "log"
+ "strings"
+)
+
+var escaper = strings.NewReplacer("\"", "\\\"", "\\", "\\\\", "\n", "\\n")
+
+// Log calls log.Println but first transforms strings so they are
+// safer to write in logs (e.g., 'foo"bar' becomes
+// '"foo\"bar"'). Non-string args are left alone.
+func Log(args ...interface{}) {
+ newargs := make([]interface{}, len(args))
+ for i, arg := range args {
+ if s, ok := arg.(string); ok {
+ newargs[i] = "\"" + escaper.Replace(s) + "\""
+ } else {
+ newargs[i] = arg
+ }
+ }
+ log.Println(newargs...)
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 5e63fb7..f2f5c8c 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -44,7 +44,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.WriteHeader(statusCode)
w.Write([]byte(statusText))
}
- log.Println(quoteStrings(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)...)
+ httpserver.Log(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)
}()
creds := auth.NewCredentialsFromHTTPRequest(r)
@@ -150,16 +150,3 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
handlerCopy.Env = append(handlerCopy.Env, "REMOTE_USER="+r.RemoteAddr) // Should be username
handlerCopy.ServeHTTP(&w, r)
}
-
-var escaper = strings.NewReplacer("\"", "\\\"", "\\", "\\\\", "\n", "\\n")
-
-// Transform strings so they are safer to write in logs (e.g.,
-// 'foo"bar' becomes '"foo\"bar"'). Non-string args are left alone.
-func quoteStrings(args ...interface{}) []interface{} {
- for i, arg := range args {
- if s, ok := arg.(string); ok {
- args[i] = "\"" + escaper.Replace(s) + "\""
- }
- }
- return args
-}
commit 65d773097bc4b67c1a3eb0f4b17f0b1659a4b1c7
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 12 01:39:27 2015 -0400
5824: Move spying ResponseWriter to SDK.
diff --git a/sdk/go/httpserver/responsewriter.go b/sdk/go/httpserver/responsewriter.go
new file mode 100644
index 0000000..1af4dc8
--- /dev/null
+++ b/sdk/go/httpserver/responsewriter.go
@@ -0,0 +1,43 @@
+package httpserver
+
+import (
+ "net/http"
+)
+
+// ResponseWriter wraps http.ResponseWriter and exposes the status
+// sent, the number of bytes sent to the client, and the last write
+// error.
+type ResponseWriter struct {
+ http.ResponseWriter
+ wroteStatus *int // Last status given to WriteHeader()
+ wroteBodyBytes *int // Bytes successfully written
+ err *error // Last error returned from Write()
+}
+
+func WrapResponseWriter(orig http.ResponseWriter) ResponseWriter {
+ return ResponseWriter{orig, new(int), new(int), new(error)}
+}
+
+func (w ResponseWriter) WriteHeader(s int) {
+ *w.wroteStatus = s
+ w.ResponseWriter.WriteHeader(s)
+}
+
+func (w ResponseWriter) Write(data []byte) (n int, err error) {
+ n, err = w.ResponseWriter.Write(data)
+ *w.wroteBodyBytes += n
+ *w.err = err
+ return
+}
+
+func (w ResponseWriter) WroteStatus() int {
+ return *w.wroteStatus
+}
+
+func (w ResponseWriter) WroteBodyBytes() int {
+ return *w.wroteBodyBytes
+}
+
+func (w ResponseWriter) Err() error {
+ return *w.err
+}
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 0a8f9fd..5e63fb7 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -11,6 +11,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
func newArvadosClient() interface{} {
@@ -24,16 +25,6 @@ func newArvadosClient() interface{} {
var connectionPool = &sync.Pool{New: newArvadosClient}
-type spyingResponseWriter struct {
- http.ResponseWriter
- wroteStatus *int
-}
-
-func (w spyingResponseWriter) WriteHeader(s int) {
- *w.wroteStatus = s
- w.ResponseWriter.WriteHeader(s)
-}
-
type authHandler struct {
handler *cgi.Handler
}
@@ -43,18 +34,17 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var statusText string
var apiToken string
var repoName string
- var wroteStatus int
- w := spyingResponseWriter{wOrig, &wroteStatus}
+ w := httpserver.WrapResponseWriter(wOrig)
defer func() {
- if wroteStatus == 0 {
+ if w.WroteStatus() == 0 {
// Nobody has called WriteHeader yet: that
// must be our job.
w.WriteHeader(statusCode)
w.Write([]byte(statusText))
}
- log.Println(quoteStrings(r.RemoteAddr, apiToken, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
+ log.Println(quoteStrings(r.RemoteAddr, apiToken, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)...)
}()
creds := auth.NewCredentialsFromHTTPRequest(r)
commit b11f4cdc04da273482b68b1e6e44156a0bb05771
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 12 01:01:52 2015 -0400
5824: Move HTTP server code to SDK.
diff --git a/services/arv-git-httpd/server.go b/sdk/go/httpserver/httpserver.go
similarity index 56%
copy from services/arv-git-httpd/server.go
copy to sdk/go/httpserver/httpserver.go
index 716b276..396fe42 100644
--- a/services/arv-git-httpd/server.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -1,14 +1,13 @@
-package main
+package httpserver
import (
"net"
"net/http"
- "net/http/cgi"
"sync"
"time"
)
-type server struct {
+type Server struct {
http.Server
Addr string // host:port where the server is listening.
err error
@@ -18,28 +17,15 @@ type server struct {
wantDown bool
}
-func (srv *server) Start() error {
- gitHandler := &cgi.Handler{
- Path: theConfig.GitCommand,
- Dir: theConfig.Root,
- Env: []string{
- "GIT_PROJECT_ROOT=" + theConfig.Root,
- "GIT_HTTP_EXPORT_ALL=",
- },
- InheritEnv: []string{"PATH"},
- Args: []string{"http-backend"},
- }
-
- // The rest of the work here is essentially
- // http.ListenAndServe() with two more features: (1) whoever
- // called Start() can discover which address:port we end up
- // listening to -- which makes listening on ":0" useful in
- // test suites -- and (2) the server can be shut down without
- // killing the process -- which is useful in test cases, and
- // makes it possible to shut down gracefully on SIGTERM
- // without killing active connections.
-
- addr, err := net.ResolveTCPAddr("tcp", theConfig.Addr)
+// Start is essentially (*http.Server)ListenAndServe() with two more
+// features: (1) by the time Start() returns, Addr is changed to the
+// address:port we ended up listening to -- which makes listening on
+// ":0" useful in test suites -- and (2) the server can be shut down
+// without killing the process -- which is useful in test cases, and
+// makes it possible to shut down gracefully on SIGTERM without
+// killing active connections.
+func (srv *Server) Start() error {
+ addr, err := net.ResolveTCPAddr("tcp", srv.Addr)
if err != nil {
return err
}
@@ -48,9 +34,6 @@ func (srv *server) Start() error {
return err
}
srv.Addr = srv.listener.Addr().String()
- mux := http.NewServeMux()
- mux.Handle("/", &authHandler{gitHandler})
- srv.Handler = mux
mutex := &sync.RWMutex{}
srv.cond = sync.NewCond(mutex.RLocker())
@@ -68,8 +51,15 @@ func (srv *server) Start() error {
return nil
}
+// Close shuts down the server and returns when it has stopped.
+func (srv *Server) Close() error {
+ srv.wantDown = true
+ srv.listener.Close()
+ return srv.Wait()
+}
+
// Wait returns when the server has shut down.
-func (srv *server) Wait() error {
+func (srv *Server) Wait() error {
if srv.cond == nil {
return nil
}
@@ -81,15 +71,7 @@ func (srv *server) Wait() error {
return srv.err
}
-// Close shuts down the server and returns when it has stopped.
-func (srv *server) Close() error {
- srv.wantDown = true
- srv.listener.Close()
- return srv.Wait()
-}
-
// tcpKeepAliveListener is copied from net/http because not exported.
-//
type tcpKeepAliveListener struct {
*net.TCPListener
}
diff --git a/services/arv-git-httpd/server.go b/services/arv-git-httpd/server.go
index 716b276..9e80481 100644
--- a/services/arv-git-httpd/server.go
+++ b/services/arv-git-httpd/server.go
@@ -1,21 +1,13 @@
package main
import (
- "net"
"net/http"
"net/http/cgi"
- "sync"
- "time"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
type server struct {
- http.Server
- Addr string // host:port where the server is listening.
- err error
- cond *sync.Cond
- running bool
- listener *net.TCPListener
- wantDown bool
+ httpserver.Server
}
func (srv *server) Start() error {
@@ -29,77 +21,9 @@ func (srv *server) Start() error {
InheritEnv: []string{"PATH"},
Args: []string{"http-backend"},
}
-
- // The rest of the work here is essentially
- // http.ListenAndServe() with two more features: (1) whoever
- // called Start() can discover which address:port we end up
- // listening to -- which makes listening on ":0" useful in
- // test suites -- and (2) the server can be shut down without
- // killing the process -- which is useful in test cases, and
- // makes it possible to shut down gracefully on SIGTERM
- // without killing active connections.
-
- addr, err := net.ResolveTCPAddr("tcp", theConfig.Addr)
- if err != nil {
- return err
- }
- srv.listener, err = net.ListenTCP("tcp", addr)
- if err != nil {
- return err
- }
- srv.Addr = srv.listener.Addr().String()
mux := http.NewServeMux()
mux.Handle("/", &authHandler{gitHandler})
srv.Handler = mux
-
- mutex := &sync.RWMutex{}
- srv.cond = sync.NewCond(mutex.RLocker())
- srv.running = true
- go func() {
- err = srv.Serve(tcpKeepAliveListener{srv.listener})
- if !srv.wantDown {
- srv.err = err
- }
- mutex.Lock()
- srv.running = false
- srv.cond.Broadcast()
- mutex.Unlock()
- }()
- return nil
-}
-
-// Wait returns when the server has shut down.
-func (srv *server) Wait() error {
- if srv.cond == nil {
- return nil
- }
- srv.cond.L.Lock()
- defer srv.cond.L.Unlock()
- for srv.running {
- srv.cond.Wait()
- }
- return srv.err
-}
-
-// Close shuts down the server and returns when it has stopped.
-func (srv *server) Close() error {
- srv.wantDown = true
- srv.listener.Close()
- return srv.Wait()
-}
-
-// tcpKeepAliveListener is copied from net/http because not exported.
-//
-type tcpKeepAliveListener struct {
- *net.TCPListener
-}
-
-func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
- tc, err := ln.AcceptTCP()
- if err != nil {
- return
- }
- tc.SetKeepAlive(true)
- tc.SetKeepAlivePeriod(3 * time.Minute)
- return tc, nil
+ srv.Addr = theConfig.Addr
+ return srv.Server.Start()
}
commit 489911981539dcbde1e6b37264cc2a952316f114
Author: Tom Clegg <tom at curoverse.com>
Date: Thu May 21 00:33:56 2015 -0400
5824: Move request auth code into an SDK package. Support more ways of passing tokens.
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
new file mode 100644
index 0000000..4a719e9
--- /dev/null
+++ b/sdk/go/auth/auth.go
@@ -0,0 +1,61 @@
+package auth
+
+import (
+ "net/http"
+ "net/url"
+ "strings"
+)
+
+type Credentials struct {
+ Tokens []string
+}
+
+func NewCredentials() *Credentials {
+ return &Credentials{Tokens: []string{}}
+}
+
+func NewCredentialsFromHTTPRequest(r *http.Request) *Credentials {
+ c := NewCredentials()
+ c.LoadTokensFromHTTPRequest(r)
+ return c
+}
+
+// LoadTokensFromHttpRequest loads all tokens it can find in the
+// headers and query string of an http query.
+func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
+ // Load plain token from "Authorization: OAuth2 ..." header
+ // (typically used by smart API clients)
+ if toks := strings.SplitN(r.Header.Get("Authorization"), " ", 2); len(toks) == 2 && toks[0] == "OAuth2" {
+ a.Tokens = append(a.Tokens, toks[1])
+ }
+
+ // Load base64-encoded token from "Authorization: Basic ..."
+ // header (typically used by git via credential helper)
+ if _, password, ok := BasicAuth(r); ok {
+ a.Tokens = append(a.Tokens, password)
+ }
+
+ // Load tokens from query string. It's generally not a good
+ // idea to pass tokens around this way, but passing a narrowly
+ // scoped token is a reasonable way to implement "secret link
+ // to an object" in a generic way.
+ //
+ // ParseQuery always returns a non-nil map which might have
+ // valid parameters, even when a decoding error causes it to
+ // return a non-nil err. We ignore err; hopefully the caller
+ // will also need to parse the query string for
+ // application-specific purposes and will therefore
+ // find/report decoding errors in a suitable way.
+ qvalues, _ := url.ParseQuery(r.URL.RawQuery)
+ if val, ok := qvalues["api_token"]; ok {
+ a.Tokens = append(a.Tokens, val...)
+ }
+
+ // TODO: Load token from Rails session cookie (if Rails site
+ // secret is known)
+}
+
+// TODO: LoadTokensFromHttpRequestBody(). We can't assume in
+// LoadTokensFromHttpRequest() that [or how] we should read and parse
+// the request body. This has to be requested explicitly by the
+// application.
diff --git a/services/arv-git-httpd/basic_auth_go13.go b/sdk/go/auth/basic_auth_go13.go
similarity index 97%
rename from services/arv-git-httpd/basic_auth_go13.go
rename to sdk/go/auth/basic_auth_go13.go
index 087f2c8..c0fe5fc 100644
--- a/services/arv-git-httpd/basic_auth_go13.go
+++ b/sdk/go/auth/basic_auth_go13.go
@@ -1,6 +1,6 @@
// +build !go1.4
-package main
+package auth
import (
"encoding/base64"
diff --git a/services/arv-git-httpd/basic_auth_go14.go b/sdk/go/auth/basic_auth_go14.go
similarity index 91%
rename from services/arv-git-httpd/basic_auth_go14.go
rename to sdk/go/auth/basic_auth_go14.go
index 6a0079a..aeedb06 100644
--- a/services/arv-git-httpd/basic_auth_go14.go
+++ b/sdk/go/auth/basic_auth_go14.go
@@ -1,6 +1,6 @@
// +build go1.4
-package main
+package auth
import (
"net/http"
diff --git a/services/arv-git-httpd/basic_auth_test.go b/sdk/go/auth/basic_auth_test.go
similarity index 98%
rename from services/arv-git-httpd/basic_auth_test.go
rename to sdk/go/auth/basic_auth_test.go
index 2bd84dc..935f696 100644
--- a/services/arv-git-httpd/basic_auth_test.go
+++ b/sdk/go/auth/basic_auth_test.go
@@ -1,4 +1,4 @@
-package main
+package auth
import (
"net/http"
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 6313d50..0a8f9fd 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -9,6 +9,7 @@ import (
"sync"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
)
@@ -40,7 +41,7 @@ type authHandler struct {
func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var statusCode int
var statusText string
- var username, password string
+ var apiToken string
var repoName string
var wroteStatus int
@@ -48,21 +49,21 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
defer func() {
if wroteStatus == 0 {
- // Nobody has called WriteHeader yet: that must be our job.
+ // Nobody has called WriteHeader yet: that
+ // must be our job.
w.WriteHeader(statusCode)
w.Write([]byte(statusText))
}
- log.Println(quoteStrings(r.RemoteAddr, username, password, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
+ log.Println(quoteStrings(r.RemoteAddr, apiToken, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
}()
- // HTTP request username is logged, but unused. Password is an
- // Arvados API token.
- username, password, ok := BasicAuth(r)
- if !ok || username == "" || password == "" {
+ creds := auth.NewCredentialsFromHTTPRequest(r)
+ if len(creds.Tokens) == 0 {
statusCode, statusText = http.StatusUnauthorized, "no credentials provided"
w.Header().Add("WWW-Authenticate", "Basic realm=\"git\"")
return
}
+ apiToken = creds.Tokens[0]
// Access to paths "/foo/bar.git/*" and "/foo/bar/.git/*" are
// protected by the permissions on the repository named
@@ -84,7 +85,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
- arv.ApiToken = password
+ arv.ApiToken = apiToken
reposFound := arvadosclient.Dict{}
if err := arv.List("repositories", arvadosclient.Dict{
"filters": [][]string{{"name", "=", repoName}},
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list