[ARVADOS] updated: d161e2280a03b4a1c9675f5ed1946310a9942acd
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 10 14:56:23 EDT 2014
Summary of changes:
sdk/go/src/arvados.org/keepclient/keepclient.go | 42 +---
.../src/arvados.org/keepclient/keepclient_test.go | 76 +++----
sdk/go/src/arvados.org/keepclient/support.go | 51 +----
sdk/go/src/arvados.org/sdk/sdk.go | 232 +++++++++++++++++++++
sdk/go/src/arvados.org/sdk/sdk_test.go | 92 ++++++++
.../keep/src/arvados.org/keepproxy/keepproxy.go | 32 +--
.../src/arvados.org/keepproxy/keepproxy_test.go | 9 +-
7 files changed, 399 insertions(+), 135 deletions(-)
create mode 100644 sdk/go/src/arvados.org/sdk/sdk.go
create mode 100644 sdk/go/src/arvados.org/sdk/sdk_test.go
via d161e2280a03b4a1c9675f5ed1946310a9942acd (commit)
from 23fd8b137488baeb3138e2127294e8e14917d3ed (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d161e2280a03b4a1c9675f5ed1946310a9942acd
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 10 14:56:01 2014 -0400
Merge branch '2826-simple-go-sdk' closes #2826
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient.go b/sdk/go/src/arvados.org/keepclient/keepclient.go
index ee91d6f..d43a215 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient.go
@@ -2,16 +2,15 @@
package keepclient
import (
+ "arvados.org/sdk"
"arvados.org/streamer"
"crypto/md5"
- "crypto/tls"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
- "os"
"regexp"
"sort"
"strings"
@@ -34,41 +33,22 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
// Information about Arvados and Keep servers.
type KeepClient struct {
- ApiServer string
- ApiToken string
- ApiInsecure bool
+ Arvados *sdk.ArvadosClient
Want_replicas int
- Client *http.Client
Using_proxy bool
- External bool
service_roots *[]string
lock sync.Mutex
+ Client *http.Client
}
-// Create a new KeepClient, initialized with standard Arvados environment
-// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
-// ARVADOS_API_HOST_INSECURE. This will contact the API server to discover
-// Keep servers.
-func MakeKeepClient() (kc KeepClient, err error) {
- insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true")
- external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true")
-
+// Create a new KeepClient. This will contact the API server to discover Keep
+// servers.
+func MakeKeepClient(arv *sdk.ArvadosClient) (kc KeepClient, err error) {
kc = KeepClient{
- ApiServer: os.Getenv("ARVADOS_API_HOST"),
- ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
- ApiInsecure: insecure,
+ Arvados: arv,
Want_replicas: 2,
- Client: &http.Client{Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
- Using_proxy: false,
- External: external}
-
- if os.Getenv("ARVADOS_API_HOST") == "" {
- return kc, MissingArvadosApiHost
- }
- if os.Getenv("ARVADOS_API_TOKEN") == "" {
- return kc, MissingArvadosApiToken
- }
+ Using_proxy: false,
+ Client: &http.Client{Transport: &http.Transport{}}}
err = (&kc).DiscoverKeepServers()
@@ -169,7 +149,7 @@ func (this KeepClient) AuthorizedGet(hash string,
continue
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
@@ -211,7 +191,7 @@ func (this KeepClient) AuthorizedAsk(hash string, signature string,
continue
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
index 753a0ac..5327fb5 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
@@ -1,6 +1,7 @@
package keepclient
import (
+ "arvados.org/sdk"
"arvados.org/streamer"
"crypto/md5"
"flag"
@@ -62,20 +63,12 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
os.Setenv("ARVADOS_API_HOST", "localhost:3001")
os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "")
-
- kc, err := MakeKeepClient()
- c.Check(kc.ApiServer, Equals, "localhost:3001")
- c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- c.Check(kc.ApiInsecure, Equals, false)
-
os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
- kc, err = MakeKeepClient()
- c.Check(kc.ApiServer, Equals, "localhost:3001")
- c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- c.Check(kc.ApiInsecure, Equals, true)
- c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
+ arv, err := sdk.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
+
+ kc, err := MakeKeepClient(&arv)
c.Assert(err, Equals, nil)
c.Check(len(kc.ServiceRoots()), Equals, 2)
@@ -133,8 +126,10 @@ func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
listener, url := RunBogusKeepServer(st, 2990)
defer listener.Close()
- kc, _ := MakeKeepClient()
- kc.ApiToken = "abc123"
+ arv, _ := sdk.MakeArvadosClient()
+ arv.ApiToken = "abc123"
+
+ kc, _ := MakeKeepClient(&arv)
reader, writer := io.Pipe()
upload_status := make(chan uploadStatus)
@@ -265,10 +260,11 @@ func (s *StandaloneSuite) TestPutB(c *C) {
"foo",
make(chan string, 2)}
- kc, _ := MakeKeepClient()
+ arv, _ := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 5)
ks := RunSomeFakeKeepServers(st, 5, 2990)
@@ -306,10 +302,11 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
"foo",
make(chan string, 2)}
- kc, _ := MakeKeepClient()
+ arv, _ := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 5)
ks := RunSomeFakeKeepServers(st, 5, 2990)
@@ -359,10 +356,11 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
fh := FailHandler{
make(chan string, 1)}
- kc, _ := MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 5)
ks1 := RunSomeFakeKeepServers(st, 4, 2990)
@@ -407,10 +405,11 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
fh := FailHandler{
make(chan string, 4)}
- kc, _ := MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 5)
ks1 := RunSomeFakeKeepServers(st, 1, 2990)
@@ -466,8 +465,9 @@ func (s *StandaloneSuite) TestGet(c *C) {
listener, url := RunBogusKeepServer(st, 2990)
defer listener.Close()
- kc, _ := MakeKeepClient()
- kc.ApiToken = "abc123"
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
kc.SetServiceRoots([]string{url})
r, n, url2, err := kc.Get(hash)
@@ -491,8 +491,9 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
listener, url := RunBogusKeepServer(st, 2990)
defer listener.Close()
- kc, _ := MakeKeepClient()
- kc.ApiToken = "abc123"
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
kc.SetServiceRoots([]string{url})
r, n, url2, err := kc.Get(hash)
@@ -520,8 +521,9 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
listener, url := RunBogusKeepServer(st, 2990)
defer listener.Close()
- kc, _ := MakeKeepClient()
- kc.ApiToken = "abc123"
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
kc.SetServiceRoots([]string{url})
r, n, _, err := kc.Get(barhash)
@@ -552,8 +554,9 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
"abc123",
[]byte("foo")}
- kc, _ := MakeKeepClient()
- kc.ApiToken = "abc123"
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
+ arv.ApiToken = "abc123"
service_roots := make([]string, 5)
ks1 := RunSomeFakeKeepServers(st, 1, 2990)
@@ -586,7 +589,8 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
- kc, err := MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, err := MakeKeepClient(&arv)
c.Assert(err, Equals, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -634,11 +638,12 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
st := StubProxyHandler{make(chan string, 1)}
- kc, _ := MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
kc.Using_proxy = true
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 1)
ks1 := RunSomeFakeKeepServers(st, 1, 2990)
@@ -664,11 +669,12 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
st := StubProxyHandler{make(chan string, 1)}
- kc, _ := MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 3
kc.Using_proxy = true
- kc.ApiToken = "abc123"
+ arv.ApiToken = "abc123"
service_roots := make([]string, 1)
ks1 := RunSomeFakeKeepServers(st, 1, 2990)
diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go
index 5aeec12..f3e47f9 100644
--- a/sdk/go/src/arvados.org/keepclient/support.go
+++ b/sdk/go/src/arvados.org/keepclient/support.go
@@ -3,7 +3,6 @@ package keepclient
import (
"arvados.org/streamer"
- "encoding/json"
"errors"
"fmt"
"io"
@@ -29,53 +28,17 @@ func (this *KeepClient) DiscoverKeepServers() error {
return nil
}
- // Construct request of keep disk list
- var req *http.Request
- var err error
-
- if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible?format=json", this.ApiServer), nil); err != nil {
- return err
- }
-
- // Add api token header
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
- if this.External {
- req.Header.Add("X-External-Client", "1")
+ type svcList struct {
+ Items []keepDisk `json:"items"`
}
+ var m svcList
- // Make the request
- var resp *http.Response
- if resp, err = this.Client.Do(req); err != nil {
- return err
- }
+ err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &m)
- if resp.StatusCode != http.StatusOK {
- // fall back on keep disks
- if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_disks", this.ApiServer), nil); err != nil {
- return err
- }
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
- if resp, err = this.Client.Do(req); err != nil {
+ if err != nil {
+ if err := this.Arvados.List("keep_disks", nil, &m); err != nil {
return err
}
- if resp.StatusCode != http.StatusOK {
- return errors.New(resp.Status)
- }
- }
-
- // 'defer' is a stack, so it will drain the Body before closing it.
- defer resp.Body.Close()
- defer io.Copy(ioutil.Discard, resp.Body)
-
- type svcList struct {
- Items []keepDisk `json:"items"`
- }
-
- // Decode json reply
- dec := json.NewDecoder(resp.Body)
- var m svcList
- if err := dec.Decode(&m); err != nil {
- return err
}
listed := make(map[string]bool)
@@ -181,7 +144,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
req.ContentLength = expectedLength
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
req.Header.Add("Content-Type", "application/octet-stream")
if this.Using_proxy {
diff --git a/sdk/go/src/arvados.org/sdk/sdk.go b/sdk/go/src/arvados.org/sdk/sdk.go
new file mode 100644
index 0000000..9f1880f
--- /dev/null
+++ b/sdk/go/src/arvados.org/sdk/sdk.go
@@ -0,0 +1,232 @@
+/* Simple Arvados Go SDK for communicating with API server. */
+
+package sdk
+
+import (
+ "bytes"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+)
+
+// Errors
+var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
+var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
+var ArvadosErrorForbidden = errors.New("Forbidden")
+var ArvadosErrorNotFound = errors.New("Not found")
+var ArvadosErrorBadRequest = errors.New("Bad request")
+var ArvadosErrorServerError = errors.New("Server error")
+
+// Helper type so we don't have to write out 'map[string]interface{}' every time.
+type Dict map[string]interface{}
+
+// Information about how to contact the Arvados server
+type ArvadosClient struct {
+ // Arvados API server, form "host:port"
+ ApiServer string
+
+ // Arvados API token for authentication
+ ApiToken string
+
+ // Whether to require a valid SSL certificate or not
+ ApiInsecure bool
+
+ // Client object shared by client requests. Supports HTTP KeepAlive.
+ Client *http.Client
+
+ // If true, sets the X-External-Client header to indicate
+ // the client is outside the cluster.
+ External bool
+}
+
+// Create a new KeepClient, initialized with standard Arvados environment
+// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
+// ARVADOS_API_HOST_INSECURE.
+func MakeArvadosClient() (kc ArvadosClient, err error) {
+ insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true")
+ external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true")
+
+ kc = ArvadosClient{
+ ApiServer: os.Getenv("ARVADOS_API_HOST"),
+ ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
+ ApiInsecure: insecure,
+ Client: &http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
+ External: external}
+
+ if os.Getenv("ARVADOS_API_HOST") == "" {
+ return kc, MissingArvadosApiHost
+ }
+ if os.Getenv("ARVADOS_API_TOKEN") == "" {
+ return kc, MissingArvadosApiToken
+ }
+
+ return kc, err
+}
+
+// Low-level access to a resource.
+//
+// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE
+// resource - the arvados resource to act on
+// uuid - the uuid of the specific item to access (may be empty)
+// action - sub-action to take on the resource or uuid (may be empty)
+// parameters - method parameters
+//
+// return
+// reader - the body reader, or nil if there was an error
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) CallRaw(method string, resource string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) {
+ var req *http.Request
+
+ u := url.URL{
+ Scheme: "https",
+ Host: this.ApiServer}
+
+ u.Path = "/arvados/v1"
+
+ if resource != "" {
+ u.Path = u.Path + "/" + resource
+ }
+ if uuid != "" {
+ u.Path = u.Path + "/" + uuid
+ }
+ if action != "" {
+ u.Path = u.Path + "/" + action
+ }
+
+ if parameters == nil {
+ parameters = make(Dict)
+ }
+
+ parameters["format"] = "json"
+
+ vals := make(url.Values)
+ for k, v := range parameters {
+ m, err := json.Marshal(v)
+ if err == nil {
+ vals.Set(k, string(m))
+ }
+ }
+
+ if method == "GET" || method == "HEAD" {
+ u.RawQuery = vals.Encode()
+ if req, err = http.NewRequest(method, u.String(), nil); err != nil {
+ return nil, err
+ }
+ } else {
+ if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
+ return nil, err
+ }
+ req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+ }
+
+ // Add api token header
+ req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken))
+ if this.External {
+ req.Header.Add("X-External-Client", "1")
+ }
+
+ // Make the request
+ var resp *http.Response
+ if resp, err = this.Client.Do(req); err != nil {
+ return nil, err
+ }
+
+ switch resp.StatusCode {
+ case http.StatusOK:
+ return resp.Body, nil
+ case http.StatusForbidden:
+ resp.Body.Close()
+ return nil, ArvadosErrorForbidden
+ case http.StatusNotFound:
+ resp.Body.Close()
+ return nil, ArvadosErrorNotFound
+ default:
+ resp.Body.Close()
+ if resp.StatusCode >= 400 && resp.StatusCode <= 499 {
+ return nil, ArvadosErrorBadRequest
+ } else {
+ return nil, ArvadosErrorServerError
+ }
+ }
+}
+
+// Access to a resource.
+//
+// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE
+// resource - the arvados resource to act on
+// uuid - the uuid of the specific item to access (may be empty)
+// action - sub-action to take on the resource or uuid (may be empty)
+// parameters - method parameters
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder
+// return
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) Call(method string, resource string, uuid string, action string, parameters Dict, output interface{}) (err error) {
+ var reader io.ReadCloser
+ reader, err = this.CallRaw(method, resource, uuid, action, parameters)
+ if reader != nil {
+ defer reader.Close()
+ }
+ if err != nil {
+ return err
+ }
+
+ if output != nil {
+ dec := json.NewDecoder(reader)
+ if err = dec.Decode(output); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Create a new instance of a resource.
+//
+// resource - the arvados resource on which to create an item
+// parameters - method parameters
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder
+// return
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) Create(resource string, parameters Dict, output interface{}) (err error) {
+ return this.Call("POST", resource, "", "", parameters, output)
+}
+
+// Delete an instance of a resource.
+//
+// resource - the arvados resource on which to delete an item
+// uuid - the item to delete
+// parameters - method parameters
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder
+// return
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) {
+ return this.Call("DELETE", resource, uuid, "", parameters, output)
+}
+
+// Update fields of an instance of a resource.
+//
+// resource - the arvados resource on which to update the item
+// uuid - the item to update
+// parameters - method parameters
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder
+// return
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) Update(resource string, uuid string, parameters Dict, output interface{}) (err error) {
+ return this.Call("PUT", resource, uuid, "", parameters, output)
+}
+
+// List the instances of a resource
+//
+// resource - the arvados resource on which to list
+// parameters - method parameters
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder
+// return
+// err - error accessing the resource, or nil if no error
+func (this ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) {
+ return this.Call("GET", resource, "", "", parameters, output)
+}
diff --git a/sdk/go/src/arvados.org/sdk/sdk_test.go b/sdk/go/src/arvados.org/sdk/sdk_test.go
new file mode 100644
index 0000000..8af848f
--- /dev/null
+++ b/sdk/go/src/arvados.org/sdk/sdk_test.go
@@ -0,0 +1,92 @@
+package sdk
+
+import (
+ "fmt"
+ . "gopkg.in/check.v1"
+ "net/http"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+var _ = Suite(&ServerRequiredSuite{})
+
+// Tests that require the Keep server running
+type ServerRequiredSuite struct{}
+
+func pythonDir() string {
+ gopath := os.Getenv("GOPATH")
+ return fmt.Sprintf("%s/../python/tests", strings.Split(gopath, ":")[0])
+}
+
+func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+ os.Chdir(pythonDir())
+ if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil {
+ panic("'python run_test_server.py start' returned error")
+ }
+ if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil {
+ panic("'python run_test_server.py start_keep' returned error")
+ }
+}
+
+func (s *ServerRequiredSuite) TestMakeArvadosClient(c *C) {
+ os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+ os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+
+ kc, err := MakeArvadosClient()
+ c.Check(kc.ApiServer, Equals, "localhost:3001")
+ c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ c.Check(kc.ApiInsecure, Equals, false)
+
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+
+ kc, err = MakeArvadosClient()
+ c.Check(kc.ApiServer, Equals, "localhost:3001")
+ c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ c.Check(kc.ApiInsecure, Equals, true)
+ c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
+
+ c.Assert(err, Equals, nil)
+}
+
+func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
+ os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+ os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+
+ arv, err := MakeArvadosClient()
+
+ getback := make(Dict)
+ err = arv.Create("pipeline_templates",
+ Dict{"pipeline_template": Dict{
+ "name": "tmp",
+ "components": Dict{
+ "c1": map[string]string{"script": "script1"},
+ "c2": map[string]string{"script": "script2"}}}},
+ &getback)
+ c.Assert(err, Equals, nil)
+ c.Assert(getback["name"], Equals, "tmp")
+ c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2")
+
+ uuid := getback["uuid"].(string)
+ getback = make(Dict)
+ err = arv.Update("pipeline_templates", uuid,
+ Dict{
+ "pipeline_template": Dict{"name": "tmp2"}},
+ &getback)
+ c.Assert(err, Equals, nil)
+ c.Assert(getback["name"], Equals, "tmp2")
+
+ c.Assert(getback["uuid"].(string), Equals, uuid)
+ getback = make(Dict)
+ err = arv.Delete("pipeline_templates", uuid, nil, &getback)
+ c.Assert(err, Equals, nil)
+ c.Assert(getback["name"], Equals, "tmp2")
+}
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index 42f5a78..56de1e1 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -2,6 +2,7 @@ package main
import (
"arvados.org/keepclient"
+ "arvados.org/sdk"
"flag"
"fmt"
"github.com/gorilla/mux"
@@ -67,7 +68,12 @@ func main() {
flagset.Parse(os.Args[1:])
- kc, err := keepclient.MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %s", err.Error())
+ }
+
+ kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
log.Fatalf("Error setting up keep client %s", err.Error())
}
@@ -205,31 +211,13 @@ func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, re
return true
}
- var usersreq *http.Request
-
- if usersreq, err = http.NewRequest("HEAD", fmt.Sprintf("https://%s/arvados/v1/users/current", kc.ApiServer), nil); err != nil {
- // Can't construct the request
+ arv := *kc.Arvados
+ arv.ApiToken = tok
+ if err := arv.Call("HEAD", "users", "", "current", nil, nil); err != nil {
log.Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
return false
}
- // Add api token header
- usersreq.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", tok))
-
- // Actually make the request
- var resp *http.Response
- if resp, err = kc.Client.Do(usersreq); err != nil {
- // Something else failed
- log.Printf("%s: CheckAuthorizationHeader error connecting to API server: %v", GetRemoteAddress(req), err.Error())
- return false
- }
-
- if resp.StatusCode != http.StatusOK {
- // Bad status
- log.Printf("%s: CheckAuthorizationHeader API server responded: %v", GetRemoteAddress(req), resp.Status)
- return false
- }
-
// Success! Update cache
cache.RememberToken(tok)
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index 47c33b4..4bf3478 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -2,6 +2,7 @@ package main
import (
"arvados.org/keepclient"
+ "arvados.org/sdk"
"crypto/md5"
"crypto/tls"
"fmt"
@@ -108,7 +109,8 @@ func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient
os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
os.Setenv("ARVADOS_API_TOKEN", token)
- kc, err := keepclient.MakeKeepClient()
+ arv, err := sdk.MakeArvadosClient()
+ kc, err := keepclient.MakeKeepClient(&arv)
c.Check(kc.Using_proxy, Equals, true)
c.Check(len(kc.ServiceRoots()), Equals, 1)
c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port))
@@ -129,8 +131,9 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
setupProxyService()
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
- kc, err := keepclient.MakeKeepClient()
- c.Check(kc.External, Equals, true)
+ arv, err := sdk.MakeArvadosClient()
+ kc, err := keepclient.MakeKeepClient(&arv)
+ c.Check(kc.Arvados.External, Equals, true)
c.Check(kc.Using_proxy, Equals, true)
c.Check(len(kc.ServiceRoots()), Equals, 1)
c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list