[ARVADOS] created: 1.3.0-1236-gaa9518473

Git user git at public.curoverse.com
Mon Jul 1 16:37:55 UTC 2019


        at  aa9518473622cb4b72a33dcb2127b7c5ccfdee60 (commit)


commit aa9518473622cb4b72a33dcb2127b7c5ccfdee60
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Jul 1 12:37:34 2019 -0400

    14287: Merge list results from multiple backends.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index e094953fc..0afc8c261 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -12,7 +12,9 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
+	"sort"
 	"strings"
+	"sync"
 
 	"git.curoverse.com/arvados.git/lib/controller/railsproxy"
 	"git.curoverse.com/arvados.git/lib/controller/rpc"
@@ -27,7 +29,7 @@ type Conn struct {
 	remotes map[string]backend
 }
 
-func New(cluster *arvados.Cluster) arvados.API {
+func New(cluster *arvados.Cluster) *Conn {
 	local := railsproxy.NewConn(cluster)
 	remotes := map[string]backend{}
 	for id, remote := range cluster.RemoteClusters {
@@ -219,7 +221,28 @@ func (conn *Conn) CollectionGet(ctx context.Context, options arvados.GetOptions)
 }
 
 func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
-	return conn.local.CollectionList(ctx, options)
+	var mtx sync.Mutex
+	var merged arvados.CollectionList
+	err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+		cl, err := backend.CollectionList(ctx, options)
+		if err != nil {
+			return nil, err
+		}
+		mtx.Lock()
+		defer mtx.Unlock()
+		if len(merged.Items) == 0 {
+			merged = cl
+		} else {
+			merged.Items = append(merged.Items, cl.Items...)
+		}
+		uuids := make([]string, 0, len(cl.Items))
+		for _, item := range cl.Items {
+			uuids = append(uuids, item.UUID)
+		}
+		return uuids, nil
+	})
+	sort.Slice(merged.Items, func(i, j int) bool { return merged.Items[i].UUID < merged.Items[j].UUID })
+	return merged, err
 }
 
 func (conn *Conn) CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
diff --git a/lib/controller/federation/list.go b/lib/controller/federation/list.go
new file mode 100644
index 000000000..8f7acf068
--- /dev/null
+++ b/lib/controller/federation/list.go
@@ -0,0 +1,198 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package federation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+// Call fn on one or more local/remote backends if opts indicates a
+// federation-wide list query, i.e.:
+//
+// * There is at least one filter of the form
+//   ["uuid","in",[a,b,c,...]] or ["uuid","=",a]
+//
+// * One or more of the supplied UUIDs (a,b,c,...) has a non-local
+//   prefix.
+//
+// * There are no other filters
+//
+// (If opts doesn't indicate a federation-wide list query, fn is just
+// called once with the local backend.)
+//
+// fn is called more than once only if the query meets the following
+// restrictions:
+//
+// * Count=="none"
+//
+// * Limit<0
+//
+// * len(Order)==0
+//
+// * there are no filters other than the "uuid = ..." and "uuid in
+//   ..." filters mentioned above.
+//
+// * The maximum possible response size (total number of objects that
+//   could potentially be matched by all of the specified filters)
+//   exceeds the local cluster's response page size limit.
+//
+// If the query involves multiple backends but doesn't meet these
+// restrictions, an error is returned without calling fn.
+//
+// Thus, the caller can assume that either:
+//
+// * splitListRequest() returns an error, or
+//
+// * fn is called exactly once, or
+//
+// * fn is called more than once, and the options satisfy the above
+//   restrictions.
+//
+// Each call to fn indicates a single (local or remote) backend and a
+// corresponding options argument suitable for sending to that
+// backend.
+func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions, fn func(context.Context, string, arvados.API, arvados.ListOptions) ([]string, error)) error {
+	nUUIDs := 0
+	cannotSplit := false
+	uuidsByRemote := map[string][]string{}
+	addUUIDs := func(uuids ...string) {
+		for _, uuid := range uuids {
+			if len(uuid) != 27 {
+				// Cannot match anything, just drop it
+			} else {
+				uuidsByRemote[uuid[:5]] = append(uuidsByRemote[uuid[:5]], uuid)
+				nUUIDs++
+			}
+		}
+	}
+	for _, f := range opts.Filters {
+		if f.Attr != "uuid" {
+			cannotSplit = true
+			continue
+		}
+		if f.Operator == "=" {
+			if uuid, ok := f.Operand.(string); ok {
+				addUUIDs(uuid)
+			} else {
+				return fmt.Errorf("invalid operand type %T for filter %q", f.Operand, f)
+			}
+		} else if f.Operator == "in" {
+			var uuids []string
+			if operand, ok := f.Operand.([]interface{}); ok {
+				// Convert []interface{} to []string,
+				// dropping any elements that aren't
+				// strings (and therefore can't affect
+				// the result by matching UUIDs).
+				uuids = make([]string, 0, len(operand))
+				for _, v := range operand {
+					if uuid, ok := v.(string); ok {
+						uuids = append(uuids, uuid)
+					}
+				}
+			} else if strings, ok := f.Operand.([]string); ok {
+				uuids = strings
+			} else {
+				return fmt.Errorf("invalid operand type %T in filter %q", f.Operand, f)
+			}
+			addUUIDs(uuids...)
+		} else {
+			cannotSplit = true
+			continue
+		}
+	}
+	if len(uuidsByRemote) > 1 {
+		if cannotSplit {
+			return errors.New("cannot execute federated list query with filters other than 'uuid = ...' and 'uuid in [...]'")
+		}
+		if opts.Count != "none" {
+			return errors.New("cannot execute federated list query unless count==\"none\"")
+		}
+		if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
+			return errors.New("cannot execute federated list query with limit, offset, or order parameter")
+		}
+		if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
+			return fmt.Errorf("cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
+		}
+		selectingUUID := false
+		for _, attr := range opts.Select {
+			if attr == "uuid" {
+				selectingUUID = true
+			}
+		}
+		if opts.Select != nil && !selectingUUID {
+			return fmt.Errorf("cannot execute federated list query with a select parameter that does not include uuid")
+		}
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	errs := make(chan error, len(uuidsByRemote))
+	for clusterID, batch := range uuidsByRemote {
+		clusterID, batch := clusterID, batch
+		todo := map[string]struct{}{}
+		for _, uuid := range batch {
+			todo[uuid] = struct{}{}
+		}
+		go func() {
+			// This goroutine sends exactly one value to
+			// errs.
+			var backend arvados.API
+			if clusterID == conn.cluster.ClusterID {
+				backend = conn.local
+			} else if backend = conn.remotes[clusterID]; backend == nil {
+				errs <- httpserver.ErrorWithStatus(fmt.Errorf("cannot execute federated list query: no proxy available for cluster %q", clusterID), http.StatusNotFound)
+				return
+			}
+			remoteOpts := opts
+			for len(todo) > 0 {
+				if len(batch) > len(todo) {
+					// Reduce batch to just the todo's
+					batch = batch[:0]
+					for uuid := range todo {
+						batch = append(batch, uuid)
+					}
+				}
+				remoteOpts.Filters = []arvados.Filter{{"uuid", "in", batch}}
+
+				done, err := fn(ctx, clusterID, backend, remoteOpts)
+				if err != nil {
+					errs <- err
+					return
+				}
+				progress := false
+				for _, uuid := range done {
+					if _, ok := todo[uuid]; ok {
+						progress = true
+						delete(todo, uuid)
+					}
+				}
+				if !progress {
+					errs <- fmt.Errorf("cannot make progress in federated list query: cluster %q returned none of the requested UUIDs", clusterID)
+					return
+				}
+			}
+			errs <- nil
+		}()
+	}
+
+	// Wait for all goroutines to return, then return the first
+	// non-nil error, if any.
+	var firstErr error
+	for i := 0; i < len(uuidsByRemote); i++ {
+		if err := <-errs; err != nil && firstErr == nil {
+			firstErr = err
+			// Signal to any remaining fn() calls that
+			// further effort is futile.
+			cancel()
+		}
+	}
+	return firstErr
+}
diff --git a/lib/controller/federation/list_test.go b/lib/controller/federation/list_test.go
new file mode 100644
index 000000000..990efa1e6
--- /dev/null
+++ b/lib/controller/federation/list_test.go
@@ -0,0 +1,268 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package federation
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+
+	"git.curoverse.com/arvados.git/lib/controller/router"
+	"git.curoverse.com/arvados.git/lib/controller/rpc"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"git.curoverse.com/arvados.git/sdk/go/auth"
+	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+	check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+var (
+	_ = check.Suite(&FederationSuite{})
+	_ = check.Suite(&CollectionListSuite{})
+)
+
+type FederationSuite struct {
+	cluster *arvados.Cluster
+	ctx     context.Context
+	fed     *Conn
+}
+
+func (s *FederationSuite) SetUpTest(c *check.C) {
+	s.cluster = &arvados.Cluster{
+		ClusterID: "aaaaa",
+		RemoteClusters: map[string]arvados.RemoteCluster{
+			"aaaaa": arvados.RemoteCluster{
+				Host: os.Getenv("ARVADOS_API_HOST"),
+			},
+		},
+	}
+	arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+	s.cluster.TLS.Insecure = true
+	s.cluster.API.MaxItemsPerResponse = 3
+
+	ctx := context.Background()
+	ctx = ctxlog.Context(ctx, ctxlog.TestLogger(c))
+	ctx = auth.NewContext(ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+	s.ctx = ctx
+
+	s.fed = New(s.cluster)
+}
+
+func (s *FederationSuite) addDirectRemote(c *check.C, id string, backend arvados.API) {
+	s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
+		Host: "in-process.local",
+	}
+	s.fed.remotes[id] = backend
+}
+
+func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend arvados.API) {
+	srv := httpserver.Server{Addr: ":"}
+	srv.Handler = router.New(backend)
+	c.Check(srv.Start(), check.IsNil)
+	s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
+		Host:  srv.Addr,
+		Proxy: true,
+	}
+	s.fed.remotes[id] = rpc.NewConn(id, &url.URL{Scheme: "http", Host: srv.Addr}, true, saltedTokenProvider(s.fed.local, id))
+}
+
+type collectionLister struct {
+	arvadostest.APIStub
+	ItemsToReturn []arvados.Collection
+	MaxPageSize   int
+}
+
+func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
+nextfilter:
+	for _, f := range filters {
+		if f.Attr == "uuid" && f.Operator == "=" {
+			s, ok := f.Operand.(string)
+			if ok && s == c.UUID {
+				continue nextfilter
+			}
+		} else if f.Attr == "uuid" && f.Operator == "in" {
+			if operand, ok := f.Operand.([]string); ok {
+				for _, s := range operand {
+					if s == c.UUID {
+						continue nextfilter
+					}
+				}
+			} else if operand, ok := f.Operand.([]interface{}); ok {
+				for _, s := range operand {
+					if s, ok := s.(string); ok && s == c.UUID {
+						continue nextfilter
+					}
+				}
+			}
+		}
+		return false
+	}
+	return true
+}
+
+func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
+	cl.APIStub.CollectionList(ctx, options)
+	for _, c := range cl.ItemsToReturn {
+		if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
+			break
+		}
+		if cl.matchFilters(c, options.Filters) {
+			resp.Items = append(resp.Items, c)
+		}
+	}
+	return
+}
+
+type CollectionListSuite struct {
+	FederationSuite
+	ids      []string   // aaaaa, bbbbb, ccccc
+	uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
+	backends []*collectionLister
+}
+
+func (s *CollectionListSuite) SetUpTest(c *check.C) {
+	s.FederationSuite.SetUpTest(c)
+
+	s.ids = nil
+	s.uuids = nil
+	s.backends = nil
+	for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
+		cl := &collectionLister{}
+		s.ids = append(s.ids, id)
+		s.uuids = append(s.uuids, nil)
+		for j := 0; j < 5; j++ {
+			uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
+			s.uuids[i] = append(s.uuids[i], uuid)
+			cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
+				UUID: uuid,
+			})
+		}
+		s.backends = append(s.backends, cl)
+		if i == 0 {
+			s.fed.local = cl
+		} else if i%1 == 0 {
+			// call some backends directly via API
+			s.addDirectRemote(c, id, cl)
+		} else {
+			// call some backends through rpc->router->API
+			// to ensure nothing is lost in translation
+			s.addHTTPRemote(c, id, cl)
+		}
+	}
+}
+
+type listTrial struct {
+	label       string
+	filters     []arvados.Filter
+	expectUUIDs []string
+	expectCalls []int // number of API calls to backends
+}
+
+func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
+		expectUUIDs: []string{s.uuids[0][0]},
+		expectCalls: []int{1, 0, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
+		expectUUIDs: []string{s.uuids[1][0]},
+		expectCalls: []int{0, 1, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
+		expectUUIDs: []string{s.uuids[0][0]},
+		expectCalls: []int{1, 0, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
+		expectUUIDs: []string{s.uuids[1][1]},
+		expectCalls: []int{0, 1, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
+		expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
+		expectCalls: []int{1, 1, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
+		expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
+		expectCalls: []int{0, 1, 1},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", []string{}}},
+		expectUUIDs: []string{},
+		expectCalls: []int{0, 0, 0},
+	})
+}
+
+func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
+	for i := range s.backends {
+		s.uuids[i] = s.uuids[i][:3]
+		s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
+	}
+	s.cluster.API.MaxItemsPerResponse = 9
+	for _, stub := range s.backends {
+		stub.MaxPageSize = 2
+	}
+	allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
+	s.test(c, listTrial{
+		filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
+		expectUUIDs: allUUIDs,
+		expectCalls: []int{2, 2, 2},
+	})
+}
+
+func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
+	resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
+		Limit:   -1,
+		Filters: trial.filters,
+		Count:   "none",
+	})
+	c.Check(err, check.IsNil)
+	var expectItems []arvados.Collection
+	for _, uuid := range trial.expectUUIDs {
+		expectItems = append(expectItems, arvados.Collection{UUID: uuid})
+	}
+	c.Check(resp, check.DeepEquals, arvados.CollectionList{
+		Items: expectItems,
+	})
+
+	for i, stub := range s.backends {
+		calls := stub.Calls(nil)
+		c.Check(calls, check.HasLen, trial.expectCalls[i])
+		if len(calls) == 0 {
+			return
+		}
+		opts := calls[0].Options.(arvados.ListOptions)
+		c.Check(opts.Limit, check.Equals, -1)
+	}
+}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index d524195e4..852327fd8 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/lib/config"
+	"git.curoverse.com/arvados.git/lib/controller/federation"
 	"git.curoverse.com/arvados.git/lib/controller/railsproxy"
 	"git.curoverse.com/arvados.git/lib/controller/router"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -91,7 +92,7 @@ func (h *Handler) setup() {
 	}))
 
 	if h.Cluster.EnableBetaController14287 {
-		rtr := router.New(h.Cluster)
+		rtr := router.New(federation.New(h.Cluster))
 		mux.Handle("/arvados/v1/collections", rtr)
 		mux.Handle("/arvados/v1/collections/", rtr)
 	}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index f37c7ea90..9c2c1f3a1 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -10,7 +10,6 @@ import (
 	"net/http"
 	"strings"
 
-	"git.curoverse.com/arvados.git/lib/controller/federation"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/auth"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
@@ -24,10 +23,10 @@ type router struct {
 	fed arvados.API
 }
 
-func New(cluster *arvados.Cluster) *router {
+func New(fed arvados.API) *router {
 	rtr := &router{
 		mux: httprouter.New(),
-		fed: federation.New(cluster),
+		fed: fed,
 	}
 	rtr.addRoutes()
 	return rtr
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 4e6b16173..3a7045aa4 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -10,11 +10,13 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"git.curoverse.com/arvados.git/lib/controller/rpc"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	"github.com/julienschmidt/httprouter"
@@ -158,7 +160,8 @@ func (s *RouterIntegrationSuite) SetUpTest(c *check.C) {
 	cluster := &arvados.Cluster{}
 	cluster.TLS.Insecure = true
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
-	s.rtr = New(cluster)
+	url, _ := url.Parse("https://" + os.Getenv("ARVADOS_TEST_API_HOST"))
+	s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider))
 }
 
 func (s *RouterIntegrationSuite) TearDownSuite(c *check.C) {
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index e07eaf40a..ea3d6fb2d 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -8,6 +8,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -17,10 +18,19 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/auth"
 )
 
 type TokenProvider func(context.Context) ([]string, error)
 
+func PassthroughTokenProvider(ctx context.Context) ([]string, error) {
+	if incoming, ok := auth.FromContext(ctx); !ok {
+		return nil, errors.New("no token provided")
+	} else {
+		return incoming.Tokens, nil
+	}
+}
+
 type Conn struct {
 	clusterID     string
 	httpClient    http.Client
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index a3cacf3f6..77a26bcba 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -7,6 +7,8 @@ package arvadostest
 import (
 	"context"
 	"errors"
+	"reflect"
+	"runtime"
 	"sync"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -121,7 +123,9 @@ func (as *APIStub) Calls(method interface{}) []APIStubCall {
 	defer as.mtx.Unlock()
 	var calls []APIStubCall
 	for _, call := range as.calls {
-		if method == nil || call.Method == method {
+
+		if method == nil || (runtime.FuncForPC(reflect.ValueOf(call.Method).Pointer()).Name() ==
+			runtime.FuncForPC(reflect.ValueOf(method).Pointer()).Name()) {
 			calls = append(calls, call)
 		}
 	}
diff --git a/sdk/go/httpserver/error.go b/sdk/go/httpserver/error.go
index b222e18ea..f1817d337 100644
--- a/sdk/go/httpserver/error.go
+++ b/sdk/go/httpserver/error.go
@@ -9,6 +9,19 @@ import (
 	"net/http"
 )
 
+func ErrorWithStatus(err error, status int) error {
+	return errorWithStatus{err, status}
+}
+
+type errorWithStatus struct {
+	error
+	Status int
+}
+
+func (ews errorWithStatus) HTTPStatus() int {
+	return ews.Status
+}
+
 type ErrorResponse struct {
 	Errors []string `json:"errors"`
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list