[ARVADOS] updated: 715bd31a183097f4cab5a0f23cef053cd5f2f5e8

git at public.curoverse.com git at public.curoverse.com
Wed Apr 8 11:42:25 EDT 2015


Summary of changes:
 sdk/go/keepclient/keepclient.go      | 57 ++++++++++++--------------
 sdk/go/keepclient/keepclient_test.go | 79 ++++++++++++++++++++++++++++++++++--
 services/keepproxy/keepproxy.go      | 14 +++++--
 3 files changed, 110 insertions(+), 40 deletions(-)

       via  715bd31a183097f4cab5a0f23cef053cd5f2f5e8 (commit)
       via  df7b325565bd555989710b4d101fd07a99304fb2 (commit)
      from  9e9117914e998143d68e3083670a8de20799331a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 715bd31a183097f4cab5a0f23cef053cd5f2f5e8
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Apr 1 11:48:39 2015 -0400

    5414: Add TestGetWithLocalServiceHint.

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index bab7364..7054bba 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -528,6 +528,57 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 	c.Check(content, DeepEquals, []byte("foo"))
 }
 
+// Use a service hint to fetch from a local disk service, overriding
+// rendezvous probe order.
+func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
+	uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+	// This one shouldn't be used, although it appears first in
+	// rendezvous probe order:
+	ks0 := RunFakeKeepServer(StubGetHandler{
+		c,
+		"error if used",
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks0.listener.Close()
+	// This one should be used:
+	ks := RunFakeKeepServer(StubGetHandler{
+		c,
+		hash+"+K@"+uuid,
+		"abc123",
+		http.StatusOK,
+		[]byte("foo")})
+	defer ks.listener.Close()
+
+	arv, err := arvadosclient.MakeArvadosClient()
+	kc, _ := MakeKeepClient(&arv)
+	arv.ApiToken = "abc123"
+	kc.SetServiceRoots(
+		map[string]string{
+			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
+			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
+			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
+			uuid: ks.url},
+		map[string]string{
+			"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
+			"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
+			"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
+			uuid: ks.url},
+	)
+
+	r, n, uri, err := kc.Get(hash+"+K@"+uuid)
+	defer r.Close()
+	c.Check(err, Equals, nil)
+	c.Check(n, Equals, int64(3))
+	c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
+
+	content, err := ioutil.ReadAll(r)
+	c.Check(err, Equals, nil)
+	c.Check(content, DeepEquals, []byte("foo"))
+}
+
 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 	uuid := "zzzzz-bi6l4-123451234512345"
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))

commit df7b325565bd555989710b4d101fd07a99304fb2
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Apr 1 11:48:12 2015 -0400

    5414: Clean up Locator / MakeLocator() API.

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 29352b4..853f4c6 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -27,6 +27,7 @@ var InsufficientReplicasError = errors.New("Could not write sufficient replicas"
 var OversizeBlockError = errors.New("Exceeded maximum block size ("+strconv.Itoa(BLOCKSIZE)+")")
 var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
+var InvalidLocatorError = errors.New("Invalid locator")
 
 const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
 const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
@@ -244,44 +245,36 @@ func (kc *KeepClient) getSortedRoots(locator string) []string {
 }
 
 type Locator struct {
-	Hash      string
-	Size      int
-	Signature string
-	Timestamp string
+	Hash  string
+	Size  int		// -1 if data size is not known
+	Hints []string		// Including the size hint, if any
 }
 
-func MakeLocator2(hash string, hints string) (locator Locator) {
-	locator.Hash = hash
-	if hints != "" {
-		signature_pat, _ := regexp.Compile("^A([[:xdigit:]]+)@([[:xdigit:]]{8})$")
-		for _, hint := range strings.Split(hints, "+") {
-			if hint != "" {
-				if match, _ := regexp.MatchString("^[[:digit:]]+$", hint); match {
-					fmt.Sscanf(hint, "%d", &locator.Size)
-				} else if m := signature_pat.FindStringSubmatch(hint); m != nil {
-					locator.Signature = m[1]
-					locator.Timestamp = m[2]
-				} else if match, _ := regexp.MatchString("^[:upper:]", hint); match {
-					// Any unknown hint that starts with an uppercase letter is
-					// presumed to be valid and ignored, to permit forward compatibility.
-				} else {
-					// Unknown format; not a valid locator.
-					return Locator{"", 0, "", ""}
-				}
-			}
-		}
+func (loc *Locator) String() string {
+	s := loc.Hash
+	if len(loc.Hints) > 0 {
+		s = s + "+" + strings.Join(loc.Hints, "+")
 	}
-	return locator
+	return s
 }
 
-var pathpattern = regexp.MustCompile("^([0-9a-f]{32})([+].*)?$")
+var locatorMatcher = regexp.MustCompile("^([0-9a-f]{32})([+](.*))?$")
 
-func MakeLocator(path string) Locator {
-	sm := pathpattern.FindStringSubmatch(path)
+func MakeLocator(path string) (*Locator, error) {
+	sm := locatorMatcher.FindStringSubmatch(path)
 	if sm == nil {
-		log.Print("Failed match ", path)
-		return Locator{"", 0, "", ""}
+		return nil, InvalidLocatorError
 	}
-
-	return MakeLocator2(sm[1], sm[2])
+	loc := Locator{Hash: sm[1], Size: -1}
+	if sm[2] != "" {
+		loc.Hints = strings.Split(sm[3], "+")
+	} else {
+		loc.Hints = []string{}
+	}
+	if len(loc.Hints) > 0 {
+		if size, err := strconv.Atoi(loc.Hints[0]); err == nil {
+			loc.Size = size
+		}
+	}
+	return &loc, nil
 }
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 825696b..bab7364 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -765,10 +765,30 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 }
 
 func (s *StandaloneSuite) TestMakeLocator(c *C) {
-	l := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde at 12345678")
-
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde at 12345678")
+	c.Check(err, Equals, nil)
 	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 	c.Check(l.Size, Equals, 3)
-	c.Check(l.Signature, Equals, "abcde")
-	c.Check(l.Timestamp, Equals, "12345678")
+	c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde at 12345678"})
+}
+
+func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(err, Equals, nil)
+	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(l.Size, Equals, -1)
+	c.Check(l.Hints, DeepEquals, []string{})
+}
+
+func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
+	l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde at 12345678")
+	c.Check(err, Equals, nil)
+	c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+	c.Check(l.Size, Equals, -1)
+	c.Check(l.Hints, DeepEquals, []string{"Aabcde at 12345678"})
+}
+
+func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
+	_, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
+	c.Check(err, Equals, InvalidLocatorError)
 }
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index af81ba2..7ba2480 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -400,10 +400,16 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 		return
 	}
 
-	if loc := keepclient.MakeLocator(locatorIn); loc.Size > 0 && int64(loc.Size) != expectLength {
-		err = LengthMismatchError
-		status = http.StatusBadRequest
-		return
+	if locatorIn != "" {
+		var loc *keepclient.Locator
+		if loc, err = keepclient.MakeLocator(locatorIn); err != nil {
+			status = http.StatusBadRequest
+			return
+		} else if loc.Size > 0 && int64(loc.Size) != expectLength {
+			err = LengthMismatchError
+			status = http.StatusBadRequest
+			return
+		}
 	}
 
 	var pass bool

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list