[ARVADOS] updated: 33d63c6d42e824744305df3e720f8e9cbcc87d78
git at public.curoverse.com
git at public.curoverse.com
Sun May 11 15:12:07 EDT 2014
Summary of changes:
sdk/go/src/arvados.org/keepclient/keepclient.go | 49 +++++++++++++++++++---
.../src/arvados.org/keepclient/keepclient_test.go | 8 ++--
sdk/python/arvados/keep.py | 2 +
sdk/python/run_test_server.py | 6 +++
4 files changed, 57 insertions(+), 8 deletions(-)
via 33d63c6d42e824744305df3e720f8e9cbcc87d78 (commit)
from 6022ddd1b07217b9b4f31e73493fc259bd19b696 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 33d63c6d42e824744305df3e720f8e9cbcc87d78
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Sun May 11 15:11:59 2014 -0400
2798: Fixed inconsistencies with Python implementation of ShuffledServiceRoots.
Started working on KeepPut.
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient.go b/sdk/go/src/arvados.org/keepclient/keepclient.go
index 6ca0eea..9c1210d 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient.go
@@ -4,11 +4,16 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
+ "io"
"net/http"
"sort"
"strconv"
)
+type KeepClient struct {
+ Service_roots []string
+}
+
type KeepDisk struct {
Hostname string `json:"service_host"`
Port int `json:"service_port"`
@@ -54,7 +59,15 @@ func KeepDisks() (service_roots []string, err error) {
return service_roots, nil
}
-func ShuffledServiceRoots(service_roots []string, hash string) (pseq []string) {
+func MakeKeepClient() (kc *KeepClient, err error) {
+ sv, err := KeepDisks()
+ if err != nil {
+ return nil, err
+ }
+ return &KeepClient{sv}, nil
+}
+
+func (this KeepClient) ShuffledServiceRoots(hash string) (pseq []string) {
// Build an ordering with which to query the Keep servers based on the
// contents of the hash. "hash" is a hex-encoded number at least 8
// digits (32 bits) long
@@ -64,10 +77,11 @@ func ShuffledServiceRoots(service_roots []string, hash string) (pseq []string) {
seed := hash
// Keep servers still to be added to the ordering
- pool := service_roots[:]
+ pool := make([]string, len(this.Service_roots))
+ copy(pool, this.Service_roots)
// output probe sequence
- pseq = make([]string, 0, len(service_roots))
+ pseq = make([]string, 0, len(this.Service_roots))
// iterate while there are servers left to be assigned
for len(pool) > 0 {
@@ -87,8 +101,8 @@ func ShuffledServiceRoots(service_roots []string, hash string) (pseq []string) {
// Take the next 8 digits (32 bytes) and interpret as an integer,
// then modulus with the size of the remaining pool to get the next
// selected server.
- probe, _ := strconv.ParseInt(seed[0:8], 16, 32)
- probe %= int64(len(pool))
+ probe, _ := strconv.ParseUint(seed[0:8], 16, 32)
+ probe %= uint64(len(pool))
// Append the selected server to the probe sequence and remove it
// from the pool.
@@ -100,3 +114,28 @@ func ShuffledServiceRoots(service_roots []string, hash string) (pseq []string) {
}
return pseq
}
+
+func Fill(buffer []byte, r io.Reader, c chan []byte, errorchan chan error) {
+ ptr := buffer[:]
+
+ for {
+ n, err := r.Read(ptr)
+ if err != nil {
+ errorchan <- err
+ return
+ }
+ c <- ptr[:n]
+ ptr = ptr[n:]
+ }
+}
+
+func (this KeepClient) KeepPut(hash string, r io.Reader) {
+ //sv := this.ShuffledServiceRoots(hash)
+ //n := 0
+ buffer := make([]byte, 0, 1024*1024*64)
+ //success := make(chan int)
+ reads := make(chan []byte)
+ errorchan := make(chan error)
+
+ go Fill(buffer, r, reads, errorchan)
+}
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
index 4152eb3..bc719c0 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
@@ -1,6 +1,7 @@
package keepclient
import (
+ "fmt"
. "gopkg.in/check.v1"
"testing"
)
@@ -22,10 +23,11 @@ func (s *MySuite) TestGetKeepDisks(c *C) {
service_roots := []string{"http://localhost:25107", "http://localhost:25108", "http://localhost:25109", "http://localhost:25110", "http://localhost:25111", "http://localhost:25112", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25116", "http://localhost:25117", "http://localhost:25118", "http://localhost:25119", "http://localhost:25120", "http://localhost:25121", "http://localhost:25122", "http://localhost:25123"}
// "foo" acbd18db4cc2f85cedef654fccc4a4d8
- //foo_shuffle := []string{"http://localhost:25116", "http://localhost:25120", "http://localhost:25119", "http://localhost:25122", "http://localhost:25108", "http://localhost:25114", "http://localhost:25112", "http://localhost:25107", "http://localhost:25118", "http://localhost:25111", "http://localhost:25113", "http://localhost:25121", "http://localhost:25110", "http://localhost:25117", "http://localhost:25109", "http://localhost:25115", "http://localhost:25123"}
- ShuffledServiceRoots(service_roots, "acbd18db4cc2f85cedef654fccc4a4d8")
+ foo_shuffle := []string{"http://localhost:25116", "http://localhost:25120", "http://localhost:25119", "http://localhost:25122", "http://localhost:25108", "http://localhost:25114", "http://localhost:25112", "http://localhost:25107", "http://localhost:25118", "http://localhost:25111", "http://localhost:25113", "http://localhost:25121", "http://localhost:25110", "http://localhost:25117", "http://localhost:25109", "http://localhost:25115", "http://localhost:25123"}
+ c.Check(ShuffledServiceRoots(service_roots, "acbd18db4cc2f85cedef654fccc4a4d8"), DeepEquals, foo_shuffle)
// "bar" 37b51d194a7513e45b56f6524f2d51f2
- //bar_shuffle := []string{"http://localhost:25108", "http://localhost:25112", "http://localhost:25119", "http://localhost:25107", "http://localhost:25110", "http://localhost:25116", "http://localhost:25122", "http://localhost:25120", "http://localhost:25121", "http://localhost:25117", "http://localhost:25111", "http://localhost:25123", "http://localhost:25118", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25109"}
+ bar_shuffle := []string{"http://localhost:25108", "http://localhost:25112", "http://localhost:25119", "http://localhost:25107", "http://localhost:25110", "http://localhost:25116", "http://localhost:25122", "http://localhost:25120", "http://localhost:25121", "http://localhost:25117", "http://localhost:25111", "http://localhost:25123", "http://localhost:25118", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25109"}
+ c.Check(ShuffledServiceRoots(service_roots, "37b51d194a7513e45b56f6524f2d51f2"), DeepEquals, bar_shuffle)
}
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index fcb59ec..e414d26 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -193,6 +193,8 @@ class KeepClient(object):
# selected server.
probe = int(seed[0:8], 16) % len(pool)
+ print seed[0:8], int(seed[0:8], 16), len(pool), probe
+
# Append the selected server to the probe sequence and remove it
# from the pool.
pseq += [pool[probe]]
diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index 3c9d55b..9901e14 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -149,6 +149,10 @@ def run_keep():
_start_keep(0)
_start_keep(1)
+
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
+
authorize_with("admin")
api = arvados.api('v1', cache=False)
a = api.keep_disks().list().execute()
@@ -210,3 +214,5 @@ if __name__ == "__main__":
run_keep()
elif args.action == 'stop_keep':
stop_keep()
+ else:
+ print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list