[ARVADOS] updated: ed101805b04c70bcbf1b070dceba2aee72170377

git at public.curoverse.com git at public.curoverse.com
Fri May 23 15:27:17 EDT 2014


Summary of changes:
 .../app/controllers/application_controller.rb      |  21 +-
 .../app/controllers/collections_controller.rb      |  48 +-
 .../app/controllers/keep_services_controller.rb    |   2 +
 .../app/models/{keep_disk.rb => keep_service.rb}   |   2 +-
 .../app/views/collections/_show_files.html.erb     |   2 +-
 .../app/views/collections/show_file_links.html.erb |  82 +++
 .../app/views/layouts/application.html.erb         |   6 +-
 apps/workbench/config/routes.rb                    |   4 +
 apps/workbench/public/robots.txt                   |   2 -
 .../test/functional/collections_controller_test.rb |  34 +-
 .../workbench/test/integration/collections_test.rb |  27 +-
 apps/workbench/test/integration_helper.rb          |   5 +
 apps/workbench/test/test_helper.rb                 |   1 +
 doc/_config.yml                                    |   2 +
 doc/api/methods/keep_services.html.textile.liquid  |  75 +++
 doc/api/schema/KeepDisk.html.textile.liquid        |   4 +-
 doc/api/schema/KeepService.html.textile.liquid     |  24 +
 sdk/go/go.sh                                       |  17 +
 sdk/go/src/arvados.org/keepclient/hashcheck.go     |  77 +++
 .../src/arvados.org/keepclient/hashcheck_test.go   |  85 +++
 sdk/go/src/arvados.org/keepclient/keepclient.go    | 245 ++++++++
 .../src/arvados.org/keepclient/keepclient_test.go  | 689 +++++++++++++++++++++
 sdk/go/src/arvados.org/keepclient/support.go       | 258 ++++++++
 sdk/go/src/arvados.org/streamer/streamer.go        | 130 ++++
 sdk/go/src/arvados.org/streamer/streamer_test.go   | 366 +++++++++++
 sdk/go/src/arvados.org/streamer/transfer.go        | 308 +++++++++
 sdk/python/arvados/keep.py                         |  32 +-
 sdk/python/run_test_server.py                      |  60 +-
 sdk/python/test_keep_client.py                     |  18 +
 .../arvados/v1/collections_controller.rb           |  62 +-
 .../app/controllers/arvados/v1/jobs_controller.rb  |   4 +-
 .../arvados/v1/keep_services_controller.rb         |  21 +
 .../api/app/models/api_client_authorization.rb     |   2 +-
 services/api/app/models/keep_disk.rb               |  18 +-
 services/api/app/models/keep_service.rb            |  15 +
 services/api/app/models/locator.rb                 |  84 +++
 services/api/config/application.default.yml        |   5 +
 services/api/config/application.yml.example        |  13 +
 services/api/config/routes.rb                      |   3 +
 .../migrate/20140519205916_create_keep_services.rb |  51 ++
 services/api/db/schema.rb                          |  23 +-
 .../test/fixtures/api_client_authorizations.yml    |   7 +
 services/api/test/fixtures/jobs.yml                |  20 +
 services/api/test/fixtures/keep_disks.yml          |  12 +-
 services/api/test/fixtures/keep_services.yml       |  23 +
 services/api/test/fixtures/links.yml               |  14 +
 .../arvados/v1/collections_controller_test.rb      | 216 +++++++
 .../functional/arvados/v1/jobs_controller_test.rb  |  15 +
 .../arvados/v1/keep_disks_controller_test.rb       |  34 +-
 .../arvados/v1/keep_services_controller_test.rb    |  23 +
 services/api/test/integration/keep_proxy_test.rb   |  25 +
 .../api/test/unit/keep_service_test.rb             |   2 +-
 services/keep/build.sh                             |  36 --
 services/keep/go.sh                                |  17 +
 .../keep/src/arvados.org/keepproxy/keepproxy.go    | 397 ++++++++++++
 .../src/arvados.org/keepproxy/keepproxy_test.go    | 205 ++++++
 services/keep/src/keep/keep.go                     |  22 +
 57 files changed, 3822 insertions(+), 173 deletions(-)
 create mode 100644 apps/workbench/app/controllers/keep_services_controller.rb
 copy apps/workbench/app/models/{keep_disk.rb => keep_service.rb} (70%)
 create mode 100644 apps/workbench/app/views/collections/show_file_links.html.erb
 create mode 100644 doc/api/methods/keep_services.html.textile.liquid
 create mode 100644 doc/api/schema/KeepService.html.textile.liquid
 create mode 100755 sdk/go/go.sh
 create mode 100644 sdk/go/src/arvados.org/keepclient/hashcheck.go
 create mode 100644 sdk/go/src/arvados.org/keepclient/hashcheck_test.go
 create mode 100644 sdk/go/src/arvados.org/keepclient/keepclient.go
 create mode 100644 sdk/go/src/arvados.org/keepclient/keepclient_test.go
 create mode 100644 sdk/go/src/arvados.org/keepclient/support.go
 create mode 100644 sdk/go/src/arvados.org/streamer/streamer.go
 create mode 100644 sdk/go/src/arvados.org/streamer/streamer_test.go
 create mode 100644 sdk/go/src/arvados.org/streamer/transfer.go
 create mode 100644 services/api/app/controllers/arvados/v1/keep_services_controller.rb
 create mode 100644 services/api/app/models/keep_service.rb
 create mode 100644 services/api/app/models/locator.rb
 create mode 100644 services/api/db/migrate/20140519205916_create_keep_services.rb
 create mode 100644 services/api/test/fixtures/keep_services.yml
 create mode 100644 services/api/test/functional/arvados/v1/keep_services_controller_test.rb
 create mode 100644 services/api/test/integration/keep_proxy_test.rb
 copy apps/workbench/test/unit/job_task_test.rb => services/api/test/unit/keep_service_test.rb (61%)
 delete mode 100755 services/keep/build.sh
 create mode 100755 services/keep/go.sh
 create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy.go
 create mode 100644 services/keep/src/arvados.org/keepproxy/keepproxy_test.go

       via  ed101805b04c70bcbf1b070dceba2aee72170377 (commit)
       via  27ffca811e6f43225fc82d582d1962eebbd1ab6e (commit)
       via  b646cec74484bf07a54f4be2de712f50dc387aa0 (commit)
       via  0d35501f448a8e7ca44152429d3d38edaa9bb30c (commit)
       via  f3dd3608f2c32fd4dcd9551bfa1b871d816954d5 (commit)
       via  2099a0a3f0ef3371ba30ed0955bbc75ffa332701 (commit)
       via  12f8c88325daf4c6af8cbf091ea64cc5d64566c0 (commit)
       via  575457ac8645c61ca71e94ce074291ec002b4c24 (commit)
       via  d5823126a51b7c31915e01fee100abe9468014e5 (commit)
       via  be92cf19aa7b61576ddcee0fd360103ca43ef5ea (commit)
       via  86dffe03ec74387d14da9ceb17934bbdab1239b9 (commit)
       via  fff404a0e2ebe2582f616526c486ad0dbecce3c8 (commit)
       via  66f1be77149feb033e3adc17c37a09a391eec6e9 (commit)
       via  c2e70e05178c9569e9de2ac5b3ef47440c49efc5 (commit)
       via  77504f6d369bd7bd323748d5347d6d99ed9c75ca (commit)
       via  f8743ac8fa1ed9dc8c7c4f7a23803ffe8721cfa6 (commit)
       via  3ba5aa15ea0156a4fce63dcb43b7f972b4c760df (commit)
       via  d54a48fa6fe94e9b80bf32c1d357e4dc3b3d67c3 (commit)
       via  1b3750c9ffbec7a76b374023188c5c13882dc651 (commit)
       via  25bdeb97230abe28b9d5d7b13aa72549d93df87b (commit)
       via  45216b83b108a3c48cdd8b1aaf898ccd93188507 (commit)
       via  32eafceeb044696ac7db49bbe1d6329e0e3785c0 (commit)
       via  64d339fa3eb693908ea107a10a4c376ba3b51d7b (commit)
       via  1f43dd857829bd1134837c43a8ac5d653c0c68b1 (commit)
       via  123ce64a39849224481a67cc400a04c5022b639f (commit)
       via  52056c365c8936e34bc6b22a01bcacb556737914 (commit)
       via  83277602806099428b859b88745ad85836cd6ee4 (commit)
       via  a2c261dada5b47077c7930a2133bb000631fab66 (commit)
       via  dffa580d39b977746f6950b835b78b949a862c9d (commit)
       via  1924b28c8f715bc08752fb219d1fd8145ccfe84f (commit)
       via  0bec8ec8a130cb3af013b6097cf322938fce5671 (commit)
       via  fa6b0b65fb71b66e36f982f4f25e9751672d0834 (commit)
       via  661b2bcd6269fae643fd7d0c46649715716959eb (commit)
       via  f876a32276a73d89ab695e5954c9ea3816b38db9 (commit)
       via  2dc7f15f81ea7f460114482614d8ec5814c36fbf (commit)
       via  35cd1ab03e06910a2536bb473529511b8af20b94 (commit)
       via  49d953403625ca145081f9ccd914e70d49e51d49 (commit)
       via  c6a6693dc36615effca5e3363b81199362007c59 (commit)
       via  38cef39fbdfeb8176e4c755d12e43a450e868439 (commit)
       via  2f32f3483d18e9a89a8b5c13e022495c8681db04 (commit)
       via  3986815ae5e7e61c48f3ed979c32358710ef7e20 (commit)
       via  a2273675f29a0f85cb80e62b3742e82d63c365e9 (commit)
       via  64aac6153e1819738d9d80e156572aeb9bf07f97 (commit)
       via  040f1c94dfa1a53356d24e77afdbc0f2e5cfe91c (commit)
       via  ab710b74b2751f9a0d4eb5c5f950c6bb04865e79 (commit)
       via  10bcbe8343f6f91f3c8e338a4a53766f7ba26ef1 (commit)
       via  286c3627d978a7862b7f7eea366aa48ab930786d (commit)
       via  2f27ee200f6cc4c97d30084e34330efef2068ebf (commit)
       via  9c3beb526fe3bdfd6769abd0def5a1634074d353 (commit)
       via  ce2d2f5807822534756e52ac04bc272dd572d13d (commit)
       via  50a277569ad6277d95b7e3be83ddefc7b92661f4 (commit)
       via  aad9cd74e61cff985944c400c40fe3f85907a1e7 (commit)
       via  efb58099eebde5bebb4cbfba9ce4ba70a142fc56 (commit)
       via  941bcf698f1cfb498510a13f23d3c9d403b0435f (commit)
       via  9482cf9f326ce64f9ea100aa34680278bdd6018e (commit)
       via  d3b11ddc2506de37b8e6538be69237d6d2a60a4a (commit)
       via  c3a88cbf511aa0954dac271ce6bda9c6e4f3191c (commit)
       via  4ec57745d2106e955fea4442c9eccb2fce7246c4 (commit)
       via  cd85fc2c07a0e9d8c34b81e4523ee3d1ebd696e9 (commit)
       via  48ffdd5ac196771381c8dc9ab47cfad5f1929720 (commit)
       via  b12f667daa270a4e3c656d16f30620ca763f9578 (commit)
       via  27f5c1635d56c3f3cb6c5ef069c28db939eec2a1 (commit)
       via  b534e4a2b167185e59ca657369fee06e5effa6cc (commit)
       via  e678f2a0e5d223ddd5b2a6c6a8d14a8afa6d463d (commit)
       via  2b19cf9f9522dd0e8774031a54ce695e73fb72fe (commit)
       via  fb7f238945e33b07f1c80b0623315c1ecf86bca2 (commit)
       via  09559ebbb9f52c8b8a47e21d0a9c5720f3a2b7c6 (commit)
       via  99cd188a0cbd9143b690750e21abc8d8d5e6dbad (commit)
       via  2a493a9215f604c63ab7bc6f0e0956d10af8ef10 (commit)
       via  66d5cdb1f34d614e5ecf1da5ef6efcad3a5a51ab (commit)
       via  90bf8cae5f1ba931c7cc3be7864cad8ecca1ca5e (commit)
       via  33d63c6d42e824744305df3e720f8e9cbcc87d78 (commit)
       via  6022ddd1b07217b9b4f31e73493fc259bd19b696 (commit)
       via  28195ca73b4c86e0f17ce2db74f3d5669d344e23 (commit)
       via  717bee7044a2cd44678e13e116ec1be2a83cfe60 (commit)
       via  952599738f3c9f8901c895858865fc90b5e30c1d (commit)
       via  6132d8efbc522b71d0084160abaaa87031678bdc (commit)
      from  77d3ed33f525bd6859580954c6d7b9b36b394b98 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit ed101805b04c70bcbf1b070dceba2aee72170377
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri May 23 15:27:13 2014 -0400

    2751: Teaching run_test_server how to run the proxy server for testing.  Added -pid option to keep server.

diff --git a/sdk/python/run_test_server.py b/sdk/python/run_test_server.py
index dbb4ff0..7bf432a 100644
--- a/sdk/python/run_test_server.py
+++ b/sdk/python/run_test_server.py
@@ -125,9 +125,7 @@ def stop():
 
 def _start_keep(n):
     keep0 = tempfile.mkdtemp()
-    kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])
-    with open("tmp/keep{}.pid".format(n), 'w') as f:
-        f.write(str(kp0.pid))
+    kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n), "-pid={}".format("tmp/keep{}.pid".format(n))])
     with open("tmp/keep{}.volume".format(n), 'w') as f:
         f.write(keep0)
 
@@ -141,7 +139,7 @@ def run_keep():
     else:
         os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
 
-    subprocess.call(["go", "install", "keep"])
+    subprocess.call(["./go.sh", "install", "keep"])
 
     if not os.path.exists("tmp"):
         os.mkdir("tmp")
@@ -149,7 +147,6 @@ def run_keep():
     _start_keep(0)
     _start_keep(1)
 
-
     os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
     os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
 
@@ -172,6 +169,7 @@ def _stop_keep(n):
     if os.path.exists("tmp/keep{}.volume".format(n)):
         with open("tmp/keep{}.volume".format(n), 'r') as r:
             shutil.rmtree(r.read(), True)
+        os.unlink("tmp/keep{}.volume".format(n))
 
 def stop_keep():
     cwd = os.getcwd()
@@ -180,8 +178,41 @@ def stop_keep():
     _stop_keep(0)
     _stop_keep(1)
 
-    shutil.rmtree("tmp", True)
+    os.chdir(cwd)
+
+def run_keep_proxy(auth):
+    stop_keep_proxy()
+
+    cwd = os.getcwd()
+    os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+    if os.environ.get('GOPATH') == None:
+        os.environ["GOPATH"] = os.getcwd()
+    else:
+        os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
+
+    subprocess.call(["./go.sh", "install", "arvados.org/keepproxy"])
+
+    if not os.path.exists("tmp"):
+        os.mkdir("tmp")
+
+    os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+    os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
+    os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
 
+    kp0 = subprocess.Popen(["bin/keepproxy", "-pid=tmp/keepproxy.pid", "-listen=:{}".format(25101)])
+
+    authorize_with("admin")
+    api = arvados.api('v1', cache=False)
+    api.keep_services().create(body={"keep_service": {"service_host": "localhost",  "service_port": 25101, "service_type": "proxy"} }).execute()
+
+    arvados.config.settings()["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+
+    os.chdir(cwd)
+
+def stop_keep_proxy():
+    cwd = os.getcwd()
+    os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+    kill_server_pid("tmp/keepproxy.pid", 0)
     os.chdir(cwd)
 
 def fixture(fix):
@@ -217,5 +248,9 @@ if __name__ == "__main__":
         run_keep()
     elif args.action == 'stop_keep':
         stop_keep()
+    elif args.action == 'start_keep_proxy':
+        run_keep_proxy("admin")
+    elif args.action == 'stop_keep_proxy':
+        stop_keep_proxy()
     else:
         print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))
diff --git a/sdk/python/test_keep_client.py b/sdk/python/test_keep_client.py
index aa79b0d..bed1d4e 100644
--- a/sdk/python/test_keep_client.py
+++ b/sdk/python/test_keep_client.py
@@ -10,6 +10,7 @@ import run_test_server
 class KeepTestCase(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
+        super(KeepTestCase, cls).setUpClass()
         try:
             del os.environ['KEEP_LOCAL_STORE']
         except KeyError:
@@ -19,6 +20,7 @@ class KeepTestCase(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
+        super(KeepTestCase, cls).tearDownClass()
         run_test_server.stop()
         run_test_server.stop_keep()
 
@@ -65,3 +67,19 @@ class KeepTestCase(unittest.TestCase):
         self.assertEqual(arvados.Keep.get(blob_locator),
                          blob_str,
                          'wrong content from Keep.get(md5(<binarydata>))')
+
+    def test_KeepProxyTest(self):
+        try:
+            run_test_server.run_keep_proxy("admin")
+
+            baz_locator = arvados.Keep.put('baz')
+            self.assertEqual(baz_locator,
+                             '73feffa4b7f6bb68e44cf984c85f6e88+3',
+                             'wrong md5 hash from Keep.put("foo"): ' + baz_locator)
+            self.assertEqual(arvados.Keep.get(baz_locator),
+                             'baz',
+                             'wrong content from Keep.get(md5("baz"))')
+
+            self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
+        finally:
+            run_test_server.stop_keep_proxy()
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go
index 2d7e276..13da958 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go
@@ -10,7 +10,9 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/signal"
 	"sync"
+	"syscall"
 	"time"
 )
 
@@ -68,6 +70,25 @@ func main() {
 		log.Fatalf("Error setting up keep client %s", err.Error())
 	}
 
+	kc.Want_replicas = default_replicas
+
+	listener, err = net.Listen("tcp", listen)
+	if err != nil {
+		log.Fatalf("Could not listen on %v", listen)
+	}
+
+	go RefreshServicesList(&kc)
+
+	// Shut down the server gracefully (by closing the listener)
+	// if SIGTERM is received.
+	term := make(chan os.Signal, 1)
+	go func(sig <-chan os.Signal) {
+		s := <-sig
+		log.Println("caught signal:", s)
+		listener.Close()
+	}(term)
+	signal.Notify(term, syscall.SIGTERM)
+
 	if pidfile != "" {
 		f, err := os.Create(pidfile)
 		if err == nil {
@@ -78,19 +99,16 @@ func main() {
 		}
 	}
 
-	kc.Want_replicas = default_replicas
-
-	listener, err = net.Listen("tcp", listen)
-	if err != nil {
-		log.Fatalf("Could not listen on %v", listen)
-	}
-
-	go RefreshServicesList(&kc)
-
 	log.Printf("Arvados Keep proxy started listening on %v with server list %v", listener.Addr(), kc.ServiceRoots())
 
 	// Start listening for requests.
 	http.Serve(listener, MakeRESTRouter(!no_get, !no_put, &kc))
+
+	log.Println("shutting down")
+
+	if pidfile != "" {
+		os.Remove(pidfile)
+	}
 }
 
 type ApiTokenCache struct {
diff --git a/services/keep/src/keep/keep.go b/services/keep/src/keep/keep.go
index d65e445..3f1d5de 100644
--- a/services/keep/src/keep/keep.go
+++ b/services/keep/src/keep/keep.go
@@ -123,6 +123,7 @@ func main() {
 		permission_ttl_sec      int
 		serialize_io            bool
 		volumearg               string
+		pidfile                 string
 	)
 	flag.StringVar(
 		&data_manager_token_file,
@@ -168,6 +169,13 @@ func main() {
 			"e.g. -volumes=/var/keep1,/var/keep2. If empty or not "+
 			"supplied, Keep will scan mounted filesystems for volumes "+
 			"with a /keep top-level directory.")
+
+	flag.StringVar(
+		&pidfile,
+		"pid",
+		"",
+		"Path to write pid file")
+
 	flag.Parse()
 
 	// Look for local keep volumes.
@@ -255,11 +263,25 @@ func main() {
 	}(term)
 	signal.Notify(term, syscall.SIGTERM)
 
+	if pidfile != "" {
+		f, err := os.Create(pidfile)
+		if err == nil {
+			fmt.Fprint(f, os.Getpid())
+			f.Close()
+		} else {
+			log.Printf("Error writing pid file (%s): %s", pidfile, err.Error())
+		}
+	}
+
 	// Start listening for requests.
 	srv := &http.Server{Addr: listen}
 	srv.Serve(listener)
 
 	log.Println("shutting down")
+
+	if pidfile != "" {
+		os.Remove(pidfile)
+	}
 }
 
 // MakeRESTRouter

commit 27ffca811e6f43225fc82d582d1962eebbd1ab6e
Merge: 77d3ed3 b646cec
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri May 23 14:16:42 2014 -0400

    Merge branch '1885-keep-proxy' into 2751-python-sdk-keep-proxy-support

diff --cc sdk/python/arvados/keep.py
index d1e2255,e414d26..7618cff
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@@ -166,50 -147,60 +166,80 @@@ class KeepClient(object)
      def shuffled_service_roots(self, hash):
          if self.service_roots == None:
              self.lock.acquire()
 -            try:
 -                keep_disks = arvados.api().keep_disks().list().execute()['items']
 -                roots = (("http%s://%s:%d/" %
 -                          ('s' if f['service_ssl_flag'] else '',
 -                           f['service_host'],
 -                           f['service_port']))
 -                         for f in keep_disks)
 -                self.service_roots = sorted(set(roots))
 -                logging.debug(str(self.service_roots))
 -            finally:
 -                self.lock.release()
 +
 +            # Override normal keep disk lookup with an explict proxy
 +            # configuration.
 +            keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
 +            if keep_proxy_env != None:
 +                if keep_proxy_env[-1:] != '/':
 +                    keep_proxy_env += "/"
 +                self.service_roots = [keep_proxy_env]
 +                self.using_proxy = True
 +            else:
 +                try:
 +                    try:
 +                        keep_services = arvados.api().keep_services().accessible().execute()['items']
 +                    except:
 +                        keep_services = arvados.api().keep_disks().list().execute()['items']
 +
 +                    if len(keep_services) == 0:
 +                        raise arvados.errors.NoKeepServersError()
 +
 +                    if 'service_type' in keep_services[0] and keep_services[0]['service_type'] == 'proxy':
 +                        self.using_proxy = True
 +
 +                    roots = (("http%s://%s:%d/" %
 +                              ('s' if f['service_ssl_flag'] else '',
 +                               f['service_host'],
 +                               f['service_port']))
 +                             for f in keep_services)
 +                    self.service_roots = sorted(set(roots))
 +                    logging.debug(str(self.service_roots))
 +                finally:
 +                    self.lock.release()
  
+         # Build an ordering with which to query the Keep servers based on the
+         # contents of the hash.
+         # "hash" is a hex-encoded number at least 8 digits
+         # (32 bits) long
+ 
+         # seed used to calculate the next keep server from 'pool'
+         # to be added to 'pseq'
          seed = hash
+ 
+         # Keep servers still to be added to the ordering
          pool = self.service_roots[:]
+ 
+         # output probe sequence
          pseq = []
+ 
+         # iterate while there are servers left to be assigned
          while len(pool) > 0:
              if len(seed) < 8:
-                 if len(pseq) < len(hash) / 4: # first time around
+                 # ran out of digits in the seed
+                 if len(pseq) < len(hash) / 4:
+                     # the number of servers added to the probe sequence is less
+                     # than the number of 4-digit slices in 'hash' so refill the
+                     # seed with the last 4 digits and then append the contents
+                     # of 'hash'.
                      seed = hash[-4:] + hash
                  else:
+                     # refill the seed with the contents of 'hash'
                      seed += hash
+ 
+             # Take the next 8 digits (32 bytes) and interpret as an integer,
+             # then modulus with the size of the remaining pool to get the next
+             # selected server.
              probe = int(seed[0:8], 16) % len(pool)
+ 
+             print seed[0:8], int(seed[0:8], 16), len(pool), probe
+ 
+             # Append the selected server to the probe sequence and remove it
+             # from the pool.
              pseq += [pool[probe]]
              pool = pool[:probe] + pool[probe+1:]
+ 
+             # Remove the digits just used from the seed
              seed = seed[8:]
          logging.debug(str(pseq))
          return pseq

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list