[ARVADOS] updated: c8b37d59c47da700a983569c7689b178da210918
git at public.curoverse.com
git at public.curoverse.com
Wed Oct 14 12:14:04 EDT 2015
Summary of changes:
apps/workbench/app/helpers/version_helper.rb | 28 +----
.../views/application/_report_issue_popup.html.erb | 15 +--
apps/workbench/config/application.default.yml | 48 +-------
apps/workbench/config/application.rb | 1 +
apps/workbench/lib/app_version.rb | 53 +++++++++
sdk/cwl/setup.py | 2 +-
sdk/go/keepclient/perms.go | 99 ++++++++++++++++
.../keepstore => sdk/go/keepclient}/perms_test.go | 57 +++------
sdk/python/arvados/commands/put.py | 4 +-
sdk/python/tests/test_arv_put.py | 17 ++-
.../controllers/arvados/v1/schema_controller.rb | 2 +-
services/api/config/application.default.yml | 48 +-------
services/api/config/initializers/app_version.rb | 1 +
services/api/lib/app_version.rb | 52 +++++++++
services/api/lib/tasks/config_check.rake | 3 +-
.../arvados/v1/schema_controller_test.rb | 22 ++++
services/api/test/unit/app_version_test.rb | 43 +++++++
services/keepstore/azure_blob_volume.go | 52 +++++++--
services/keepstore/azure_blob_volume_test.go | 129 ++++++++++++++++++++-
services/keepstore/keepstore_test.go | 2 +-
services/keepstore/perms.go | 100 ++--------------
services/keepstore/perms_test.go | 107 ++++-------------
services/keepstore/volume_generic_test.go | 32 +++++
.../arvnodeman/computenode/dispatch/__init__.py | 13 ++-
.../arvnodeman/computenode/dispatch/slurm.py | 15 ++-
.../tests/test_computenode_dispatch_slurm.py | 13 ++-
26 files changed, 597 insertions(+), 361 deletions(-)
create mode 100644 apps/workbench/lib/app_version.rb
create mode 100644 sdk/go/keepclient/perms.go
copy {services/keepstore => sdk/go/keepclient}/perms_test.go (62%)
create mode 100644 services/api/config/initializers/app_version.rb
create mode 100644 services/api/lib/app_version.rb
create mode 100644 services/api/test/unit/app_version_test.rb
discards 7a6a9670682aa6532457b9e0bba7e21a3e4cfb3d (commit)
discards ee5d4be791ff85a5eb446d3e0dbaaae87f69e868 (commit)
discards 3a7dea8c7a0f0cd98d258a3815b0f012a9ca08a0 (commit)
via c8b37d59c47da700a983569c7689b178da210918 (commit)
via 40c9b26a39c773e806e0a1430774f1787820376f (commit)
via fb0f0a8eade18142e28e2691d656f262b0cdc01a (commit)
via 97ece5619eb80acce46ec5c7b521c08ecaaf0e86 (commit)
via 5b8507cca827ef8de5d2bdc74dd74659f2c67e39 (commit)
via cf85808b2e97062450565c93ace5025bbd6db365 (commit)
via e028e5daadb83ef695b7e0b1ef6bd45eef4cfc29 (commit)
via 634d2e25d36b20961d660df0e69628e60099d893 (commit)
via 14a47e56d3afc1d3f3d7be3bf5ad9d9f6d60b6a2 (commit)
via e55f6e9fcb652a2b1505364b34d9d48e79adaeaf (commit)
via e9a7e33f90106db0d36d8d45f1929483662c7168 (commit)
via f814b4b5d0c6168c95d2eda87a45c87df07c72c9 (commit)
via 5e2bf20f75a532625719f0f779d1e26f9d4466c4 (commit)
via 5b80941723d6be1fd22589026635a43b33e6cf20 (commit)
via 5089c3bb94101d32028d0fc06283f43fd70af53c (commit)
via 38576187f1f1198afe0a8b6a62f7bf1388bd339d (commit)
via 20c5b73d598463a7dfb4ce711993480d56e23838 (commit)
via 3ee62426a5ebfc056f58cd0655ba1e9f7ed0a722 (commit)
via b8afbe378247ba5cb81b067f37a35350726f038b (commit)
via 8ff3305a11a7effb73ad9cfe2c55ba572e56f6b0 (commit)
via a084ddc8af3f05ffe35392a86cf2400c5bbe2176 (commit)
via 1d73ebb1fb2f2f50144de4f74b3aa77616a5d4f2 (commit)
via 15e0b0671b04a81a8e8fa289002655d28d7efb50 (commit)
via b598910bb715d9f4803a2ca1c0f84492c5761ed7 (commit)
via 966b17d66eafb2d1707c5b5119ba0c87fcdae44a (commit)
via 8a3fb7961809da1a2aeca4bfd26c012f794379e3 (commit)
via e88cbab376ec5e26266ed021193b9d387b14dfa7 (commit)
via 7c97dd88e541a0245272b8e93a33e4d2fe4e32cd (commit)
via ba019ddb35781404a62924374ec3c0046323ead5 (commit)
via 094b60e60cc4e3b20a015daecda8bcafdc3bdb43 (commit)
via 2a94b125b93a3aba204f55c37ecdc2876d81d642 (commit)
via ccdb53bf1ea5067edbbbbe71ecb89425bafa86b1 (commit)
via 9c95a4c2dcd650627d524513e1e18596c8533ac0 (commit)
via 8325a8860ac6c758e6c8005b4e31fa95a9412992 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (7a6a9670682aa6532457b9e0bba7e21a3e4cfb3d)
\
N -- N -- N (c8b37d59c47da700a983569c7689b178da210918)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c8b37d59c47da700a983569c7689b178da210918
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 2 19:54:36 2015 -0400
7410: Add keepexercise
diff --git a/tools/keepexercise/.gitignore b/tools/keepexercise/.gitignore
new file mode 100644
index 0000000..f8f29eb
--- /dev/null
+++ b/tools/keepexercise/.gitignore
@@ -0,0 +1 @@
+keepexercise
diff --git a/tools/keepexercise/keepexercise.go b/tools/keepexercise/keepexercise.go
new file mode 100644
index 0000000..fda0d5f
--- /dev/null
+++ b/tools/keepexercise/keepexercise.go
@@ -0,0 +1,154 @@
+// Testing tool for Keep services.
+//
+// keepexercise helps measure throughput and test reliability under
+// various usage patterns.
+//
+// By default, it reads and writes blocks containing 2^26 NUL
+// bytes. This generates network traffic without consuming much disk
+// space.
+//
+// For a more realistic test, enable -vary-request. Warning: this will
+// fill your storage volumes with random data if you leave it running,
+// which can cost you money or leave you with too little room for
+// useful data.
+//
+package main
+
+import (
+ "crypto/rand"
+ "encoding/binary"
+ "flag"
+ "io"
+ "io/ioutil"
+ "log"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+)
+
+var BlockSize = flag.Int("block-size", keepclient.BLOCKSIZE, "bytes per read/write op")
+var ReadThreads = flag.Int("rthreads", 1, "number of concurrent readers")
+var WriteThreads = flag.Int("wthreads", 1, "number of concurrent writers")
+var VaryRequest = flag.Bool("vary-request", false, "vary the data for each request: consumes disk space, exercises write behavior")
+var VaryThread = flag.Bool("vary-thread", false, "use -wthreads different data blocks")
+var Replicas = flag.Int("replicas", 1, "replication level for writing")
+var StatsInterval = flag.Duration("stats-interval", time.Second, "time interval between IO stats reports, or 0 to disable")
+
+func main() {
+ flag.Parse()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ log.Fatal(err)
+ }
+ kc, err := keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ log.Fatal(err)
+ }
+ kc.Want_replicas = *Replicas
+ kc.Client.Timeout = 10 * time.Minute
+
+ nextBuf := make(chan []byte, *WriteThreads)
+ nextLocator := make(chan string, *ReadThreads+*WriteThreads)
+
+ go countBeans(nextLocator)
+ for i := 0; i < *WriteThreads; i++ {
+ go makeBufs(nextBuf, i)
+ go doWrites(kc, nextBuf, nextLocator)
+ }
+ for i := 0; i < *ReadThreads; i++ {
+ go doReads(kc, nextLocator)
+ }
+ <-make(chan struct{})
+}
+
+// Send 1234 to bytesInChan when we receive 1234 bytes from keepstore.
+var bytesInChan = make(chan uint64)
+var bytesOutChan = make(chan uint64)
+
+// Send struct{}{} to errorsChan when an error happens.
+var errorsChan = make(chan struct{})
+
+func countBeans(nextLocator chan string) {
+ t0 := time.Now()
+ var tickChan <-chan time.Time
+ if *StatsInterval > 0 {
+ tickChan = time.NewTicker(*StatsInterval).C
+ }
+ var bytesIn uint64
+ var bytesOut uint64
+ var errors uint64
+ for {
+ select {
+ case <-tickChan:
+ elapsed := time.Since(t0)
+ log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
+ elapsed,
+ bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
+ bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
+ errors,
+ )
+ case i := <-bytesInChan:
+ bytesIn += i
+ case o := <-bytesOutChan:
+ bytesOut += o
+ case <-errorsChan:
+ errors++
+ }
+ }
+}
+
+func makeBufs(nextBuf chan []byte, threadID int) {
+ buf := make([]byte, *BlockSize)
+ if *VaryThread {
+ binary.PutVarint(buf, int64(threadID))
+ }
+ for {
+ if *VaryRequest {
+ if _, err := io.ReadFull(rand.Reader, buf); err != nil {
+ log.Fatal(err)
+ }
+ }
+ nextBuf <- buf
+ }
+}
+
+func doWrites(kc *keepclient.KeepClient, nextBuf chan []byte, nextLocator chan string) {
+ for buf := range nextBuf {
+ locator, _, err := kc.PutB(buf)
+ if err != nil {
+ log.Print(err)
+ errorsChan <- struct{}{}
+ continue
+ }
+ bytesOutChan <- uint64(len(buf))
+ for cap(nextLocator) > len(nextLocator)+*WriteThreads {
+ // Give the readers something to do, unless
+ // they have lots queued up already.
+ nextLocator <- locator
+ }
+ }
+}
+
+func doReads(kc *keepclient.KeepClient, nextLocator chan string) {
+ for locator := range nextLocator {
+ rdr, size, url, err := kc.Get(locator)
+ if err != nil {
+ log.Print(err)
+ errorsChan <- struct{}{}
+ continue
+ }
+ n, err := io.Copy(ioutil.Discard, rdr)
+ rdr.Close()
+ if n != size || err != nil {
+ log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
+ errorsChan <- struct{}{}
+ continue
+ // Note we don't count the bytes received in
+ // partial/corrupt responses: we are measuring
+ // throughput, not resource consumption.
+ }
+ bytesInChan <- uint64(n)
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list