[ARVADOS] updated: c4c44a33aabe63ad95d3bd02899390ae269cd1c8
Git user
git at public.curoverse.com
Tue May 24 15:59:12 EDT 2016
Summary of changes:
build/package-build-dockerfiles/Makefile | 8 +++---
build/package-build-dockerfiles/centos6/Dockerfile | 2 +-
build/package-build-dockerfiles/debian7/Dockerfile | 2 +-
build/package-build-dockerfiles/debian8/Dockerfile | 2 +-
.../ubuntu1204/Dockerfile | 2 +-
.../ubuntu1404/Dockerfile | 2 +-
build/run-tests.sh | 2 ++
sdk/go/arvados/keep_service.go | 4 +--
services/keep-balance/balance.go | 4 +--
services/keep-balance/balance_run_test.go | 31 +++++++++++++++++++++-
services/keep-balance/balance_test.go | 18 ++++++++++---
services/keep-balance/block_state.go | 7 ++---
services/keep-balance/duration.go | 23 ++++++++++++++++
services/keep-balance/keep_service.go | 21 ++++++++++++---
services/keep-balance/main.go | 25 +++++------------
services/keep-balance/time_me.go | 2 +-
16 files changed, 113 insertions(+), 42 deletions(-)
create mode 100644 services/keep-balance/duration.go
via c4c44a33aabe63ad95d3bd02899390ae269cd1c8 (commit)
via 530ab920f030bc3990a79506ede5b8ee9f21188d (commit)
via 0e011fe94001dcf93758cc740e8a9998c64a3558 (commit)
via 605ca9ec67ca4587f9385bc1753875c9e702a17e (commit)
via cf2d28d648724016e578b57f5c5fb92d3f2b2c19 (commit)
via 917ff758d05b39c88d75affabb00b25fc65846d9 (commit)
from 6717b69f19af5dd4489bcc97c3e328fc499fa311 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c4c44a33aabe63ad95d3bd02899390ae269cd1c8
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:57:39 2016 -0400
9162: Expand "unbalanced" test case a bit.
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index b74c5d2..13c8081 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -131,12 +131,24 @@ func (bal *balancerSuite) TestFixUnbalanced(c *check.C) {
desired: 2,
current: slots{2, 7},
shouldPull: slots{0, 1}})
- // unbalanced + excessive replication => pull + trash
+ // if only one of the pulls succeeds, we'll see this next:
bal.try(c, tester{
- desired: 1,
- current: slots{2, 7},
+ desired: 2,
+ current: slots{2, 1, 7},
shouldPull: slots{0},
shouldTrash: slots{7}})
+ // if both pulls succeed, we'll see this next:
+ bal.try(c, tester{
+ desired: 2,
+ current: slots{2, 0, 1, 7},
+ shouldTrash: slots{2, 7}})
+
+ // unbalanced + excessive replication => pull + trash
+ bal.try(c, tester{
+ desired: 2,
+ current: slots{2, 5, 7},
+ shouldPull: slots{0, 1},
+ shouldTrash: slots{7}})
}
func (bal *balancerSuite) TestIncreaseReplTimestampCollision(c *check.C) {
commit 530ab920f030bc3990a79506ede5b8ee9f21188d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:50:35 2016 -0400
9162: Add RunForever test case.
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 15e3c18..a0ba995 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -121,7 +121,7 @@ func (bal *Balancer) CheckSanityEarly(c *arvados.Client) error {
return fmt.Errorf("CurrentUser(): %v", err)
}
if !u.IsActive || !u.IsAdmin {
- return fmt.Errorf("Current user (%s) is not an active admin user", u.UUID)
+ return fmt.Errorf("current user (%s) is not an active admin user", u.UUID)
}
for _, srv := range bal.KeepServices {
if srv.ServiceType == "proxy" {
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 6e9b872..e6c2ea0 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -10,6 +10,7 @@ import (
"net/http/httptest"
"strings"
"sync"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -250,7 +251,7 @@ func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
trashReqs := s.stub.serveKeepstoreTrash()
pullReqs := s.stub.serveKeepstorePull()
err := (&Balancer{}).Run(s.config, opts)
- c.Check(err, check.ErrorMatches, "Current user .* is not .* admin user")
+ c.Check(err, check.ErrorMatches, "current user .* is not .* admin user")
c.Check(trashReqs.Count(), check.Equals, 0)
c.Check(pullReqs.Count(), check.Equals, 0)
}
@@ -301,3 +302,31 @@ func (s *runSuite) TestCommit(c *check.C) {
// in a poor rendezvous position
c.Check(stats.pulls, check.Equals, 2)
}
+
+func (s *runSuite) TestRunForever(c *check.C) {
+ opts := RunOptions{
+ CommitPulls: true,
+ CommitTrash: true,
+ Logger: s.logger(c),
+ Dumper: s.logger(c),
+ }
+ s.stub.serveCurrentUserAdmin()
+ s.stub.serveFooBarFileCollections()
+ s.stub.serveFourDiskKeepServices()
+ s.stub.serveKeepstoreIndexFoo4Bar1()
+ trashReqs := s.stub.serveKeepstoreTrash()
+ pullReqs := s.stub.serveKeepstorePull()
+
+ stop := make(chan interface{})
+ s.config.RunPeriod = duration(time.Millisecond)
+ go RunForever(s.config, opts, stop)
+
+ // Each run should send 4 pull + 4 trash lists. We should
+ // complete four runs in much less than 1 minute.
+ for t0 := time.Now(); trashReqs.Count() < 16 && time.Since(t0) < time.Minute; {
+ time.Sleep(time.Millisecond)
+ }
+ stop <- true
+ c.Check(trashReqs.Count() >= 16, check.Equals, true)
+ c.Check(pullReqs.Count(), check.Equals, trashReqs.Count())
+}
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 76f64e4..88b7eaf 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -87,7 +87,7 @@ func main() {
if *once {
err = (&Balancer{}).Run(config, runOptions)
} else {
- err = RunForever(config, runOptions)
+ err = RunForever(config, runOptions, nil)
}
if err != nil {
log.Fatal(err)
@@ -102,7 +102,9 @@ func mustReadJSON(dst interface{}, path string) {
}
}
-func RunForever(config Config, runOptions RunOptions) error {
+// RunForever runs forever, or (for testing purposes) until the given
+// stop channel is ready to receive.
+func RunForever(config Config, runOptions RunOptions, stop <-chan interface{}) error {
ticker := time.NewTicker(time.Duration(config.RunPeriod))
// The unbuffered channel here means we only hear SIGUSR1 if
@@ -126,6 +128,9 @@ func RunForever(config Config, runOptions RunOptions) error {
}
select {
+ case <-stop:
+ signal.Stop(sigUSR1)
+ return nil
case <-ticker.C:
case <-sigUSR1:
log.Print("received SIGUSR1, resetting timer")
commit 0e011fe94001dcf93758cc740e8a9998c64a3558
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:26:28 2016 -0400
9162: Improve error message
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 32a4b21..15e3c18 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -125,7 +125,7 @@ func (bal *Balancer) CheckSanityEarly(c *arvados.Client) error {
}
for _, srv := range bal.KeepServices {
if srv.ServiceType == "proxy" {
- return fmt.Errorf("Balancing a proxy service (%s) must be a config error", srv)
+ return fmt.Errorf("config error: %s: proxy servers cannot be balanced", srv)
}
}
return nil
commit 605ca9ec67ca4587f9385bc1753875c9e702a17e
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:22:00 2016 -0400
9162: Require Go >= 1.6
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 9216f82..3482886 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -20,10 +20,12 @@ ubuntu1404/generated: common-generated-all
test -d ubuntu1404/generated || mkdir ubuntu1404/generated
cp -rlt ubuntu1404/generated common-generated/*
-common-generated-all: common-generated/golang-amd64.tar.gz
+GOTARBALL=go1.6.2.linux-amd64.tar.gz
-common-generated/golang-amd64.tar.gz: common-generated
- wget -cqO common-generated/golang-amd64.tar.gz http://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
+common-generated-all: common-generated/$(GOTARBALL)
+
+common-generated/$(GOTARBALL): common-generated
+ wget -cqO common-generated/$(GOTARBALL) http://storage.googleapis.com/golang/$(GOTARBALL)
common-generated:
mkdir common-generated
diff --git a/build/package-build-dockerfiles/centos6/Dockerfile b/build/package-build-dockerfiles/centos6/Dockerfile
index 679f68e..397758d 100644
--- a/build/package-build-dockerfiles/centos6/Dockerfile
+++ b/build/package-build-dockerfiles/centos6/Dockerfile
@@ -5,7 +5,7 @@ MAINTAINER Brett Smith <brett at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-SCL postgresql-devel
# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
+ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install RVM
diff --git a/build/package-build-dockerfiles/debian7/Dockerfile b/build/package-build-dockerfiles/debian7/Dockerfile
index a62a9fe..122ffc0 100644
--- a/build/package-build-dockerfiles/debian7/Dockerfile
+++ b/build/package-build-dockerfiles/debian7/Dockerfile
@@ -12,7 +12,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler fpm
# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
+ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index f1f23f4..5262dce 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -12,7 +12,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler fpm
# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
+ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1204/Dockerfile b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
index 8f3997a..3eb4d16 100644
--- a/build/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
@@ -12,7 +12,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler fpm
# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
+ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index edd1207..8a2cfac 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -12,7 +12,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler fpm
# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
+ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
ENV WORKSPACE /arvados
diff --git a/build/run-tests.sh b/build/run-tests.sh
index bf1cee2..30a80f5 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -152,6 +152,8 @@ sanity_checks() {
echo -n 'go: '
go version \
|| fatal "No go binary. See http://golang.org/doc/install"
+ [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 6 ]] \
+ || fatal "Go >= 1.6 required. See http://golang.org/doc/install"
echo -n 'gcc: '
gcc --version | egrep ^gcc \
|| fatal "No gcc. Try: apt-get install build-essential"
commit cf2d28d648724016e578b57f5c5fb92d3f2b2c19
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:20:57 2016 -0400
9162: Improve comments
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index ff4a699..6a5c0f7 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -24,8 +24,8 @@ type KeepServiceList struct {
Limit int `json:"limit"`
}
-// IndexEntry is what keepstore's index response tells us about a
-// stored block.
+// KeepServiceIndexEntry is what a keep service's index response tells
+// us about a stored block.
type KeepServiceIndexEntry struct {
SizedDigest
Mtime int64
diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index e18674c..9d15fdd 100644
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -6,15 +6,16 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
)
-// Replica is a file on disk (or equivalent), as reported in a
-// keepstore index response.
+// Replica is a file on disk (or object in an S3 bucket, or blob in an
+// Azure storage container, etc.) as reported in a keepstore index
+// response.
type Replica struct {
*KeepService
Mtime int64
}
// BlockState indicates the number of desired replicas (according to
-// the collections we know about) and the replicas found on disk
+// the collections we know about) and the replicas actually stored
// (according to the keepstore indexes we know about).
type BlockState struct {
Replicas []Replica
diff --git a/services/keep-balance/keep_service.go b/services/keep-balance/keep_service.go
index 8ee58eb..6b7c431 100644
--- a/services/keep-balance/keep_service.go
+++ b/services/keep-balance/keep_service.go
@@ -40,24 +40,37 @@ func (srv *KeepService) CommitTrash(c *arvados.Client) error {
return srv.put(c, "trash", srv.ChangeSet.Trashes)
}
+// Perform a PUT request at path, with data (as JSON) in the request
+// body.
func (srv *KeepService) put(c *arvados.Client, path string, data interface{}) error {
- errC := make(chan error, 1)
+ // We'll start a goroutine to do the JSON encoding, so we can
+ // stream it to the http client through a Pipe, rather than
+ // keeping the entire encoded version in memory.
jsonR, jsonW := io.Pipe()
+
+ // errC communicates any encoding errors back to our main
+ // goroutine.
+ errC := make(chan error, 1)
+
go func() {
enc := json.NewEncoder(jsonW)
errC <- enc.Encode(data)
jsonW.Close()
}()
+
url := srv.URLBase() + "/" + path
req, err := http.NewRequest("PUT", url, ioutil.NopCloser(jsonR))
if err != nil {
return fmt.Errorf("building request for %s: %v", url, err)
}
err = c.DoAndDecode(nil, req)
+
+ // If there was an error encoding the request body, report
+ // that instead of the response: obviously we won't get a
+ // useful response if our request wasn't properly encoded.
if encErr := <-errC; encErr != nil {
return fmt.Errorf("encoding data for %s: %v", url, encErr)
- } else if err != nil {
- return err
}
- return nil
+
+ return err
}
diff --git a/services/keep-balance/time_me.go b/services/keep-balance/time_me.go
index bbc5252..e5f16b7 100644
--- a/services/keep-balance/time_me.go
+++ b/services/keep-balance/time_me.go
@@ -9,6 +9,6 @@ func timeMe(logger *log.Logger, label string) func() {
t0 := time.Now()
logger.Printf("%s: start", label)
return func() {
- logger.Printf("%s: %v", label, time.Since(t0))
+ logger.Printf("%s: took %v", label, time.Since(t0))
}
}
commit 917ff758d05b39c88d75affabb00b25fc65846d9
Author: Tom Clegg <tom at curoverse.com>
Date: Tue May 24 15:20:21 2016 -0400
9162: Move duration type out to its own file
diff --git a/services/keep-balance/duration.go b/services/keep-balance/duration.go
new file mode 100644
index 0000000..0bf0fb5
--- /dev/null
+++ b/services/keep-balance/duration.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+ "fmt"
+ "time"
+)
+
+// duration is time.Duration but can be given in a config file as
+// "600s" rather than a number of nanoseconds.
+type duration time.Duration
+
+func (d *duration) UnmarshalJSON(data []byte) error {
+ if data[0] == '"' {
+ dur, err := time.ParseDuration(string(data[1 : len(data)-1]))
+ *d = duration(dur)
+ return err
+ }
+ return fmt.Errorf("duration must be given as a string like \"600s\" or \"1h30m\"")
+}
+
+func (d duration) String() string {
+ return time.Duration(d).String()
+}
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 9404398..76f64e4 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"flag"
- "fmt"
"io/ioutil"
"log"
"os"
@@ -138,18 +137,3 @@ func RunForever(config Config, runOptions RunOptions) error {
log.Print("waking up")
}
}
-
-type duration time.Duration
-
-func (d *duration) UnmarshalJSON(data []byte) error {
- if data[0] == '"' {
- dur, err := time.ParseDuration(string(data[1 : len(data)-1]))
- *d = duration(dur)
- return err
- }
- return fmt.Errorf("duration must be given as a string like \"600s\" or \"1h30m\"")
-}
-
-func (d duration) String() string {
- return time.Duration(d).String()
-}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list