[ARVADOS] updated: 069704ebbd82ff84106e228b158be0fd78fb5c89
git at public.curoverse.com
git at public.curoverse.com
Wed Dec 9 16:01:23 EST 2015
Summary of changes:
sdk/go/keepclient/discover.go | 4 ++--
sdk/go/keepclient/keepclient.go | 3 +--
sdk/go/keepclient/keepclient_test.go | 37 +-----------------------------------
sdk/go/keepclient/support.go | 15 +++++++--------
services/keepstore/s3_volume.go | 20 ++++++++++++++-----
tools/keep-exercise/keep-exercise.go | 25 ++++++++++++++++++++++++
6 files changed, 51 insertions(+), 53 deletions(-)
discards ce16f83188ba880b4c09723937be29552b4fc2e9 (commit)
via 069704ebbd82ff84106e228b158be0fd78fb5c89 (commit)
via f13f1e9691b3155585a812a77ecee691707a1246 (commit)
via 303b0667bf53a97ac2c248908d654d59401947ca (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (ce16f83188ba880b4c09723937be29552b4fc2e9)
\
N -- N -- N (069704ebbd82ff84106e228b158be0fd78fb5c89)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 069704ebbd82ff84106e228b158be0fd78fb5c89
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Dec 9 16:00:34 2015 -0500
7393: Add -uuid and -url options, fix memory sharing in -vary-request.
diff --git a/tools/keep-exercise/keep-exercise.go b/tools/keep-exercise/keep-exercise.go
index a94c01e..9dc8f94 100644
--- a/tools/keep-exercise/keep-exercise.go
+++ b/tools/keep-exercise/keep-exercise.go
@@ -36,6 +36,8 @@ var (
VaryThread = flag.Bool("vary-thread", false, "use -wthreads different data blocks")
Replicas = flag.Int("replicas", 1, "replication level for writing")
StatsInterval = flag.Duration("stats-interval", time.Second, "time interval between IO stats reports, or 0 to disable")
+ ServiceURL = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
+ ServiceUUID = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
)
func main() {
@@ -52,6 +54,8 @@ func main() {
kc.Want_replicas = *Replicas
kc.Client.Timeout = 10 * time.Minute
+ overrideServices(kc)
+
nextBuf := make(chan []byte, *WriteThreads)
nextLocator := make(chan string, *ReadThreads+*WriteThreads)
@@ -109,6 +113,7 @@ func makeBufs(nextBuf chan []byte, threadID int) {
}
for {
if *VaryRequest {
+ buf = make([]byte, *BlockSize)
if _, err := io.ReadFull(rand.Reader, buf); err != nil {
log.Fatal(err)
}
@@ -155,3 +160,23 @@ func doReads(kc *keepclient.KeepClient, nextLocator chan string) {
bytesInChan <- uint64(n)
}
}
+
+func overrideServices(kc *keepclient.KeepClient) {
+ roots := make(map[string]string)
+ if *ServiceURL != "" {
+ roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
+ } else if *ServiceUUID != "" {
+ for uuid, url := range kc.GatewayRoots() {
+ if uuid == *ServiceUUID {
+ roots[uuid] = url
+ break
+ }
+ }
+ if len(roots) == 0 {
+ log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
+ }
+ } else {
+ return
+ }
+ kc.SetServiceRoots(roots, roots, roots)
+}
commit f13f1e9691b3155585a812a77ecee691707a1246
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Dec 9 15:59:54 2015 -0500
7393: Quiet excessive debug printfs.
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 099c56f..f039c21 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -57,14 +57,14 @@ func (kc *KeepClient) RefreshServices(interval, errInterval time.Duration) {
timer.Reset(interval)
if err := kc.DiscoverKeepServers(); err != nil {
- log.Println("Error retrieving services list: %v (retrying in %v)", err, errInterval)
+ log.Printf("WARNING: Error retrieving services list: %v (retrying in %v)", err, errInterval)
timer.Reset(errInterval)
continue
}
newRoots := []map[string]string{kc.LocalRoots(), kc.GatewayRoots()}
if !reflect.DeepEqual(previousRoots, newRoots) {
- log.Printf("Updated services list: locals %v gateways %v", newRoots[0], newRoots[1])
+ DebugPrintf("DEBUG: Updated services list: locals %v gateways %v", newRoots[0], newRoots[1])
previousRoots = newRoots
}
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index f15a6b2..26aa717 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -11,7 +11,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"net/http"
"regexp"
"strconv"
@@ -233,7 +232,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
}
serversToTry = retryList
}
- log.Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+ DebugPrintf("DEBUG: %s %s failed: %v", method, locator, errs)
var err error
if count404 == numServers {
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 87b9b1d..4ba1d7c 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -155,13 +155,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
status := <-upload_status
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
-
- log.Printf("TestUploadToStubKeepServer done")
}
func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
- log.Printf("TestUploadToStubKeepServerBufferReader")
-
st := StubPutHandler{
c,
"acbd18db4cc2f85cedef654fccc4a4d8",
@@ -188,8 +184,6 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
status := <-upload_status
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
-
- log.Printf("TestUploadToStubKeepServerBufferReader done")
}
type FailHandler struct {
@@ -227,8 +221,6 @@ func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
}
func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
- log.Printf("TestFailedUploadToStubKeepServer")
-
st := FailHandler{
make(chan string)}
@@ -249,7 +241,6 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
c.Check(status.statusCode, Equals, 500)
})
- log.Printf("TestFailedUploadToStubKeepServer done")
}
type KeepServer struct {
@@ -268,8 +259,6 @@ func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
}
func (s *StandaloneSuite) TestPutB(c *C) {
- log.Printf("TestPutB")
-
hash := Md5String("foo")
st := StubPutHandler{
@@ -308,13 +297,9 @@ func (s *StandaloneSuite) TestPutB(c *C) {
(s1 == shuff[1] && s2 == shuff[0]),
Equals,
true)
-
- log.Printf("TestPutB done")
}
func (s *StandaloneSuite) TestPutHR(c *C) {
- log.Printf("TestPutHR")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
@@ -352,7 +337,6 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
kc.PutHR(hash, reader, 3)
shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
- log.Print(shuff)
s1 := <-st.handled
s2 := <-st.handled
@@ -361,13 +345,9 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
(s1 == shuff[1] && s2 == shuff[0]),
Equals,
true)
-
- log.Printf("TestPutHR done")
}
func (s *StandaloneSuite) TestPutWithFail(c *C) {
- log.Printf("TestPutWithFail")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
@@ -406,6 +386,7 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
shuff := NewRootSorter(
kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
+ c.Logf("%+v", shuff)
phash, replicas, err := kc.PutB([]byte("foo"))
@@ -425,8 +406,6 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
}
func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
- log.Printf("TestPutWithTooManyFail")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
@@ -469,8 +448,6 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
c.Check(err, Equals, InsufficientReplicasError)
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, ks1[0].url)
-
- log.Printf("TestPutWithTooManyFail done")
}
type StubGetHandler struct {
@@ -490,8 +467,6 @@ func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
}
func (s *StandaloneSuite) TestGet(c *C) {
- log.Printf("TestGet")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubGetHandler{
@@ -518,8 +493,6 @@ func (s *StandaloneSuite) TestGet(c *C) {
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
c.Check(content, DeepEquals, []byte("foo"))
-
- log.Printf("TestGet done")
}
func (s *StandaloneSuite) TestGet404(c *C) {
@@ -887,8 +860,6 @@ func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reque
}
func (s *StandaloneSuite) TestPutProxy(c *C) {
- log.Printf("TestPutProxy")
-
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
@@ -914,13 +885,9 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
c.Check(err, Equals, nil)
c.Check(replicas, Equals, 2)
-
- log.Printf("TestPutProxy done")
}
func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
- log.Printf("TestPutProxy")
-
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
@@ -945,8 +912,6 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
c.Check(err, Equals, InsufficientReplicasError)
c.Check(replicas, Equals, 2)
-
- log.Printf("TestPutProxy done")
}
func (s *StandaloneSuite) TestMakeLocator(c *C) {
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index b904b09..b12f512 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -7,7 +7,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"math/rand"
"net"
"net/http"
@@ -101,7 +100,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
- log.Printf("[%08x] Error creating request PUT %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%08x] Error creating request PUT %v error: %v", requestID, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
body.Close()
return
@@ -126,7 +125,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
- log.Printf("[%08x] Upload failed %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%08x] Upload failed %v error: %v", requestID, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
@@ -142,13 +141,13 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
- log.Printf("[%08x] Upload %v error: %v response: %v", requestID, url, err2.Error(), response)
+ DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, err2.Error(), response)
upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
- log.Printf("[%08x] Upload %v success", requestID, url)
+ DebugPrintf("DEBUG: [%08x] Upload %v success", requestID, url)
upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
- log.Printf("[%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
+ DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
@@ -205,7 +204,7 @@ func (this *KeepClient) putReplicas(
for active*replicasPerThread < remaining_replicas {
// Start some upload requests
if next_server < len(sv) {
- log.Printf("[%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
+ DebugPrintf("DEBUG: [%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestID)
next_server += 1
active += 1
@@ -217,7 +216,7 @@ func (this *KeepClient) putReplicas(
}
}
}
- log.Printf("[%08x] Replicas remaining to write: %v active uploads: %v",
+ DebugPrintf("DEBUG: [%08x] Replicas remaining to write: %v active uploads: %v",
requestID, remaining_replicas, active)
// Now wait for something to happen.
commit 303b0667bf53a97ac2c248908d654d59401947ca
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Dec 9 12:43:50 2015 -0500
7393: Add S3 volume type.
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 3256ec2..47b75b3 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -24,3 +24,12 @@ const PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a751
`./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:b\141z\040w\141z` + "\n" +
"./foo acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero 0:3:foo\n" +
". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo/zero 0:3:foo/foo\n"
+
+// An MD5 collision.
+var (
+ MD5CollisionData = [][]byte{
+ []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9epO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\\\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef"),
+ []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9etO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\xdc\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef"),
+ }
+ MD5CollisionMD5 = "cee9a457e790cf20d4bdaa6d69f01e41"
+)
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index e9fda2a..0f98e6e 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -189,7 +189,7 @@ func (v *AzureBlobVolume) Compare(loc string, expect []byte) error {
return compareReaderWithBuf(rdr, expect, loc[:32])
}
-// Put sotres a Keep block as a block blob in the container.
+// Put stores a Keep block as a block blob in the container.
func (v *AzureBlobVolume) Put(loc string, block []byte) error {
if v.readonly {
return MethodDisabledError
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index a240c23..b8bf5cb 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -292,10 +292,10 @@ type TestableAzureBlobVolume struct {
*AzureBlobVolume
azHandler *azStubHandler
azStub *httptest.Server
- t *testing.T
+ t TB
}
-func NewTestableAzureBlobVolume(t *testing.T, readonly bool, replication int) *TestableAzureBlobVolume {
+func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableAzureBlobVolume {
azHandler := newAzStubHandler()
azStub := httptest.NewServer(azHandler)
@@ -341,7 +341,7 @@ func TestAzureBlobVolumeWithGeneric(t *testing.T) {
}
azureWriteRaceInterval = time.Millisecond
azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
})
}
@@ -355,7 +355,7 @@ func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
}
azureWriteRaceInterval = time.Millisecond
azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
return NewTestableAzureBlobVolume(t, true, azureStorageReplication)
})
}
diff --git a/services/keepstore/bufferpool_test.go b/services/keepstore/bufferpool_test.go
index 8726a19..7b51b64 100644
--- a/services/keepstore/bufferpool_test.go
+++ b/services/keepstore/bufferpool_test.go
@@ -2,15 +2,9 @@ package main
import (
. "gopkg.in/check.v1"
- "testing"
"time"
)
-// Gocheck boilerplate
-func TestBufferPool(t *testing.T) {
- TestingT(t)
-}
-
var _ = Suite(&BufferPoolSuite{})
type BufferPoolSuite struct{}
diff --git a/services/keepstore/collision_test.go b/services/keepstore/collision_test.go
index 379dadd..d9b7e61 100644
--- a/services/keepstore/collision_test.go
+++ b/services/keepstore/collision_test.go
@@ -2,17 +2,11 @@ package main
import (
"bytes"
- "testing"
"testing/iotest"
check "gopkg.in/check.v1"
)
-// Gocheck boilerplate
-func Test(t *testing.T) {
- check.TestingT(t)
-}
-
var _ = check.Suite(&CollisionSuite{})
type CollisionSuite struct{}
diff --git a/services/keepstore/gocheck_test.go b/services/keepstore/gocheck_test.go
new file mode 100644
index 0000000..133ed6e
--- /dev/null
+++ b/services/keepstore/gocheck_test.go
@@ -0,0 +1,10 @@
+package main
+
+import (
+ "gopkg.in/check.v1"
+ "testing"
+)
+
+func TestGocheck(t *testing.T) {
+ check.TestingT(t)
+}
diff --git a/services/keepstore/handlers_with_generic_volume_test.go b/services/keepstore/handlers_with_generic_volume_test.go
index 9f31f5f..c5349d3 100644
--- a/services/keepstore/handlers_with_generic_volume_test.go
+++ b/services/keepstore/handlers_with_generic_volume_test.go
@@ -2,19 +2,18 @@ package main
import (
"bytes"
- "testing"
)
// A TestableVolumeManagerFactory creates a volume manager with at least two TestableVolume instances.
// The factory function, and the TestableVolume instances it returns, can use "t" to write
// logs, fail the current test, etc.
-type TestableVolumeManagerFactory func(t *testing.T) (*RRVolumeManager, []TestableVolume)
+type TestableVolumeManagerFactory func(t TB) (*RRVolumeManager, []TestableVolume)
// DoHandlersWithGenericVolumeTests runs a set of handler tests with a
// Volume Manager comprised of TestableVolume instances.
// It calls factory to create a volume manager with TestableVolume
// instances for each test case, to avoid leaking state between tests.
-func DoHandlersWithGenericVolumeTests(t *testing.T, factory TestableVolumeManagerFactory) {
+func DoHandlersWithGenericVolumeTests(t TB, factory TestableVolumeManagerFactory) {
testGetBlock(t, factory, TestHash, TestBlock)
testGetBlock(t, factory, EmptyHash, EmptyBlock)
testPutRawBadDataGetBlock(t, factory, TestHash, TestBlock, []byte("baddata"))
@@ -26,7 +25,7 @@ func DoHandlersWithGenericVolumeTests(t *testing.T, factory TestableVolumeManage
}
// Setup RRVolumeManager with TestableVolumes
-func setupHandlersWithGenericVolumeTest(t *testing.T, factory TestableVolumeManagerFactory) []TestableVolume {
+func setupHandlersWithGenericVolumeTest(t TB, factory TestableVolumeManagerFactory) []TestableVolume {
vm, testableVolumes := factory(t)
KeepVM = vm
@@ -39,7 +38,7 @@ func setupHandlersWithGenericVolumeTest(t *testing.T, factory TestableVolumeMana
}
// Put a block using PutRaw in just one volume and Get it using GetBlock
-func testGetBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
+func testGetBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
// Put testBlock in one volume
@@ -56,7 +55,7 @@ func testGetBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash s
}
// Put a bad block using PutRaw and get it.
-func testPutRawBadDataGetBlock(t *testing.T, factory TestableVolumeManagerFactory,
+func testPutRawBadDataGetBlock(t TB, factory TestableVolumeManagerFactory,
testHash string, testBlock []byte, badData []byte) {
testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
@@ -72,7 +71,7 @@ func testPutRawBadDataGetBlock(t *testing.T, factory TestableVolumeManagerFactor
}
// Invoke PutBlock twice to ensure CompareAndTouch path is tested.
-func testPutBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
+func testPutBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
setupHandlersWithGenericVolumeTest(t, factory)
// PutBlock
@@ -95,7 +94,7 @@ func testPutBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash s
}
// Put a bad block using PutRaw, overwrite it using PutBlock and get it.
-func testPutBlockCorrupt(t *testing.T, factory TestableVolumeManagerFactory,
+func testPutBlockCorrupt(t TB, factory TestableVolumeManagerFactory,
testHash string, testBlock []byte, badData []byte) {
testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
diff --git a/services/keepstore/keepstore_test.go b/services/keepstore/keepstore_test.go
index 8a004b7..2a1c3d2 100644
--- a/services/keepstore/keepstore_test.go
+++ b/services/keepstore/keepstore_test.go
@@ -10,6 +10,8 @@ import (
"sort"
"strings"
"testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
)
var TestBlock = []byte("The quick brown fox jumps over the lazy dog.")
@@ -229,9 +231,9 @@ func TestPutBlockCollision(t *testing.T) {
defer teardown()
// These blocks both hash to the MD5 digest cee9a457e790cf20d4bdaa6d69f01e41.
- var b1 = []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9epO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\\\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef")
- var b2 = []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9etO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\xdc\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef")
- var locator = "cee9a457e790cf20d4bdaa6d69f01e41"
+ b1 := arvadostest.MD5CollisionData[0]
+ b2 := arvadostest.MD5CollisionData[1]
+ locator := arvadostest.MD5CollisionMD5
// Prepare two test Keep volumes.
KeepVM = MakeTestVolumeManager(2)
diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go
index c6a4195..5076b85 100644
--- a/services/keepstore/pull_worker_test.go
+++ b/services/keepstore/pull_worker_test.go
@@ -8,20 +8,13 @@ import (
. "gopkg.in/check.v1"
"io"
"net/http"
- "testing"
"time"
)
-type PullWorkerTestSuite struct{}
-
-// Gocheck boilerplate
-func TestPullWorker(t *testing.T) {
- TestingT(t)
-}
-
-// Gocheck boilerplate
var _ = Suite(&PullWorkerTestSuite{})
+type PullWorkerTestSuite struct{}
+
var testPullLists map[string]string
var readContent string
var readError error
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
new file mode 100644
index 0000000..572ee46
--- /dev/null
+++ b/services/keepstore/s3_volume.go
@@ -0,0 +1,312 @@
+package main
+
+import (
+ "encoding/base64"
+ "encoding/hex"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "regexp"
+ "time"
+
+ "github.com/AdRoll/goamz/aws"
+ "github.com/AdRoll/goamz/s3"
+)
+
+var (
+ ErrS3DeleteNotAvailable = fmt.Errorf("delete without -s3-unsafe-delete is not implemented")
+
+ s3AccessKeyFile string
+ s3SecretKeyFile string
+ s3RegionName string
+ s3Endpoint string
+ s3Replication int
+ s3UnsafeDelete bool
+
+ s3ACL = s3.Private
+)
+
+const (
+ maxClockSkew = 600 * time.Second
+ nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
+)
+
+type s3VolumeAdder struct {
+ *volumeSet
+}
+
+func (s *s3VolumeAdder) Set(bucketName string) error {
+ if bucketName == "" {
+ return fmt.Errorf("no container name given")
+ }
+ if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
+ return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
+ }
+ region, ok := aws.Regions[s3RegionName]
+ if s3Endpoint == "" {
+ if !ok {
+ return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", s3RegionName)
+ }
+ } else {
+ if ok {
+ return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
+ "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", s3RegionName, s3Endpoint)
+ }
+ region = aws.Region{
+ Name: s3RegionName,
+ S3Endpoint: s3Endpoint,
+ }
+ }
+ var err error
+ var auth aws.Auth
+ auth.AccessKey, err = readKeyFromFile(s3AccessKeyFile)
+ if err != nil {
+ return err
+ }
+ auth.SecretKey, err = readKeyFromFile(s3SecretKeyFile)
+ if err != nil {
+ return err
+ }
+ if flagSerializeIO {
+ log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
+ }
+ v := NewS3Volume(auth, region, bucketName, flagReadonly, s3Replication)
+ if err := v.Check(); err != nil {
+ return err
+ }
+ *s.volumeSet = append(*s.volumeSet, v)
+ return nil
+}
+
+func s3regions() (okList []string) {
+ for r, _ := range aws.Regions {
+ okList = append(okList, r)
+ }
+ return
+}
+
+func init() {
+ flag.Var(&s3VolumeAdder{&volumes},
+ "s3-bucket-volume",
+ "Use the given bucket as a storage volume. Can be given multiple times.")
+ flag.StringVar(
+ &s3RegionName,
+ "s3-region",
+ "",
+ fmt.Sprintf("AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are %+q.", s3regions()))
+ flag.StringVar(
+ &s3Endpoint,
+ "s3-endpoint",
+ "",
+ "Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use \"https://storage.googleapis.com\".")
+ flag.StringVar(
+ &s3AccessKeyFile,
+ "s3-access-key-file",
+ "",
+ "File containing the access key used for subsequent -s3-bucket-volume arguments.")
+ flag.StringVar(
+ &s3SecretKeyFile,
+ "s3-secret-key-file",
+ "",
+ "File containing the secret key used for subsequent -s3-bucket-volume arguments.")
+ flag.IntVar(
+ &s3Replication,
+ "s3-replication",
+ 2,
+ "Replication level reported to clients for subsequent -s3-bucket-volume arguments.")
+ flag.BoolVar(
+ &s3UnsafeDelete,
+ "s3-unsafe-delete",
+ false,
+ "EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.")
+}
+
+type S3Volume struct {
+ *s3.Bucket
+ readonly bool
+ replication int
+ indexPageSize int
+}
+
+// NewS3Volume returns a new S3Volume using the given auth, region,
+// and bucket name. The replication argument specifies the replication
+// level to report when writing data.
+func NewS3Volume(auth aws.Auth, region aws.Region, bucket string, readonly bool, replication int) *S3Volume {
+ return &S3Volume{
+ Bucket: &s3.Bucket{
+ S3: s3.New(auth, region),
+ Name: bucket,
+ },
+ readonly: readonly,
+ replication: replication,
+ indexPageSize: 1000,
+ }
+}
+
+func (v *S3Volume) Check() error {
+ return nil
+}
+
+func (v *S3Volume) Get(loc string) ([]byte, error) {
+ rdr, err := v.Bucket.GetReader(loc)
+ if err != nil {
+ return nil, v.translateError(err)
+ }
+ defer rdr.Close()
+ buf := bufs.Get(BlockSize)
+ n, err := io.ReadFull(rdr, buf)
+ switch err {
+ case nil, io.EOF, io.ErrUnexpectedEOF:
+ return buf[:n], nil
+ default:
+ bufs.Put(buf)
+ return nil, v.translateError(err)
+ }
+}
+
+func (v *S3Volume) Compare(loc string, expect []byte) error {
+ rdr, err := v.Bucket.GetReader(loc)
+ if err != nil {
+ return v.translateError(err)
+ }
+ defer rdr.Close()
+ return v.translateError(compareReaderWithBuf(rdr, expect, loc[:32]))
+}
+
+func (v *S3Volume) Put(loc string, block []byte) error {
+ if v.readonly {
+ return MethodDisabledError
+ }
+ var opts s3.Options
+ if len(block) > 0 {
+ md5, err := hex.DecodeString(loc)
+ if err != nil {
+ return err
+ }
+ opts.ContentMD5 = base64.StdEncoding.EncodeToString(md5)
+ }
+ return v.translateError(
+ v.Bucket.Put(
+ loc, block, "application/octet-stream", s3ACL, opts))
+}
+
+func (v *S3Volume) Touch(loc string) error {
+ if v.readonly {
+ return MethodDisabledError
+ }
+ result, err := v.Bucket.PutCopy(loc, s3ACL, s3.CopyOptions{
+ ContentType: "application/octet-stream",
+ MetadataDirective: "REPLACE",
+ }, v.Bucket.Name+"/"+loc)
+ if err != nil {
+ return v.translateError(err)
+ }
+ t, err := time.Parse(time.RFC3339, result.LastModified)
+ if err != nil {
+ return err
+ }
+ if time.Since(t) > maxClockSkew {
+ return fmt.Errorf("PutCopy returned old LastModified %s => %s (%s ago)", result.LastModified, t, time.Since(t))
+ }
+ return nil
+}
+
+func (v *S3Volume) Mtime(loc string) (time.Time, error) {
+ resp, err := v.Bucket.Head(loc, nil)
+ if err != nil {
+ return zeroTime, v.translateError(err)
+ }
+ hdr := resp.Header.Get("Last-Modified")
+ t, err := time.Parse(time.RFC1123, hdr)
+ if err != nil && hdr != "" {
+ // AWS example is "Sun, 1 Jan 2006 12:00:00 GMT",
+ // which isn't quite "Sun, 01 Jan 2006 12:00:00 GMT"
+ // as required by HTTP spec. If it's not a valid HTTP
+ // header value, it's probably AWS (or s3test) giving
+ // us a nearly-RFC1123 timestamp.
+ t, err = time.Parse(nearlyRFC1123, hdr)
+ }
+ return t, err
+}
+
+func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
+ nextMarker := ""
+ for {
+ listResp, err := v.Bucket.List(prefix, "", nextMarker, v.indexPageSize)
+ if err != nil {
+ return err
+ }
+ for _, key := range listResp.Contents {
+ t, err := time.Parse(time.RFC3339, key.LastModified)
+ if err != nil {
+ return err
+ }
+ if !v.isKeepBlock(key.Key) {
+ continue
+ }
+ fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.Unix())
+ }
+ if !listResp.IsTruncated {
+ break
+ }
+ nextMarker = listResp.NextMarker
+ }
+ return nil
+}
+
+func (v *S3Volume) Delete(loc string) error {
+ if v.readonly {
+ return MethodDisabledError
+ }
+ if t, err := v.Mtime(loc); err != nil {
+ return err
+ } else if time.Since(t) < blobSignatureTTL {
+ return nil
+ }
+ if !s3UnsafeDelete {
+ return ErrS3DeleteNotAvailable
+ }
+ return v.Bucket.Del(loc)
+}
+
+func (v *S3Volume) Status() *VolumeStatus {
+ return &VolumeStatus{
+ DeviceNum: 1,
+ BytesFree: BlockSize * 1000,
+ BytesUsed: 1,
+ }
+}
+
+func (v *S3Volume) String() string {
+ return fmt.Sprintf("s3-bucket:%+q", v.Bucket.Name)
+}
+
+func (v *S3Volume) Writable() bool {
+ return !v.readonly
+}
+func (v *S3Volume) Replication() int {
+ return v.replication
+}
+
+var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
+func (v *S3Volume) isKeepBlock(s string) bool {
+ return s3KeepBlockRegexp.MatchString(s)
+}
+
+func (v *S3Volume) translateError(err error) error {
+ switch err := err.(type) {
+ case *s3.Error:
+ if err.StatusCode == http.StatusNotFound && err.Code == "NoSuchKey" {
+ return os.ErrNotExist
+ }
+ // Other 404 errors like NoSuchVersion and
+ // NoSuchBucket are different problems which should
+ // get called out downstream, so we don't convert them
+ // to os.ErrNotExist.
+ }
+ return err
+}
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
new file mode 100644
index 0000000..e58b66c
--- /dev/null
+++ b/services/keepstore/s3_volume_test.go
@@ -0,0 +1,143 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "log"
+ "strings"
+ "time"
+
+ "github.com/AdRoll/goamz/aws"
+ "github.com/AdRoll/goamz/s3"
+ "github.com/AdRoll/goamz/s3/s3test"
+ check "gopkg.in/check.v1"
+)
+
+type TestableS3Volume struct {
+ *S3Volume
+ server *s3test.Server
+ c *check.C
+ serverClock *fakeClock
+}
+
+const (
+ TestBucketName = "testbucket"
+)
+
+type fakeClock struct {
+ now *time.Time
+}
+
+func (c *fakeClock) Now() time.Time {
+ if c.now == nil {
+ return time.Now()
+ }
+ return *c.now
+}
+
+func init() {
+ // Deleting isn't safe from races, but if it's turned on
+ // anyway we do expect it to pass the generic volume tests.
+ s3UnsafeDelete = true
+}
+
+func NewTestableS3Volume(c *check.C, readonly bool, replication int) *TestableS3Volume {
+ clock := &fakeClock{}
+ srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
+ c.Assert(err, check.IsNil)
+ auth := aws.Auth{}
+ region := aws.Region{
+ Name: "test-region-1",
+ S3Endpoint: srv.URL(),
+ S3LocationConstraint: true,
+ }
+ bucket := &s3.Bucket{
+ S3: s3.New(auth, region),
+ Name: TestBucketName,
+ }
+ err = bucket.PutBucket(s3.ACL("private"))
+ c.Assert(err, check.IsNil)
+
+ return &TestableS3Volume{
+ S3Volume: NewS3Volume(auth, region, TestBucketName, readonly, replication),
+ server: srv,
+ serverClock: clock,
+ }
+}
+
+var _ = check.Suite(&StubbedS3Suite{})
+
+type StubbedS3Suite struct {
+ volumes []*TestableS3Volume
+}
+
+// func (s *StubbedS3Suite) SetUpTest(c *check.C) {
+// s.volumes = append(s.volumes[:0], NewTestableS3Volume(c, false, 2))
+// }
+
+// func (s *StubbedS3Suite) TearDownTest(c *check.C) {
+// for _, v := range s.volumes {
+// v.Teardown()
+// }
+// }
+
+func (s *StubbedS3Suite) TestGeneric(c *check.C) {
+ DoGenericVolumeTests(c, func(t TB) TestableVolume {
+ return NewTestableS3Volume(c, false, 2)
+ })
+}
+
+func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
+ DoGenericVolumeTests(c, func(t TB) TestableVolume {
+ return NewTestableS3Volume(c, true, 2)
+ })
+}
+
+func (s *StubbedS3Suite) TestIndex(c *check.C) {
+ v := NewTestableS3Volume(c, false, 2)
+ v.indexPageSize = 3
+ for i := 0; i < 256; i++ {
+ v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111})
+ }
+ for _, spec := range []struct {
+ prefix string
+ expectMatch int
+ }{
+ {"", 256},
+ {"c", 16},
+ {"bc", 1},
+ {"abc", 0},
+ } {
+ buf := new(bytes.Buffer)
+ err := v.IndexTo(spec.prefix, buf)
+ c.Check(err, check.IsNil)
+
+ idx := bytes.SplitAfter(buf.Bytes(), []byte{10})
+ c.Check(len(idx), check.Equals, spec.expectMatch+1)
+ c.Check(len(idx[len(idx)-1]), check.Equals, 0)
+ }
+}
+
+// PutRaw skips the ContentMD5 test
+func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
+ err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
+ if err != nil {
+ log.Printf("PutRaw: %+v", err)
+ }
+}
+
+// TouchWithDate turns back the clock while doing a Touch(). We assume
+// there are no other operations happening on the same s3test server
+// while we do this.
+func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) {
+ v.serverClock.now = &lastPut
+ err := v.Touch(locator)
+ if err != nil && !strings.Contains(err.Error(), "PutCopy returned old LastModified") {
+ log.Printf("Touch: %+v", err)
+ }
+ v.serverClock.now = nil
+}
+
+func (v *TestableS3Volume) Teardown() {
+ v.server.Quit()
+}
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index 61088f1..fae4a9e 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -8,19 +8,32 @@ import (
"regexp"
"sort"
"strings"
- "testing"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
)
+type TB interface {
+ Error(args ...interface{})
+ Errorf(format string, args ...interface{})
+ Fail()
+ FailNow()
+ Failed() bool
+ Fatal(args ...interface{})
+ Fatalf(format string, args ...interface{})
+ Log(args ...interface{})
+ Logf(format string, args ...interface{})
+}
+
// A TestableVolumeFactory returns a new TestableVolume. The factory
// function, and the TestableVolume it returns, can use "t" to write
// logs, fail the current test, etc.
-type TestableVolumeFactory func(t *testing.T) TestableVolume
+type TestableVolumeFactory func(t TB) TestableVolume
// DoGenericVolumeTests runs a set of tests that every TestableVolume
// is expected to pass. It calls factory to create a new TestableVolume
// for each test case, to avoid leaking state between tests.
-func DoGenericVolumeTests(t *testing.T, factory TestableVolumeFactory) {
+func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
testGet(t, factory)
testGetNoSuchBlock(t, factory)
@@ -36,10 +49,10 @@ func DoGenericVolumeTests(t *testing.T, factory TestableVolumeFactory) {
testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, TestHash, TestBlock, TestBlock2)
- testPutBlockWithDifferentContent(t, factory, TestHash, EmptyBlock, TestBlock)
- testPutBlockWithDifferentContent(t, factory, TestHash, TestBlock, EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, TestBlock)
+ testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
+ testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
+ testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
+ testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
testPutMultipleBlocks(t, factory)
testPutAndTouch(t, factory)
@@ -67,7 +80,7 @@ func DoGenericVolumeTests(t *testing.T, factory TestableVolumeFactory) {
// Put a test block, get it and verify content
// Test should pass for both writable and read-only volumes
-func testGet(t *testing.T, factory TestableVolumeFactory) {
+func testGet(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -87,7 +100,7 @@ func testGet(t *testing.T, factory TestableVolumeFactory) {
// Invoke get on a block that does not exist in volume; should result in error
// Test should pass for both writable and read-only volumes
-func testGetNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
+func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -99,7 +112,7 @@ func testGetNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
// Compare() should return os.ErrNotExist if the block does not exist.
// Otherwise, writing new data causes CompareAndTouch() to generate
// error logs even though everything is working fine.
-func testCompareNonexistent(t *testing.T, factory TestableVolumeFactory) {
+func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -111,7 +124,7 @@ func testCompareNonexistent(t *testing.T, factory TestableVolumeFactory) {
// Put a test block and compare the locator with same content
// Test should pass for both writable and read-only volumes
-func testCompareSameContent(t *testing.T, factory TestableVolumeFactory, testHash string, testData []byte) {
+func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
v := factory(t)
defer v.Teardown()
@@ -129,7 +142,7 @@ func testCompareSameContent(t *testing.T, factory TestableVolumeFactory, testHas
// testHash = md5(testDataA).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCollision(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
v := factory(t)
defer v.Teardown()
@@ -146,7 +159,7 @@ func testCompareWithCollision(t *testing.T, factory TestableVolumeFactory, testH
// corrupted. Requires testHash = md5(testDataA) != md5(testDataB).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCorruptStoredData(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
v := factory(t)
defer v.Teardown()
@@ -160,7 +173,7 @@ func testCompareWithCorruptStoredData(t *testing.T, factory TestableVolumeFactor
// Put a block and put again with same content
// Test is intended for only writable volumes
-func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory, testHash string, testData []byte) {
+func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
v := factory(t)
defer v.Teardown()
@@ -181,7 +194,7 @@ func testPutBlockWithSameContent(t *testing.T, factory TestableVolumeFactory, te
// Put a block and put again with different content
// Test is intended for only writable volumes
-func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
v := factory(t)
defer v.Teardown()
@@ -189,10 +202,7 @@ func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactor
return
}
- err := v.Put(testHash, testDataA)
- if err != nil {
- t.Errorf("Got err putting block %q: %q, expected nil", testDataA, err)
- }
+ v.PutRaw(testHash, testDataA)
putErr := v.Put(testHash, testDataB)
buf, getErr := v.Get(testHash)
@@ -217,7 +227,7 @@ func testPutBlockWithDifferentContent(t *testing.T, factory TestableVolumeFactor
// Put and get multiple blocks
// Test is intended for only writable volumes
-func testPutMultipleBlocks(t *testing.T, factory TestableVolumeFactory) {
+func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -275,7 +285,7 @@ func testPutMultipleBlocks(t *testing.T, factory TestableVolumeFactory) {
// Test that when applying PUT to a block that already exists,
// the block's modification time is updated.
// Test is intended for only writable volumes
-func testPutAndTouch(t *testing.T, factory TestableVolumeFactory) {
+func testPutAndTouch(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -317,7 +327,7 @@ func testPutAndTouch(t *testing.T, factory TestableVolumeFactory) {
// Touching a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testTouchNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
+func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -328,7 +338,7 @@ func testTouchNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
// Invoking Mtime on a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testMtimeNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
+func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -342,7 +352,7 @@ func testMtimeNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
// * with a prefix
// * with no such prefix
// Test should pass for both writable and read-only volumes
-func testIndexTo(t *testing.T, factory TestableVolumeFactory) {
+func testIndexTo(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -399,7 +409,7 @@ func testIndexTo(t *testing.T, factory TestableVolumeFactory) {
// Calling Delete() for a block immediately after writing it (not old enough)
// should neither delete the data nor return an error.
// Test is intended for only writable volumes
-func testDeleteNewBlock(t *testing.T, factory TestableVolumeFactory) {
+func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
blobSignatureTTL = 300 * time.Second
@@ -427,7 +437,7 @@ func testDeleteNewBlock(t *testing.T, factory TestableVolumeFactory) {
// Calling Delete() for a block with a timestamp older than
// blobSignatureTTL seconds in the past should delete the data.
// Test is intended for only writable volumes
-func testDeleteOldBlock(t *testing.T, factory TestableVolumeFactory) {
+func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
blobSignatureTTL = 300 * time.Second
@@ -449,7 +459,7 @@ func testDeleteOldBlock(t *testing.T, factory TestableVolumeFactory) {
// Calling Delete() for a block that does not exist should result in error.
// Test should pass for both writable and read-only volumes
-func testDeleteNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
+func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -460,7 +470,7 @@ func testDeleteNoSuchBlock(t *testing.T, factory TestableVolumeFactory) {
// Invoke Status and verify that VolumeStatus is returned
// Test should pass for both writable and read-only volumes
-func testStatus(t *testing.T, factory TestableVolumeFactory) {
+func testStatus(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -481,7 +491,7 @@ func testStatus(t *testing.T, factory TestableVolumeFactory) {
// Invoke String for the volume; expect non-empty result
// Test should pass for both writable and read-only volumes
-func testString(t *testing.T, factory TestableVolumeFactory) {
+func testString(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -492,7 +502,7 @@ func testString(t *testing.T, factory TestableVolumeFactory) {
// Putting, updating, touching, and deleting blocks from a read-only volume result in error.
// Test is intended for only read-only volumes
-func testUpdateReadOnly(t *testing.T, factory TestableVolumeFactory) {
+func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -539,7 +549,7 @@ func testUpdateReadOnly(t *testing.T, factory TestableVolumeFactory) {
// Launch concurrent Gets
// Test should pass for both writable and read-only volumes
-func testGetConcurrent(t *testing.T, factory TestableVolumeFactory) {
+func testGetConcurrent(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -592,7 +602,7 @@ func testGetConcurrent(t *testing.T, factory TestableVolumeFactory) {
// Launch concurrent Puts
// Test is intended for only writable volumes
-func testPutConcurrent(t *testing.T, factory TestableVolumeFactory) {
+func testPutConcurrent(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
@@ -660,7 +670,7 @@ func testPutConcurrent(t *testing.T, factory TestableVolumeFactory) {
}
// Write and read back a full size block
-func testPutFullBlock(t *testing.T, factory TestableVolumeFactory) {
+func testPutFullBlock(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
diff --git a/services/keepstore/volume_unix_test.go b/services/keepstore/volume_unix_test.go
index 924637f..b216810 100644
--- a/services/keepstore/volume_unix_test.go
+++ b/services/keepstore/volume_unix_test.go
@@ -16,10 +16,10 @@ import (
type TestableUnixVolume struct {
UnixVolume
- t *testing.T
+ t TB
}
-func NewTestableUnixVolume(t *testing.T, serialize bool, readonly bool) *TestableUnixVolume {
+func NewTestableUnixVolume(t TB, serialize bool, readonly bool) *TestableUnixVolume {
d, err := ioutil.TempDir("", "volume_test")
if err != nil {
t.Fatal(err)
@@ -66,28 +66,28 @@ func (v *TestableUnixVolume) Teardown() {
// serialize = false; readonly = false
func TestUnixVolumeWithGenericTests(t *testing.T) {
- DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
return NewTestableUnixVolume(t, false, false)
})
}
// serialize = false; readonly = true
func TestUnixVolumeWithGenericTestsReadOnly(t *testing.T) {
- DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
return NewTestableUnixVolume(t, false, true)
})
}
// serialize = true; readonly = false
func TestUnixVolumeWithGenericTestsSerialized(t *testing.T) {
- DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
return NewTestableUnixVolume(t, true, false)
})
}
// serialize = false; readonly = false
func TestUnixVolumeHandlersWithGenericVolumeTests(t *testing.T) {
- DoHandlersWithGenericVolumeTests(t, func(t *testing.T) (*RRVolumeManager, []TestableVolume) {
+ DoHandlersWithGenericVolumeTests(t, func(t TB) (*RRVolumeManager, []TestableVolume) {
vols := make([]Volume, 2)
testableUnixVols := make([]TestableVolume, 2)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list