[ARVADOS] updated: 46b11ba2ed71e2c074e9e6c8f5b9f7a003e7067f
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 1 21:44:58 EDT 2015
Summary of changes:
apps/workbench/config/application.default.yml | 47 +++-
doc/_includes/_install_ruby_and_bundler.liquid | 7 +-
doc/install/install-api-server.html.textile.liquid | 2 +-
doc/install/install-sso.html.textile.liquid | 255 +++++++++++----------
.../install-workbench-app.html.textile.liquid | 4 +-
docker/jobs/Dockerfile | 2 +-
sdk/cli/bin/crunch-job | 17 +-
sdk/go/keepclient/keepclient.go | 17 +-
sdk/go/keepclient/support.go | 44 ++--
sdk/python/arvados/arvfile.py | 20 +-
sdk/python/arvados/keep.py | 15 +-
sdk/python/tests/test_arvfile.py | 15 ++
services/api/config/application.default.yml | 43 +++-
services/api/script/crunch-dispatch.rb | 5 +
services/api/script/salvage_collection.rb | 1 +
services/datamanager/keep/keep.go | 19 +-
.../keepproxy/pkg-extras/etc/default/keepproxy | 7 +
services/keepproxy/pkg-extras/etc/init.d/keepproxy | 156 +++++++++++++
services/keepstore/collision.go | 2 +-
.../keepstore/handlers_with_generic_volume_test.go | 119 ++++++++++
services/keepstore/keepstore.go | 6 +-
services/keepstore/keepstore_test.go | 4 +
services/keepstore/mock_mutex_for_test.go | 6 +-
services/keepstore/volume_generic_test.go | 81 ++++---
services/keepstore/volume_unix_test.go | 16 ++
services/login-sync/bin/arvados-login-sync | 1 +
26 files changed, 689 insertions(+), 222 deletions(-)
create mode 100644 services/keepproxy/pkg-extras/etc/default/keepproxy
create mode 100755 services/keepproxy/pkg-extras/etc/init.d/keepproxy
create mode 100644 services/keepstore/handlers_with_generic_volume_test.go
via 46b11ba2ed71e2c074e9e6c8f5b9f7a003e7067f (commit)
via 4e8c321b344d8ec23de7ee7c68b2553d87defb67 (commit)
via 3a4b7ec9daecbff438c83380c6beccaf88d9e47c (commit)
via f3f86fcf67775df91937392dd74a527dcbcf1886 (commit)
via f235c17c253d9ece933c972039e4872ef7157533 (commit)
via d084c7b404fa0cb597fbba4dcb007c087da49ba5 (commit)
via f9620be04d0197fe88e3b4d1a1ee54231632cae6 (commit)
via 8752f4d9f578b8898eefd739464c35c1f34fe1f9 (commit)
via 9446aefa68d0304c368e60359a2376fc72a5b1b4 (commit)
via c6d10ad822ac94453d74371af0bf57072c43f018 (commit)
via 88b4f320eb101cdac88d2b7ee15135dd67703d20 (commit)
via 3677f1aea53f1a04811270f98a681dbcd6002e67 (commit)
via 4ad7111513fabbdce01794c283791c8dc9459aa4 (commit)
via 6b5e785498db0a3e32faa0959886d2685a81a0ef (commit)
via a5b2fa0f7dbefe103ce2bc87f1136ee01e915141 (commit)
via 77a313cf888a98705193012057da7ae1b9860acc (commit)
via b4112f2e4220312c71c5c3d288f888133a5ba41a (commit)
via e31d1ec3c19d88042b47516fad9ed952d6561fd6 (commit)
via c2a9349561ec0a0c5c7d3f39e1953ae41e1c76cd (commit)
via 72748d17a195be1b65f847b078d1924f4648d45a (commit)
via fc37dc82cd5ca98df0ac1f939d96dbc6b4e98c7c (commit)
via 57694f580feff10b274d6b13e9b4b013bfdad937 (commit)
via b8ad249763a75f3eb7bea758b88cdc87389639f1 (commit)
via 0ba145115071473ba46a3fac30c2dc98b74f49ad (commit)
via 0fdfc23c640c831cb2c0ee1e04fb639eb032de6f (commit)
via 73ce0cf7675e060d33e75488edfa4f533c177f82 (commit)
via 32a29ee7d171dccdb424990ff9c73e4b893dc3e4 (commit)
via 5a55113805af906145449be18ec86b1a95a5017b (commit)
via 285c0092f8ae4d757e31754c91314baa4db5af44 (commit)
via 559dc0ac07ea3d6820f5e220b9f6faf850ae63bf (commit)
via ee9c164a23be2aabc22da20737bce362fad8ed7e (commit)
via 743022a885b563e47bc5d31e144b5165ab4688c9 (commit)
via 5a974b0568f7645a7e4377394da04940fa1ec137 (commit)
via 2b20a0ec9a32abfd85c09474e044cfd4c09f4963 (commit)
via 99349abd0ee7347b5bac3d4a9638853c6d4b97ab (commit)
via acd1241cec9260d54c2dca55785e309644334c41 (commit)
via c21c6330cc8a3c8cc96d83ff9a315fe10dff7c8f (commit)
via e45aa811c8489cae3ac83836aceca7f2ce6ba398 (commit)
via 9db403a87830f7afec86dae5bb279cf87df8fd0e (commit)
via 20ad8adb02808a85ac38f865fd870b00fb400b74 (commit)
via af901a9a0434c9b5ed8b9349835a1749f45f95d3 (commit)
via 67f730557e062608f842fa3abc612fb717fbce2f (commit)
via 33596fce13c3bd5c368f5efeb89d625f684a622d (commit)
via e668f9e2424cb1224b540865374b12fa9e1e3a8f (commit)
via 127c47f949fb71738186b2ae7b98adfd4bdef25e (commit)
via fdbf2ead3452bf7c8ba8f3f275cf486339b5406e (commit)
via 501325d396c1ac30637b5a48ecda17f77cd73206 (commit)
via a3e32a2f4a702c076d46a5b19305dd20a1ee3012 (commit)
via b2ed333572a4db15e82ba23a162b0f065d24c2b7 (commit)
via 6d352c4bfe2d0e9a405a52863c28b00de47f4be5 (commit)
via 009c08b43affdefa6f2825059a971c750b90e222 (commit)
via 43748be08c5d67171cb2f5565670dd0eef889b07 (commit)
via 87892d98eac028934480c378748cfb65a780d1cc (commit)
via d4f15342fcaabb0bec0165b2abdf040c88c08229 (commit)
via 16edd7c54ac58cda991d3e9f0a1b48cea3dd985c (commit)
via b16358fa5145d3d4bcd0ac37bd81d7605eac040f (commit)
via 0fc6e0671e822cd4c5671519c7e7c58cff06c91f (commit)
via 7fdfbc3eae8de4fe7f6912c160e408ef57a89487 (commit)
via 0333c360cdd24b8fdd28a88d6d99bebe7e7a9bed (commit)
via c7b4e4802762eada81652224878a74b538a1d704 (commit)
via 67c6f66d2b954f1f0f887011b7b6d5153b2d7bf1 (commit)
via 9bd1f604e8f5ad0dd33b4501c535d9915924e8bd (commit)
via 0219fab6b0eb8b4710af2ae03c5231bf4d932a28 (commit)
via 6cca3e129b6de3fc4b34a1664f494fcbbbd940d1 (commit)
via bb663993cc851971d9444d834acfc416363931b2 (commit)
via c447d4a79f653bbf2c172a0a715d30db896a4a32 (commit)
from e9f437d9e590cc37ada8534401d254bd5e0a5e85 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 46b11ba2ed71e2c074e9e6c8f5b9f7a003e7067f
Merge: e9f437d 4e8c321
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Sep 29 14:22:22 2015 -0400
Merge branch 'master' into 7241-azure-blob-volume
Conflicts:
services/keepstore/volume_generic_test.go
services/keepstore/handlers_with_generic_volume_test.go
diff --cc services/keepstore/handlers_with_generic_volume_test.go
index 0000000,90094f3..9f31f5f
mode 000000,100644..100644
--- a/services/keepstore/handlers_with_generic_volume_test.go
+++ b/services/keepstore/handlers_with_generic_volume_test.go
@@@ -1,0 -1,119 +1,119 @@@
+ package main
+
+ import (
+ "bytes"
+ "testing"
+ )
+
+ // A TestableVolumeManagerFactory creates a volume manager with at least two TestableVolume instances.
+ // The factory function, and the TestableVolume instances it returns, can use "t" to write
+ // logs, fail the current test, etc.
+ type TestableVolumeManagerFactory func(t *testing.T) (*RRVolumeManager, []TestableVolume)
+
+ // DoHandlersWithGenericVolumeTests runs a set of handler tests with a
+ // Volume Manager comprised of TestableVolume instances.
+ // It calls factory to create a volume manager with TestableVolume
+ // instances for each test case, to avoid leaking state between tests.
+ func DoHandlersWithGenericVolumeTests(t *testing.T, factory TestableVolumeManagerFactory) {
+ testGetBlock(t, factory, TestHash, TestBlock)
+ testGetBlock(t, factory, EmptyHash, EmptyBlock)
+ testPutRawBadDataGetBlock(t, factory, TestHash, TestBlock, []byte("baddata"))
+ testPutRawBadDataGetBlock(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
+ testPutBlock(t, factory, TestHash, TestBlock)
+ testPutBlock(t, factory, EmptyHash, EmptyBlock)
+ testPutBlockCorrupt(t, factory, TestHash, TestBlock, []byte("baddata"))
+ testPutBlockCorrupt(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
+ }
+
+ // Setup RRVolumeManager with TestableVolumes
+ func setupHandlersWithGenericVolumeTest(t *testing.T, factory TestableVolumeManagerFactory) []TestableVolume {
+ vm, testableVolumes := factory(t)
+ KeepVM = vm
+
+ for _, v := range testableVolumes {
+ defer v.Teardown()
+ }
+ defer KeepVM.Close()
+
+ return testableVolumes
+ }
+
+ // Put a block using PutRaw in just one volume and Get it using GetBlock
+ func testGetBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
+ testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
+
+ // Put testBlock in one volume
+ testableVolumes[1].PutRaw(testHash, testBlock)
+
+ // Get should pass
+ buf, err := GetBlock(testHash)
+ if err != nil {
+ t.Fatalf("Error while getting block %s", err)
+ }
+ if bytes.Compare(buf, testBlock) != 0 {
+ t.Errorf("Put succeeded but Get returned %+v, expected %+v", buf, testBlock)
+ }
+ }
+
+ // Put a bad block using PutRaw and get it.
+ func testPutRawBadDataGetBlock(t *testing.T, factory TestableVolumeManagerFactory,
+ testHash string, testBlock []byte, badData []byte) {
+ testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
+
+ // Put bad data for testHash in both volumes
+ testableVolumes[0].PutRaw(testHash, badData)
+ testableVolumes[1].PutRaw(testHash, badData)
+
+ // Get should fail
+ _, err := GetBlock(testHash)
+ if err == nil {
+ t.Fatalf("Expected error while getting corrupt block %v", testHash)
+ }
+ }
+
+ // Invoke PutBlock twice to ensure CompareAndTouch path is tested.
+ func testPutBlock(t *testing.T, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
+ setupHandlersWithGenericVolumeTest(t, factory)
+
+ // PutBlock
- if err := PutBlock(testBlock, testHash); err != nil {
++ if _, err := PutBlock(testBlock, testHash); err != nil {
+ t.Fatalf("Error during PutBlock: %s", err)
+ }
+
+ // Check that PutBlock succeeds again even after CompareAndTouch
- if err := PutBlock(testBlock, testHash); err != nil {
++ if _, err := PutBlock(testBlock, testHash); err != nil {
+ t.Fatalf("Error during PutBlock: %s", err)
+ }
+
+ // Check that PutBlock stored the data as expected
+ buf, err := GetBlock(testHash)
+ if err != nil {
+ t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
+ } else if bytes.Compare(buf, testBlock) != 0 {
+ t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf)
+ }
+ }
+
+ // Put a bad block using PutRaw, overwrite it using PutBlock and get it.
+ func testPutBlockCorrupt(t *testing.T, factory TestableVolumeManagerFactory,
+ testHash string, testBlock []byte, badData []byte) {
+ testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
+
+ // Put bad data for testHash in both volumes
+ testableVolumes[0].PutRaw(testHash, badData)
+ testableVolumes[1].PutRaw(testHash, badData)
+
+ // Check that PutBlock with good data succeeds
- if err := PutBlock(testBlock, testHash); err != nil {
++ if _, err := PutBlock(testBlock, testHash); err != nil {
+ t.Fatalf("Error during PutBlock for %q: %s", testHash, err)
+ }
+
+ // Put succeeded and overwrote the badData in one volume,
+ // and Get should return the testBlock now, ignoring the bad data.
+ buf, err := GetBlock(testHash)
+ if err != nil {
+ t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
+ } else if bytes.Compare(buf, testBlock) != 0 {
+ t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf)
+ }
+ }
diff --cc services/keepstore/volume_generic_test.go
index 503e6b9,c08c3f5..6dca74e
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@@ -170,15 -181,15 +181,15 @@@ func testPutBlockWithDifferentContent(
if putErr == nil {
// Put must not return a nil error unless it has
// overwritten the existing data.
- if bytes.Compare(buf, TestBlock2) != 0 {
- t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf, TestBlock2)
+ if bytes.Compare(buf, testDataB) != 0 {
- t.Errorf("Put succeeded but Get returned %+v, expected %+v", buf, testDataB)
++ t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf, testDataB)
}
} else {
// It is permissible for Put to fail, but it must
// leave us with either the original data, the new
// data, or nothing at all.
- if getErr == nil && bytes.Compare(buf, TestBlock) != 0 && bytes.Compare(buf, TestBlock2) != 0 {
- t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf, TestBlock, TestBlock2)
+ if getErr == nil && bytes.Compare(buf, testDataA) != 0 && bytes.Compare(buf, testDataB) != 0 {
- t.Errorf("Put failed but Get returned %+v, which is neither %+v nor %+v", buf, testDataA, testDataB)
++ t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf, testDataA, testDataB)
}
}
if getErr == nil {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list