[ARVADOS] updated: 6823f2d65a48bc989b819b85b6428c230b223c82
git at public.curoverse.com
git at public.curoverse.com
Mon Oct 5 09:27:20 EDT 2015
Summary of changes:
sdk/python/tests/run_test_server.py | 7 +-
services/keepstore/azure_blob_volume.go | 243 ++++++++++++++
services/keepstore/azure_blob_volume_test.go | 353 +++++++++++++++++++++
services/keepstore/collision.go | 35 ++
services/keepstore/handler_test.go | 16 +
services/keepstore/handlers.go | 38 +--
.../keepstore/handlers_with_generic_volume_test.go | 6 +-
services/keepstore/keepstore.go | 101 +-----
services/keepstore/keepstore_test.go | 30 +-
services/keepstore/pull_worker.go | 2 +-
services/keepstore/volume.go | 5 +
services/keepstore/volume_generic_test.go | 44 ++-
services/keepstore/volume_test.go | 4 +
services/keepstore/volume_unix.go | 135 ++++++--
tools/keep-rsync/keep-rsync.go | 42 +--
tools/keep-rsync/keep-rsync_test.go | 26 +-
16 files changed, 866 insertions(+), 221 deletions(-)
create mode 100644 services/keepstore/azure_blob_volume.go
create mode 100644 services/keepstore/azure_blob_volume_test.go
via 6823f2d65a48bc989b819b85b6428c230b223c82 (commit)
via cb48eb95d516b1d4a1ffe18be34703005531117a (commit)
via f81f84e19902e37c28fd1610999cfefa1c4a0b71 (commit)
via 2c07efe6ac7455059f2fccd558ea796f9c315e19 (commit)
via 46b11ba2ed71e2c074e9e6c8f5b9f7a003e7067f (commit)
via e9f437d9e590cc37ada8534401d254bd5e0a5e85 (commit)
via 852eadc79b7103b3889eed53a851a1c26c4daeab (commit)
via 4eac79ebafa9b7979bbd295c2da85acbb3981bac (commit)
via b7f7878f8f0648ba5a53e24abb109ce9ad59bfc3 (commit)
via b2bcd45082d2df2b5a17645eb60473cc17c76e88 (commit)
via da74a60c2d276ed8612f138d73e73787f450ea2e (commit)
via 96c3fcd2d013af7747f20fea55f460ca2d2dd637 (commit)
via 109cd685ecbfb5b685347731340c6dd69e630617 (commit)
via 8626abb0a44cfc303bef3552a7bc57163c79231a (commit)
from 86df40e33e586ccb4dc506e00f773392d454804c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6823f2d65a48bc989b819b85b6428c230b223c82
Author: radhika <radhika at curoverse.com>
Date: Mon Oct 5 09:25:34 2015 -0400
7167: args not avaialble in all tests; hence store keep_existing argument in a variable rather than accessing it directly from args.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d5d1874..62a2a82 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -43,6 +43,7 @@ if not os.path.exists(TEST_TMPDIR):
my_api_host = None
_cached_config = {}
+keep_existing = None
def find_server_pid(PID_PATH, wait=10):
now = time.time()
@@ -324,7 +325,7 @@ def _start_keep(n, keep_args):
return port
def run_keep(blob_signing_key=None, enforce_permissions=False):
- if args.keep_existing is None:
+ if keep_existing is None:
stop_keep()
keep_args = {}
@@ -352,7 +353,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
api.keep_disks().delete(uuid=d['uuid']).execute()
start_index = 0
- if args.keep_existing is not None:
+ if keep_existing is not None:
start_index = 2
for d in range(start_index, start_index+2):
port = _start_keep(d, keep_args)
@@ -606,6 +607,8 @@ if __name__ == "__main__":
parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
args = parser.parse_args()
+ keep_existing = args.keep_existing
+
if args.action not in actions:
print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
sys.exit(1)
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index eff8b9c..c9fd77a 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -11,17 +11,17 @@ import (
// keep-rsync arguments
var (
- srcConfig map[string]string
- destConfig map[string]string
- srcKeepServicesJSON string
- destKeepServicesJSON string
- replications int
- prefix string
+ srcConfig map[string]string
+ dstConfig map[string]string
+ srcKeepServicesJSON string
+ dstKeepServicesJSON string
+ replications int
+ prefix string
)
func main() {
var srcConfigFile string
- var destConfigFile string
+ var dstConfigFile string
flag.StringVar(
&srcConfigFile,
@@ -32,8 +32,8 @@ func main() {
"ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
flag.StringVar(
- &destConfigFile,
- "dest-config-file",
+ &dstConfigFile,
+ "dst-config-file",
"",
"Destination configuration filename with full path that contains "+
"an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
@@ -47,11 +47,11 @@ func main() {
"If not provided, this list is obtained from api server configured in src-config-file.")
flag.StringVar(
- &destKeepServicesJSON,
- "dest-keep-services-json",
+ &dstKeepServicesJSON,
+ "dst-keep-services-json",
"",
"An optional list of available destination keepservices. "+
- "If not provided, this list is obtained from api server configured in dest-config-file.")
+ "If not provided, this list is obtained from api server configured in dst-config-file.")
flag.IntVar(
&replications,
@@ -77,10 +77,10 @@ func main() {
log.Fatal("Error reading source configuration: %s", err.Error())
}
- if destConfigFile == "" {
- log.Fatal("-dest-config-file must be specified.")
+ if dstConfigFile == "" {
+ log.Fatal("-dst-config-file must be specified.")
}
- destConfig, err = readConfigFromFile(destConfigFile)
+ dstConfig, err = readConfigFromFile(dstConfigFile)
if err != nil {
log.Fatal("Error reading destination configuration: %s", err.Error())
}
@@ -112,10 +112,10 @@ func readConfigFromFile(filename string) (map[string]string, error) {
// keep-rsync source and destination clients
var (
- arvSrc arvadosclient.ArvadosClient
- arvDest arvadosclient.ArvadosClient
- kcSrc *keepclient.KeepClient
- kcDest *keepclient.KeepClient
+ arvSrc arvadosclient.ArvadosClient
+ arvDst arvadosclient.ArvadosClient
+ kcSrc *keepclient.KeepClient
+ kcDst *keepclient.KeepClient
)
// Initializes keep-rsync using the config provided
@@ -125,7 +125,7 @@ func initializeKeepRsync() (err error) {
return
}
- arvDest, err = arvadosclient.MakeArvadosClientWithConfig(destConfig)
+ arvDst, err = arvadosclient.MakeArvadosClientWithConfig(dstConfig)
if err != nil {
return
}
@@ -135,7 +135,7 @@ func initializeKeepRsync() (err error) {
return
}
- kcDest, err = keepclient.MakeKeepClient(&arvDest)
+ kcDst, err = keepclient.MakeKeepClient(&arvDst)
return
}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index e2b1f0f..c45fbf8 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -36,7 +36,7 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopAPI()
}
-// Testing keep-rsync needs two sets of keep services: src and dest.
+// Testing keep-rsync needs two sets of keep services: src and dst.
// The test setup hence tweaks keep-rsync initialzation to achieve this.
// First invoke initializeKeepRsync and then invoke StartKeepAdditional
// to create the keep servers to be used as destination.
@@ -47,11 +47,11 @@ func setupRsync(c *C) {
srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
- // destConfig
- destConfig = make(map[string]string)
- destConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
- destConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
- destConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ // dstConfig
+ dstConfig = make(map[string]string)
+ dstConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ dstConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ dstConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
arvadostest.StartAPI()
arvadostest.StartKeep()
@@ -63,8 +63,8 @@ func setupRsync(c *C) {
// Create two more keep servers to be used as destination
arvadostest.StartKeepAdditional(true)
- // load kcDest
- kcDest, err = keepclient.MakeKeepClient(&arvDest)
+ // load kcDst
+ kcDst, err = keepclient.MakeKeepClient(&arvDst)
c.Assert(err, Equals, nil)
}
@@ -90,10 +90,10 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
c.Assert(config["EXTERNAL_CLIENT"], Equals, "")
}
-// Test keep-rsync initialization, with src and dest keep servers.
+// Test keep-rsync initialization, with src and dst keep servers.
// Do a Put and Get in src, both of which should succeed.
-// Do a Get in dest for the same hash, which should raise block not found error.
-func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDestShouldFail(c *C) {
+// Do a Get in dst for the same hash, which should raise block not found error.
+func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDstShouldFail(c *C) {
setupRsync(c)
// Put a block in src using kcSrc and Get it
@@ -111,7 +111,7 @@ func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDestShouldFail(c *C) {
all, err := ioutil.ReadAll(reader)
c.Check(all, DeepEquals, data)
- // Get using kcDest should fail with NotFound error
- _, _, _, err = kcDest.Get(hash)
+ // Get using kcDst should fail with NotFound error
+ _, _, _, err = kcDst.Get(hash)
c.Assert(err.Error(), Equals, "Block not found")
}
commit cb48eb95d516b1d4a1ffe18be34703005531117a
Merge: 86df40e f81f84e
Author: radhika <radhika at curoverse.com>
Date: Mon Oct 5 07:48:47 2015 -0400
Merge branch 'master' into 7167-keep-rsync-test-setup
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list