[ARVADOS] created: 86df40e33e586ccb4dc506e00f773392d454804c
git at public.curoverse.com
git at public.curoverse.com
Mon Oct 5 07:48:26 EDT 2015
at 86df40e33e586ccb4dc506e00f773392d454804c (commit)
commit 86df40e33e586ccb4dc506e00f773392d454804c
Author: radhika <radhika at curoverse.com>
Date: Mon Oct 5 07:46:56 2015 -0400
7167: keep-rsync parameter loading and intialization. Update test framework to allow creating two sets of keep servers, source and destination.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 1cce0a7..ab2d9b2 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -82,13 +82,25 @@ type ArvadosClient struct {
// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
// ARVADOS_API_HOST_INSECURE.
func MakeArvadosClient() (ac ArvadosClient, err error) {
+ config := make(map[string]string)
+ config["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ config["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ config["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ config["ARVADOS_EXTERNAL_CLIENT"] = os.Getenv("ARVADOS_EXTERNAL_CLIENT")
+
+ return MakeArvadosClientWithConfig(config)
+}
+
+// Create a new ArvadosClient, using the given input parameters.
+func MakeArvadosClientWithConfig(config map[string]string) (ac ArvadosClient, err error) {
var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
- insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
- external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
+
+ insecure := matchTrue.MatchString(config["ARVADOS_API_HOST_INSECURE"])
+ external := matchTrue.MatchString(config["ARVADOS_EXTERNAL_CLIENT"])
ac = ArvadosClient{
- ApiServer: os.Getenv("ARVADOS_API_HOST"),
- ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
+ ApiServer: config["ARVADOS_API_HOST"],
+ ApiToken: config["ARVADOS_API_TOKEN"],
ApiInsecure: insecure,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index cad1691..a175136 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -99,11 +99,18 @@ func StopAPI() {
}
func StartKeep() {
+ StartKeepAdditional(false)
+}
+
+func StartKeepAdditional(keepExisting bool) {
cwd, _ := os.Getwd()
defer os.Chdir(cwd)
chdirToPythonTests()
cmd := exec.Command("python", "run_test_server.py", "start_keep")
+ if keepExisting {
+ cmd = exec.Command("python", "run_test_server.py", "start_keep", "--keep_existing", "true")
+ }
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatalf("Setting up stderr pipe: %s", err)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 5d0c42a..d5d1874 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -324,7 +324,8 @@ def _start_keep(n, keep_args):
return port
def run_keep(blob_signing_key=None, enforce_permissions=False):
- stop_keep()
+ if args.keep_existing is None:
+ stop_keep()
keep_args = {}
if not blob_signing_key:
@@ -344,12 +345,16 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
host=os.environ['ARVADOS_API_HOST'],
token=os.environ['ARVADOS_API_TOKEN'],
insecure=True)
+
for d in api.keep_services().list().execute()['items']:
api.keep_services().delete(uuid=d['uuid']).execute()
for d in api.keep_disks().list().execute()['items']:
api.keep_disks().delete(uuid=d['uuid']).execute()
- for d in range(0, 2):
+ start_index = 0
+ if args.keep_existing is not None:
+ start_index = 2
+ for d in range(start_index, start_index+2):
port = _start_keep(d, keep_args)
svc = api.keep_services().create(body={'keep_service': {
'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
@@ -374,6 +379,9 @@ def _stop_keep(n):
def stop_keep():
_stop_keep(0)
_stop_keep(1)
+ # We may have created 2 additional keep servers when keep_existing is used
+ _stop_keep(2)
+ _stop_keep(3)
def run_keep_proxy():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
@@ -595,6 +603,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
+ parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
args = parser.parse_args()
if args.action not in actions:
diff --git a/tools/keep-rsync/.gitignore b/tools/keep-rsync/.gitignore
new file mode 100644
index 0000000..5ee7f3b
--- /dev/null
+++ b/tools/keep-rsync/.gitignore
@@ -0,0 +1 @@
+keep-rsync
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
new file mode 100644
index 0000000..eff8b9c
--- /dev/null
+++ b/tools/keep-rsync/keep-rsync.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+ "flag"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "io/ioutil"
+ "log"
+ "strings"
+)
+
+// keep-rsync arguments
+var (
+ srcConfig map[string]string
+ destConfig map[string]string
+ srcKeepServicesJSON string
+ destKeepServicesJSON string
+ replications int
+ prefix string
+)
+
+func main() {
+ var srcConfigFile string
+ var destConfigFile string
+
+ flag.StringVar(
+ &srcConfigFile,
+ "src-config-file",
+ "",
+ "Source configuration filename with full path that contains "+
+ "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+
+ flag.StringVar(
+ &destConfigFile,
+ "dest-config-file",
+ "",
+ "Destination configuration filename with full path that contains "+
+ "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+
+ flag.StringVar(
+ &srcKeepServicesJSON,
+ "src-keep-services-json",
+ "",
+ "An optional list of available source keepservices. "+
+ "If not provided, this list is obtained from api server configured in src-config-file.")
+
+ flag.StringVar(
+ &destKeepServicesJSON,
+ "dest-keep-services-json",
+ "",
+ "An optional list of available destination keepservices. "+
+ "If not provided, this list is obtained from api server configured in dest-config-file.")
+
+ flag.IntVar(
+ &replications,
+ "replications",
+ 3,
+ "Number of replications to write to the destination.")
+
+ flag.StringVar(
+ &prefix,
+ "prefix",
+ "",
+ "Index prefix")
+
+ flag.Parse()
+
+ var err error
+
+ if srcConfigFile == "" {
+ log.Fatal("-src-config-file must be specified.")
+ }
+ srcConfig, err = readConfigFromFile(srcConfigFile)
+ if err != nil {
+ log.Fatal("Error reading source configuration: %s", err.Error())
+ }
+
+ if destConfigFile == "" {
+ log.Fatal("-dest-config-file must be specified.")
+ }
+ destConfig, err = readConfigFromFile(destConfigFile)
+ if err != nil {
+ log.Fatal("Error reading destination configuration: %s", err.Error())
+ }
+
+ err = initializeKeepRsync()
+ if err != nil {
+ log.Fatal("Error configurating keep-rsync: %s", err.Error())
+ }
+}
+
+// Reads config from file
+func readConfigFromFile(filename string) (map[string]string, error) {
+ content, err := ioutil.ReadFile(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ config := make(map[string]string)
+ lines := strings.Split(string(content), "\n")
+ for _, line := range lines {
+ if line == "" {
+ continue
+ }
+ kv := strings.Split(line, "=")
+ config[kv[0]] = kv[1]
+ }
+ return config, nil
+}
+
+// keep-rsync source and destination clients
+var (
+ arvSrc arvadosclient.ArvadosClient
+ arvDest arvadosclient.ArvadosClient
+ kcSrc *keepclient.KeepClient
+ kcDest *keepclient.KeepClient
+)
+
+// Initializes keep-rsync using the config provided
+func initializeKeepRsync() (err error) {
+ arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig)
+ if err != nil {
+ return
+ }
+
+ arvDest, err = arvadosclient.MakeArvadosClientWithConfig(destConfig)
+ if err != nil {
+ return
+ }
+
+ kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
+ if err != nil {
+ return
+ }
+
+ kcDest, err = keepclient.MakeKeepClient(&arvDest)
+
+ return
+}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
new file mode 100644
index 0000000..e2b1f0f
--- /dev/null
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -0,0 +1,117 @@
+package main
+
+import (
+ "crypto/md5"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredSuite{})
+
+// Tests that require the Keep server running
+type ServerRequiredSuite struct{}
+
+func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+}
+
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
+}
+
+// Testing keep-rsync needs two sets of keep services: src and dest.
+// The test setup hence tweaks keep-rsync initialzation to achieve this.
+// First invoke initializeKeepRsync and then invoke StartKeepAdditional
+// to create the keep servers to be used as destination.
+func setupRsync(c *C) {
+ // srcConfig
+ srcConfig = make(map[string]string)
+ srcConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+
+ // destConfig
+ destConfig = make(map[string]string)
+ destConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ destConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ destConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+
+ // initialize keep-rsync
+ err := initializeKeepRsync()
+ c.Assert(err, Equals, nil)
+
+ // Create two more keep servers to be used as destination
+ arvadostest.StartKeepAdditional(true)
+
+ // load kcDest
+ kcDest, err = keepclient.MakeKeepClient(&arvDest)
+ c.Assert(err, Equals, nil)
+}
+
+// Test readConfigFromFile method
+func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
+ // Setup a test config file
+ file, err := ioutil.TempFile(os.TempDir(), "config")
+ c.Assert(err, Equals, nil)
+ defer os.Remove(file.Name())
+
+ fileContent := "ARVADOS_API_HOST=testhost\n"
+ fileContent += "ARVADOS_API_TOKEN=testtoken\n"
+ fileContent += "ARVADOS_API_HOST_INSECURE=true"
+
+ _, err = file.Write([]byte(fileContent))
+
+ // Invoke readConfigFromFile method with this test filename
+ config, err := readConfigFromFile(file.Name())
+ c.Assert(err, Equals, nil)
+ c.Assert(config["ARVADOS_API_HOST"], Equals, "testhost")
+ c.Assert(config["ARVADOS_API_TOKEN"], Equals, "testtoken")
+ c.Assert(config["ARVADOS_API_HOST_INSECURE"], Equals, "true")
+ c.Assert(config["EXTERNAL_CLIENT"], Equals, "")
+}
+
+// Test keep-rsync initialization, with src and dest keep servers.
+// Do a Put and Get in src, both of which should succeed.
+// Do a Get in dest for the same hash, which should raise block not found error.
+func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDestShouldFail(c *C) {
+ setupRsync(c)
+
+ // Put a block in src using kcSrc and Get it
+ data := []byte("test-data")
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcSrc.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+9(\+.+)?$`, hash))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kcSrc.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(9))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ // Get using kcDest should fail with NotFound error
+ _, _, _, err = kcDest.Get(hash)
+ c.Assert(err.Error(), Equals, "Block not found")
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list