[ARVADOS] updated: c9018ad3b7f67f2f15a6a6671bf271858926586e
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 15 22:57:39 EDT 2015
Summary of changes:
...nstall-manual-prerequisites.html.textile.liquid | 3 +-
doc/sdk/cli/subcommands.html.textile.liquid | 14 +
sdk/cli/bin/arv | 104 +++++--
sdk/cli/test/test_arv-get.rb | 328 +++++++++------------
.../test/{test_arv-get.rb => test_arv-keep-get.rb} | 2 +-
.../test/{test_arv-put.rb => test_arv-keep-put.rb} | 2 +-
tools/keep-exercise/.gitignore | 1 +
tools/keep-exercise/keep-exercise.go | 157 ++++++++++
tools/keep-rsync/keep-rsync.go | 54 ++--
tools/keep-rsync/keep-rsync_test.go | 81 ++++-
10 files changed, 483 insertions(+), 263 deletions(-)
copy sdk/cli/test/{test_arv-get.rb => test_arv-keep-get.rb} (99%)
rename sdk/cli/test/{test_arv-put.rb => test_arv-keep-put.rb} (99%)
create mode 100644 tools/keep-exercise/.gitignore
create mode 100644 tools/keep-exercise/keep-exercise.go
via c9018ad3b7f67f2f15a6a6671bf271858926586e (commit)
via 807e4cd4abee760736fa32704785d673b0e908cd (commit)
via 494de455828983fe827c96cf425fac6492618308 (commit)
via ce30948199736d45112ee9103642c22f59f84997 (commit)
via 86087cc5ea76bd498d9f615fc314c14c08e721b9 (commit)
via e88a0a3ff00fd5448f893909d08ee11dc301776e (commit)
via db868a3c02cb84f068de77b438b4f2cb498826ae (commit)
via fc17d3c31cc2074a63f6db8698cfcc00a85ec449 (commit)
via c262d3c07f5e8025d51646877c86315996022bae (commit)
via a50278e3d0e26bb5d513d0af5da2fb559b112388 (commit)
via 16082e400046005f0b785ff4c63eda5801258415 (commit)
via dd4d5ce9b5bbcf0ec18698f604a4d666f37fb9b9 (commit)
via e738f6d9d628d01ed2e6ff9800979bb27cfc44f5 (commit)
via 76bb593e5b16879a5efc85515cb47e179b31601f (commit)
via 1b5e6506170075e5c01dda7489d814dd4c3f7774 (commit)
via 59ee5d1b20c9d4b06c194b33b781d353d8e0ba3c (commit)
via 7e6304bf646aaeb38196d24ce98c8bafc212e144 (commit)
via 9e845fe4a5ff18f66c3850b25aae5f77680ac7cd (commit)
via 1a4b37d333d312a97c7979b1203c6246a28e601b (commit)
via fd0bb03ede4bf503fc599f9bac468adb620b2142 (commit)
via 044804a54376e69baccb23a241f7e7bffd7b529b (commit)
via ba0d6a92c642ca86dae5ab807b3ad7fe04e335d2 (commit)
via be9b62b5186bd9aa961f08f66bc104200acf760a (commit)
via 489caef97d5185d91e7441980125480f98428c79 (commit)
via 80572f8164b46079617f0248b1372cf860ca9957 (commit)
via 89e2ddbacad459856fc15f56bfa2b5c036b090a1 (commit)
via 5ba12c5e842c926a38fad477d890ccbf9e96278d (commit)
via 45f10d80d1b584808a6e375214b5be6bc7d2a730 (commit)
via 8e2ad737e429bc263620859468da3835708258e2 (commit)
via ae0f6b92c3c302757ff7e81239bd7b41a4430e70 (commit)
via df9e166a5ffc4aa79658bec1a5d552a3b413f0d8 (commit)
from 22928daa55bc0de7d539793ff27503affac8a39a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c9018ad3b7f67f2f15a6a6671bf271858926586e
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 15 22:55:03 2015 -0400
7167: Break all the code from keep-rsync main method into a separate func so that arg parsing can also be tested.
Rather than using default flag parsing, use FlagSet so that flags can be set multiple times from multiple tests.
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 705025b..7cd795e 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -18,77 +18,79 @@ import (
)
func main() {
- var srcConfigFile, dstConfigFile, srcKeepServicesJSON, dstKeepServicesJSON, prefix string
- var replications int
- var srcBlobSigningKey string
+ err := doMain()
+ if err != nil {
+ log.Fatalf("%v", err)
+ }
+}
- flag.StringVar(
- &srcConfigFile,
+func doMain() error {
+ flags := flag.NewFlagSet("keep-rsync", flag.ExitOnError)
+
+ srcConfigFile := flags.String(
"src",
"",
"Source configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf")
- flag.StringVar(
- &dstConfigFile,
+ dstConfigFile := flags.String(
"dst",
"",
"Destination configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf")
- flag.StringVar(
- &srcKeepServicesJSON,
+ srcKeepServicesJSON := flags.String(
"src-keep-services-json",
"",
"An optional list of available source keepservices. "+
"If not provided, this list is obtained from api server configured in src-config-file.")
- flag.StringVar(
- &dstKeepServicesJSON,
+ dstKeepServicesJSON := flags.String(
"dst-keep-services-json",
"",
"An optional list of available destination keepservices. "+
"If not provided, this list is obtained from api server configured in dst-config-file.")
- flag.IntVar(
- &replications,
+ replications := flags.Int(
"replications",
0,
"Number of replications to write to the destination. If replications not specified, "+
"default replication level configured on destination server will be used.")
- flag.StringVar(
- &prefix,
+ prefix := flags.String(
"prefix",
"",
"Index prefix")
- flag.Parse()
+ // Parse args; omit the first arg which is the command name
+ flags.Parse(os.Args[1:])
- srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile)
+ srcConfig, srcBlobSigningKey, err := loadConfig(*srcConfigFile)
if err != nil {
- log.Fatalf("Error loading src configuration from file: %s", err.Error())
+ return fmt.Errorf("Error loading src configuration from file: %s", err.Error())
}
- dstConfig, _, err := loadConfig(dstConfigFile)
+ dstConfig, _, err := loadConfig(*dstConfigFile)
if err != nil {
- log.Fatalf("Error loading dst configuration from file: %s", err.Error())
+ return fmt.Errorf("Error loading dst configuration from file: %s", err.Error())
}
// setup src and dst keepclients
- kcSrc, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0)
+ kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0)
if err != nil {
- log.Fatalf("Error configuring src keepclient: %s", err.Error())
+ return fmt.Errorf("Error configuring src keepclient: %s", err.Error())
}
- kcDst, err := setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications)
+ kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications)
if err != nil {
- log.Fatalf("Error configuring dst keepclient: %s", err.Error())
+ return fmt.Errorf("Error configuring dst keepclient: %s", err.Error())
}
// Copy blocks not found in dst from src
- err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, prefix)
+ err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, *prefix)
if err != nil {
- log.Fatalf("Error while syncing data: %s", err.Error())
+ return fmt.Errorf("Error while syncing data: %s", err.Error())
}
+
+ return nil
}
type apiConfig struct {
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 299df5a..f109188 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -23,10 +23,12 @@ func Test(t *testing.T) {
// Gocheck boilerplate
var _ = Suite(&ServerRequiredSuite{})
var _ = Suite(&ServerNotRequiredSuite{})
+var _ = Suite(&DoMainTestSuite{})
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
type ServerNotRequiredSuite struct{}
+type DoMainTestSuite struct{}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
// Start API server
@@ -38,6 +40,12 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.ResetEnv()
}
+var initialArgs []string
+
+func (s *DoMainTestSuite) SetUpSuite(c *C) {
+ initialArgs = os.Args
+}
+
var kcSrc, kcDst *keepclient.KeepClient
var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
@@ -54,6 +62,15 @@ func (s *ServerRequiredSuite) TearDownTest(c *C) {
arvadostest.StopKeepWithParams(3)
}
+func (s *DoMainTestSuite) SetUpTest(c *C) {
+ args := []string{"keep-rsync"}
+ os.Args = args
+}
+
+func (s *DoMainTestSuite) TearDownTest(c *C) {
+ os.Args = initialArgs
+}
+
var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
// Testing keep-rsync needs two sets of keep services: src and dst.
@@ -77,7 +94,6 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
}
// Start Keep servers
- arvadostest.StartAPI()
arvadostest.StartKeepWithParams(3, enforcePermissions)
// setup keepclients
@@ -369,17 +385,17 @@ func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile)
c.Check(err, IsNil)
- c.Assert(srcConfig.APIHost, Equals, "testhost")
- c.Assert(srcConfig.APIToken, Equals, "testtoken")
- c.Assert(srcConfig.APIHostInsecure, Equals, true)
+ c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
+ c.Assert(srcConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+ c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
c.Assert(srcConfig.ExternalClient, Equals, false)
dstConfig, _, err := loadConfig(dstConfigFile)
c.Check(err, IsNil)
- c.Assert(dstConfig.APIHost, Equals, "testhost")
- c.Assert(dstConfig.APIToken, Equals, "testtoken")
- c.Assert(dstConfig.APIHostInsecure, Equals, true)
+ c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
+ c.Assert(dstConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+ c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
c.Assert(dstConfig.ExternalClient, Equals, false)
c.Assert(srcBlobSigningKey, Equals, "abcdefg")
@@ -402,9 +418,9 @@ func setupConfigFile(c *C, name string) *os.File {
file, err := ioutil.TempFile(os.TempDir(), name)
c.Check(err, IsNil)
- fileContent := "ARVADOS_API_HOST=testhost\n"
- fileContent += "ARVADOS_API_TOKEN=testtoken\n"
- fileContent += "ARVADOS_API_HOST_INSECURE=true\n"
+ fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
+ fileContent += "ARVADOS_API_TOKEN=" + os.Getenv("ARVADOS_API_TOKEN") + "\n"
+ fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
_, err = file.Write([]byte(fileContent))
@@ -412,3 +428,48 @@ func setupConfigFile(c *C, name string) *os.File {
return file
}
+
+func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) {
+ err := doMain()
+ c.Check(err, NotNil)
+ c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
+}
+
+func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) {
+ srcConfig := setupConfigFile(c, "src")
+ args := []string{"-replications", "3", "-src", srcConfig.Name()}
+ os.Args = append(os.Args, args...)
+ err := doMain()
+ c.Check(err, NotNil)
+ c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified")
+}
+
+func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) {
+ args := []string{"-src", "abcd"}
+ os.Args = append(os.Args, args...)
+ err := doMain()
+ c.Check(err, NotNil)
+ c.Assert(strings.HasPrefix(err.Error(), "Error loading src configuration from file: Error reading config file"), Equals, true)
+}
+
+func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) {
+ args := []string{"-replications", "3"}
+ os.Args = append(os.Args, args...)
+ err := doMain()
+ c.Check(err, NotNil)
+ c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
+}
+
+func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
+ srcConfig := setupConfigFile(c, "src")
+ dstConfig := setupConfigFile(c, "dst")
+ args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
+ os.Args = append(os.Args, args...)
+
+ // Start keepservers. Since we are not doing any tweaking as in setupRsync func,
+ // kcSrc and kcDst will be the same and no actual copying to dst will happen, but that's ok.
+ arvadostest.StartKeep()
+
+ err := doMain()
+ c.Check(err, IsNil)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list