[ARVADOS] updated: 11e3caaba692bd76d2d12b0c0e1d8e531d1a0910

git at public.curoverse.com git at public.curoverse.com
Tue Oct 6 16:55:28 EDT 2015


Summary of changes:
 sdk/go/arvadosclient/arvadosclient.go              | 47 ++++++++++++--------
 .../nodemanager/arvnodeman/computenode/__init__.py | 13 ++++++
 .../arvnodeman/computenode/dispatch/__init__.py    | 21 ++++++---
 .../arvnodeman/computenode/dispatch/slurm.py       |  8 +++-
 .../arvnodeman/computenode/driver/__init__.py      |  4 ++
 .../arvnodeman/computenode/driver/azure.py         |  7 +++
 services/nodemanager/arvnodeman/daemon.py          | 13 +++++-
 services/nodemanager/setup.py                      |  4 +-
 .../nodemanager/tests/test_computenode_dispatch.py | 51 +++++++++++++++++-----
 .../tests/test_computenode_dispatch_slurm.py       |  2 +-
 services/nodemanager/tests/test_daemon.py          | 48 +++++++++++++++++++-
 tools/keep-rsync/keep-rsync.go                     | 26 ++++++++---
 tools/keep-rsync/keep-rsync_test.go                | 23 +++++-----
 13 files changed, 204 insertions(+), 63 deletions(-)

       via  11e3caaba692bd76d2d12b0c0e1d8e531d1a0910 (commit)
       via  99ca2640c855e88c7b08c3509b21be9e160ccac8 (commit)
       via  956c3a765e3794a291376cbb5a2ee9ec6ef4560b (commit)
       via  05b52b297b30d075ef2409a123f7d096c1156cf8 (commit)
       via  1f8d81d0eeda07c3cedcaad3e942ec8dedd461cb (commit)
       via  d5f5f869d46f9096c7c680d608c1cc654d1d7fa0 (commit)
       via  11df73b96ae395fca11b4006253475046e3b74cc (commit)
       via  2e919859109fe27d552b81b13d47aed61e80eca6 (commit)
       via  be81c03a3c26f365eba35b91e4f0827244a02ef7 (commit)
       via  c0f33379c7fd062fc097ecef92808334e821cb6b (commit)
       via  f6aa7c0c8c84b85b550d73117c6fdbd663a38c4c (commit)
       via  560c318fdc49835b03f96af35774fbbfa7984fe7 (commit)
       via  72e3566f2cdacd44f095183ebf88f7aab8b0d8dc (commit)
      from  bd1580a11a377270a5a7eed5abc8a6dfe6f9547d (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 11e3caaba692bd76d2d12b0c0e1d8e531d1a0910
Merge: bd1580a 99ca264
Author: radhika <radhika at curoverse.com>
Date:   Tue Oct 6 16:54:01 2015 -0400

    Merge branch '7167-keep-rsync-test-setup' into 7167-keep-rsync

diff --cc tools/keep-rsync/keep-rsync.go
index c06fb8a,1f00a9f..72f4356
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@@ -92,19 -92,19 +93,22 @@@ func main() 
  	if err != nil {
  		log.Fatal("Error configurating keep-rsync: %s", err.Error())
  	}
 +
 +	// Copy blocks not found in dst from src
 +	performKeepRsync()
  }
  
+ var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+ 
  // Reads config from file
- func readConfigFromFile(filename string) (map[string]string, error) {
+ func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
+ 	var config arvadosclient.APIConfig
+ 
  	content, err := ioutil.ReadFile(filename)
  	if err != nil {
- 		return nil, err
+ 		return config, err
  	}
  
- 	config := make(map[string]string)
  	lines := strings.Split(string(content), "\n")
  	for _, line := range lines {
  		if line == "" {

commit 99ca2640c855e88c7b08c3509b21be9e160ccac8
Author: radhika <radhika at curoverse.com>
Date:   Tue Oct 6 16:51:36 2015 -0400

    7167: Use struct instead of map for APIConfig

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index ab2d9b2..cc99efd 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -78,33 +78,42 @@ type ArvadosClient struct {
 	DiscoveryDoc Dict
 }
 
-// Create a new ArvadosClient, initialized with standard Arvados environment
-// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
-// ARVADOS_API_HOST_INSECURE.
+// APIConfig struct consists of:
+//    APIToken        string
+//    APIHost         string
+//    APIHostInsecure bool
+//    ExternalClient  bool
+type APIConfig struct {
+	APIToken        string
+	APIHost         string
+	APIHostInsecure bool
+	ExternalClient  bool
+}
+
+// Create a new ArvadosClient, initialized with standard Arvados environment variables
+// ARVADOS_API_HOST, ARVADOS_API_TOKEN, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT.
 func MakeArvadosClient() (ac ArvadosClient, err error) {
-	config := make(map[string]string)
-	config["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
-	config["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
-	config["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
-	config["ARVADOS_EXTERNAL_CLIENT"] = os.Getenv("ARVADOS_EXTERNAL_CLIENT")
+	var config APIConfig
+	config.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+	config.APIHost = os.Getenv("ARVADOS_API_HOST")
+
+	var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+
+	config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+	config.ExternalClient = matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
 
 	return MakeArvadosClientWithConfig(config)
 }
 
 // Create a new ArvadosClient, using the given input parameters.
-func MakeArvadosClientWithConfig(config map[string]string) (ac ArvadosClient, err error) {
-	var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
-
-	insecure := matchTrue.MatchString(config["ARVADOS_API_HOST_INSECURE"])
-	external := matchTrue.MatchString(config["ARVADOS_EXTERNAL_CLIENT"])
-
+func MakeArvadosClientWithConfig(config APIConfig) (ac ArvadosClient, err error) {
 	ac = ArvadosClient{
-		ApiServer:   config["ARVADOS_API_HOST"],
-		ApiToken:    config["ARVADOS_API_TOKEN"],
-		ApiInsecure: insecure,
+		ApiServer:   config.APIHost,
+		ApiToken:    config.APIToken,
+		ApiInsecure: config.APIHostInsecure,
 		Client: &http.Client{Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
-		External: external}
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
+		External: config.ExternalClient}
 
 	if ac.ApiServer == "" {
 		return ac, MissingArvadosApiHost
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 3761cc7..1f00a9f 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -6,13 +6,14 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"io/ioutil"
 	"log"
+	"regexp"
 	"strings"
 )
 
 // keep-rsync arguments
 var (
-	srcConfig           map[string]string
-	dstConfig           map[string]string
+	srcConfig           arvadosclient.APIConfig
+	dstConfig           arvadosclient.APIConfig
 	srcKeepServicesJSON string
 	dstKeepServicesJSON string
 	replications        int
@@ -69,6 +70,7 @@ func main() {
 
 	var err error
 
+	// Load config
 	if srcConfigFile == "" {
 		log.Fatal("-src-config-file must be specified.")
 	}
@@ -85,27 +87,41 @@ func main() {
 		log.Fatal("Error reading destination configuration: %s", err.Error())
 	}
 
+	// Initialize keep-rsync
 	err = initializeKeepRsync()
 	if err != nil {
 		log.Fatal("Error configurating keep-rsync: %s", err.Error())
 	}
 }
 
+var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+
 // Reads config from file
-func readConfigFromFile(filename string) (map[string]string, error) {
+func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
+	var config arvadosclient.APIConfig
+
 	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return nil, err
+		return config, err
 	}
 
-	config := make(map[string]string)
 	lines := strings.Split(string(content), "\n")
 	for _, line := range lines {
 		if line == "" {
 			continue
 		}
 		kv := strings.Split(line, "=")
-		config[kv[0]] = kv[1]
+
+		switch kv[0] {
+		case "ARVADOS_API_TOKEN":
+			config.APIToken = kv[1]
+		case "ARVADOS_API_HOST":
+			config.APIHost = kv[1]
+		case "ARVADOS_API_HOST_INSECURE":
+			config.APIHostInsecure = matchTrue.MatchString(kv[1])
+		case "ARVADOS_EXTERNAL_CLIENT":
+			config.ExternalClient = matchTrue.MatchString(kv[1])
+		}
 	}
 	return config, nil
 }
@@ -157,6 +173,7 @@ func initializeKeepRsync() (err error) {
 			return
 		}
 	}
+	kcDst.Want_replicas = replications
 
 	return
 }
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 97db571..ae80f45 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"testing"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -44,17 +45,18 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
 // to create the keep servers to be used as destination.
 func setupRsync(c *C) {
 	// srcConfig
-	srcConfig = make(map[string]string)
-	srcConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
-	srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
-	srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+	srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+	srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+	srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
 	// dstConfig
-	dstConfig = make(map[string]string)
-	dstConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
-	dstConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
-	dstConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+	dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+	dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+	dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
+	replications = 1
+
+	// Start API and Keep servers
 	arvadostest.StartAPI()
 	arvadostest.StartKeep()
 
@@ -87,10 +89,10 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
 	// Invoke readConfigFromFile method with this test filename
 	config, err := readConfigFromFile(file.Name())
 	c.Assert(err, Equals, nil)
-	c.Assert(config["ARVADOS_API_HOST"], Equals, "testhost")
-	c.Assert(config["ARVADOS_API_TOKEN"], Equals, "testtoken")
-	c.Assert(config["ARVADOS_API_HOST_INSECURE"], Equals, "true")
-	c.Assert(config["EXTERNAL_CLIENT"], Equals, "")
+	c.Assert(config.APIHost, Equals, "testhost")
+	c.Assert(config.APIToken, Equals, "testtoken")
+	c.Assert(config.APIHostInsecure, Equals, true)
+	c.Assert(config.ExternalClient, Equals, false)
 }
 
 // Test keep-rsync initialization, with src and dst keep servers.
@@ -150,7 +152,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
 	c.Check(localRoots != nil, Equals, true)
 
 	foundIt := false
-	for k, _ := range localRoots {
+	for k := range localRoots {
 		if k == "zzzzz-bi6l4-123456789012340" {
 			foundIt = true
 		}
@@ -158,7 +160,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
 	c.Check(foundIt, Equals, true)
 
 	foundIt = false
-	for k, _ := range localRoots {
+	for k := range localRoots {
 		if k == "zzzzz-bi6l4-123456789012341" {
 			foundIt = true
 		}

commit 956c3a765e3794a291376cbb5a2ee9ec6ef4560b
Merge: 2902472 05b52b2
Author: radhika <radhika at curoverse.com>
Date:   Tue Oct 6 13:56:12 2015 -0400

    Merge branch 'master' into 7167-keep-rsync-test-setup


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list