[ARVADOS] updated: 99ca2640c855e88c7b08c3509b21be9e160ccac8
git at public.curoverse.com
git at public.curoverse.com
Tue Oct 6 16:53:47 EDT 2015
Summary of changes:
sdk/go/arvadosclient/arvadosclient.go | 47 ++++++++++++--------
.../nodemanager/arvnodeman/computenode/__init__.py | 13 ++++++
.../arvnodeman/computenode/dispatch/__init__.py | 21 ++++++---
.../arvnodeman/computenode/dispatch/slurm.py | 8 +++-
.../arvnodeman/computenode/driver/__init__.py | 4 ++
.../arvnodeman/computenode/driver/azure.py | 7 +++
services/nodemanager/arvnodeman/daemon.py | 13 +++++-
services/nodemanager/setup.py | 4 +-
.../nodemanager/tests/test_computenode_dispatch.py | 51 +++++++++++++++++-----
.../tests/test_computenode_dispatch_slurm.py | 2 +-
services/nodemanager/tests/test_daemon.py | 48 +++++++++++++++++++-
tools/keep-rsync/keep-rsync.go | 29 +++++++++---
tools/keep-rsync/keep-rsync_test.go | 30 +++++++------
13 files changed, 212 insertions(+), 65 deletions(-)
via 99ca2640c855e88c7b08c3509b21be9e160ccac8 (commit)
via 956c3a765e3794a291376cbb5a2ee9ec6ef4560b (commit)
via 05b52b297b30d075ef2409a123f7d096c1156cf8 (commit)
via 1f8d81d0eeda07c3cedcaad3e942ec8dedd461cb (commit)
via d5f5f869d46f9096c7c680d608c1cc654d1d7fa0 (commit)
via 11df73b96ae395fca11b4006253475046e3b74cc (commit)
via 2e919859109fe27d552b81b13d47aed61e80eca6 (commit)
via be81c03a3c26f365eba35b91e4f0827244a02ef7 (commit)
via c0f33379c7fd062fc097ecef92808334e821cb6b (commit)
via f6aa7c0c8c84b85b550d73117c6fdbd663a38c4c (commit)
via 560c318fdc49835b03f96af35774fbbfa7984fe7 (commit)
via 72e3566f2cdacd44f095183ebf88f7aab8b0d8dc (commit)
from 290247250e0c5702a341f98dfdc380e29f1b45c7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 99ca2640c855e88c7b08c3509b21be9e160ccac8
Author: radhika <radhika at curoverse.com>
Date: Tue Oct 6 16:51:36 2015 -0400
7167: Use struct instead of map for APIConfig
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index ab2d9b2..cc99efd 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -78,33 +78,42 @@ type ArvadosClient struct {
DiscoveryDoc Dict
}
-// Create a new ArvadosClient, initialized with standard Arvados environment
-// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
-// ARVADOS_API_HOST_INSECURE.
+// APIConfig struct consists of:
+// APIToken string
+// APIHost string
+// APIHostInsecure bool
+// ExternalClient bool
+type APIConfig struct {
+ APIToken string
+ APIHost string
+ APIHostInsecure bool
+ ExternalClient bool
+}
+
+// Create a new ArvadosClient, initialized with standard Arvados environment variables
+// ARVADOS_API_HOST, ARVADOS_API_TOKEN, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT.
func MakeArvadosClient() (ac ArvadosClient, err error) {
- config := make(map[string]string)
- config["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
- config["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
- config["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
- config["ARVADOS_EXTERNAL_CLIENT"] = os.Getenv("ARVADOS_EXTERNAL_CLIENT")
+ var config APIConfig
+ config.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+ config.APIHost = os.Getenv("ARVADOS_API_HOST")
+
+ var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+
+ config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+ config.ExternalClient = matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
return MakeArvadosClientWithConfig(config)
}
// Create a new ArvadosClient, using the given input parameters.
-func MakeArvadosClientWithConfig(config map[string]string) (ac ArvadosClient, err error) {
- var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
-
- insecure := matchTrue.MatchString(config["ARVADOS_API_HOST_INSECURE"])
- external := matchTrue.MatchString(config["ARVADOS_EXTERNAL_CLIENT"])
-
+func MakeArvadosClientWithConfig(config APIConfig) (ac ArvadosClient, err error) {
ac = ArvadosClient{
- ApiServer: config["ARVADOS_API_HOST"],
- ApiToken: config["ARVADOS_API_TOKEN"],
- ApiInsecure: insecure,
+ ApiServer: config.APIHost,
+ ApiToken: config.APIToken,
+ ApiInsecure: config.APIHostInsecure,
Client: &http.Client{Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
- External: external}
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
+ External: config.ExternalClient}
if ac.ApiServer == "" {
return ac, MissingArvadosApiHost
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 3761cc7..1f00a9f 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -6,13 +6,14 @@ import (
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"io/ioutil"
"log"
+ "regexp"
"strings"
)
// keep-rsync arguments
var (
- srcConfig map[string]string
- dstConfig map[string]string
+ srcConfig arvadosclient.APIConfig
+ dstConfig arvadosclient.APIConfig
srcKeepServicesJSON string
dstKeepServicesJSON string
replications int
@@ -69,6 +70,7 @@ func main() {
var err error
+ // Load config
if srcConfigFile == "" {
log.Fatal("-src-config-file must be specified.")
}
@@ -85,27 +87,41 @@ func main() {
log.Fatal("Error reading destination configuration: %s", err.Error())
}
+ // Initialize keep-rsync
err = initializeKeepRsync()
if err != nil {
log.Fatal("Error configurating keep-rsync: %s", err.Error())
}
}
+var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+
// Reads config from file
-func readConfigFromFile(filename string) (map[string]string, error) {
+func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
+ var config arvadosclient.APIConfig
+
content, err := ioutil.ReadFile(filename)
if err != nil {
- return nil, err
+ return config, err
}
- config := make(map[string]string)
lines := strings.Split(string(content), "\n")
for _, line := range lines {
if line == "" {
continue
}
kv := strings.Split(line, "=")
- config[kv[0]] = kv[1]
+
+ switch kv[0] {
+ case "ARVADOS_API_TOKEN":
+ config.APIToken = kv[1]
+ case "ARVADOS_API_HOST":
+ config.APIHost = kv[1]
+ case "ARVADOS_API_HOST_INSECURE":
+ config.APIHostInsecure = matchTrue.MatchString(kv[1])
+ case "ARVADOS_EXTERNAL_CLIENT":
+ config.ExternalClient = matchTrue.MatchString(kv[1])
+ }
}
return config, nil
}
@@ -157,6 +173,7 @@ func initializeKeepRsync() (err error) {
return
}
}
+ kcDst.Want_replicas = replications
return
}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 97db571..ae80f45 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"os"
+ "regexp"
"testing"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -44,17 +45,18 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
// to create the keep servers to be used as destination.
func setupRsync(c *C) {
// srcConfig
- srcConfig = make(map[string]string)
- srcConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
- srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
- srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+ srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+ srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
// dstConfig
- dstConfig = make(map[string]string)
- dstConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
- dstConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
- dstConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+ dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+ dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+ replications = 1
+
+ // Start API and Keep servers
arvadostest.StartAPI()
arvadostest.StartKeep()
@@ -87,10 +89,10 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
// Invoke readConfigFromFile method with this test filename
config, err := readConfigFromFile(file.Name())
c.Assert(err, Equals, nil)
- c.Assert(config["ARVADOS_API_HOST"], Equals, "testhost")
- c.Assert(config["ARVADOS_API_TOKEN"], Equals, "testtoken")
- c.Assert(config["ARVADOS_API_HOST_INSECURE"], Equals, "true")
- c.Assert(config["EXTERNAL_CLIENT"], Equals, "")
+ c.Assert(config.APIHost, Equals, "testhost")
+ c.Assert(config.APIToken, Equals, "testtoken")
+ c.Assert(config.APIHostInsecure, Equals, true)
+ c.Assert(config.ExternalClient, Equals, false)
}
// Test keep-rsync initialization, with src and dst keep servers.
@@ -150,7 +152,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
c.Check(localRoots != nil, Equals, true)
foundIt := false
- for k, _ := range localRoots {
+ for k := range localRoots {
if k == "zzzzz-bi6l4-123456789012340" {
foundIt = true
}
@@ -158,7 +160,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
c.Check(foundIt, Equals, true)
foundIt = false
- for k, _ := range localRoots {
+ for k := range localRoots {
if k == "zzzzz-bi6l4-123456789012341" {
foundIt = true
}
commit 956c3a765e3794a291376cbb5a2ee9ec6ef4560b
Merge: 2902472 05b52b2
Author: radhika <radhika at curoverse.com>
Date: Tue Oct 6 13:56:12 2015 -0400
Merge branch 'master' into 7167-keep-rsync-test-setup
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list