[ARVADOS] updated: 3bbe1cd9310cfdbdbc63b27bee029cd29c5647e9
Git user
git at public.curoverse.com
Tue Sep 27 14:00:45 EDT 2016
Summary of changes:
services/keep-balance/main_test.go | 7 +++----
services/keep-balance/usage.go | 22 +++++++++-------------
2 files changed, 12 insertions(+), 17 deletions(-)
discards 588587a3200d920fa6ca67340b79ed4de0958a16 (commit)
via 3bbe1cd9310cfdbdbc63b27bee029cd29c5647e9 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (588587a3200d920fa6ca67340b79ed4de0958a16)
\
N -- N -- N (3bbe1cd9310cfdbdbc63b27bee029cd29c5647e9)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3bbe1cd9310cfdbdbc63b27bee029cd29c5647e9
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Sep 27 14:00:21 2016 -0400
9912: Support YAML config files. Change default config file paths to /etc/arvados/foo/foo.yml.
diff --git a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
index 1b4edc9..1d07873 100644
--- a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
@@ -47,15 +47,12 @@ Set up crunch-dispatch-slurm's configuration directory:
</code></pre>
</notextile>
-Edit @/etc/arvados/crunch-dispatch-slurm/config.json@ to authenticate to your Arvados API server, using the token you generated in the previous step. Follow this JSON format:
+Edit @/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml@ to authenticate to your Arvados API server, using the token you generated in the previous step. Follow this YAML format:
<notextile>
-<pre><code class="userinput">{
- "Client": {
- "APIHost": <b>"zzzzz.arvadosapi.com"</b>,
- "AuthToken": <b>"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"</b>
- }
-}
+<pre><code class="userinput">Client:
+ APIHost: <b>zzzzz.arvadosapi.com</b>
+ AuthToken: <b>zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</b>
</code></pre>
</notextile>
diff --git a/doc/install/crunch2-slurm/install-test.html.textile.liquid b/doc/install/crunch2-slurm/install-test.html.textile.liquid
index 4d101ee..d51cfce 100644
--- a/doc/install/crunch2-slurm/install-test.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-test.html.textile.liquid
@@ -57,7 +57,7 @@ This command should return a record with a @container_uuid@ field. Once crunch-
</code></pre>
</notextile>
-If you do not see crunch-dispatch-slurm try to dispatch the container, double-check that it is running and that the API hostname and token in @/etc/arvados/crunch-dispatch-slurm/config.json@ are correct.
+If you do not see crunch-dispatch-slurm try to dispatch the container, double-check that it is running and that the API hostname and token in @/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml@ are correct.
Before the container finishes, SLURM's @squeue@ command will show the new job in the list of queued and running jobs. For example, you might see:
diff --git a/sdk/go/config/load.go b/sdk/go/config/load.go
index 143be8b..9c65d65 100644
--- a/sdk/go/config/load.go
+++ b/sdk/go/config/load.go
@@ -1,21 +1,22 @@
package config
import (
- "encoding/json"
"fmt"
"io/ioutil"
+
+ "github.com/ghodss/yaml"
)
// LoadFile loads configuration from the file given by configPath and
// decodes it into cfg.
//
-// Currently, only JSON is supported. Support for YAML is anticipated.
+// YAML and JSON formats are supported.
func LoadFile(cfg interface{}, configPath string) error {
buf, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
- err = json.Unmarshal(buf, cfg)
+ err = yaml.Unmarshal(buf, cfg)
if err != nil {
return fmt.Errorf("Error decoding config %q: %v", configPath, err)
}
diff --git a/services/arv-git-httpd/main.go b/services/arv-git-httpd/main.go
index 40dcf32..26d2914 100644
--- a/services/arv-git-httpd/main.go
+++ b/services/arv-git-httpd/main.go
@@ -35,7 +35,7 @@ func defaultConfig() *Config {
}
func init() {
- const defaultCfgPath = "/etc/arvados/arv-git-httpd/config.json"
+ const defaultCfgPath = "/etc/arvados/arv-git-httpd/arv-git-httpd.yml"
const deprecated = " (DEPRECATED -- use config file instead)"
flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
"Address to listen on, \"host:port\" or \":port\"."+deprecated)
diff --git a/services/arv-git-httpd/usage.go b/services/arv-git-httpd/usage.go
index a4a9900..666edc0 100644
--- a/services/arv-git-httpd/usage.go
+++ b/services/arv-git-httpd/usage.go
@@ -20,7 +20,7 @@ arv-git-httpd provides authenticated access to Arvados-hosted git repositories.
See http://doc.arvados.org/install/install-arv-git-httpd.html.
-Usage: arv-git-httpd [-config path/to/config.json]
+Usage: arv-git-httpd [-config path/to/arv-git-httpd.yml]
Options:
`)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index b33dc64..5a1ebc5 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -3,11 +3,11 @@ package main
// Dispatcher service for Crunch that submits containers to the slurm queue.
import (
- "encoding/json"
"flag"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/config"
"git.curoverse.com/arvados.git/sdk/go/dispatch"
"github.com/coreos/go-systemd/daemon"
"io"
@@ -37,16 +37,16 @@ type Config struct {
func main() {
err := doMain()
if err != nil {
- log.Fatalf("%q", err)
+ log.Fatal(err)
}
}
var (
- config Config
+ theConfig Config
squeueUpdater Squeue
)
-const defaultConfigPath = "/etc/arvados/crunch-dispatch-slurm/config.json"
+const defaultConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
func doMain() error {
flags := flag.NewFlagSet("crunch-dispatch-slurm", flag.ExitOnError)
@@ -55,33 +55,32 @@ func doMain() error {
configPath := flags.String(
"config",
defaultConfigPath,
- "`path` to json configuration file")
+ "`path` to JSON or YAML configuration file")
// Parse args; omit the first arg which is the command name
flags.Parse(os.Args[1:])
- err := readConfig(&config, *configPath)
+ err := readConfig(&theConfig, *configPath)
if err != nil {
- log.Printf("Error reading configuration: %v", err)
return err
}
- if config.CrunchRunCommand == nil {
- config.CrunchRunCommand = []string{"crunch-run"}
+ if theConfig.CrunchRunCommand == nil {
+ theConfig.CrunchRunCommand = []string{"crunch-run"}
}
- if config.PollPeriod == 0 {
- config.PollPeriod = arvados.Duration(10 * time.Second)
+ if theConfig.PollPeriod == 0 {
+ theConfig.PollPeriod = arvados.Duration(10 * time.Second)
}
- if config.Client.APIHost != "" || config.Client.AuthToken != "" {
+ if theConfig.Client.APIHost != "" || theConfig.Client.AuthToken != "" {
// Copy real configs into env vars so [a]
// MakeArvadosClient() uses them, and [b] they get
// propagated to crunch-run via SLURM.
- os.Setenv("ARVADOS_API_HOST", config.Client.APIHost)
- os.Setenv("ARVADOS_API_TOKEN", config.Client.AuthToken)
+ os.Setenv("ARVADOS_API_HOST", theConfig.Client.APIHost)
+ os.Setenv("ARVADOS_API_TOKEN", theConfig.Client.AuthToken)
os.Setenv("ARVADOS_API_INSECURE", "")
- if config.Client.Insecure {
+ if theConfig.Client.Insecure {
os.Setenv("ARVADOS_API_INSECURE", "1")
}
os.Setenv("ARVADOS_KEEP_SERVICES", "")
@@ -97,13 +96,13 @@ func doMain() error {
}
arv.Retries = 25
- squeueUpdater.StartMonitor(time.Duration(config.PollPeriod))
+ squeueUpdater.StartMonitor(time.Duration(theConfig.PollPeriod))
defer squeueUpdater.Done()
dispatcher := dispatch.Dispatcher{
Arv: arv,
RunContainer: run,
- PollInterval: time.Duration(config.PollPeriod),
+ PollInterval: time.Duration(theConfig.PollPeriod),
DoneProcessing: make(chan struct{})}
if _, err := daemon.SdNotify("READY=1"); err != nil {
@@ -124,7 +123,7 @@ func sbatchFunc(container arvados.Container) *exec.Cmd {
var sbatchArgs []string
sbatchArgs = append(sbatchArgs, "--share")
- sbatchArgs = append(sbatchArgs, config.SbatchArguments...)
+ sbatchArgs = append(sbatchArgs, theConfig.SbatchArguments...)
sbatchArgs = append(sbatchArgs, fmt.Sprintf("--job-name=%s", container.UUID))
sbatchArgs = append(sbatchArgs, fmt.Sprintf("--mem-per-cpu=%d", int(memPerCPU)))
sbatchArgs = append(sbatchArgs, fmt.Sprintf("--cpus-per-task=%d", container.RuntimeConstraints.VCPUs))
@@ -240,7 +239,7 @@ func monitorSubmitOrCancel(dispatcher *dispatch.Dispatcher, container arvados.Co
log.Printf("About to submit queued container %v", container.UUID)
- if err := submit(dispatcher, container, config.CrunchRunCommand); err != nil {
+ if err := submit(dispatcher, container, theConfig.CrunchRunCommand); err != nil {
log.Printf("Error submitting container %s to slurm: %v",
container.UUID, err)
// maybe sbatch is broken, put it back to queued
@@ -320,16 +319,10 @@ func run(dispatcher *dispatch.Dispatcher,
}
func readConfig(dst interface{}, path string) error {
- if buf, err := ioutil.ReadFile(path); err != nil && os.IsNotExist(err) {
- if path == defaultConfigPath {
- log.Printf("Config not specified. Continue with default configuration.")
- } else {
- return fmt.Errorf("Config file not found %q: %v", path, err)
- }
- } else if err != nil {
- return fmt.Errorf("Error reading config %q: %v", path, err)
- } else if err = json.Unmarshal(buf, dst); err != nil {
- return fmt.Errorf("Error decoding config %q: %v", path, err)
+ err := config.LoadFile(dst, path)
+ if err != nil && os.IsNotExist(err) && path == defaultConfigPath {
+ log.Printf("Config not specified. Continue with default configuration.")
+ err = nil
}
- return nil
+ return err
}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 6692f7f..39ec15d 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -143,7 +143,7 @@ func (s *TestSuite) integrationTest(c *C,
c.Check(err, IsNil)
c.Check(len(containers.Items), Equals, 1)
- config.CrunchRunCommand = []string{"echo"}
+ theConfig.CrunchRunCommand = []string{"echo"}
doneProcessing := make(chan struct{})
dispatcher := dispatch.Dispatcher{
@@ -205,7 +205,7 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon
log.SetOutput(io.MultiWriter(buf, os.Stderr))
defer log.SetOutput(os.Stderr)
- config.CrunchRunCommand = []string{crunchCmd}
+ theConfig.CrunchRunCommand = []string{crunchCmd}
doneProcessing := make(chan struct{})
dispatcher := dispatch.Dispatcher{
@@ -303,14 +303,14 @@ func (s *MockArvadosServerSuite) TestSbatchFuncWithConfigArgs(c *C) {
}
func testSbatchFuncWithArgs(c *C, args []string) {
- config.SbatchArguments = append(config.SbatchArguments, args...)
+ theConfig.SbatchArguments = append(theConfig.SbatchArguments, args...)
container := arvados.Container{UUID: "123", RuntimeConstraints: arvados.RuntimeConstraints{RAM: 1000000, VCPUs: 2}}
sbatchCmd := sbatchFunc(container)
var expected []string
expected = append(expected, "sbatch", "--share")
- expected = append(expected, config.SbatchArguments...)
+ expected = append(expected, theConfig.SbatchArguments...)
expected = append(expected, "--job-name=123", "--mem-per-cpu=1", "--cpus-per-task=2")
c.Check(sbatchCmd.Args, DeepEquals, expected)
diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 7fcc5ad..634b44b 100644
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -5,7 +5,7 @@ After=network.target
[Service]
Type=simple
-ExecStart=/usr/bin/keep-balance -config /etc/arvados/keep-balance/config.json -commit-pulls -commit-trash
+ExecStart=/usr/bin/keep-balance -config /etc/arvados/keep-balance/keep-balance.yml -commit-pulls -commit-trash
Restart=always
RestartSec=10s
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index da4fb62..60349e9 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"flag"
- "io/ioutil"
"log"
"os"
"os/signal"
@@ -11,6 +10,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/config"
)
// Config specifies site configuration, like API credentials and the
@@ -66,9 +66,9 @@ func main() {
var runOptions RunOptions
configPath := flag.String("config", "",
- "`path` of json configuration file")
+ "`path` of JSON or YAML configuration file")
serviceListPath := flag.String("config.KeepServiceList", "",
- "`path` of json file with list of keep services to balance, as given by \"arv keep_service list\" "+
+ "`path` of JSON or YAML file with list of keep services to balance, as given by \"arv keep_service list\" "+
"(default: config[\"KeepServiceList\"], or if none given, get all available services and filter by config[\"KeepServiceTypes\"])")
flag.BoolVar(&runOptions.Once, "once", false,
"balance once and then exit")
@@ -84,9 +84,9 @@ func main() {
if *configPath == "" {
log.Fatal("You must specify a config file (see `keep-balance -help`)")
}
- mustReadJSON(&config, *configPath)
+ mustReadConfig(&config, *configPath)
if *serviceListPath != "" {
- mustReadJSON(&config.KeepServiceList, *serviceListPath)
+ mustReadConfig(&config.KeepServiceList, *serviceListPath)
}
if *debugFlag {
@@ -113,11 +113,9 @@ func main() {
}
}
-func mustReadJSON(dst interface{}, path string) {
- if buf, err := ioutil.ReadFile(path); err != nil {
- log.Fatalf("Reading %q: %v", path, err)
- } else if err = json.Unmarshal(buf, dst); err != nil {
- log.Fatalf("Decoding %q: %v", path, err)
+func mustReadConfig(dst interface{}, path string) {
+ if err := config.LoadFile(dst, path); err != nil {
+ log.Fatal(err)
}
}
diff --git a/services/keep-balance/main_test.go b/services/keep-balance/main_test.go
index 4a56098..157ee45 100644
--- a/services/keep-balance/main_test.go
+++ b/services/keep-balance/main_test.go
@@ -1,9 +1,9 @@
package main
import (
- "encoding/json"
"time"
+ "github.com/ghodss/yaml"
check "gopkg.in/check.v1"
)
@@ -13,7 +13,7 @@ type mainSuite struct{}
func (s *mainSuite) TestExampleJSON(c *check.C) {
var config Config
- c.Check(json.Unmarshal(exampleConfigFile, &config), check.IsNil)
+ c.Check(yaml.Unmarshal(exampleConfigFile, &config), check.IsNil)
c.Check(config.KeepServiceTypes, check.DeepEquals, []string{"disk"})
c.Check(config.Client.AuthToken, check.Equals, "xyzzy")
c.Check(time.Duration(config.RunPeriod), check.Equals, 600*time.Second)
@@ -21,8 +21,7 @@ func (s *mainSuite) TestExampleJSON(c *check.C) {
func (s *mainSuite) TestConfigJSONWithKeepServiceList(c *check.C) {
var config Config
- c.Check(json.Unmarshal([]byte(`
- {
+ c.Check(yaml.Unmarshal([]byte(`{
"Client": {
"APIHost": "zzzzz.arvadosapi.com:443",
"AuthToken": "xyzzy",
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index b521c65..d112010 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -7,19 +7,15 @@ import (
)
var exampleConfigFile = []byte(`
- {
- "Client": {
- "APIHost": "zzzzz.arvadosapi.com:443",
- "AuthToken": "xyzzy",
- "Insecure": false
- },
- "KeepServiceTypes": [
- "disk"
- ],
- "RunPeriod": "600s",
- "CollectionBatchSize": 100000,
- "CollectionBuffers": 1000
- }`)
+Client:
+ APIHost: zzzzz.arvadosapi.com:443
+ AuthToken: xyzzy
+ Insecure: false
+KeepServiceTypes:
+ - disk
+RunPeriod: 600s
+CollectionBatchSize: 100000
+CollectionBuffers: 1000`)
func usage() {
fmt.Fprintf(os.Stderr, `
@@ -30,7 +26,7 @@ overreplicated and unreferenced blocks, and moves blocks to better
positions (according to the rendezvous hash algorithm) so clients find
them faster.
-Usage: keep-balance -config path/to/config.json [options]
+Usage: keep-balance -config path/to/keep-balance.yml [options]
Options:
`)
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 37f81b1..3326dd1 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -9,32 +9,30 @@
// Configuration
//
// The default configuration file location is
-// /etc/arvados/keep-web/config.json.
+// /etc/arvados/keep-web/keep-web.yml.
//
// Example configuration file
//
-// {
-// "Client": {
-// "APIHost": "zzzzz.arvadosapi.com:443",
-// "AuthToken": "",
-// "Insecure": false
-// },
-// "Listen":":1234",
-// "AnonymousTokens":["xxxxxxxxxxxxxxxxxxxx"],
-// "AttachmentOnlyHost":"",
-// "TrustAllContent":false
-// }
+// Client:
+// APIHost: "zzzzz.arvadosapi.com:443"
+// AuthToken: ""
+// Insecure: false
+// Listen: :1234
+// AnonymousTokens:
+// - xxxxxxxxxxxxxxxxxxxx
+// AttachmentOnlyHost: ""
+// TrustAllContent: false
//
// Starting the server
//
// Start a server using the default config file
-// /etc/arvados/keep-web/config.json:
+// /etc/arvados/keep-web/keep-web.yml:
//
// keep-web
//
-// Start a server using the config file /path/to/config.json:
+// Start a server using the config file /path/to/keep-web.yml:
//
-// keep-web -config /path/to/config.json
+// keep-web -config /path/to/keep-web.yml
//
// Proxy configuration
//
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 2d563bd..13d8c1b 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -11,7 +11,7 @@ import (
)
var (
- defaultConfigPath = "/etc/arvados/keep-web/config.json"
+ defaultConfigPath = "/etc/arvados/keep-web/keep-web.yml"
)
// Config specifies server configuration.
@@ -53,7 +53,7 @@ func main() {
var configPath string
deprecated := " (DEPRECATED -- use config file instead)"
flag.StringVar(&configPath, "config", defaultConfigPath,
- "`path` to json configuration file")
+ "`path` to JSON or YAML configuration file")
flag.StringVar(&cfg.Listen, "listen", "",
"address:port or :port to listen on"+deprecated)
flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false,
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
index 1cd4d56..a36bf58 100644
--- a/services/keep-web/usage.go
+++ b/services/keep-web/usage.go
@@ -21,7 +21,7 @@ Keep-web provides read-only HTTP access to files stored in Keep; see
https://godoc.org/github.com/curoverse/arvados/services/keep-web and
http://doc.arvados.org/install/install-keep-web.html
-Usage: keep-web -config path/to/config.json
+Usage: keep-web -config path/to/keep-web.yml
Options:
`)
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 4666493..816de29 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -60,7 +60,7 @@ func main() {
timeoutSeconds := flagset.Int("timeout", int(time.Duration(cfg.Timeout)/time.Second), "Timeout (in seconds) on requests to internal Keep services."+deprecated)
var cfgPath string
- const defaultCfgPath = "/etc/arvados/keepproxy/config.json"
+ const defaultCfgPath = "/etc/arvados/keepproxy/keepproxy.yml"
flagset.StringVar(&cfgPath, "config", defaultCfgPath, "Configuration file `path`")
flagset.Parse(os.Args[1:])
diff --git a/services/keepproxy/usage.go b/services/keepproxy/usage.go
index 18bf0ad..403c6bf 100644
--- a/services/keepproxy/usage.go
+++ b/services/keepproxy/usage.go
@@ -19,7 +19,7 @@ func usage() {
Keepproxy forwards GET and PUT requests to keepstore servers. See
http://doc.arvados.org/install/install-keepproxy.html
-Usage: keepproxy [-config path/to/config.json]
+Usage: keepproxy [-config path/to/keepproxy.yml]
Options:
`)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list