[ARVADOS] created: 588587a3200d920fa6ca67340b79ed4de0958a16

Git user git at public.curoverse.com
Tue Sep 27 13:53:19 EDT 2016


        at  588587a3200d920fa6ca67340b79ed4de0958a16 (commit)


commit 588587a3200d920fa6ca67340b79ed4de0958a16
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Sep 27 11:15:18 2016 -0400

    9912: Support YAML config files. Change default config file paths to /etc/arvados/foo/foo.yml.

diff --git a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
index 1b4edc9..1d07873 100644
--- a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
@@ -47,15 +47,12 @@ Set up crunch-dispatch-slurm's configuration directory:
 </code></pre>
 </notextile>
 
-Edit @/etc/arvados/crunch-dispatch-slurm/config.json@ to authenticate to your Arvados API server, using the token you generated in the previous step.  Follow this JSON format:
+Edit @/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml@ to authenticate to your Arvados API server, using the token you generated in the previous step.  Follow this YAML format:
 
 <notextile>
-<pre><code class="userinput">{
-  "Client": {
-    "APIHost": <b>"zzzzz.arvadosapi.com"</b>,
-    "AuthToken": <b>"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"</b>
-  }
-}
+<pre><code class="userinput">Client:
+  APIHost: <b>zzzzz.arvadosapi.com</b>
+  AuthToken: <b>zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</b>
 </code></pre>
 </notextile>
 
diff --git a/doc/install/crunch2-slurm/install-test.html.textile.liquid b/doc/install/crunch2-slurm/install-test.html.textile.liquid
index 4d101ee..d51cfce 100644
--- a/doc/install/crunch2-slurm/install-test.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-test.html.textile.liquid
@@ -57,7 +57,7 @@ This command should return a record with a @container_uuid@ field.  Once crunch-
 </code></pre>
 </notextile>
 
-If you do not see crunch-dispatch-slurm try to dispatch the container, double-check that it is running and that the API hostname and token in @/etc/arvados/crunch-dispatch-slurm/config.json@ are correct.
+If you do not see crunch-dispatch-slurm try to dispatch the container, double-check that it is running and that the API hostname and token in @/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml@ are correct.
 
 Before the container finishes, SLURM's @squeue@ command will show the new job in the list of queued and running jobs.  For example, you might see:
 
diff --git a/sdk/go/config/load.go b/sdk/go/config/load.go
index 143be8b..9c65d65 100644
--- a/sdk/go/config/load.go
+++ b/sdk/go/config/load.go
@@ -1,21 +1,22 @@
 package config
 
 import (
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
+
+	"github.com/ghodss/yaml"
 )
 
 // LoadFile loads configuration from the file given by configPath and
 // decodes it into cfg.
 //
-// Currently, only JSON is supported. Support for YAML is anticipated.
+// YAML and JSON formats are supported.
 func LoadFile(cfg interface{}, configPath string) error {
 	buf, err := ioutil.ReadFile(configPath)
 	if err != nil {
 		return err
 	}
-	err = json.Unmarshal(buf, cfg)
+	err = yaml.Unmarshal(buf, cfg)
 	if err != nil {
 		return fmt.Errorf("Error decoding config %q: %v", configPath, err)
 	}
diff --git a/services/arv-git-httpd/main.go b/services/arv-git-httpd/main.go
index 40dcf32..26d2914 100644
--- a/services/arv-git-httpd/main.go
+++ b/services/arv-git-httpd/main.go
@@ -35,7 +35,7 @@ func defaultConfig() *Config {
 }
 
 func init() {
-	const defaultCfgPath = "/etc/arvados/arv-git-httpd/config.json"
+	const defaultCfgPath = "/etc/arvados/arv-git-httpd/arv-git-httpd.yml"
 	const deprecated = " (DEPRECATED -- use config file instead)"
 	flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
 		"Address to listen on, \"host:port\" or \":port\"."+deprecated)
diff --git a/services/arv-git-httpd/usage.go b/services/arv-git-httpd/usage.go
index a4a9900..666edc0 100644
--- a/services/arv-git-httpd/usage.go
+++ b/services/arv-git-httpd/usage.go
@@ -20,7 +20,7 @@ arv-git-httpd provides authenticated access to Arvados-hosted git repositories.
 
 See http://doc.arvados.org/install/install-arv-git-httpd.html.
 
-Usage: arv-git-httpd [-config path/to/config.json]
+Usage: arv-git-httpd [-config path/to/arv-git-httpd.yml]
 
 Options:
 `)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index b33dc64..5a1ebc5 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -3,11 +3,11 @@ package main
 // Dispatcher service for Crunch that submits containers to the slurm queue.
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/config"
 	"git.curoverse.com/arvados.git/sdk/go/dispatch"
 	"github.com/coreos/go-systemd/daemon"
 	"io"
@@ -37,16 +37,16 @@ type Config struct {
 func main() {
 	err := doMain()
 	if err != nil {
-		log.Fatalf("%q", err)
+		log.Fatal(err)
 	}
 }
 
 var (
-	config        Config
+	theConfig     Config
 	squeueUpdater Squeue
 )
 
-const defaultConfigPath = "/etc/arvados/crunch-dispatch-slurm/config.json"
+const defaultConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
 
 func doMain() error {
 	flags := flag.NewFlagSet("crunch-dispatch-slurm", flag.ExitOnError)
@@ -55,33 +55,32 @@ func doMain() error {
 	configPath := flags.String(
 		"config",
 		defaultConfigPath,
-		"`path` to json configuration file")
+		"`path` to JSON or YAML configuration file")
 
 	// Parse args; omit the first arg which is the command name
 	flags.Parse(os.Args[1:])
 
-	err := readConfig(&config, *configPath)
+	err := readConfig(&theConfig, *configPath)
 	if err != nil {
-		log.Printf("Error reading configuration: %v", err)
 		return err
 	}
 
-	if config.CrunchRunCommand == nil {
-		config.CrunchRunCommand = []string{"crunch-run"}
+	if theConfig.CrunchRunCommand == nil {
+		theConfig.CrunchRunCommand = []string{"crunch-run"}
 	}
 
-	if config.PollPeriod == 0 {
-		config.PollPeriod = arvados.Duration(10 * time.Second)
+	if theConfig.PollPeriod == 0 {
+		theConfig.PollPeriod = arvados.Duration(10 * time.Second)
 	}
 
-	if config.Client.APIHost != "" || config.Client.AuthToken != "" {
+	if theConfig.Client.APIHost != "" || theConfig.Client.AuthToken != "" {
 		// Copy real configs into env vars so [a]
 		// MakeArvadosClient() uses them, and [b] they get
 		// propagated to crunch-run via SLURM.
-		os.Setenv("ARVADOS_API_HOST", config.Client.APIHost)
-		os.Setenv("ARVADOS_API_TOKEN", config.Client.AuthToken)
+		os.Setenv("ARVADOS_API_HOST", theConfig.Client.APIHost)
+		os.Setenv("ARVADOS_API_TOKEN", theConfig.Client.AuthToken)
 		os.Setenv("ARVADOS_API_INSECURE", "")
-		if config.Client.Insecure {
+		if theConfig.Client.Insecure {
 			os.Setenv("ARVADOS_API_INSECURE", "1")
 		}
 		os.Setenv("ARVADOS_KEEP_SERVICES", "")
@@ -97,13 +96,13 @@ func doMain() error {
 	}
 	arv.Retries = 25
 
-	squeueUpdater.StartMonitor(time.Duration(config.PollPeriod))
+	squeueUpdater.StartMonitor(time.Duration(theConfig.PollPeriod))
 	defer squeueUpdater.Done()
 
 	dispatcher := dispatch.Dispatcher{
 		Arv:            arv,
 		RunContainer:   run,
-		PollInterval:   time.Duration(config.PollPeriod),
+		PollInterval:   time.Duration(theConfig.PollPeriod),
 		DoneProcessing: make(chan struct{})}
 
 	if _, err := daemon.SdNotify("READY=1"); err != nil {
@@ -124,7 +123,7 @@ func sbatchFunc(container arvados.Container) *exec.Cmd {
 
 	var sbatchArgs []string
 	sbatchArgs = append(sbatchArgs, "--share")
-	sbatchArgs = append(sbatchArgs, config.SbatchArguments...)
+	sbatchArgs = append(sbatchArgs, theConfig.SbatchArguments...)
 	sbatchArgs = append(sbatchArgs, fmt.Sprintf("--job-name=%s", container.UUID))
 	sbatchArgs = append(sbatchArgs, fmt.Sprintf("--mem-per-cpu=%d", int(memPerCPU)))
 	sbatchArgs = append(sbatchArgs, fmt.Sprintf("--cpus-per-task=%d", container.RuntimeConstraints.VCPUs))
@@ -240,7 +239,7 @@ func monitorSubmitOrCancel(dispatcher *dispatch.Dispatcher, container arvados.Co
 
 			log.Printf("About to submit queued container %v", container.UUID)
 
-			if err := submit(dispatcher, container, config.CrunchRunCommand); err != nil {
+			if err := submit(dispatcher, container, theConfig.CrunchRunCommand); err != nil {
 				log.Printf("Error submitting container %s to slurm: %v",
 					container.UUID, err)
 				// maybe sbatch is broken, put it back to queued
@@ -320,16 +319,10 @@ func run(dispatcher *dispatch.Dispatcher,
 }
 
 func readConfig(dst interface{}, path string) error {
-	if buf, err := ioutil.ReadFile(path); err != nil && os.IsNotExist(err) {
-		if path == defaultConfigPath {
-			log.Printf("Config not specified. Continue with default configuration.")
-		} else {
-			return fmt.Errorf("Config file not found %q: %v", path, err)
-		}
-	} else if err != nil {
-		return fmt.Errorf("Error reading config %q: %v", path, err)
-	} else if err = json.Unmarshal(buf, dst); err != nil {
-		return fmt.Errorf("Error decoding config %q: %v", path, err)
+	err := config.LoadFile(dst, path)
+	if err != nil && os.IsNotExist(err) && path == defaultConfigPath {
+		log.Printf("Config not specified. Continue with default configuration.")
+		err = nil
 	}
-	return nil
+	return err
 }
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 6692f7f..39ec15d 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -143,7 +143,7 @@ func (s *TestSuite) integrationTest(c *C,
 	c.Check(err, IsNil)
 	c.Check(len(containers.Items), Equals, 1)
 
-	config.CrunchRunCommand = []string{"echo"}
+	theConfig.CrunchRunCommand = []string{"echo"}
 
 	doneProcessing := make(chan struct{})
 	dispatcher := dispatch.Dispatcher{
@@ -205,7 +205,7 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon
 	log.SetOutput(io.MultiWriter(buf, os.Stderr))
 	defer log.SetOutput(os.Stderr)
 
-	config.CrunchRunCommand = []string{crunchCmd}
+	theConfig.CrunchRunCommand = []string{crunchCmd}
 
 	doneProcessing := make(chan struct{})
 	dispatcher := dispatch.Dispatcher{
@@ -303,14 +303,14 @@ func (s *MockArvadosServerSuite) TestSbatchFuncWithConfigArgs(c *C) {
 }
 
 func testSbatchFuncWithArgs(c *C, args []string) {
-	config.SbatchArguments = append(config.SbatchArguments, args...)
+	theConfig.SbatchArguments = append(theConfig.SbatchArguments, args...)
 
 	container := arvados.Container{UUID: "123", RuntimeConstraints: arvados.RuntimeConstraints{RAM: 1000000, VCPUs: 2}}
 	sbatchCmd := sbatchFunc(container)
 
 	var expected []string
 	expected = append(expected, "sbatch", "--share")
-	expected = append(expected, config.SbatchArguments...)
+	expected = append(expected, theConfig.SbatchArguments...)
 	expected = append(expected, "--job-name=123", "--mem-per-cpu=1", "--cpus-per-task=2")
 
 	c.Check(sbatchCmd.Args, DeepEquals, expected)
diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 7fcc5ad..634b44b 100644
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -5,7 +5,7 @@ After=network.target
 
 [Service]
 Type=simple
-ExecStart=/usr/bin/keep-balance -config /etc/arvados/keep-balance/config.json -commit-pulls -commit-trash
+ExecStart=/usr/bin/keep-balance -config /etc/arvados/keep-balance/keep-balance.yml -commit-pulls -commit-trash
 Restart=always
 RestartSec=10s
 
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index da4fb62..60349e9 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"flag"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/signal"
@@ -11,6 +10,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/config"
 )
 
 // Config specifies site configuration, like API credentials and the
@@ -66,9 +66,9 @@ func main() {
 	var runOptions RunOptions
 
 	configPath := flag.String("config", "",
-		"`path` of json configuration file")
+		"`path` of JSON or YAML configuration file")
 	serviceListPath := flag.String("config.KeepServiceList", "",
-		"`path` of json file with list of keep services to balance, as given by \"arv keep_service list\" "+
+		"`path` of JSON or YAML file with list of keep services to balance, as given by \"arv keep_service list\" "+
 			"(default: config[\"KeepServiceList\"], or if none given, get all available services and filter by config[\"KeepServiceTypes\"])")
 	flag.BoolVar(&runOptions.Once, "once", false,
 		"balance once and then exit")
@@ -84,9 +84,9 @@ func main() {
 	if *configPath == "" {
 		log.Fatal("You must specify a config file (see `keep-balance -help`)")
 	}
-	mustReadJSON(&config, *configPath)
+	mustReadConfig(&config, *configPath)
 	if *serviceListPath != "" {
-		mustReadJSON(&config.KeepServiceList, *serviceListPath)
+		mustReadConfig(&config.KeepServiceList, *serviceListPath)
 	}
 
 	if *debugFlag {
@@ -113,11 +113,9 @@ func main() {
 	}
 }
 
-func mustReadJSON(dst interface{}, path string) {
-	if buf, err := ioutil.ReadFile(path); err != nil {
-		log.Fatalf("Reading %q: %v", path, err)
-	} else if err = json.Unmarshal(buf, dst); err != nil {
-		log.Fatalf("Decoding %q: %v", path, err)
+func mustReadConfig(dst interface{}, path string) {
+	if err := config.LoadFile(dst, path); err != nil {
+		log.Fatal(err)
 	}
 }
 
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index b521c65..8d2ae6f 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -30,7 +30,7 @@ overreplicated and unreferenced blocks, and moves blocks to better
 positions (according to the rendezvous hash algorithm) so clients find
 them faster.
 
-Usage: keep-balance -config path/to/config.json [options]
+Usage: keep-balance -config path/to/keep-balance.yml [options]
 
 Options:
 `)
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 37f81b1..3326dd1 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -9,32 +9,30 @@
 // Configuration
 //
 // The default configuration file location is
-// /etc/arvados/keep-web/config.json.
+// /etc/arvados/keep-web/keep-web.yml.
 //
 // Example configuration file
 //
-//   {
-//     "Client": {
-//       "APIHost": "zzzzz.arvadosapi.com:443",
-//       "AuthToken": "",
-//       "Insecure": false
-//     },
-//     "Listen":":1234",
-//     "AnonymousTokens":["xxxxxxxxxxxxxxxxxxxx"],
-//     "AttachmentOnlyHost":"",
-//     "TrustAllContent":false
-//   }
+//     Client:
+//       APIHost: "zzzzz.arvadosapi.com:443"
+//       AuthToken: ""
+//       Insecure: false
+//     Listen: :1234
+//     AnonymousTokens:
+//       - xxxxxxxxxxxxxxxxxxxx
+//     AttachmentOnlyHost: ""
+//     TrustAllContent: false
 //
 // Starting the server
 //
 // Start a server using the default config file
-// /etc/arvados/keep-web/config.json:
+// /etc/arvados/keep-web/keep-web.yml:
 //
 //   keep-web
 //
-// Start a server using the config file /path/to/config.json:
+// Start a server using the config file /path/to/keep-web.yml:
 //
-//   keep-web -config /path/to/config.json
+//   keep-web -config /path/to/keep-web.yml
 //
 // Proxy configuration
 //
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 2d563bd..13d8c1b 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	defaultConfigPath = "/etc/arvados/keep-web/config.json"
+	defaultConfigPath = "/etc/arvados/keep-web/keep-web.yml"
 )
 
 // Config specifies server configuration.
@@ -53,7 +53,7 @@ func main() {
 	var configPath string
 	deprecated := " (DEPRECATED -- use config file instead)"
 	flag.StringVar(&configPath, "config", defaultConfigPath,
-		"`path` to json configuration file")
+		"`path` to JSON or YAML configuration file")
 	flag.StringVar(&cfg.Listen, "listen", "",
 		"address:port or :port to listen on"+deprecated)
 	flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false,
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
index 1cd4d56..a36bf58 100644
--- a/services/keep-web/usage.go
+++ b/services/keep-web/usage.go
@@ -21,7 +21,7 @@ Keep-web provides read-only HTTP access to files stored in Keep; see
 https://godoc.org/github.com/curoverse/arvados/services/keep-web and
 http://doc.arvados.org/install/install-keep-web.html
 
-Usage: keep-web -config path/to/config.json
+Usage: keep-web -config path/to/keep-web.yml
 
 Options:
 `)
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 4666493..816de29 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -60,7 +60,7 @@ func main() {
 	timeoutSeconds := flagset.Int("timeout", int(time.Duration(cfg.Timeout)/time.Second), "Timeout (in seconds) on requests to internal Keep services."+deprecated)
 
 	var cfgPath string
-	const defaultCfgPath = "/etc/arvados/keepproxy/config.json"
+	const defaultCfgPath = "/etc/arvados/keepproxy/keepproxy.yml"
 	flagset.StringVar(&cfgPath, "config", defaultCfgPath, "Configuration file `path`")
 	flagset.Parse(os.Args[1:])
 
diff --git a/services/keepproxy/usage.go b/services/keepproxy/usage.go
index 18bf0ad..403c6bf 100644
--- a/services/keepproxy/usage.go
+++ b/services/keepproxy/usage.go
@@ -19,7 +19,7 @@ func usage() {
 Keepproxy forwards GET and PUT requests to keepstore servers.  See
 http://doc.arvados.org/install/install-keepproxy.html
 
-Usage: keepproxy [-config path/to/config.json]
+Usage: keepproxy [-config path/to/keepproxy.yml]
 
 Options:
 `)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list