[ARVADOS] created: 1.3.0-1402-g59a2d537f

Git user git at public.curoverse.com
Thu Jul 25 13:39:01 UTC 2019


        at  59a2d537f3450407aa48e32645d92a5246c046fe (commit)


commit 59a2d537f3450407aa48e32645d92a5246c046fe
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Jul 24 16:25:06 2019 -0400

    14717: Delete obsolete functions from run_test_server.py
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 987c67303..b073eff40 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -758,35 +758,6 @@ def stop_nginx():
 def _pidfile(program):
     return os.path.join(TEST_TMPDIR, program + '.pid')
 
-def _dbconfig(key):
-    global _cached_db_config
-    if not _cached_db_config:
-        if "ARVADOS_CONFIG" in os.environ:
-            _cached_db_config = list(yaml.safe_load(open(os.environ["ARVADOS_CONFIG"]))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
-        else:
-            _cached_db_config = yaml.safe_load(open(os.path.join(
-                SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))["test"]
-            _cached_db_config["dbname"] = _cached_db_config["database"]
-            _cached_db_config["user"] = _cached_db_config["username"]
-    return _cached_db_config[key]
-
-def _apiconfig(key):
-    global _cached_config
-    if _cached_config:
-        return _cached_config[key]
-    def _load(f, required=True):
-        fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
-        if not required and not os.path.exists(fullpath):
-            return {}
-        return yaml.safe_load(fullpath)
-    cdefault = _load('application.default.yml')
-    csite = _load('application.yml', required=False)
-    _cached_config = {}
-    for section in [cdefault.get('common',{}), cdefault.get('test',{}),
-                    csite.get('common',{}), csite.get('test',{})]:
-        _cached_config.update(section)
-    return _cached_config[key]
-
 def fixture(fix):
     '''load a fixture yaml file'''
     with open(os.path.join(SERVICES_SRC_DIR, 'api', "test", "fixtures",

commit 7b4ec6d3c3ed209a42f542e1b646b8e672847fea
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Jul 24 15:58:40 2019 -0400

    14717: Refactor run_test_server.py and run_test.sh to use config.yml
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 626ea974a..0e38b484e 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -385,12 +385,8 @@ checkpidfile() {
 
 checkhealth() {
     svc="$1"
-    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
-    scheme=http
-    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
-        scheme=https
-    fi
-    url="$scheme://localhost:${port}/_health/ping"
+    base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]")
+    url="$base/_health/ping"
     if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
         echo "${url} failed"
         return 1
@@ -428,22 +424,22 @@ start_services() {
         && export ARVADOS_TEST_API_INSTALLED="$$" \
         && checkpidfile api \
         && checkdiscoverydoc $ARVADOS_API_HOST \
+        && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+        && checkpidfile nginx \
         && python sdk/python/tests/run_test_server.py start_controller \
         && checkpidfile controller \
-        && checkhealth controller \
+        && checkhealth Controller \
+        && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && checkpidfile keep-web \
-        && checkhealth keep-web \
+        && checkhealth WebDAV \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && checkpidfile arv-git-httpd \
-        && checkhealth arv-git-httpd \
+        && checkhealth GitHTTP \
         && python sdk/python/tests/run_test_server.py start_ws \
         && checkpidfile ws \
-        && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
-        && checkdiscoverydoc $ARVADOS_API_HOST \
-        && checkpidfile nginx \
         && export ARVADOS_TEST_PROXY_SERVICES=1 \
         && (env | egrep ^ARVADOS) \
         && fail=0
@@ -627,25 +623,6 @@ initialize() {
     # whine a lot.
     setup_ruby_environment
 
-    if [[ -s "$CONFIGSRC/config.yml" ]] ; then
-	echo "Getting database configuration from $CONFIGSRC/config.yml"
-	cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
-    else
-	if [[ -s /etc/arvados/config.yml ]] ; then
-	    echo "Getting database configuration from /etc/arvados/config.yml"
-	    python > "$temp/test-config.yml" <<EOF
-import yaml
-import json
-v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0]['PostgreSQL']
-v['Connection']['dbname'] = 'arvados_test'
-print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
-EOF
-	else
-	    fatal "Please provide a config.yml file for the test suite in CONFIGSRC or /etc/arvados"
-	fi
-    fi
-    export ARVADOS_CONFIG="$temp/test-config.yml"
-
     echo "PATH is $PATH"
 }
 
@@ -681,6 +658,8 @@ install_env() {
     pip install --no-cache-dir PyYAML \
         || fatal "pip install PyYAML failed"
 
+    eval $(python sdk/python/tests/run_test_server.py setup_config)
+
     # Preinstall libcloud if using a fork; otherwise nodemanager "pip
     # install" won't pick it up by default.
     if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 0b86aea13..987c67303 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -313,15 +313,9 @@ def run(leave_running_atexit=False):
         os.makedirs(gitdir)
     subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
 
-    # The nginx proxy isn't listening here yet, but we need to choose
-    # the wss:// port now so we can write the API server config file.
-    wss_port = find_available_port()
-    _setport('wss', wss_port)
-
-    port = find_available_port()
+    port = internal_port_from_config("RailsAPI")
     env = os.environ.copy()
     env['RAILS_ENV'] = 'test'
-    env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
     env.pop('ARVADOS_WEBSOCKETS', None)
     env.pop('ARVADOS_TEST_API_HOST', None)
     env.pop('ARVADOS_API_HOST', None)
@@ -375,10 +369,7 @@ def reset():
 
     os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
     os.environ['ARVADOS_API_TOKEN'] = token
-    if _wait_until_port_listens(_getport('controller-ssl'), timeout=0.5, warn=False):
-        os.environ['ARVADOS_API_HOST'] = '0.0.0.0:'+str(_getport('controller-ssl'))
-    else:
-        os.environ['ARVADOS_API_HOST'] = existing_api_host
+    os.environ['ARVADOS_API_HOST'] = existing_api_host
 
 def stop(force=False):
     """Stop the API server, if one is running.
@@ -399,58 +390,28 @@ def stop(force=False):
         kill_server_pid(_pidfile('api'))
         my_api_host = None
 
+def get_config():
+    with open(os.environ["ARVADOS_CONFIG"]) as f:
+        return yaml.safe_load(f)
+
+def internal_port_from_config(service):
+    return int(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0].split(":")[2])
+
+def external_port_from_config(service):
+    return int(get_config()["Clusters"]["zzzzz"]["Services"][service]["ExternalURL"].split(":")[2])
+
 def run_controller():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
         return
     stop_controller()
-    rails_api_port = int(string.split(os.environ.get('ARVADOS_TEST_API_HOST', my_api_host), ':')[-1])
-    port = find_available_port()
-    conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
-    with open(conf, 'w') as f:
-        f.write("""
-Clusters:
-  zzzzz:
-    EnableBetaController14287: {beta14287}
-    ManagementToken: e687950a23c3a9bceec28c6223a06c79
-    API:
-      RequestTimeout: 30s
-    Logging:
-        Level: "{loglevel}"
-    HTTPRequestTimeout: 30s
-    PostgreSQL:
-      ConnectionPool: 32
-      Connection:
-        host: {dbhost}
-        dbname: {dbname}
-        user: {dbuser}
-        password: {dbpass}
-    TLS:
-      Insecure: true
-    Services:
-      Controller:
-        InternalURLs:
-          "http://localhost:{controllerport}": {{}}
-      RailsAPI:
-        InternalURLs:
-          "https://localhost:{railsport}": {{}}
-        """.format(
-            beta14287=('true' if '14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '') else 'false'),
-            loglevel=('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
-            dbhost=_dbconfig('host'),
-            dbname=_dbconfig('dbname'),
-            dbuser=_dbconfig('user'),
-            dbpass=_dbconfig('password'),
-            controllerport=port,
-            railsport=rails_api_port,
-        ))
     logf = open(_logfilename('controller'), 'a')
+    port = internal_port_from_config("Controller")
     controller = subprocess.Popen(
-        ["arvados-server", "controller", "-config", conf],
+        ["arvados-server", "controller"],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
     with open(_pidfile('controller'), 'w') as f:
         f.write(str(controller.pid))
     _wait_until_port_listens(port)
-    _setport('controller', port)
     return port
 
 def stop_controller():
@@ -462,36 +423,13 @@ def run_ws():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
         return
     stop_ws()
-    port = find_available_port()
-    conf = os.path.join(TEST_TMPDIR, 'ws.yml')
-    with open(conf, 'w') as f:
-        f.write("""
-Client:
-  APIHost: {}
-  Insecure: true
-Listen: :{}
-LogLevel: {}
-Postgres:
-  host: {}
-  dbname: {}
-  user: {}
-  password: {}
-  sslmode: require
-        """.format(os.environ['ARVADOS_API_HOST'],
-                   port,
-                   ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
-                   _dbconfig('host'),
-                   _dbconfig('dbname'),
-                   _dbconfig('user'),
-                   _dbconfig('password')))
+    port = internal_port_from_config("Websocket")
     logf = open(_logfilename('ws'), 'a')
-    ws = subprocess.Popen(
-        ["ws", "-config", conf],
+    ws = subprocess.Popen(["ws"],
         stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
     with open(_pidfile('ws'), 'w') as f:
         f.write(str(ws.pid))
     _wait_until_port_listens(port)
-    _setport('ws', port)
     return port
 
 def stop_ws():
@@ -590,11 +528,11 @@ def stop_keep(num_servers=2):
 
 def run_keep_proxy():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
-        os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
+        os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(internal_port_from_config('Keepproxy'))
         return
     stop_keep_proxy()
 
-    port = find_available_port()
+    port = internal_port_from_config("Keepproxy")
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
     logf = open(_logfilename('keepproxy'), 'a')
@@ -604,6 +542,7 @@ def run_keep_proxy():
          '-listen=:{}'.format(port)],
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
 
+    print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
     api = arvados.api(
         version='v1',
         host=os.environ['ARVADOS_API_HOST'],
@@ -619,7 +558,6 @@ def run_keep_proxy():
         'service_ssl_flag': False,
     }}).execute()
     os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
-    _setport('keepproxy', port)
     _wait_until_port_listens(port)
 
 def stop_keep_proxy():
@@ -633,7 +571,7 @@ def run_arv_git_httpd():
     stop_arv_git_httpd()
 
     gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
-    gitport = find_available_port()
+    gitport = internal_port_from_config("GitHTTP")
     env = os.environ.copy()
     env.pop('ARVADOS_API_TOKEN', None)
     logf = open(_logfilename('arv-git-httpd'), 'a')
@@ -645,7 +583,6 @@ def run_arv_git_httpd():
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('arv-git-httpd'), 'w') as f:
         f.write(str(agh.pid))
-    _setport('arv-git-httpd', gitport)
     _wait_until_port_listens(gitport)
 
 def stop_arv_git_httpd():
@@ -658,7 +595,7 @@ def run_keep_web():
         return
     stop_keep_web()
 
-    keepwebport = find_available_port()
+    keepwebport = internal_port_from_config("WebDAV")
     env = os.environ.copy()
     env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
     logf = open(_logfilename('keep-web'), 'a')
@@ -671,7 +608,6 @@ def run_keep_web():
         env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
     with open(_pidfile('keep-web'), 'w') as f:
         f.write(str(keepweb.pid))
-    _setport('keep-web', keepwebport)
     _wait_until_port_listens(keepwebport)
 
 def stop_keep_web():
@@ -684,17 +620,17 @@ def run_nginx():
         return
     stop_nginx()
     nginxconf = {}
-    nginxconf['CONTROLLERPORT'] = _getport('controller')
-    nginxconf['CONTROLLERSSLPORT'] = find_available_port()
-    nginxconf['KEEPWEBPORT'] = _getport('keep-web')
-    nginxconf['KEEPWEBDLSSLPORT'] = find_available_port()
-    nginxconf['KEEPWEBSSLPORT'] = find_available_port()
-    nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
-    nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
-    nginxconf['GITPORT'] = _getport('arv-git-httpd')
-    nginxconf['GITSSLPORT'] = find_available_port()
-    nginxconf['WSPORT'] = _getport('ws')
-    nginxconf['WSSPORT'] = _getport('wss')
+    nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
+    nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
+    nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
+    nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
+    nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
+    nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
+    nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
+    nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
+    nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
+    nginxconf['WSPORT'] = internal_port_from_config("Websocket")
+    nginxconf['WSSPORT'] = external_port_from_config("Websocket")
     nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
     nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
     nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
@@ -718,11 +654,101 @@ def run_nginx():
          '-g', 'pid '+_pidfile('nginx')+';',
          '-c', conffile],
         env=env, stdin=open('/dev/null'), stdout=sys.stderr)
-    _setport('controller-ssl', nginxconf['CONTROLLERSSLPORT'])
-    _setport('keep-web-dl-ssl', nginxconf['KEEPWEBDLSSLPORT'])
-    _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
-    _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
-    _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
+
+def setup_config():
+    rails_api_port = find_available_port()
+    controller_port = find_available_port()
+    controller_external_port = find_available_port()
+    websocket_port = find_available_port()
+    websocket_external_port = find_available_port()
+    git_httpd_port = find_available_port()
+    git_httpd_external_port = find_available_port()
+    keepproxy_port = find_available_port()
+    keepproxy_external_port = find_available_port()
+    keep_web_port = find_available_port()
+    keep_web_external_port = find_available_port()
+    keep_web_dl_port = find_available_port()
+    keep_web_dl_external_port = find_available_port()
+
+    if "CONFIGSRC" in os.environ:
+        dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
+    else:
+        dbconf = "/etc/arvados/config.yml"
+
+    print("Getting config from %s" % dbconf, file=sys.stderr)
+
+    pgconnection = list(yaml.safe_load(open(dbconf))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
+
+    if "test" not in pgconnection["dbname"]:
+        pgconnection["dbname"] = "arvados_test"
+
+    services = {
+        "RailsAPI": {
+            "InternalURLs": { }
+        },
+        "Controller": {
+            "ExternalURL": "https://localhost:%s" % controller_external_port,
+            "InternalURLs": { }
+        },
+        "Websocket": {
+            "ExternalURL": "https://localhost:%s" % websocket_external_port,
+            "InternalURLs": { }
+        },
+        "GitHTTP": {
+            "ExternalURL": "https://localhost:%s" % git_httpd_external_port,
+            "InternalURLs": { }
+        },
+        "Keepproxy": {
+            "ExternalURL": "https://localhost:%s" % keepproxy_external_port,
+            "InternalURLs": { }
+        },
+        "WebDAV": {
+            "ExternalURL": "https://localhost:%s" % keep_web_external_port,
+            "InternalURLs": { }
+        },
+        "WebDAVDownload": {
+            "ExternalURL": "https://localhost:%s" % keep_web_dl_external_port,
+            "InternalURLs": { }
+        }
+    }
+    services["RailsAPI"]["InternalURLs"]["https://localhost:%s"%rails_api_port] = {}
+    services["Controller"]["InternalURLs"]["http://localhost:%s"%controller_port] = {}
+    services["Websocket"]["InternalURLs"]["http://localhost:%s"%websocket_port] = {}
+    services["GitHTTP"]["InternalURLs"]["http://localhost:%s"%git_httpd_port] = {}
+    services["Keepproxy"]["InternalURLs"]["http://localhost:%s"%keepproxy_port] = {}
+    services["WebDAV"]["InternalURLs"]["http://localhost:%s"%keep_web_port] = {}
+    services["WebDAVDownload"]["InternalURLs"]["http://localhost:%s"%keep_web_dl_port] = {}
+
+    config = {
+        "Clusters": {
+            "zzzzz": {
+                "EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
+                "ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
+                "API": {
+                    "RequestTimeout": "30s"
+                },
+                "SystemLogs": {
+                    "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
+                },
+                "PostgreSQL": {
+                    "Connection": pgconnection,
+                },
+                "TLS": {
+                    "Insecure": True
+                },
+                "Services": services
+            }
+        }
+    }
+
+    conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
+    with open(conf, 'w') as f:
+        yaml.safe_dump(config, f)
+
+    ex = "export ARVADOS_CONFIG="+conf
+    print(ex, file=sys.stderr)
+    print(ex)
+
 
 def stop_nginx():
     if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
@@ -732,21 +758,6 @@ def stop_nginx():
 def _pidfile(program):
     return os.path.join(TEST_TMPDIR, program + '.pid')
 
-def _portfile(program):
-    return os.path.join(TEST_TMPDIR, program + '.port')
-
-def _setport(program, port):
-    with open(_portfile(program), 'w') as f:
-        f.write(str(port))
-
-# Returns 9 if program is not up.
-def _getport(program):
-    try:
-        with open(_portfile(program)) as prog:
-            return int(prog.read())
-    except IOError:
-        return 9
-
 def _dbconfig(key):
     global _cached_db_config
     if not _cached_db_config:
@@ -868,7 +879,7 @@ if __name__ == "__main__":
         'start_keep_proxy', 'stop_keep_proxy',
         'start_keep-web', 'stop_keep-web',
         'start_arv-git-httpd', 'stop_arv-git-httpd',
-        'start_nginx', 'stop_nginx',
+        'start_nginx', 'stop_nginx', 'setup_config',
     ]
     parser = argparse.ArgumentParser()
     parser.add_argument('action', type=str, help="one of {}".format(actions))
@@ -922,8 +933,10 @@ if __name__ == "__main__":
         stop_keep_web()
     elif args.action == 'start_nginx':
         run_nginx()
-        print("export ARVADOS_API_HOST=0.0.0.0:{}".format(_getport('controller-ssl')))
+        print("export ARVADOS_API_HOST=0.0.0.0:{}".format(external_port_from_config('Controller')))
     elif args.action == 'stop_nginx':
         stop_nginx()
+    elif args.action == 'setup_config':
+        setup_config()
     else:
         raise Exception("action recognized but not implemented!?")
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 8b43cef37..1b9a50ca6 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -190,8 +190,7 @@ ManagementToken: qqqqq
 		"sslmode":                   "require",
 		"user":                      "arvados"})
 	c.Check(cluster.PostgreSQL.ConnectionPool, check.Equals, 63)
-	c.Check(cluster.Services.Websocket.InternalURLs, check.DeepEquals, map[arvados.URL]arvados.ServiceInstance{
-		arvados.URL{Host: ":8765"}: arvados.ServiceInstance{}})
+	c.Check(cluster.Services.Websocket.InternalURLs[arvados.URL{Host: ":8765"}], check.NotNil)
 	c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 	c.Check(cluster.SystemLogs.Format, check.Equals, "text")
 	c.Check(cluster.API.SendTimeout, check.Equals, arvados.Duration(61*time.Second))

commit bebedf99016c2a784bbeb4c64ce4a579b8649b13
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Jul 23 16:32:28 2019 -0400

    14717: Remove LegacyComponentConfig behavior
    
    run-tests.sh now requires a minimal config.yml with database information.
    
    Remove database.yml from run-tests.sh
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index a46f2ec76..626ea974a 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -628,10 +628,11 @@ initialize() {
     setup_ruby_environment
 
     if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+	echo "Getting database configuration from $CONFIGSRC/config.yml"
 	cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
-	export ARVADOS_CONFIG="$temp/test-config.yml"
     else
 	if [[ -s /etc/arvados/config.yml ]] ; then
+	    echo "Getting database configuration from /etc/arvados/config.yml"
 	    python > "$temp/test-config.yml" <<EOF
 import yaml
 import json
@@ -639,13 +640,11 @@ v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0
 v['Connection']['dbname'] = 'arvados_test'
 print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
 EOF
-	    export ARVADOS_CONFIG="$temp/test-config.yml"
 	else
-	    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-		fatal "Please provide a database.yml file for the test suite"
-	    fi
+	    fatal "Please provide a config.yml file for the test suite in CONFIGSRC or /etc/arvados"
 	fi
     fi
+    export ARVADOS_CONFIG="$temp/test-config.yml"
 
     echo "PATH is $PATH"
 }
@@ -951,19 +950,11 @@ install_services/api() {
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
 
-    if [ -n "$CONFIGSRC" ]
-    then
-        for f in database.yml
-        do
-            cp "$CONFIGSRC/$f" config/ || fatal "$f"
-        done
-    fi
-
     # Clear out any lingering postgresql connections to the test
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
-        && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
+        && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 845e5113f..3e1ec7278 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -127,10 +127,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
 }
 
 // update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 	var oc oldKeepstoreConfig
 	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
-	if os.IsNotExist(err) && !required {
+	if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
 		return nil
 	} else if err != nil {
 		return err
@@ -197,10 +197,10 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 }
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 	var oc oldCrunchDispatchSlurmConfig
 	err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
-	if os.IsNotExist(err) && !required {
+	if os.IsNotExist(err) && (ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath) {
 		return nil
 	} else if err != nil {
 		return err
@@ -262,10 +262,10 @@ type oldWsConfig struct {
 const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error {
 	var oc oldWsConfig
 	err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
-	if os.IsNotExist(err) && !required {
+	if os.IsNotExist(err) && ldr.WebsocketPath == defaultWebsocketConfigPath {
 		return nil
 	} else if err != nil {
 		return err
diff --git a/lib/config/load.go b/lib/config/load.go
index f9ee6989d..2dacd5c26 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -33,12 +33,6 @@ type Loader struct {
 	CrunchDispatchSlurmPath string
 	WebsocketPath           string
 
-	// Legacy config file for the current component (will be the
-	// same as one of the above files).  If set, not being able to
-	// load the 'main' config.yml will not be a fatal error, but
-	// the the legacy file will be required instead.
-	LegacyComponentConfig string
-
 	configdata []byte
 }
 
@@ -144,15 +138,10 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 	if ldr.configdata == nil {
 		buf, err := ldr.loadBytes(ldr.Path)
 		if err != nil {
-			if ldr.LegacyComponentConfig != "" && os.IsNotExist(err) && !ldr.SkipDeprecated {
-				buf = []byte(`Clusters: {zzzzz: {}}`)
-			} else {
-				return nil, err
-			}
+			return nil, err
 		}
 		ldr.configdata = buf
 	}
-	noConfigLoaded := bytes.Compare(ldr.configdata, []byte(`Clusters: {zzzzz: {}}`)) == 0
 
 	// Load the config into a dummy map to get the cluster ID
 	// keys, discarding the values; then set up defaults for each
@@ -223,14 +212,9 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 		// * no primary config was loaded, and this is the
 		// legacy config file for the current component
 		for _, err := range []error{
-			ldr.loadOldKeepstoreConfig(&cfg, (ldr.KeepstorePath != defaultKeepstoreConfigPath) ||
-				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.KeepstorePath)),
-
-			ldr.loadOldCrunchDispatchSlurmConfig(&cfg, (ldr.CrunchDispatchSlurmPath != defaultCrunchDispatchSlurmConfigPath) ||
-				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.CrunchDispatchSlurmPath)),
-
-			ldr.loadOldWebsocketConfig(&cfg, (ldr.WebsocketPath != defaultWebsocketConfigPath) ||
-				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.WebsocketPath)),
+			ldr.loadOldKeepstoreConfig(&cfg),
+			ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
+			ldr.loadOldWebsocketConfig(&cfg),
 		} {
 			if err != nil {
 				return nil, err
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 75e6146f5..1a7ad6fac 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -109,7 +109,6 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
 	disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
-	loader.LegacyComponentConfig = loader.CrunchDispatchSlurmPath
 	cfg, err := loader.Load()
 	if err != nil {
 		return err
diff --git a/services/ws/main.go b/services/ws/main.go
index 2ea1a987d..0556c77d6 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -36,7 +36,6 @@ func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
 		return nil
 	}
 
-	loader.LegacyComponentConfig = loader.WebsocketPath
 	cfg, err := loader.Load()
 	if err != nil {
 		log.Fatal(err)
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 1d6231fe8..8b43cef37 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -36,7 +36,6 @@ func (s *serverSuite) SetUpTest(c *check.C) {
 
 func (*serverSuite) testConfig() (*arvados.Cluster, error) {
 	ldr := config.NewLoader(nil, nil)
-	ldr.LegacyComponentConfig = "ws-test"
 	cfg, err := ldr.Load()
 	if err != nil {
 		return nil, err

commit b47472f012130d22bd30969b2a273b91ffe41b51
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Jul 22 14:53:18 2019 -0400

    14717: Fix ws tests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index ca57f9faa..1d6231fe8 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -182,13 +182,20 @@ ManagementToken: qqqqq
 	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
 	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 
-	c.Check(cluster.PostgreSQL.Connection.String(), check.Equals, "connect_timeout='30' dbname='arvados_production' fallback_application_name='arvados-ws' host='localhost' password='xyzzy' sslmode='require' user='arvados' ")
+	c.Check(cluster.PostgreSQL.Connection, check.DeepEquals, arvados.PostgreSQLConnection{
+		"connect_timeout":           "30",
+		"dbname":                    "arvados_production",
+		"fallback_application_name": "arvados-ws",
+		"host":                      "localhost",
+		"password":                  "xyzzy",
+		"sslmode":                   "require",
+		"user":                      "arvados"})
 	c.Check(cluster.PostgreSQL.ConnectionPool, check.Equals, 63)
 	c.Check(cluster.Services.Websocket.InternalURLs, check.DeepEquals, map[arvados.URL]arvados.ServiceInstance{
 		arvados.URL{Host: ":8765"}: arvados.ServiceInstance{}})
 	c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
 	c.Check(cluster.SystemLogs.Format, check.Equals, "text")
-	c.Check(cluster.API.WebsocketKeepaliveTimeout, check.Equals, arvados.Duration(61*time.Second))
+	c.Check(cluster.API.SendTimeout, check.Equals, arvados.Duration(61*time.Second))
 	c.Check(cluster.API.WebsocketClientEventQueue, check.Equals, 62)
 	c.Check(cluster.API.WebsocketServerEventQueue, check.Equals, 5)
 	c.Check(cluster.ManagementToken, check.Equals, "qqqqq")

commit 3fbea8f4814e1bbc6ec650576daf63f72d121250
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Jul 22 14:14:39 2019 -0400

    14717: Rename WebsocketKeepaliveTimeout to SendTimeout and add a comment
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 1a1f1c5ac..595b05f8c 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -198,7 +198,11 @@ Clusters:
       # Maximum wall clock time to spend handling an incoming request.
       RequestTimeout: 5m
 
-      WebsocketKeepaliveTimeout: 60s
+      # Websocket will send a periodic empty event after 'SendTimeout'
+      # if there is no other activity to maintain the connection /
+      # detect dropped connections.
+      SendTimeout: 60s
+
       WebsocketClientEventQueue: 64
       WebsocketServerEventQueue: 4
 
diff --git a/lib/config/export.go b/lib/config/export.go
index 1cb84a2a1..25d1b7a2c 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -65,7 +65,7 @@ var whitelist = map[string]bool{
 	"API.RailsSessionSecretToken":                  false,
 	"API.RequestTimeout":                           true,
 	"API.WebsocketClientEventQueue":                false,
-	"API.WebsocketKeepaliveTimeout":                true,
+	"API.SendTimeout":                              true,
 	"API.WebsocketServerEventQueue":                false,
 	"AuditLogs":                                    false,
 	"AuditLogs.MaxAge":                             false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 7bcb38441..7704a3fae 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -204,7 +204,11 @@ Clusters:
       # Maximum wall clock time to spend handling an incoming request.
       RequestTimeout: 5m
 
-      WebsocketKeepaliveTimeout: 60s
+      # Websocket will send a periodic empty event after 'SendTimeout'
+      # if there is no other activity to maintain the connection /
+      # detect dropped connections.
+      SendTimeout: 60s
+
       WebsocketClientEventQueue: 64
       WebsocketServerEventQueue: 4
 

commit 72a8b3582d925ea30fe78697ff76bafb20d8bd9e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Jul 22 13:59:36 2019 -0400

    14717: Fix fallback behavior for component config vs main config
    
    For backwards compatability, we need to be able to start with only a
    component config and not the main config, and it is a fatal error if
    it doesn't exist.
    
    However, if there is a main config, and then it is okay if the
    component config doesn't exist.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 4526706a3..845e5113f 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -127,10 +127,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
 }
 
 // update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config, required bool) error {
 	var oc oldKeepstoreConfig
 	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
-	if os.IsNotExist(err) && ldr.KeepstorePath == defaultKeepstoreConfigPath {
+	if os.IsNotExist(err) && !required {
 		return nil
 	} else if err != nil {
 		return err
@@ -197,10 +197,10 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 }
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config, required bool) error {
 	var oc oldCrunchDispatchSlurmConfig
 	err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
-	if os.IsNotExist(err) && ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath {
+	if os.IsNotExist(err) && !required {
 		return nil
 	} else if err != nil {
 		return err
@@ -259,13 +259,13 @@ type oldWsConfig struct {
 	ManagementToken *string
 }
 
-const defaultWebsocketsConfigPath = "/etc/arvados/ws/ws.yml"
+const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config, required bool) error {
 	var oc oldWsConfig
-	err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketsPath, &oc)
-	if os.IsNotExist(err) && ldr.WebsocketsPath == defaultWebsocketsConfigPath {
+	err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
+	if os.IsNotExist(err) && !required {
 		return nil
 	} else if err != nil {
 		return err
@@ -277,7 +277,6 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
 	}
 
 	loadOldClientConfig(cluster, oc.Client)
-	fmt.Printf("Clllllllllllient %v %v", *oc.Client, cluster.Services.Controller.ExternalURL)
 
 	if oc.Postgres != nil {
 		cluster.PostgreSQL.Connection = *oc.Postgres
@@ -295,7 +294,7 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
 		cluster.SystemLogs.Format = *oc.LogFormat
 	}
 	if oc.PingTimeout != nil {
-		cluster.API.WebsocketKeepaliveTimeout = *oc.PingTimeout
+		cluster.API.SendTimeout = *oc.PingTimeout
 	}
 	if oc.ClientEventQueue != nil {
 		cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue
diff --git a/lib/config/load.go b/lib/config/load.go
index 63b6ac7d9..f9ee6989d 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -31,7 +31,13 @@ type Loader struct {
 	Path                    string
 	KeepstorePath           string
 	CrunchDispatchSlurmPath string
-	WebsocketsPath          string
+	WebsocketPath           string
+
+	// Legacy config file for the current component (will be the
+	// same as one of the above files).  If set, not being able to
+	// load the 'main' config.yml will not be a fatal error, but
+	// the the legacy file will be required instead.
+	LegacyComponentConfig string
 
 	configdata []byte
 }
@@ -60,7 +66,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
 	flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
 	flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
 	flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
-	flagset.StringVar(&ldr.WebsocketsPath, "legacy-ws-config", defaultWebsocketsConfigPath, "Legacy arvados-ws configuration `file`")
+	flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
 }
 
 // MungeLegacyConfigArgs checks args for a -config flag whose argument
@@ -134,20 +140,19 @@ func (ldr *Loader) loadBytes(path string) ([]byte, error) {
 	return ioutil.ReadAll(f)
 }
 
-func (ldr *Loader) LoadDefaults() (*arvados.Config, error) {
-	ldr.configdata = []byte(`Clusters: {zzzzz: {}}`)
-	defer func() { ldr.configdata = nil }()
-	return ldr.Load()
-}
-
 func (ldr *Loader) Load() (*arvados.Config, error) {
 	if ldr.configdata == nil {
 		buf, err := ldr.loadBytes(ldr.Path)
 		if err != nil {
-			return nil, err
+			if ldr.LegacyComponentConfig != "" && os.IsNotExist(err) && !ldr.SkipDeprecated {
+				buf = []byte(`Clusters: {zzzzz: {}}`)
+			} else {
+				return nil, err
+			}
 		}
 		ldr.configdata = buf
 	}
+	noConfigLoaded := bytes.Compare(ldr.configdata, []byte(`Clusters: {zzzzz: {}}`)) == 0
 
 	// Load the config into a dummy map to get the cluster ID
 	// keys, discarding the values; then set up defaults for each
@@ -213,10 +218,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 		if err != nil {
 			return nil, err
 		}
+		// legacy file is required when either:
+		// * a non-default location was specified
+		// * no primary config was loaded, and this is the
+		// legacy config file for the current component
 		for _, err := range []error{
-			ldr.loadOldKeepstoreConfig(&cfg),
-			ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
-			ldr.loadOldWebsocketsConfig(&cfg),
+			ldr.loadOldKeepstoreConfig(&cfg, (ldr.KeepstorePath != defaultKeepstoreConfigPath) ||
+				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.KeepstorePath)),
+
+			ldr.loadOldCrunchDispatchSlurmConfig(&cfg, (ldr.CrunchDispatchSlurmPath != defaultCrunchDispatchSlurmConfigPath) ||
+				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.CrunchDispatchSlurmPath)),
+
+			ldr.loadOldWebsocketConfig(&cfg, (ldr.WebsocketPath != defaultWebsocketConfigPath) ||
+				(noConfigLoaded && ldr.LegacyComponentConfig == ldr.WebsocketPath)),
 		} {
 			if err != nil {
 				return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 9072b7319..d0ab58ae4 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -76,7 +76,7 @@ type Cluster struct {
 		MaxRequestSize                 int
 		RailsSessionSecretToken        string
 		RequestTimeout                 Duration
-		WebsocketKeepaliveTimeout      Duration
+		SendTimeout                    Duration
 		WebsocketClientEventQueue      int
 		WebsocketServerEventQueue      int
 	}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1a7ad6fac..75e6146f5 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -109,6 +109,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
 	disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
+	loader.LegacyComponentConfig = loader.CrunchDispatchSlurmPath
 	cfg, err := loader.Load()
 	if err != nil {
 		return err
diff --git a/services/ws/main.go b/services/ws/main.go
index 0556c77d6..2ea1a987d 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -36,6 +36,7 @@ func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
 		return nil
 	}
 
+	loader.LegacyComponentConfig = loader.WebsocketPath
 	cfg, err := loader.Load()
 	if err != nil {
 		log.Fatal(err)
diff --git a/services/ws/router.go b/services/ws/router.go
index 5a5b7c53b..14dc63ec3 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -54,7 +54,7 @@ type debugStatuser interface {
 
 func (rtr *router) setup() {
 	rtr.handler = &handler{
-		PingTimeout: time.Duration(rtr.cluster.API.WebsocketKeepaliveTimeout),
+		PingTimeout: time.Duration(rtr.cluster.API.SendTimeout),
 		QueueSize:   rtr.cluster.API.WebsocketClientEventQueue,
 	}
 	rtr.mux = http.NewServeMux()
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 097889c98..ca57f9faa 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -36,7 +36,8 @@ func (s *serverSuite) SetUpTest(c *check.C) {
 
 func (*serverSuite) testConfig() (*arvados.Cluster, error) {
 	ldr := config.NewLoader(nil, nil)
-	cfg, err := ldr.LoadDefaults()
+	ldr.LegacyComponentConfig = "ws-test"
+	cfg, err := ldr.Load()
 	if err != nil {
 		return nil, err
 	}

commit 0e5ee27bb7bef018395a73f1fa2617050dc18d7a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Jul 18 16:25:50 2019 -0400

    14717: Migrate websockets to new config
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 15bae9af8..1a1f1c5ac 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -198,6 +198,10 @@ Clusters:
       # Maximum wall clock time to spend handling an incoming request.
       RequestTimeout: 5m
 
+      WebsocketKeepaliveTimeout: 60s
+      WebsocketClientEventQueue: 64
+      WebsocketServerEventQueue: 4
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 8f3f3d7ed..4526706a3 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -178,6 +178,24 @@ type oldCrunchDispatchSlurmConfig struct {
 
 const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
 
+func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
+	if client == nil {
+		return
+	}
+	if client.APIHost != "" {
+		cluster.Services.Controller.ExternalURL.Host = client.APIHost
+	}
+	if client.Scheme != "" {
+		cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
+	} else {
+		cluster.Services.Controller.ExternalURL.Scheme = "https"
+	}
+	if client.AuthToken != "" {
+		cluster.SystemRootToken = client.AuthToken
+	}
+	cluster.TLS.Insecure = client.Insecure
+}
+
 // update config using values from an crunch-dispatch-slurm config file.
 func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 	var oc oldCrunchDispatchSlurmConfig
@@ -193,18 +211,7 @@ func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 		return err
 	}
 
-	if oc.Client != nil {
-		u := arvados.URL{}
-		u.Host = oc.Client.APIHost
-		if oc.Client.Scheme != "" {
-			u.Scheme = oc.Client.Scheme
-		} else {
-			u.Scheme = "https"
-		}
-		cluster.Services.Controller.ExternalURL = u
-		cluster.SystemRootToken = oc.Client.AuthToken
-		cluster.TLS.Insecure = oc.Client.Insecure
-	}
+	loadOldClientConfig(cluster, oc.Client)
 
 	if oc.SbatchArguments != nil {
 		cluster.Containers.SLURM.SbatchArgumentsList = *oc.SbatchArguments
@@ -236,3 +243,70 @@ func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 	cfg.Clusters[cluster.ClusterID] = *cluster
 	return nil
 }
+
+type oldWsConfig struct {
+	Client       *arvados.Client
+	Postgres     *arvados.PostgreSQLConnection
+	PostgresPool *int
+	Listen       *string
+	LogLevel     *string
+	LogFormat    *string
+
+	PingTimeout      *arvados.Duration
+	ClientEventQueue *int
+	ServerEventQueue *int
+
+	ManagementToken *string
+}
+
+const defaultWebsocketsConfigPath = "/etc/arvados/ws/ws.yml"
+
+// update config using values from an crunch-dispatch-slurm config file.
+func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
+	var oc oldWsConfig
+	err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketsPath, &oc)
+	if os.IsNotExist(err) && ldr.WebsocketsPath == defaultWebsocketsConfigPath {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
+	}
+
+	loadOldClientConfig(cluster, oc.Client)
+	fmt.Printf("Clllllllllllient %v %v", *oc.Client, cluster.Services.Controller.ExternalURL)
+
+	if oc.Postgres != nil {
+		cluster.PostgreSQL.Connection = *oc.Postgres
+	}
+	if oc.PostgresPool != nil {
+		cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
+	}
+	if oc.Listen != nil {
+		cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+	}
+	if oc.LogLevel != nil {
+		cluster.SystemLogs.LogLevel = *oc.LogLevel
+	}
+	if oc.LogFormat != nil {
+		cluster.SystemLogs.Format = *oc.LogFormat
+	}
+	if oc.PingTimeout != nil {
+		cluster.API.WebsocketKeepaliveTimeout = *oc.PingTimeout
+	}
+	if oc.ClientEventQueue != nil {
+		cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue
+	}
+	if oc.ServerEventQueue != nil {
+		cluster.API.WebsocketServerEventQueue = *oc.ServerEventQueue
+	}
+	if oc.ManagementToken != nil {
+		cluster.ManagementToken = *oc.ManagementToken
+	}
+
+	cfg.Clusters[cluster.ClusterID] = *cluster
+	return nil
+}
diff --git a/lib/config/export.go b/lib/config/export.go
index dbbaac127..1cb84a2a1 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -64,6 +64,9 @@ var whitelist = map[string]bool{
 	"API.MaxRequestSize":                           true,
 	"API.RailsSessionSecretToken":                  false,
 	"API.RequestTimeout":                           true,
+	"API.WebsocketClientEventQueue":                false,
+	"API.WebsocketKeepaliveTimeout":                true,
+	"API.WebsocketServerEventQueue":                false,
 	"AuditLogs":                                    false,
 	"AuditLogs.MaxAge":                             false,
 	"AuditLogs.MaxDeleteBatch":                     false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 58a7690f4..7bcb38441 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -204,6 +204,10 @@ Clusters:
       # Maximum wall clock time to spend handling an incoming request.
       RequestTimeout: 5m
 
+      WebsocketKeepaliveTimeout: 60s
+      WebsocketClientEventQueue: 64
+      WebsocketServerEventQueue: 4
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want
diff --git a/lib/config/load.go b/lib/config/load.go
index bce57d759..63b6ac7d9 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -31,6 +31,7 @@ type Loader struct {
 	Path                    string
 	KeepstorePath           string
 	CrunchDispatchSlurmPath string
+	WebsocketsPath          string
 
 	configdata []byte
 }
@@ -59,6 +60,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
 	flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
 	flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
 	flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
+	flagset.StringVar(&ldr.WebsocketsPath, "legacy-ws-config", defaultWebsocketsConfigPath, "Legacy arvados-ws configuration `file`")
 }
 
 // MungeLegacyConfigArgs checks args for a -config flag whose argument
@@ -132,6 +134,12 @@ func (ldr *Loader) loadBytes(path string) ([]byte, error) {
 	return ioutil.ReadAll(f)
 }
 
+func (ldr *Loader) LoadDefaults() (*arvados.Config, error) {
+	ldr.configdata = []byte(`Clusters: {zzzzz: {}}`)
+	defer func() { ldr.configdata = nil }()
+	return ldr.Load()
+}
+
 func (ldr *Loader) Load() (*arvados.Config, error) {
 	if ldr.configdata == nil {
 		buf, err := ldr.loadBytes(ldr.Path)
@@ -208,6 +216,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 		for _, err := range []error{
 			ldr.loadOldKeepstoreConfig(&cfg),
 			ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
+			ldr.loadOldWebsocketsConfig(&cfg),
 		} {
 			if err != nil {
 				return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 12ec8e6b2..9072b7319 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -76,6 +76,9 @@ type Cluster struct {
 		MaxRequestSize                 int
 		RailsSessionSecretToken        string
 		RequestTimeout                 Duration
+		WebsocketKeepaliveTimeout      Duration
+		WebsocketClientEventQueue      int
+		WebsocketServerEventQueue      int
 	}
 	AuditLogs struct {
 		MaxAge             Duration
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index ca3944d76..6007c6d4a 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -395,7 +395,7 @@ func (s *StubbedSuite) TestLoadLegacyConfig(c *C) {
 	content := []byte(`
 Client:
   APIHost: example.com
-  APIToken: abcdefg
+  AuthToken: abcdefg
 SbatchArguments: ["--foo", "bar"]
 PollPeriod: 12s
 PrioritySpread: 42
@@ -422,6 +422,7 @@ BatchSize: 99
 	c.Check(err, IsNil)
 
 	c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg")
 	c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
 	c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
 	c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42))
diff --git a/services/ws/config.go b/services/ws/config.go
deleted file mode 100644
index ead1ec20c..000000000
--- a/services/ws/config.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
-	"time"
-
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
-)
-
-type wsConfig struct {
-	Client       arvados.Client
-	Postgres     arvados.PostgreSQLConnection
-	PostgresPool int
-	Listen       string
-	LogLevel     string
-	LogFormat    string
-
-	PingTimeout      arvados.Duration
-	ClientEventQueue int
-	ServerEventQueue int
-
-	ManagementToken string
-}
-
-func defaultConfig() wsConfig {
-	return wsConfig{
-		Client: arvados.Client{
-			APIHost: "localhost:443",
-		},
-		Postgres: arvados.PostgreSQLConnection{
-			"dbname":                    "arvados_production",
-			"user":                      "arvados",
-			"password":                  "xyzzy",
-			"host":                      "localhost",
-			"connect_timeout":           "30",
-			"sslmode":                   "require",
-			"fallback_application_name": "arvados-ws",
-		},
-		PostgresPool:     64,
-		LogLevel:         "info",
-		LogFormat:        "json",
-		PingTimeout:      arvados.Duration(time.Minute),
-		ClientEventQueue: 64,
-		ServerEventQueue: 4,
-	}
-}
diff --git a/services/ws/main.go b/services/ws/main.go
index a0006a4f8..0556c77d6 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -7,47 +7,71 @@ package main
 import (
 	"flag"
 	"fmt"
+	"os"
 
-	"git.curoverse.com/arvados.git/sdk/go/config"
+	"git.curoverse.com/arvados.git/lib/config"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 var logger = ctxlog.FromContext
 var version = "dev"
 
-func main() {
-	log := logger(nil)
+func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
+	flags := flag.NewFlagSet(args[0], flag.ExitOnError)
+	dumpConfig := flags.Bool("dump-config", false, "show current configuration and exit")
+	getVersion := flags.Bool("version", false, "Print version information and exit.")
+
+	loader := config.NewLoader(nil, log)
+	loader.SetupFlags(flags)
+	args = loader.MungeLegacyConfigArgs(log, args[1:], "-legacy-ws-config")
 
-	configPath := flag.String("config", "/etc/arvados/ws/ws.yml", "`path` to config file")
-	dumpConfig := flag.Bool("dump-config", false, "show current configuration and exit")
-	getVersion := flag.Bool("version", false, "Print version information and exit.")
-	cfg := defaultConfig()
-	flag.Parse()
+	flags.Parse(args)
 
 	// Print version information if requested
 	if *getVersion {
 		fmt.Printf("arvados-ws %s\n", version)
-		return
+		return nil
 	}
 
-	err := config.LoadFile(&cfg, *configPath)
+	cfg, err := loader.Load()
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	ctxlog.SetLevel(cfg.LogLevel)
-	ctxlog.SetFormat(cfg.LogFormat)
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ctxlog.SetLevel(cluster.SystemLogs.LogLevel)
+	ctxlog.SetFormat(cluster.SystemLogs.Format)
 
 	if *dumpConfig {
-		txt, err := config.Dump(&cfg)
+		out, err := yaml.Marshal(cfg)
 		if err != nil {
 			log.Fatal(err)
 		}
-		fmt.Print(string(txt))
+		_, err = os.Stdout.Write(out)
+		if err != nil {
+			log.Fatal(err)
+		}
+		return nil
+	}
+	return cluster
+}
+
+func main() {
+	log := logger(nil)
+
+	cluster := configure(log, os.Args)
+	if cluster == nil {
 		return
 	}
 
 	log.Printf("arvados-ws %s started", version)
-	srv := &server{wsConfig: &cfg}
+	srv := &server{cluster: cluster}
 	log.Fatal(srv.Run())
 }
diff --git a/services/ws/router.go b/services/ws/router.go
index a408b58bd..5a5b7c53b 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"git.curoverse.com/arvados.git/sdk/go/health"
 	"github.com/sirupsen/logrus"
@@ -27,7 +28,8 @@ type wsConn interface {
 }
 
 type router struct {
-	Config         *wsConfig
+	client         arvados.Client
+	cluster        *arvados.Cluster
 	eventSource    eventSource
 	newPermChecker func() permChecker
 
@@ -52,8 +54,8 @@ type debugStatuser interface {
 
 func (rtr *router) setup() {
 	rtr.handler = &handler{
-		PingTimeout: rtr.Config.PingTimeout.Duration(),
-		QueueSize:   rtr.Config.ClientEventQueue,
+		PingTimeout: time.Duration(rtr.cluster.API.WebsocketKeepaliveTimeout),
+		QueueSize:   rtr.cluster.API.WebsocketClientEventQueue,
 	}
 	rtr.mux = http.NewServeMux()
 	rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0))
@@ -62,7 +64,7 @@ func (rtr *router) setup() {
 	rtr.mux.Handle("/status.json", rtr.jsonHandler(rtr.Status))
 
 	rtr.mux.Handle("/_health/", &health.Handler{
-		Token:  rtr.Config.ManagementToken,
+		Token:  rtr.cluster.ManagementToken,
 		Prefix: "/_health/",
 		Routes: health.Routes{
 			"db": rtr.eventSource.DBHealth,
@@ -87,7 +89,7 @@ func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
 
 			stats := rtr.handler.Handle(ws, rtr.eventSource,
 				func(ws wsConn, sendq chan<- interface{}) (session, error) {
-					return newSession(ws, sendq, rtr.eventSource.DB(), rtr.newPermChecker(), &rtr.Config.Client)
+					return newSession(ws, sendq, rtr.eventSource.DB(), rtr.newPermChecker(), &rtr.client)
 				})
 
 			log.WithFields(logrus.Fields{
diff --git a/services/ws/server.go b/services/ws/server.go
index eda7ff2a4..081ff53b3 100644
--- a/services/ws/server.go
+++ b/services/ws/server.go
@@ -10,13 +10,14 @@ import (
 	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/coreos/go-systemd/daemon"
 )
 
 type server struct {
 	httpServer  *http.Server
 	listener    net.Listener
-	wsConfig    *wsConfig
+	cluster     *arvados.Cluster
 	eventSource *pgEventSource
 	setupOnce   sync.Once
 }
@@ -40,27 +41,38 @@ func (srv *server) Run() error {
 func (srv *server) setup() {
 	log := logger(nil)
 
-	ln, err := net.Listen("tcp", srv.wsConfig.Listen)
+	var listen arvados.URL
+	for listen, _ = range srv.cluster.Services.Websocket.InternalURLs {
+		break
+	}
+	ln, err := net.Listen("tcp", listen.Host)
 	if err != nil {
-		log.WithField("Listen", srv.wsConfig.Listen).Fatal(err)
+		log.WithField("Listen", listen).Fatal(err)
 	}
 	log.WithField("Listen", ln.Addr().String()).Info("listening")
 
+	client := arvados.Client{}
+	client.APIHost = srv.cluster.Services.Controller.ExternalURL.Host
+	client.AuthToken = srv.cluster.SystemRootToken
+	client.Insecure = srv.cluster.TLS.Insecure
+
 	srv.listener = ln
 	srv.eventSource = &pgEventSource{
-		DataSource:   srv.wsConfig.Postgres.String(),
-		MaxOpenConns: srv.wsConfig.PostgresPool,
-		QueueSize:    srv.wsConfig.ServerEventQueue,
+		DataSource:   srv.cluster.PostgreSQL.Connection.String(),
+		MaxOpenConns: srv.cluster.PostgreSQL.ConnectionPool,
+		QueueSize:    srv.cluster.API.WebsocketServerEventQueue,
 	}
+
 	srv.httpServer = &http.Server{
-		Addr:           srv.wsConfig.Listen,
+		Addr:           listen.Host,
 		ReadTimeout:    time.Minute,
 		WriteTimeout:   time.Minute,
 		MaxHeaderBytes: 1 << 20,
 		Handler: &router{
-			Config:         srv.wsConfig,
+			cluster:        srv.cluster,
+			client:         client,
 			eventSource:    srv.eventSource,
-			newPermChecker: func() permChecker { return newPermChecker(srv.wsConfig.Client) },
+			newPermChecker: func() permChecker { return newPermChecker(client) },
 		},
 	}
 
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index b1f943857..097889c98 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -7,10 +7,13 @@ package main
 import (
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/lib/config"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	check "gopkg.in/check.v1"
@@ -19,29 +22,42 @@ import (
 var _ = check.Suite(&serverSuite{})
 
 type serverSuite struct {
-	cfg *wsConfig
-	srv *server
-	wg  sync.WaitGroup
+	cluster *arvados.Cluster
+	srv     *server
+	wg      sync.WaitGroup
 }
 
 func (s *serverSuite) SetUpTest(c *check.C) {
-	s.cfg = s.testConfig()
-	s.srv = &server{wsConfig: s.cfg}
+	var err error
+	s.cluster, err = s.testConfig()
+	c.Assert(err, check.IsNil)
+	s.srv = &server{cluster: s.cluster}
 }
 
-func (*serverSuite) testConfig() *wsConfig {
-	cfg := defaultConfig()
-	cfg.Client = *(arvados.NewClientFromEnv())
-	cfg.Postgres = testDBConfig()
-	cfg.Listen = ":"
-	cfg.ManagementToken = arvadostest.ManagementToken
-	return &cfg
+func (*serverSuite) testConfig() (*arvados.Cluster, error) {
+	ldr := config.NewLoader(nil, nil)
+	cfg, err := ldr.LoadDefaults()
+	if err != nil {
+		return nil, err
+	}
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return nil, err
+	}
+	client := arvados.NewClientFromEnv()
+	cluster.Services.Controller.ExternalURL.Host = client.APIHost
+	cluster.SystemRootToken = client.AuthToken
+	cluster.TLS.Insecure = client.Insecure
+	cluster.PostgreSQL.Connection = testDBConfig()
+	cluster.Services.Websocket.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":"}: arvados.ServiceInstance{}}
+	cluster.ManagementToken = arvadostest.ManagementToken
+	return cluster, nil
 }
 
 // TestBadDB ensures Run() returns an error (instead of panicking or
 // deadlocking) if it can't connect to the database server at startup.
 func (s *serverSuite) TestBadDB(c *check.C) {
-	s.cfg.Postgres["password"] = "1234"
+	s.cluster.PostgreSQL.Connection["password"] = "1234"
 
 	var wg sync.WaitGroup
 	wg.Add(1)
@@ -72,7 +88,7 @@ func (s *serverSuite) TestHealth(c *check.C) {
 	go s.srv.Run()
 	defer s.srv.Close()
 	s.srv.WaitReady()
-	for _, token := range []string{"", "foo", s.cfg.ManagementToken} {
+	for _, token := range []string{"", "foo", s.cluster.ManagementToken} {
 		req, err := http.NewRequest("GET", "http://"+s.srv.listener.Addr().String()+"/_health/ping", nil)
 		c.Assert(err, check.IsNil)
 		if token != "" {
@@ -80,7 +96,7 @@ func (s *serverSuite) TestHealth(c *check.C) {
 		}
 		resp, err := http.DefaultClient.Do(req)
 		c.Check(err, check.IsNil)
-		if token == s.cfg.ManagementToken {
+		if token == s.cluster.ManagementToken {
 			c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 			buf, err := ioutil.ReadAll(resp.Body)
 			c.Check(err, check.IsNil)
@@ -107,7 +123,7 @@ func (s *serverSuite) TestStatus(c *check.C) {
 }
 
 func (s *serverSuite) TestHealthDisabled(c *check.C) {
-	s.cfg.ManagementToken = ""
+	s.cluster.ManagementToken = ""
 
 	go s.srv.Run()
 	defer s.srv.Close()
@@ -122,3 +138,57 @@ func (s *serverSuite) TestHealthDisabled(c *check.C) {
 		c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 	}
 }
+
+func (s *serverSuite) TestLoadLegacyConfig(c *check.C) {
+	content := []byte(`
+Client:
+  APIHost: example.com
+  AuthToken: abcdefg
+Postgres:
+  "dbname": "arvados_production"
+  "user": "arvados"
+  "password": "xyzzy"
+  "host": "localhost"
+  "connect_timeout": "30"
+  "sslmode": "require"
+  "fallback_application_name": "arvados-ws"
+PostgresPool: 63
+Listen: ":8765"
+LogLevel: "debug"
+LogFormat: "text"
+PingTimeout: 61s
+ClientEventQueue: 62
+ServerEventQueue:  5
+ManagementToken: qqqqq
+`)
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	if _, err := tmpfile.Write(content); err != nil {
+		log.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		log.Fatal(err)
+
+	}
+	cluster := configure(logger(nil), []string{"arvados-ws", "-config", tmpfile.Name()})
+	c.Check(cluster, check.NotNil)
+
+	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
+
+	c.Check(cluster.PostgreSQL.Connection.String(), check.Equals, "connect_timeout='30' dbname='arvados_production' fallback_application_name='arvados-ws' host='localhost' password='xyzzy' sslmode='require' user='arvados' ")
+	c.Check(cluster.PostgreSQL.ConnectionPool, check.Equals, 63)
+	c.Check(cluster.Services.Websocket.InternalURLs, check.DeepEquals, map[arvados.URL]arvados.ServiceInstance{
+		arvados.URL{Host: ":8765"}: arvados.ServiceInstance{}})
+	c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
+	c.Check(cluster.SystemLogs.Format, check.Equals, "text")
+	c.Check(cluster.API.WebsocketKeepaliveTimeout, check.Equals, arvados.Duration(61*time.Second))
+	c.Check(cluster.API.WebsocketClientEventQueue, check.Equals, 62)
+	c.Check(cluster.API.WebsocketServerEventQueue, check.Equals, 5)
+	c.Check(cluster.ManagementToken, check.Equals, "qqqqq")
+}

commit 40e3249b1f75ff1442adbc360885c80db436c50b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Jul 18 10:25:44 2019 -0400

    14713: Fix handling default/missing keys, can't load default file
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 614f5dcf9..8f3f3d7ed 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -108,19 +108,17 @@ type oldKeepstoreConfig struct {
 	Debug *bool
 }
 
-func (ldr *Loader) loadOldConfigHelper(component, path, defaultPath string, target interface{}) error {
+func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
 	if path == "" {
 		return nil
 	}
 	buf, err := ioutil.ReadFile(path)
-	if os.IsNotExist(err) && path == defaultPath {
-		return nil
-	} else if err != nil {
+	if err != nil {
 		return err
-	} else {
-		ldr.Logger.Warnf("you should remove the legacy %v config file (%s) after migrating all config keys to the cluster configuration file (%s)", component, path, ldr.Path)
 	}
 
+	ldr.Logger.Warnf("you should remove the legacy %v config file (%s) after migrating all config keys to the cluster configuration file (%s)", component, path, ldr.Path)
+
 	err = yaml.Unmarshal(buf, target)
 	if err != nil {
 		return fmt.Errorf("%s: %s", path, err)
@@ -131,8 +129,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path, defaultPath string, targ
 // update config using values from an old-style keepstore config file.
 func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 	var oc oldKeepstoreConfig
-	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, defaultKeepstoreConfigPath, &oc)
-	if err != nil {
+	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
+	if os.IsNotExist(err) && ldr.KeepstorePath == defaultKeepstoreConfigPath {
+		return nil
+	} else if err != nil {
 		return err
 	}
 
@@ -153,27 +153,27 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 }
 
 type oldCrunchDispatchSlurmConfig struct {
-	Client arvados.Client
+	Client *arvados.Client
 
-	SbatchArguments []string
-	PollPeriod      arvados.Duration
-	PrioritySpread  int64
+	SbatchArguments *[]string
+	PollPeriod      *arvados.Duration
+	PrioritySpread  *int64
 
 	// crunch-run command to invoke. The container UUID will be
 	// appended. If nil, []string{"crunch-run"} will be used.
 	//
 	// Example: []string{"crunch-run", "--cgroup-parent-subsystem=memory"}
-	CrunchRunCommand []string
+	CrunchRunCommand *[]string
 
 	// Extra RAM to reserve (in Bytes) for SLURM job, in addition
 	// to the amount specified in the container's RuntimeConstraints
-	ReserveExtraRAM int64
+	ReserveExtraRAM *int64
 
 	// Minimum time between two attempts to run the same container
-	MinRetryPeriod arvados.Duration
+	MinRetryPeriod *arvados.Duration
 
 	// Batch size for container queries
-	BatchSize int64
+	BatchSize *int64
 }
 
 const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
@@ -181,8 +181,10 @@ const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm
 // update config using values from an crunch-dispatch-slurm config file.
 func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 	var oc oldCrunchDispatchSlurmConfig
-	err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, defaultCrunchDispatchSlurmConfigPath, &oc)
-	if err != nil {
+	err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
+	if os.IsNotExist(err) && ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath {
+		return nil
+	} else if err != nil {
 		return err
 	}
 
@@ -191,30 +193,45 @@ func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
 		return err
 	}
 
-	u := arvados.URL{}
-	u.Host = oc.Client.APIHost
-	if oc.Client.Scheme != "" {
-		u.Scheme = oc.Client.Scheme
-	} else {
-		u.Scheme = "https"
+	if oc.Client != nil {
+		u := arvados.URL{}
+		u.Host = oc.Client.APIHost
+		if oc.Client.Scheme != "" {
+			u.Scheme = oc.Client.Scheme
+		} else {
+			u.Scheme = "https"
+		}
+		cluster.Services.Controller.ExternalURL = u
+		cluster.SystemRootToken = oc.Client.AuthToken
+		cluster.TLS.Insecure = oc.Client.Insecure
 	}
-	cluster.Services.Controller.ExternalURL = u
-	cluster.SystemRootToken = oc.Client.AuthToken
-	cluster.TLS.Insecure = oc.Client.Insecure
 
-	cluster.Containers.SLURM.SbatchArgumentsList = oc.SbatchArguments
-	cluster.Containers.CloudVMs.PollInterval = oc.PollPeriod
-	cluster.Containers.SLURM.PrioritySpread = oc.PrioritySpread
-	if len(oc.CrunchRunCommand) >= 1 {
-		cluster.Containers.CrunchRunCommand = oc.CrunchRunCommand[0]
+	if oc.SbatchArguments != nil {
+		cluster.Containers.SLURM.SbatchArgumentsList = *oc.SbatchArguments
 	}
-	if len(oc.CrunchRunCommand) >= 2 {
-		cluster.Containers.CrunchRunArgumentsList = oc.CrunchRunCommand[1:]
+	if oc.PollPeriod != nil {
+		cluster.Containers.CloudVMs.PollInterval = *oc.PollPeriod
+	}
+	if oc.PrioritySpread != nil {
+		cluster.Containers.SLURM.PrioritySpread = *oc.PrioritySpread
+	}
+	if oc.CrunchRunCommand != nil {
+		if len(*oc.CrunchRunCommand) >= 1 {
+			cluster.Containers.CrunchRunCommand = (*oc.CrunchRunCommand)[0]
+		}
+		if len(*oc.CrunchRunCommand) >= 2 {
+			cluster.Containers.CrunchRunArgumentsList = (*oc.CrunchRunCommand)[1:]
+		}
+	}
+	if oc.ReserveExtraRAM != nil {
+		cluster.Containers.ReserveExtraRAM = arvados.ByteSize(*oc.ReserveExtraRAM)
+	}
+	if oc.MinRetryPeriod != nil {
+		cluster.Containers.MinRetryPeriod = *oc.MinRetryPeriod
+	}
+	if oc.BatchSize != nil {
+		cluster.API.MaxItemsPerResponse = int(*oc.BatchSize)
 	}
-	cluster.Containers.ReserveExtraRAM = arvados.ByteSize(oc.ReserveExtraRAM)
-	cluster.Containers.MinRetryPeriod = oc.MinRetryPeriod
-
-	cluster.API.MaxItemsPerResponse = int(oc.BatchSize)
 
 	cfg.Clusters[cluster.ClusterID] = *cluster
 	return nil
diff --git a/lib/config/export.go b/lib/config/export.go
index a050492b3..dbbaac127 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -101,7 +101,7 @@ var whitelist = map[string]bool{
 	"Containers.MaxDispatchAttempts":               false,
 	"Containers.MaxRetryAttempts":                  true,
 	"Containers.MinRetryPeriod":                    true,
-	"Containers.ReserveExtraRam":                   true,
+	"Containers.ReserveExtraRAM":                   true,
 	"Containers.SLURM":                             false,
 	"Containers.StaleLockTimeout":                  false,
 	"Containers.SupportedDockerImageFormats":       true,
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 340eb0a0a..fa04c0075 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -168,7 +168,9 @@ func (s *LoadSuite) TestSampleKeys(c *check.C) {
 }
 
 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
-	cfg, err := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil).Load()
+	ldr := testLoader(c, `{"Clusters":{"z1111":{},"z2222":{}}}`, nil)
+	ldr.SkipDeprecated = true
+	cfg, err := ldr.Load()
 	c.Assert(err, check.IsNil)
 	c1, err := cfg.GetCluster("z1111")
 	c.Assert(err, check.IsNil)

commit 22422ab1e539977ca730aedd46b4cd919e73d05e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Jul 17 16:54:01 2019 -0400

    14713: Migrate old crunch-dispatch-slurm config to new config
    
    Update code to use new config.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 0b0bb2668..614f5dcf9 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -108,29 +108,37 @@ type oldKeepstoreConfig struct {
 	Debug *bool
 }
 
-// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
-	path := ldr.KeepstorePath
+func (ldr *Loader) loadOldConfigHelper(component, path, defaultPath string, target interface{}) error {
 	if path == "" {
 		return nil
 	}
 	buf, err := ioutil.ReadFile(path)
-	if os.IsNotExist(err) && path == defaultKeepstoreConfigPath {
+	if os.IsNotExist(err) && path == defaultPath {
 		return nil
 	} else if err != nil {
 		return err
 	} else {
-		ldr.Logger.Warnf("you should remove the legacy keepstore config file (%s) after migrating all config keys to the cluster configuration file (%s)", path, ldr.Path)
+		ldr.Logger.Warnf("you should remove the legacy %v config file (%s) after migrating all config keys to the cluster configuration file (%s)", component, path, ldr.Path)
 	}
-	cluster, err := cfg.GetCluster("")
+
+	err = yaml.Unmarshal(buf, target)
 	if err != nil {
-		return err
+		return fmt.Errorf("%s: %s", path, err)
 	}
+	return nil
+}
 
+// update config using values from an old-style keepstore config file.
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 	var oc oldKeepstoreConfig
-	err = yaml.Unmarshal(buf, &oc)
+	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, defaultKeepstoreConfigPath, &oc)
 	if err != nil {
-		return fmt.Errorf("%s: %s", path, err)
+		return err
+	}
+
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
 	}
 
 	if v := oc.Debug; v == nil {
@@ -143,3 +151,71 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 	cfg.Clusters[cluster.ClusterID] = *cluster
 	return nil
 }
+
+type oldCrunchDispatchSlurmConfig struct {
+	Client arvados.Client
+
+	SbatchArguments []string
+	PollPeriod      arvados.Duration
+	PrioritySpread  int64
+
+	// crunch-run command to invoke. The container UUID will be
+	// appended. If nil, []string{"crunch-run"} will be used.
+	//
+	// Example: []string{"crunch-run", "--cgroup-parent-subsystem=memory"}
+	CrunchRunCommand []string
+
+	// Extra RAM to reserve (in Bytes) for SLURM job, in addition
+	// to the amount specified in the container's RuntimeConstraints
+	ReserveExtraRAM int64
+
+	// Minimum time between two attempts to run the same container
+	MinRetryPeriod arvados.Duration
+
+	// Batch size for container queries
+	BatchSize int64
+}
+
+const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
+
+// update config using values from an crunch-dispatch-slurm config file.
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
+	var oc oldCrunchDispatchSlurmConfig
+	err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, defaultCrunchDispatchSlurmConfigPath, &oc)
+	if err != nil {
+		return err
+	}
+
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
+	}
+
+	u := arvados.URL{}
+	u.Host = oc.Client.APIHost
+	if oc.Client.Scheme != "" {
+		u.Scheme = oc.Client.Scheme
+	} else {
+		u.Scheme = "https"
+	}
+	cluster.Services.Controller.ExternalURL = u
+	cluster.SystemRootToken = oc.Client.AuthToken
+	cluster.TLS.Insecure = oc.Client.Insecure
+
+	cluster.Containers.SLURM.SbatchArgumentsList = oc.SbatchArguments
+	cluster.Containers.CloudVMs.PollInterval = oc.PollPeriod
+	cluster.Containers.SLURM.PrioritySpread = oc.PrioritySpread
+	if len(oc.CrunchRunCommand) >= 1 {
+		cluster.Containers.CrunchRunCommand = oc.CrunchRunCommand[0]
+	}
+	if len(oc.CrunchRunCommand) >= 2 {
+		cluster.Containers.CrunchRunArgumentsList = oc.CrunchRunCommand[1:]
+	}
+	cluster.Containers.ReserveExtraRAM = arvados.ByteSize(oc.ReserveExtraRAM)
+	cluster.Containers.MinRetryPeriod = oc.MinRetryPeriod
+
+	cluster.API.MaxItemsPerResponse = int(oc.BatchSize)
+
+	cfg.Clusters[cluster.ClusterID] = *cluster
+	return nil
+}
diff --git a/lib/config/export.go b/lib/config/export.go
index b79dec4d9..a050492b3 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -83,6 +83,8 @@ var whitelist = map[string]bool{
 	"Collections.TrustAllContent":                  false,
 	"Containers":                                   true,
 	"Containers.CloudVMs":                          false,
+	"Containers.CrunchRunCommand":                  false,
+	"Containers.CrunchRunArgumentsList":            false,
 	"Containers.DefaultKeepCacheRAM":               true,
 	"Containers.DispatchPrivateKey":                false,
 	"Containers.JobsAPI":                           true,
@@ -98,6 +100,8 @@ var whitelist = map[string]bool{
 	"Containers.MaxComputeVMs":                     false,
 	"Containers.MaxDispatchAttempts":               false,
 	"Containers.MaxRetryAttempts":                  true,
+	"Containers.MinRetryPeriod":                    true,
+	"Containers.ReserveExtraRam":                   true,
 	"Containers.SLURM":                             false,
 	"Containers.StaleLockTimeout":                  false,
 	"Containers.SupportedDockerImageFormats":       true,
diff --git a/lib/config/load.go b/lib/config/load.go
index 168c1aa22..bce57d759 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -28,8 +28,9 @@ type Loader struct {
 	Logger         logrus.FieldLogger
 	SkipDeprecated bool // Don't load legacy/deprecated config keys/files
 
-	Path          string
-	KeepstorePath string
+	Path                    string
+	KeepstorePath           string
+	CrunchDispatchSlurmPath string
 
 	configdata []byte
 }
@@ -57,6 +58,7 @@ func NewLoader(stdin io.Reader, logger logrus.FieldLogger) *Loader {
 func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
 	flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
 	flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
+	flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
 }
 
 // MungeLegacyConfigArgs checks args for a -config flag whose argument
@@ -205,6 +207,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 		}
 		for _, err := range []error{
 			ldr.loadOldKeepstoreConfig(&cfg),
+			ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
 		} {
 			if err != nil {
 				return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index c8206c7da..12ec8e6b2 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -253,12 +253,16 @@ type InstanceType struct {
 
 type ContainersConfig struct {
 	CloudVMs                    CloudVMsConfig
+	CrunchRunCommand            string
+	CrunchRunArgumentsList      []string
 	DefaultKeepCacheRAM         ByteSize
 	DispatchPrivateKey          string
 	LogReuseDecisions           bool
 	MaxComputeVMs               int
 	MaxDispatchAttempts         int
 	MaxRetryAttempts            int
+	MinRetryPeriod              Duration
+	ReserveExtraRAM             ByteSize
 	StaleLockTimeout            Duration
 	SupportedDockerImageFormats []string
 	UsePreemptibleInstances     bool
@@ -285,7 +289,9 @@ type ContainersConfig struct {
 		LogUpdateSize                ByteSize
 	}
 	SLURM struct {
-		Managed struct {
+		PrioritySpread      int64
+		SbatchArgumentsList []string
+		Managed             struct {
 			DNSServerConfDir       string
 			DNSServerConfTemplate  string
 			DNSServerReloadCommand string
diff --git a/sdk/go/dispatch/dispatch.go b/sdk/go/dispatch/dispatch.go
index fdb52e510..587c9999c 100644
--- a/sdk/go/dispatch/dispatch.go
+++ b/sdk/go/dispatch/dispatch.go
@@ -38,7 +38,7 @@ type Dispatcher struct {
 	Logger Logger
 
 	// Batch size for container queries
-	BatchSize int64
+	BatchSize int
 
 	// Queue polling frequency
 	PollPeriod time.Duration
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 09e3d591a..1a7ad6fac 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -25,6 +25,7 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/dispatch"
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 type logger interface {
@@ -35,8 +36,7 @@ type logger interface {
 const initialNiceValue int64 = 10000
 
 var (
-	version           = "dev"
-	defaultConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
+	version = "dev"
 )
 
 type Dispatcher struct {
@@ -74,16 +74,15 @@ func (disp *Dispatcher) Run(prog string, args []string) error {
 
 // configure() loads config files. Tests skip this.
 func (disp *Dispatcher) configure(prog string, args []string) error {
+	if disp.logger == nil {
+		disp.logger = logrus.StandardLogger()
+	}
 	flags := flag.NewFlagSet(prog, flag.ExitOnError)
 	flags.Usage = func() { usage(flags) }
 
-	loader := config.NewLoader(stdin, log)
+	loader := config.NewLoader(nil, disp.logger)
 	loader.SetupFlags(flags)
 
-	configPath := flags.String(
-		"config",
-		defaultConfigPath,
-		"`path` to JSON or YAML configuration file")
 	dumpConfig := flag.Bool(
 		"dump-config",
 		false,
@@ -93,10 +92,10 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 		false,
 		"Print version information and exit.")
 
-	args = loader.MungeLegacyConfigArgs(logrus.StandardLogger(), args, "-crunch-dispatch-slurm-config")
+	args = loader.MungeLegacyConfigArgs(logrus.StandardLogger(), args, "-legacy-crunch-dispatch-slurm-config")
 
 	// Parse args; omit the first arg which is the command name
-	flags.Parse(args)
+	err := flags.Parse(args)
 
 	if err == flag.ErrHelp {
 		return nil
@@ -119,8 +118,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 		return fmt.Errorf("config error: %s", err)
 	}
 
-	disp.Client.APIHost = fmt.Sprintf("%s:%d", disp.cluster.Services.Controller.ExternalURL.Host,
-		disp.cluster.Services.Controller.ExternalURL.Port)
+	disp.Client.APIHost = disp.cluster.Services.Controller.ExternalURL.Host
 	disp.Client.AuthToken = disp.cluster.SystemRootToken
 	disp.Client.Insecure = disp.cluster.TLS.Insecure
 
@@ -141,7 +139,14 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 	}
 
 	if *dumpConfig {
-		return config.DumpAndExit(cfg)
+		out, err := yaml.Marshal(cfg)
+		if err != nil {
+			return err
+		}
+		_, err = os.Stdout.Write(out)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -149,9 +154,6 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
 // setup() initializes private fields after configure().
 func (disp *Dispatcher) setup() {
-	if disp.logger == nil {
-		disp.logger = logrus.StandardLogger()
-	}
 	arv, err := arvadosclient.MakeArvadosClient()
 	if err != nil {
 		disp.logger.Fatalf("Error making Arvados client: %v", err)
@@ -161,17 +163,17 @@ func (disp *Dispatcher) setup() {
 	disp.slurm = NewSlurmCLI()
 	disp.sqCheck = &SqueueChecker{
 		Logger:         disp.logger,
-		Period:         time.Duration(disp.PollPeriod),
-		PrioritySpread: disp.PrioritySpread,
+		Period:         time.Duration(disp.cluster.Containers.CloudVMs.PollInterval),
+		PrioritySpread: disp.cluster.Containers.SLURM.PrioritySpread,
 		Slurm:          disp.slurm,
 	}
 	disp.Dispatcher = &dispatch.Dispatcher{
 		Arv:            arv,
 		Logger:         disp.logger,
-		BatchSize:      disp.BatchSize,
+		BatchSize:      disp.cluster.API.MaxItemsPerResponse,
 		RunContainer:   disp.runContainer,
-		PollPeriod:     time.Duration(disp.PollPeriod),
-		MinRetryPeriod: time.Duration(disp.MinRetryPeriod),
+		PollPeriod:     time.Duration(disp.cluster.Containers.CloudVMs.PollInterval),
+		MinRetryPeriod: time.Duration(disp.cluster.Containers.MinRetryPeriod),
 	}
 }
 
@@ -209,7 +211,9 @@ func (disp *Dispatcher) checkSqueueForOrphans() {
 }
 
 func (disp *Dispatcher) slurmConstraintArgs(container arvados.Container) []string {
-	mem := int64(math.Ceil(float64(container.RuntimeConstraints.RAM+container.RuntimeConstraints.KeepCacheRAM+disp.ReserveExtraRAM) / float64(1048576)))
+	mem := int64(math.Ceil(float64(container.RuntimeConstraints.RAM+
+		container.RuntimeConstraints.KeepCacheRAM+
+		int64(disp.cluster.Containers.ReserveExtraRAM)) / float64(1048576)))
 
 	disk := dispatchcloud.EstimateScratchSpace(&container)
 	disk = int64(math.Ceil(float64(disk) / float64(1048576)))
@@ -222,7 +226,7 @@ func (disp *Dispatcher) slurmConstraintArgs(container arvados.Container) []strin
 
 func (disp *Dispatcher) sbatchArgs(container arvados.Container) ([]string, error) {
 	var args []string
-	args = append(args, disp.SbatchArguments...)
+	args = append(args, disp.cluster.Containers.SLURM.SbatchArgumentsList...)
 	args = append(args, "--job-name="+container.UUID, fmt.Sprintf("--nice=%d", initialNiceValue), "--no-requeue")
 
 	if disp.cluster == nil {
@@ -270,7 +274,9 @@ func (disp *Dispatcher) runContainer(_ *dispatch.Dispatcher, ctr arvados.Contain
 
 	if ctr.State == dispatch.Locked && !disp.sqCheck.HasUUID(ctr.UUID) {
 		log.Printf("Submitting container %s to slurm", ctr.UUID)
-		if err := disp.submit(ctr, disp.CrunchRunCommand); err != nil {
+		cmd := []string{disp.cluster.Containers.CrunchRunCommand}
+		cmd = append(cmd, disp.cluster.Containers.CrunchRunArgumentsList...)
+		if err := disp.submit(ctr, cmd); err != nil {
 			var text string
 			if err, ok := err.(dispatchcloud.ConstraintsNotSatisfiableError); ok {
 				var logBuf bytes.Buffer
@@ -361,12 +367,3 @@ func (disp *Dispatcher) scancel(ctr arvados.Container) {
 		time.Sleep(time.Second)
 	}
 }
-
-func (disp *Dispatcher) readConfig(path string) error {
-	err := config.LoadFile(disp, path)
-	if err != nil && os.IsNotExist(err) && path == defaultConfigPath {
-		log.Printf("Config not specified. Continue with default configuration.")
-		err = nil
-	}
-	return err
-}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index eea102012..ca3944d76 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -45,6 +46,7 @@ func (s *IntegrationSuite) SetUpTest(c *C) {
 	arvadostest.StartAPI()
 	os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token)
 	s.disp = Dispatcher{}
+	s.disp.cluster = &arvados.Cluster{}
 	s.disp.setup()
 	s.slurm = slurmFake{}
 }
@@ -118,7 +120,7 @@ func (s *IntegrationSuite) integrationTest(c *C,
 	c.Check(err, IsNil)
 	c.Assert(len(containers.Items), Equals, 1)
 
-	s.disp.CrunchRunCommand = []string{"echo"}
+	s.disp.cluster.Containers.CrunchRunCommand = "echo"
 
 	ctx, cancel := context.WithCancel(context.Background())
 	doneRun := make(chan struct{})
@@ -243,6 +245,7 @@ type StubbedSuite struct {
 
 func (s *StubbedSuite) SetUpTest(c *C) {
 	s.disp = Dispatcher{}
+	s.disp.cluster = &arvados.Cluster{}
 	s.disp.setup()
 }
 
@@ -272,7 +275,7 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva
 	logrus.SetOutput(io.MultiWriter(buf, os.Stderr))
 	defer logrus.SetOutput(os.Stderr)
 
-	s.disp.CrunchRunCommand = []string{crunchCmd}
+	s.disp.cluster.Containers.CrunchRunCommand = "crunchCmd"
 
 	ctx, cancel := context.WithCancel(context.Background())
 	dispatcher := dispatch.Dispatcher{
@@ -302,51 +305,6 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva
 	c.Check(buf.String(), Matches, `(?ms).*`+expected+`.*`)
 }
 
-func (s *StubbedSuite) TestNoSuchConfigFile(c *C) {
-	err := s.disp.readConfig("/nosuchdir89j7879/8hjwr7ojgyy7")
-	c.Assert(err, NotNil)
-}
-
-func (s *StubbedSuite) TestBadSbatchArgsConfig(c *C) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), "config")
-	c.Check(err, IsNil)
-	defer os.Remove(tmpfile.Name())
-
-	_, err = tmpfile.Write([]byte(`{"SbatchArguments": "oops this is not a string array"}`))
-	c.Check(err, IsNil)
-
-	err = s.disp.readConfig(tmpfile.Name())
-	c.Assert(err, NotNil)
-}
-
-func (s *StubbedSuite) TestNoSuchArgInConfigIgnored(c *C) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), "config")
-	c.Check(err, IsNil)
-	defer os.Remove(tmpfile.Name())
-
-	_, err = tmpfile.Write([]byte(`{"NoSuchArg": "Nobody loves me, not one tiny hunk."}`))
-	c.Check(err, IsNil)
-
-	err = s.disp.readConfig(tmpfile.Name())
-	c.Assert(err, IsNil)
-	c.Check(0, Equals, len(s.disp.SbatchArguments))
-}
-
-func (s *StubbedSuite) TestReadConfig(c *C) {
-	tmpfile, err := ioutil.TempFile(os.TempDir(), "config")
-	c.Check(err, IsNil)
-	defer os.Remove(tmpfile.Name())
-
-	args := []string{"--arg1=v1", "--arg2", "--arg3=v3"}
-	argsS := `{"SbatchArguments": ["--arg1=v1",  "--arg2", "--arg3=v3"]}`
-	_, err = tmpfile.Write([]byte(argsS))
-	c.Check(err, IsNil)
-
-	err = s.disp.readConfig(tmpfile.Name())
-	c.Assert(err, IsNil)
-	c.Check(args, DeepEquals, s.disp.SbatchArguments)
-}
-
 func (s *StubbedSuite) TestSbatchArgs(c *C) {
 	container := arvados.Container{
 		UUID:               "123",
@@ -360,7 +318,7 @@ func (s *StubbedSuite) TestSbatchArgs(c *C) {
 		{"--arg1=v1", "--arg2"},
 	} {
 		c.Logf("%#v", defaults)
-		s.disp.SbatchArguments = defaults
+		s.disp.cluster.Containers.SLURM.SbatchArgumentsList = defaults
 
 		args, err := s.disp.sbatchArgs(container)
 		c.Check(args, DeepEquals, append(defaults, "--job-name=123", "--nice=10000", "--no-requeue", "--mem=239", "--cpus-per-task=2", "--tmp=0"))
@@ -432,3 +390,44 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) {
 	})
 	c.Check(err, IsNil)
 }
+
+func (s *StubbedSuite) TestLoadLegacyConfig(c *C) {
+	content := []byte(`
+Client:
+  APIHost: example.com
+  APIToken: abcdefg
+SbatchArguments: ["--foo", "bar"]
+PollPeriod: 12s
+PrioritySpread: 42
+CrunchRunCommand: ["x-crunch-run", "--cgroup-parent-subsystem=memory"]
+ReserveExtraRAM: 12345
+MinRetryPeriod: 13s
+BatchSize: 99
+`)
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	if _, err := tmpfile.Write(content); err != nil {
+		log.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		log.Fatal(err)
+
+	}
+	err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()})
+	c.Check(err, IsNil)
+
+	c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
+	c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
+	c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42))
+	c.Check(s.disp.cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run")
+	c.Check(s.disp.cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"})
+	c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
+	c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
+	c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99)
+}

commit f5ef76001884b7b464574c51783efc352f5e7532
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Jul 15 14:02:51 2019 -0400

    Fix test
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 7e5b47191..15bae9af8 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -449,6 +449,20 @@ Clusters:
       # stale locks from a previous dispatch process.
       StaleLockTimeout: 1m
 
+      # The crunch-run command to manage the container on a node
+      CrunchRunCommand: "crunch-run"
+
+      # Extra arguments to add to crunch-run invocation
+      # Example: ["--cgroup-parent-subsystem=memory"]
+      CrunchRunArgumentsList: []
+
+      # Extra RAM to reserve on the node, in addition to
+      # the amount specified in the container's RuntimeConstraints
+      ReserveExtraRAM: 256MiB
+
+      # Minimum time between two attempts to run the same container
+      MinRetryPeriod: 0s
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -492,6 +506,8 @@ Clusters:
         LogUpdateSize: 32MiB
 
       SLURM:
+        PrioritySpread: 0
+        SbatchArgumentsList: []
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 0a9d7a5b6..58a7690f4 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -455,6 +455,20 @@ Clusters:
       # stale locks from a previous dispatch process.
       StaleLockTimeout: 1m
 
+      # The crunch-run command to manage the container on a node
+      CrunchRunCommand: "crunch-run"
+
+      # Extra arguments to add to crunch-run invocation
+      # Example: ["--cgroup-parent-subsystem=memory"]
+      CrunchRunArgumentsList: []
+
+      # Extra RAM to reserve on the node, in addition to
+      # the amount specified in the container's RuntimeConstraints
+      ReserveExtraRAM: 256MiB
+
+      # Minimum time between two attempts to run the same container
+      MinRetryPeriod: 0s
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -498,6 +512,8 @@ Clusters:
         LogUpdateSize: 32MiB
 
       SLURM:
+        PrioritySpread: 0
+        SbatchArgumentsList: []
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 889e41095..09e3d591a 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -18,10 +18,10 @@ import (
 	"strings"
 	"time"
 
+	"git.curoverse.com/arvados.git/lib/config"
 	"git.curoverse.com/arvados.git/lib/dispatchcloud"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/config"
 	"git.curoverse.com/arvados.git/sdk/go/dispatch"
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/sirupsen/logrus"
@@ -47,26 +47,6 @@ type Dispatcher struct {
 	slurm   Slurm
 
 	Client arvados.Client
-
-	SbatchArguments []string
-	PollPeriod      arvados.Duration
-	PrioritySpread  int64
-
-	// crunch-run command to invoke. The container UUID will be
-	// appended. If nil, []string{"crunch-run"} will be used.
-	//
-	// Example: []string{"crunch-run", "--cgroup-parent-subsystem=memory"}
-	CrunchRunCommand []string
-
-	// Extra RAM to reserve (in Bytes) for SLURM job, in addition
-	// to the amount specified in the container's RuntimeConstraints
-	ReserveExtraRAM int64
-
-	// Minimum time between two attempts to run the same container
-	MinRetryPeriod arvados.Duration
-
-	// Batch size for container queries
-	BatchSize int64
 }
 
 func main() {
@@ -97,6 +77,9 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 	flags := flag.NewFlagSet(prog, flag.ExitOnError)
 	flags.Usage = func() { usage(flags) }
 
+	loader := config.NewLoader(stdin, log)
+	loader.SetupFlags(flags)
+
 	configPath := flags.String(
 		"config",
 		defaultConfigPath,
@@ -109,9 +92,16 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 		"version",
 		false,
 		"Print version information and exit.")
+
+	args = loader.MungeLegacyConfigArgs(logrus.StandardLogger(), args, "-crunch-dispatch-slurm-config")
+
 	// Parse args; omit the first arg which is the command name
 	flags.Parse(args)
 
+	if err == flag.ErrHelp {
+		return nil
+	}
+
 	// Print version information if requested
 	if *getVersion {
 		fmt.Printf("crunch-dispatch-slurm %s\n", version)
@@ -120,18 +110,19 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
 	disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
-	err := disp.readConfig(*configPath)
+	cfg, err := loader.Load()
 	if err != nil {
 		return err
 	}
 
-	if disp.CrunchRunCommand == nil {
-		disp.CrunchRunCommand = []string{"crunch-run"}
+	if disp.cluster, err = cfg.GetCluster(""); err != nil {
+		return fmt.Errorf("config error: %s", err)
 	}
 
-	if disp.PollPeriod == 0 {
-		disp.PollPeriod = arvados.Duration(10 * time.Second)
-	}
+	disp.Client.APIHost = fmt.Sprintf("%s:%d", disp.cluster.Services.Controller.ExternalURL.Host,
+		disp.cluster.Services.Controller.ExternalURL.Port)
+	disp.Client.AuthToken = disp.cluster.SystemRootToken
+	disp.Client.Insecure = disp.cluster.TLS.Insecure
 
 	if disp.Client.APIHost != "" || disp.Client.AuthToken != "" {
 		// Copy real configs into env vars so [a]
@@ -150,16 +141,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 	}
 
 	if *dumpConfig {
-		return config.DumpAndExit(disp)
-	}
-
-	siteConfig, err := arvados.GetConfig(arvados.DefaultConfigFile)
-	if os.IsNotExist(err) {
-		disp.logger.Warnf("no cluster config (%s), proceeding with no node types defined", err)
-	} else if err != nil {
-		return fmt.Errorf("error loading config: %s", err)
-	} else if disp.cluster, err = siteConfig.GetCluster(""); err != nil {
-		return fmt.Errorf("config error: %s", err)
+		return config.DumpAndExit(cfg)
 	}
 
 	return nil
diff --git a/services/login-sync/Gemfile.lock b/services/login-sync/Gemfile.lock
index eff59d2f7..f0283b611 100644
--- a/services/login-sync/Gemfile.lock
+++ b/services/login-sync/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    arvados-login-sync (1.4.0.20190701162225)
+    arvados-login-sync (1.4.0.20190709140013)
       arvados (~> 1.3.0, >= 1.3.0)
 
 GEM

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list