[ARVADOS] created: 1.1.4-351-g3271e860e
Git user
git at public.curoverse.com
Fri Jul 20 09:29:29 EDT 2018
at 3271e860e871b19a2f3be68376c57c2ce88993ac (commit)
commit 3271e860e871b19a2f3be68376c57c2ce88993ac
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Jul 19 18:15:03 2018 -0400
13879: Arvbox support for arvados-controller
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 1ac0e76c3..374692689 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -89,7 +89,7 @@ ADD fuse.conf /etc/
ADD crunch-setup.sh gitolite.rc \
keep-setup.sh common.sh createusers.sh \
logger runsu.sh waitforpostgres.sh \
- application_yml_override.py api-setup.sh \
+ yml_override.py api-setup.sh \
go-setup.sh \
/usr/local/lib/arvbox/
diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index 1618c11e4..749df4ad0 100755
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -68,7 +68,7 @@ $RAILS_ENV:
keep_web_service_url: http://$localip:${services[keep-web]}/
EOF
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
if ! test -f /var/lib/arvados/api_database_pw ; then
ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 319889bae..a82a964ea 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -20,7 +20,9 @@ fi
declare -A services
services=(
[workbench]=80
- [api]=8000
+ [api]=8004
+ [controller]=8003
+ [controller-ssl]=8000
[sso]=8900
[composer]=4200
[arv-git-httpd]=9001
diff --git a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
index b3ec5cd10..a36e5891b 100755
--- a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
@@ -19,7 +19,7 @@ else
RAILS_ENV=development
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /usr/src/arvados/services/api/superuser_token)
export CRUNCH_JOB_BIN=/usr/src/arvados/sdk/cli/bin/crunch-job
diff --git a/tools/arvbox/lib/arvbox/docker/keep-setup.sh b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
index 8ef66a606..ec63027ef 100755
--- a/tools/arvbox/lib/arvbox/docker/keep-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
@@ -19,7 +19,7 @@ fi
mkdir -p /var/lib/arvados/$1
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service
index f7ab6be6a..f052b5d63 100755
--- a/tools/arvbox/lib/arvbox/docker/service/api/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service
@@ -31,6 +31,4 @@ if test "$1" = "--only-setup" ; then
exit
fi
-exec bundle exec passenger start --port=${services[api]} \
- --ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \
- --ssl-certificate-key=/var/lib/arvados/self-signed.key
+exec bundle exec passenger start --port=${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
index 1383f7140..9339f2328 100755
--- a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
@@ -16,7 +16,7 @@ if test "$1" = "--only-deps" ; then
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export PATH="$PATH:/var/lib/arvados/git/bin"
cd ~git
diff --git a/tools/arvbox/lib/arvbox/docker/service/composer/run-service b/tools/arvbox/lib/arvbox/docker/service/composer/run-service
index abd350f07..f00b7f776 100755
--- a/tools/arvbox/lib/arvbox/docker/service/composer/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/composer/run-service
@@ -18,5 +18,5 @@ if test "$1" = "--only-deps" ; then
exit
fi
-echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
+echo "apiEndPoint: https://${localip}:${services[controller-ssl]}" > /usr/src/composer/src/composer.yml
exec node_modules/.bin/ng serve --host 0.0.0.0 --port 4200 --env=webdev
diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/log/main/.gitstub b/tools/arvbox/lib/arvbox/docker/service/controller/log/main/.gitstub
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/log/run b/tools/arvbox/lib/arvbox/docker/service/controller/log/run
new file mode 120000
index 000000000..d6aef4a77
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/controller/log/run
@@ -0,0 +1 @@
+/usr/local/lib/arvbox/logger
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/run b/tools/arvbox/lib/arvbox/docker/service/controller/run
new file mode 100755
index 000000000..c2afc1783
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/controller/run
@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/cmd/arvados-server"
+install $GOPATH/bin/arvados-server /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server arvados-controller)
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+database_pw=$(cat /var/lib/arvados/api_database_pw)
+
+mkdir -p /etc/arvados
+
+cat >/var/lib/arvados/cluster_config.yml <<EOF
+Clusters:
+ ${uuid_prefix}:
+ NodeProfiles:
+ "*":
+ arvados-controller:
+ Listen: ":${services[controller]}" # choose a port
+ arvados-api-server:
+ Listen: ":${services[api]}" # must match Rails server port in your Nginx config
+ PostgreSQL:
+ ConnectionPool: 32 # max concurrent connections per arvados server daemon
+ Connection:
+ # All parameters here are passed to the PG client library in a connection string;
+ # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+ Host: localhost
+ User: arvados
+ Password: ${database_pw}
+ DBName: arvados_development
+ client_encoding: utf8
+EOF
+
+/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
+
+cp /var/lib/arvados/cluster_config.yml /etc/arvados/config.yml
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-controller
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
index decbccdde..87c427cd2 100755
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
@@ -23,7 +23,7 @@ exec /usr/local/bin/crunch-run -container-enable-networking=always -container-ne
EOF
chmod +x /usr/local/bin/crunch-run.sh
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/doc/run-service b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
index 183ff2abf..83225ed0b 100755
--- a/tools/arvbox/lib/arvbox/docker/service/doc/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
@@ -18,34 +18,7 @@ if test "$1" = "--only-deps" ; then
exit
fi
-set -u
-
-cat <<EOF >/var/lib/arvados/doc-nginx.conf
-worker_processes auto;
-pid /var/lib/arvados/doc-nginx.pid;
-
-error_log stderr;
-daemon off;
-
-events {
- worker_connections 64;
-}
-
-http {
- access_log off;
- include /etc/nginx/mime.types;
- default_type application/octet-stream;
- server {
- listen ${services[doc]} default_server;
- listen [::]:${services[doc]} default_server;
- root /usr/src/arvados/doc/.site;
- index index.html;
- server_name _;
- }
-}
-EOF
-
cd /usr/src/arvados/doc
-bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[api]} arvados_workbench_host=http://$localip
+bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
-exec nginx -c /var/lib/arvados/doc-nginx.conf
+sv stop doc >/dev/null
diff --git a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
index a38e49a0d..eea0e120b 100755
--- a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
@@ -10,7 +10,7 @@ set -eux -o pipefail
mkdir -p /var/lib/arvados/git
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
@@ -112,7 +112,7 @@ cat > config/arvados-clients.yml <<EOF
$RAILS_ENV:
gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
gitolite_tmp: /var/lib/arvados/git
- arvados_api_host: $localip:${services[api]}
+ arvados_api_host: $localip:${services[controller-ssl]}
arvados_api_token: "$ARVADOS_API_TOKEN"
arvados_api_host_insecure: true
gitolite_arvados_git_user_key: "$git_user_key"
diff --git a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
index 70f2470b9..b539b6ae1 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
@@ -16,7 +16,7 @@ if test "$1" = "--only-deps" ; then
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
index 199247b7a..bf802d45f 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
@@ -17,7 +17,7 @@ if test "$1" = "--only-deps" ; then
exit
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/log/main/.gitstub b/tools/arvbox/lib/arvbox/docker/service/nginx/log/main/.gitstub
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/log/run b/tools/arvbox/lib/arvbox/docker/service/nginx/log/run
new file mode 120000
index 000000000..d6aef4a77
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/nginx/log/run
@@ -0,0 +1 @@
+/usr/local/lib/arvbox/logger
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/run b/tools/arvbox/lib/arvbox/docker/service/nginx/run
new file mode 120000
index 000000000..a388c8b67
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/nginx/run
@@ -0,0 +1 @@
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/run-service b/tools/arvbox/lib/arvbox/docker/service/nginx/run-service
new file mode 100755
index 000000000..6e9ae2b29
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/nginx/run-service
@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cat <<EOF >/var/lib/arvados/nginx.conf
+worker_processes auto;
+pid /var/lib/arvados/nginx.pid;
+
+error_log stderr;
+daemon off;
+
+events {
+ worker_connections 64;
+}
+
+http {
+ access_log off;
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+ server {
+ listen ${services[doc]} default_server;
+ listen [::]:${services[doc]} default_server;
+ root /usr/src/arvados/doc/.site;
+ index index.html;
+ server_name _;
+ }
+
+ upstream controller {
+ server localhost:${services[controller]};
+ }
+ server {
+ listen *:${services[controller-ssl]} ssl default_server;
+ server_name keepproxy;
+ ssl_certificate "/var/lib/arvados/self-signed.pem";
+ ssl_certificate_key "/var/lib/arvados/self-signed.key";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+}
+
+EOF
+
+exec nginx -c /var/lib/arvados/nginx.conf
diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index 6d5fe243e..cfd7d5aa5 100755
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -45,7 +45,7 @@ if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
waiting="$waiting crunch-dispatch"
fi
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
vm_ok=0
diff --git a/tools/arvbox/lib/arvbox/docker/service/sso/run-service b/tools/arvbox/lib/arvbox/docker/service/sso/run-service
index ab20d5758..281405949 100755
--- a/tools/arvbox/lib/arvbox/docker/service/sso/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sso/run-service
@@ -47,7 +47,7 @@ $RAILS_ENV:
allow_account_registration: true
EOF
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
if ! test -f /var/lib/arvados/sso_database_pw ; then
ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/sso_database_pw
diff --git a/tools/arvbox/lib/arvbox/docker/service/vm/run b/tools/arvbox/lib/arvbox/docker/service/vm/run
index 2b571a820..863de7341 100755
--- a/tools/arvbox/lib/arvbox/docker/service/vm/run
+++ b/tools/arvbox/lib/arvbox/docker/service/vm/run
@@ -14,7 +14,7 @@ git config --system "credential.http://$localip:${services[arv-git-httpd]}/.help
cd /usr/src/arvados/services/login-sync
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
diff --git a/tools/arvbox/lib/arvbox/docker/service/vm/run-service b/tools/arvbox/lib/arvbox/docker/service/vm/run-service
index 134f767dc..065c55701 100755
--- a/tools/arvbox/lib/arvbox/docker/service/vm/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/vm/run-service
@@ -18,7 +18,7 @@ fi
set -u
-export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
diff --git a/tools/arvbox/lib/arvbox/docker/service/websockets/run-service b/tools/arvbox/lib/arvbox/docker/service/websockets/run-service
index 2d01d9079..ebdf266c6 100755
--- a/tools/arvbox/lib/arvbox/docker/service/websockets/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/websockets/run-service
@@ -27,7 +27,7 @@ database_pw=$(cat /var/lib/arvados/api_database_pw)
cat >/var/lib/arvados/arvados-ws.yml <<EOF
Client:
- APIHost: $localip:${services[api]}
+ APIHost: $localip:${services[controller-ssl]}
Insecure: true
Postgres:
dbname: arvados_$RAILS_ENV
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 09d77e01d..366096ace 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -40,8 +40,8 @@ fi
cat >config/application.yml <<EOF
$RAILS_ENV:
secret_token: $secret_token
- arvados_login_base: https://$localip:${services[api]}/login
- arvados_v1_base: https://$localip:${services[api]}/arvados/v1
+ arvados_login_base: https://$localip:${services[controller-ssl]}/login
+ arvados_v1_base: https://$localip:${services[controller-ssl]}/arvados/v1
arvados_insecure_https: true
keep_web_download_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
keep_web_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
@@ -52,4 +52,4 @@ EOF
bundle exec rake assets:precompile
-(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
diff --git a/tools/arvbox/lib/arvbox/docker/application_yml_override.py b/tools/arvbox/lib/arvbox/docker/yml_override.py
similarity index 79%
rename from tools/arvbox/lib/arvbox/docker/application_yml_override.py
rename to tools/arvbox/lib/arvbox/docker/yml_override.py
index bec067a88..b44acf4c3 100755
--- a/tools/arvbox/lib/arvbox/docker/application_yml_override.py
+++ b/tools/arvbox/lib/arvbox/docker/yml_override.py
@@ -4,14 +4,17 @@
# SPDX-License-Identifier: AGPL-3.0
import yaml
+import sys
+
+fn = sys.argv[1]
try:
- with open("application.yml.override") as f:
+ with open(fn+".override") as f:
b = yaml.load(f)
except IOError:
exit()
-with open("application.yml") as f:
+with open(fn) as f:
a = yaml.load(f)
def recursiveMerge(a, b):
@@ -23,5 +26,5 @@ def recursiveMerge(a, b):
else:
return b
-with open("application.yml", "w") as f:
+with open(fn, "w") as f:
yaml.dump(recursiveMerge(a, b), f)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list