[ARVADOS] created: 8cda7f9ce25adeff03b9636f70026f5e9263dbe8
git at public.curoverse.com
git at public.curoverse.com
Sun Mar 22 20:20:38 EDT 2015
at 8cda7f9ce25adeff03b9636f70026f5e9263dbe8 (commit)
commit 8cda7f9ce25adeff03b9636f70026f5e9263dbe8
Author: Ward Vandewege <ward at curoverse.com>
Date: Sun Mar 22 20:21:55 2015 -0400
Add a keepproxy docker image
refs #5529
diff --git a/docker/README.md b/docker/README.md
index f521b8c..9c03e1b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -6,16 +6,13 @@ containers, so that they can be run easily in different environments
(a dedicated server, a developer's laptop, a virtual machine,
etc).
-This is a work in progress; instructions will almost certainly be
-incomplete and possibly out of date.
-
Prerequisites
-------------
* Docker
- Docker is a Linux container management system based on LXC. It is a
- very young system but is being developed rapidly.
+ Docker is a Linux container management system. It is a very young system but
+ is being developed rapidly.
[Installation packages](http://www.docker.io/gettingstarted/)
are available for several platforms.
@@ -52,10 +49,13 @@ Building
Type `./build.sh` to configure and build the following Docker images:
* arvados/api - the Arvados API server
+ * arvados/compute - Arvados compute node image
* arvados/doc - Arvados documentation
- * arvados/warehouse - Keep, the Arvados content-addressable filesystem
- * arvados/workbench - the Arvados console
+ * arvados/keep - Keep, the Arvados content-addressable filesystem
+ * arvados/keepproxy - Keep proxy
+ * arvados/shell - Arvados shell node image
* arvados/sso - the Arvados single-signon authentication server
+ * arvados/workbench - the Arvados console
`build.sh` will generate reasonable defaults for all configuration
settings. If you want more control over the way Arvados is
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index abd2114..07acb63 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -63,6 +63,7 @@ RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid
# Create keep and compute node objects
ADD keep_server_0.json /root/
ADD keep_server_1.json /root/
+ADD keep_proxy.json /root/
# Set up update-gitolite.rb
RUN mkdir /usr/local/arvados/config -p
diff --git a/docker/api/keep_proxy.json b/docker/api/keep_proxy.json
new file mode 100644
index 0000000..117e590
--- /dev/null
+++ b/docker/api/keep_proxy.json
@@ -0,0 +1,6 @@
+{
+ "service_host": "localhost",
+ "service_port": 9902,
+ "service_ssl_flag": "false",
+ "service_type": "proxy"
+}
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index 7af6afb..0111595 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -35,6 +35,11 @@ if [[ "$?" != "0" ]]; then
arv keep_service create --keep-service "$(cat /root/keep_server_1.json)"
fi
+grep -q keep_proxy /tmp/keep_service.list
+if [[ "$?" != "0" ]]; then
+ arv keep_service create --keep-service "$(cat /root/keep_proxy.json)"
+fi
+
# User repository object
user_uuid=`arv --format=uuid user current`
repo_uuid=`arv --format=uuid repository create --repository '{"name":"@@ARVADOS_USER_NAME@@","fetch_url":"git at api:@@ARVADOS_USER_NAME@@.git","push_url":"git at api:@@ARVADOS_USER_NAME@@.git"}'`
diff --git a/docker/arvdock b/docker/arvdock
index 142ba27..ecd90c9 100755
--- a/docker/arvdock
+++ b/docker/arvdock
@@ -21,6 +21,7 @@ function usage {
echo >&2 " -v, --vm Shell server"
echo >&2 " -n, --nameserver Nameserver"
echo >&2 " -k, --keep Keep servers"
+ echo >&2 " -p, --keepproxy Keepproxy server"
echo >&2 " -h, --help Display this help and exit"
echo >&2
echo >&2 " If no options are given, the action is applied to all servers."
@@ -73,7 +74,6 @@ function start_container {
$DOCKER rm "$name" 2>/dev/null
echo "Starting container:"
- #echo " $DOCKER run --dns=127.0.0.1 $args $image"
echo " $DOCKER run $args $image"
container=`$DOCKER run $args $image`
if [ "$?" != "0" -o "$container" = "" ]; then
@@ -136,10 +136,11 @@ function do_start {
local start_vm=false
local start_nameserver=false
local start_keep=false
+ local start_keepproxy=false
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- local TEMP=`getopt -o d::s::a::cw::nkvh \
- --long doc::,sso::,api::,compute,workbench::,nameserver,keep,vm,help \
+ local TEMP=`getopt -o d::s::a::cw::nkpvh \
+ --long doc::,sso::,api::,compute,workbench::,nameserver,keep,keepproxy,vm,help \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -190,6 +191,10 @@ function do_start {
start_keep=true
shift
;;
+ -p | --keepproxy)
+ start_keepproxy=true
+ shift
+ ;;
--)
shift
break
@@ -209,7 +214,8 @@ function do_start {
$start_workbench == false &&
$start_vm == false &&
$start_nameserver == false &&
- $start_keep == false ]]
+ $start_keep == false &&
+ $start_keepproxy == false ]]
then
start_doc=9898
#the sso server is currently not used by default so don't start it unless explicitly requested
@@ -220,20 +226,7 @@ function do_start {
start_vm=true
start_nameserver=true
start_keep=true
- fi
-
- if [[ $start_sso != false ]]
- then
- start_container "$start_sso:443" "sso_server" '' '' "arvados/sso"
- fi
-
- if [[ $start_api != false ]]
- then
- if [[ $start_sso != false ]]; then
- start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
- else
- start_container "$start_api:443" "api_server" '' '' "arvados/api"
- fi
+ start_keepproxy=true
fi
if [[ $start_nameserver != false ]]
@@ -254,6 +247,20 @@ function do_start {
fi
fi
+ if [[ $start_sso != false ]]
+ then
+ start_container "$start_sso:443" "sso_server" '' '' "arvados/sso"
+ fi
+
+ if [[ $start_api != false ]]
+ then
+ if [[ $start_sso != false ]]; then
+ start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
+ else
+ start_container "$start_api:443" "api_server" '' '' "arvados/api"
+ fi
+ fi
+
if [[ $start_compute != false ]]
then
for i in `seq 0 $(($start_compute - 1))`; do
@@ -280,6 +287,13 @@ function do_start {
"arvados/keep"
fi
+ if [[ $start_keepproxy != false ]]
+ then
+ start_container "9902:9100" "keepproxy_server" '' \
+ "api_server:api" \
+ "arvados/keepproxy"
+ fi
+
if [[ $start_doc != false ]]
then
start_container "$start_doc:80" "doc_server" '' '' "arvados/doc"
@@ -320,10 +334,11 @@ function do_stop {
local stop_nameserver=""
local stop_vm=""
local stop_keep=""
+ local stop_keepproxy=""
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- local TEMP=`getopt -o dsacwnkvh \
- --long doc,sso,api,compute,workbench,nameserver,keep,vm,help \
+ local TEMP=`getopt -o dsacwnkpvh \
+ --long doc,sso,api,compute,workbench,nameserver,keep,keepproxy,vm,help \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -350,6 +365,8 @@ function do_stop {
stop_vm="shell" ; shift ;;
-k | --keep )
stop_keep="keep_server_0 keep_server_1" ; shift ;;
+ -p | --keepproxy )
+ stop_keep="keepproxy_server" ; shift ;;
--)
shift
break
@@ -369,7 +386,8 @@ function do_stop {
$stop_workbench == "" &&
$stop_vm == "" &&
$stop_nameserver == "" &&
- $stop_keep == "" ]]
+ $stop_keep == "" &&
+ $stop_keepproxy == "" ]]
then
stop_doc=doc_server
stop_sso=sso_server
@@ -379,9 +397,10 @@ function do_stop {
stop_vm=shell
stop_nameserver="skydock skydns"
stop_keep="keep_server_0 keep_server_1"
+ stop_keepproxy="keepproxy_server"
fi
- $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_vm \
+ $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_keepproxy $stop_vm \
2>/dev/null
}
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 9d93d2b..621aa04 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -4,7 +4,7 @@ OLD_SHELL := $(SHELL)
SHELL = $(warning [$@])$(OLD_SHELL) -x
endif
-all: skydns-image skydock-image api-image compute-image doc-image workbench-image keep-image sso-image shell-image
+all: skydns-image skydock-image api-image compute-image doc-image workbench-image keep-image keep-proxy-image sso-image shell-image
IMAGE_FILES := $(shell ls *-image 2>/dev/null |grep -v -E 'debian-arvados-image|skydns-image|skydock-image')
GENERATED_DIRS := $(shell ls */generated 2>/dev/null)
@@ -19,7 +19,7 @@ clean:
DEBIAN_IMAGE := $(shell $(DOCKER) images -q arvados/debian |head -n1)
-REALCLEAN_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e doc_server -e workbench_server |cut -f 1 -d' ')
+REALCLEAN_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e keep_proxy_server -e doc_server -e workbench_server |cut -f 1 -d' ')
REALCLEAN_IMAGES := $(shell $(DOCKER) images -q arvados/* |grep -v $(DEBIAN_IMAGE) 2>/dev/null)
DEEPCLEAN_IMAGES := $(shell $(DOCKER) images -q arvados/*)
SKYDNS_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e crosbymichael/skydns -e crosbymichael/skydock |cut -f 1 -d' ')
@@ -78,6 +78,8 @@ WORKBENCH_DEPS = workbench/Dockerfile \
KEEP_DEPS = keep/Dockerfile config.yml $(KEEP_GENERATED)
+KEEP_PROXY_DEPS = keepproxy/Dockerfile config.yml $(KEEP_PROXY_GENERATED)
+
SSO_DEPS = config.yml $(SSO_GENERATED)
BCBIO_NEXTGEN_DEPS = bcbio-nextgen/Dockerfile
@@ -90,6 +92,9 @@ COMPUTE_GENERATED = compute/generated/*
KEEP_GENERATED_IN = keep/*.in
KEEP_GENERATED = keep/generated/*
+KEEP_PROXY_GENERATED_IN = keepproxy/*.in
+KEEP_PROXY_GENERATED = keepproxy/generated/*
+
API_GENERATED_IN = api/*.in
API_GENERATED = api/generated/*
@@ -112,6 +117,12 @@ keep/generated/bin/%: $(wildcard build/services/%/*.go)
ln -sfn ../../../../.. keep/generated/src/git.curoverse.com/arvados.git
GOPATH=$(shell pwd)/keep/generated go get $(@:keep/generated/bin/%=git.curoverse.com/arvados.git/services/%)
+KEEP_PROXY_DEPS += keepproxy/generated/bin/keepproxy
+keepproxy/generated/bin/%: $(wildcard build/services/%/*.go)
+ mkdir -p keepproxy/generated/src/git.curoverse.com
+ ln -sfn ../../../../.. keepproxy/generated/src/git.curoverse.com/arvados.git
+ GOPATH=$(shell pwd)/keepproxy/generated go get $(@:keepproxy/generated/bin/%=git.curoverse.com/arvados.git/services/%)
+
$(BUILD):
mkdir -p build
rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
@@ -150,6 +161,9 @@ $(SSO_GENERATED): $(SSO_GENERATED_IN)
$(KEEP_GENERATED): $(KEEP_GENERATED_IN)
$(CONFIG_RB) keep
+$(KEEP_PROXY_GENERATED): $(KEEP_PROXY_GENERATED_IN)
+ $(CONFIG_RB) keepproxy
+
DOCKER_BUILD = $(DOCKER) build --rm=true
# ============================================================
@@ -185,6 +199,11 @@ keep-image: debian-arvados-image $(BUILD) $(KEEP_DEPS)
$(DOCKER_BUILD) -t arvados/keep keep
date >keep-image
+keep-proxy-image: debian-arvados-image $(BUILD) $(KEEP_PROXY_DEPS)
+ @echo "Building keep-proxy-image"
+ $(DOCKER_BUILD) -t arvados/keepproxy keepproxy
+ date >keep-proxy-image
+
jobs-image: base-image $(BUILD) $(JOBS_DEPS)
$(DOCKER_BUILD) -t arvados/jobs jobs
date >jobs-image
diff --git a/docker/keepproxy/Dockerfile b/docker/keepproxy/Dockerfile
new file mode 100644
index 0000000..78e82b9
--- /dev/null
+++ b/docker/keepproxy/Dockerfile
@@ -0,0 +1,12 @@
+# Based on Debian Wheezy
+FROM arvados/debian:wheezy
+MAINTAINER Ward Vandewege <ward at curoverse.com>
+
+RUN apt-get update -qq
+RUN apt-get install -qqy ca-certificates
+
+ADD generated/bin/keepproxy /usr/local/bin/
+ADD generated/run-keepproxy /usr/local/bin/
+
+# Start keep
+CMD ["/usr/local/bin/run-keepproxy"]
diff --git a/docker/keepproxy/run-keepproxy.in b/docker/keepproxy/run-keepproxy.in
new file mode 100755
index 0000000..08fc441
--- /dev/null
+++ b/docker/keepproxy/run-keepproxy.in
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+export ARVADOS_API_HOST=api
+export ARVADOS_API_HOST_INSECURE=yes
+# This should be an anonymous token, but we don't have a good way
+# to get one while building the images
+export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
+
+exec keepproxy -listen=':9100'
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list