[ARVADOS] created: 8cda7f9ce25adeff03b9636f70026f5e9263dbe8

git at public.curoverse.com git at public.curoverse.com
Sun Mar 22 20:20:38 EDT 2015


        at  8cda7f9ce25adeff03b9636f70026f5e9263dbe8 (commit)


commit 8cda7f9ce25adeff03b9636f70026f5e9263dbe8
Author: Ward Vandewege <ward at curoverse.com>
Date:   Sun Mar 22 20:21:55 2015 -0400

    Add a keepproxy docker image
    
    refs #5529

diff --git a/docker/README.md b/docker/README.md
index f521b8c..9c03e1b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -6,16 +6,13 @@ containers, so that they can be run easily in different environments
 (a dedicated server, a developer's laptop, a virtual machine,
 etc).
 
-This is a work in progress; instructions will almost certainly be
-incomplete and possibly out of date.
-
 Prerequisites
 -------------
 
 * Docker
 
-  Docker is a Linux container management system based on LXC. It is a
-  very young system but is being developed rapidly.
+  Docker is a Linux container management system. It is a very young system but
+  is being developed rapidly.
   [Installation packages](http://www.docker.io/gettingstarted/)
   are available for several platforms.
   
@@ -52,10 +49,13 @@ Building
 Type `./build.sh` to configure and build the following Docker images:
 
    * arvados/api       - the Arvados API server
+   * arvados/compute   - Arvados compute node image
    * arvados/doc       - Arvados documentation
-   * arvados/warehouse - Keep, the Arvados content-addressable filesystem
-   * arvados/workbench - the Arvados console
+   * arvados/keep      - Keep, the Arvados content-addressable filesystem
+   * arvados/keepproxy - Keep proxy
+   * arvados/shell     - Arvados shell node image
    * arvados/sso       - the Arvados single-signon authentication server
+   * arvados/workbench - the Arvados console
 
 `build.sh` will generate reasonable defaults for all configuration
 settings.  If you want more control over the way Arvados is
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index abd2114..07acb63 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -63,6 +63,7 @@ RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid
 # Create keep and compute node objects
 ADD keep_server_0.json /root/
 ADD keep_server_1.json /root/
+ADD keep_proxy.json /root/
 
 # Set up update-gitolite.rb
 RUN mkdir /usr/local/arvados/config -p
diff --git a/docker/api/keep_proxy.json b/docker/api/keep_proxy.json
new file mode 100644
index 0000000..117e590
--- /dev/null
+++ b/docker/api/keep_proxy.json
@@ -0,0 +1,6 @@
+{
+  "service_host": "localhost",
+  "service_port": 9902,
+  "service_ssl_flag": "false",
+  "service_type": "proxy"
+}
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index 7af6afb..0111595 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -35,6 +35,11 @@ if [[ "$?" != "0" ]]; then
   arv keep_service create --keep-service "$(cat /root/keep_server_1.json)"
 fi
 
+grep -q keep_proxy /tmp/keep_service.list
+if [[ "$?" != "0" ]]; then
+  arv keep_service create --keep-service "$(cat /root/keep_proxy.json)"
+fi
+
 # User repository object
 user_uuid=`arv --format=uuid user current`
 repo_uuid=`arv --format=uuid repository create --repository '{"name":"@@ARVADOS_USER_NAME@@","fetch_url":"git at api:@@ARVADOS_USER_NAME@@.git","push_url":"git at api:@@ARVADOS_USER_NAME@@.git"}'`
diff --git a/docker/arvdock b/docker/arvdock
index 142ba27..ecd90c9 100755
--- a/docker/arvdock
+++ b/docker/arvdock
@@ -21,6 +21,7 @@ function usage {
     echo >&2 "  -v, --vm                      Shell server"
     echo >&2 "  -n, --nameserver              Nameserver"
     echo >&2 "  -k, --keep                    Keep servers"
+    echo >&2 "  -p, --keepproxy               Keepproxy server"
     echo >&2 "  -h, --help                    Display this help and exit"
     echo >&2
     echo >&2 "  If no options are given, the action is applied to all servers."
@@ -73,7 +74,6 @@ function start_container {
     $DOCKER rm "$name" 2>/dev/null
 
     echo "Starting container:"
-    #echo "  $DOCKER run --dns=127.0.0.1 $args $image"
     echo "  $DOCKER run $args $image"
     container=`$DOCKER run $args $image`
     if [ "$?" != "0" -o "$container" = "" ]; then
@@ -136,10 +136,11 @@ function do_start {
     local start_vm=false
     local start_nameserver=false
     local start_keep=false
+    local start_keepproxy=false
 
     # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-    local TEMP=`getopt -o d::s::a::cw::nkvh \
-                  --long doc::,sso::,api::,compute,workbench::,nameserver,keep,vm,help \
+    local TEMP=`getopt -o d::s::a::cw::nkpvh \
+                  --long doc::,sso::,api::,compute,workbench::,nameserver,keep,keepproxy,vm,help \
                   -n "$0" -- "$@"`
 
     if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -190,6 +191,10 @@ function do_start {
                 start_keep=true
                 shift
                 ;;
+            -p | --keepproxy)
+                start_keepproxy=true
+                shift
+                ;;
             --)
                 shift
                 break
@@ -209,7 +214,8 @@ function do_start {
           $start_workbench == false &&
           $start_vm == false &&
           $start_nameserver == false &&
-          $start_keep == false ]]
+          $start_keep == false &&
+          $start_keepproxy == false ]]
     then
         start_doc=9898
         #the sso server is currently not used by default so don't start it unless explicitly requested
@@ -220,20 +226,7 @@ function do_start {
         start_vm=true
         start_nameserver=true
         start_keep=true
-    fi
-
-    if [[ $start_sso != false ]]
-    then
-        start_container "$start_sso:443" "sso_server" '' '' "arvados/sso"
-    fi
-
-    if [[ $start_api != false ]]
-    then
-      if [[ $start_sso != false ]]; then
-        start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
-      else
-        start_container "$start_api:443" "api_server" '' '' "arvados/api"
-      fi
+        start_keepproxy=true
     fi
 
     if [[ $start_nameserver != false ]]
@@ -254,6 +247,20 @@ function do_start {
       fi
     fi
 
+    if [[ $start_sso != false ]]
+    then
+        start_container "$start_sso:443" "sso_server" '' '' "arvados/sso"
+    fi
+
+    if [[ $start_api != false ]]
+    then
+      if [[ $start_sso != false ]]; then
+        start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
+      else
+        start_container "$start_api:443" "api_server" '' '' "arvados/api"
+      fi
+    fi
+
     if [[ $start_compute != false ]]
     then
         for i in `seq 0 $(($start_compute - 1))`; do
@@ -280,6 +287,13 @@ function do_start {
             "arvados/keep"
     fi
 
+    if [[ $start_keepproxy != false ]]
+    then
+        start_container "9902:9100" "keepproxy_server" '' \
+            "api_server:api" \
+            "arvados/keepproxy"
+    fi
+
     if [[ $start_doc != false ]]
     then
         start_container "$start_doc:80" "doc_server" '' '' "arvados/doc"
@@ -320,10 +334,11 @@ function do_stop {
     local stop_nameserver=""
     local stop_vm=""
     local stop_keep=""
+    local stop_keepproxy=""
 
     # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-    local TEMP=`getopt -o dsacwnkvh \
-                  --long doc,sso,api,compute,workbench,nameserver,keep,vm,help \
+    local TEMP=`getopt -o dsacwnkpvh \
+                  --long doc,sso,api,compute,workbench,nameserver,keep,keepproxy,vm,help \
                   -n "$0" -- "$@"`
 
     if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -350,6 +365,8 @@ function do_stop {
                 stop_vm="shell" ; shift ;;
             -k | --keep )
                 stop_keep="keep_server_0 keep_server_1" ; shift ;;
+            -p | --keepproxy )
+                stop_keep="keepproxy_server" ; shift ;;
             --)
                 shift
                 break
@@ -369,7 +386,8 @@ function do_stop {
           $stop_workbench == "" &&
           $stop_vm == "" &&
           $stop_nameserver == "" &&
-          $stop_keep == "" ]]
+          $stop_keep == "" &&
+          $stop_keepproxy == "" ]]
     then
         stop_doc=doc_server
         stop_sso=sso_server
@@ -379,9 +397,10 @@ function do_stop {
         stop_vm=shell
         stop_nameserver="skydock skydns"
         stop_keep="keep_server_0 keep_server_1"
+        stop_keepproxy="keepproxy_server"
     fi
 
-    $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_vm \
+    $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_keepproxy $stop_vm \
         2>/dev/null
 }
 
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 9d93d2b..621aa04 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -4,7 +4,7 @@ OLD_SHELL := $(SHELL)
 SHELL = $(warning [$@])$(OLD_SHELL) -x
 endif
 
-all: skydns-image skydock-image api-image compute-image doc-image workbench-image keep-image sso-image shell-image
+all: skydns-image skydock-image api-image compute-image doc-image workbench-image keep-image keep-proxy-image sso-image shell-image
 
 IMAGE_FILES := $(shell ls *-image 2>/dev/null |grep -v -E 'debian-arvados-image|skydns-image|skydock-image')
 GENERATED_DIRS := $(shell ls */generated 2>/dev/null)
@@ -19,7 +19,7 @@ clean:
 
 DEBIAN_IMAGE := $(shell $(DOCKER) images -q arvados/debian |head -n1)
 
-REALCLEAN_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e doc_server -e workbench_server |cut -f 1 -d' ')
+REALCLEAN_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e keep_proxy_server -e doc_server -e workbench_server |cut -f 1 -d' ')
 REALCLEAN_IMAGES := $(shell $(DOCKER) images -q arvados/* |grep -v $(DEBIAN_IMAGE) 2>/dev/null)
 DEEPCLEAN_IMAGES := $(shell $(DOCKER) images -q arvados/*)
 SKYDNS_CONTAINERS := $(shell $(DOCKER) ps -a |grep -e crosbymichael/skydns -e crosbymichael/skydock |cut -f 1 -d' ')
@@ -78,6 +78,8 @@ WORKBENCH_DEPS = workbench/Dockerfile \
 
 KEEP_DEPS = keep/Dockerfile config.yml $(KEEP_GENERATED)
 
+KEEP_PROXY_DEPS = keepproxy/Dockerfile config.yml $(KEEP_PROXY_GENERATED)
+
 SSO_DEPS = config.yml $(SSO_GENERATED)
 
 BCBIO_NEXTGEN_DEPS = bcbio-nextgen/Dockerfile
@@ -90,6 +92,9 @@ COMPUTE_GENERATED      = compute/generated/*
 KEEP_GENERATED_IN      = keep/*.in
 KEEP_GENERATED         = keep/generated/*
 
+KEEP_PROXY_GENERATED_IN      = keepproxy/*.in
+KEEP_PROXY_GENERATED         = keepproxy/generated/*
+
 API_GENERATED_IN       = api/*.in
 API_GENERATED          = api/generated/*
 
@@ -112,6 +117,12 @@ keep/generated/bin/%: $(wildcard build/services/%/*.go)
 	ln -sfn ../../../../.. keep/generated/src/git.curoverse.com/arvados.git
 	GOPATH=$(shell pwd)/keep/generated go get $(@:keep/generated/bin/%=git.curoverse.com/arvados.git/services/%)
 
+KEEP_PROXY_DEPS += keepproxy/generated/bin/keepproxy
+keepproxy/generated/bin/%: $(wildcard build/services/%/*.go)
+	mkdir -p keepproxy/generated/src/git.curoverse.com
+	ln -sfn ../../../../.. keepproxy/generated/src/git.curoverse.com/arvados.git
+	GOPATH=$(shell pwd)/keepproxy/generated go get $(@:keepproxy/generated/bin/%=git.curoverse.com/arvados.git/services/%)
+
 $(BUILD):
 	mkdir -p build
 	rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
@@ -150,6 +161,9 @@ $(SSO_GENERATED): $(SSO_GENERATED_IN)
 $(KEEP_GENERATED): $(KEEP_GENERATED_IN)
 	$(CONFIG_RB) keep
 
+$(KEEP_PROXY_GENERATED): $(KEEP_PROXY_GENERATED_IN)
+	$(CONFIG_RB) keepproxy
+
 DOCKER_BUILD = $(DOCKER) build --rm=true
 
 # ============================================================
@@ -185,6 +199,11 @@ keep-image: debian-arvados-image $(BUILD) $(KEEP_DEPS)
 	$(DOCKER_BUILD) -t arvados/keep keep
 	date >keep-image
 
+keep-proxy-image: debian-arvados-image $(BUILD) $(KEEP_PROXY_DEPS)
+	@echo "Building keep-proxy-image"
+	$(DOCKER_BUILD) -t arvados/keepproxy keepproxy
+	date >keep-proxy-image
+
 jobs-image: base-image $(BUILD) $(JOBS_DEPS)
 	$(DOCKER_BUILD) -t arvados/jobs jobs
 	date >jobs-image
diff --git a/docker/keepproxy/Dockerfile b/docker/keepproxy/Dockerfile
new file mode 100644
index 0000000..78e82b9
--- /dev/null
+++ b/docker/keepproxy/Dockerfile
@@ -0,0 +1,12 @@
+# Based on Debian Wheezy
+FROM arvados/debian:wheezy
+MAINTAINER Ward Vandewege <ward at curoverse.com>
+
+RUN apt-get update -qq
+RUN apt-get install -qqy ca-certificates
+
+ADD generated/bin/keepproxy /usr/local/bin/
+ADD generated/run-keepproxy /usr/local/bin/
+
+# Start keep
+CMD ["/usr/local/bin/run-keepproxy"]
diff --git a/docker/keepproxy/run-keepproxy.in b/docker/keepproxy/run-keepproxy.in
new file mode 100755
index 0000000..08fc441
--- /dev/null
+++ b/docker/keepproxy/run-keepproxy.in
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+export ARVADOS_API_HOST=api
+export ARVADOS_API_HOST_INSECURE=yes
+# This should be an anonymous token, but we don't have a good way
+# to get one while building the images
+export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
+
+exec keepproxy -listen=':9100'

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list