[ARVADOS] updated: 33f52182ea5d0ca602ebe4dff81822ed57a3a847
git at public.curoverse.com
git at public.curoverse.com
Sat Jul 19 10:22:29 EDT 2014
Summary of changes:
doc/_includes/_alert_docker.liquid | 4 --
doc/install/index.html.textile.liquid | 2 -
doc/install/install-docker.html.textile.liquid | 5 +--
.../check-environment.html.textile.liquid | 2 +-
docker/api/Dockerfile | 8 +---
docker/api/application.yml.in | 4 +-
.../{setup-gitolite.sh => setup-gitolite.sh.in} | 24 +++++++++-
docker/api/setup.sh.in | 25 +++++++++--
docker/api/update-gitolite.rb | 8 +---
docker/arvdock | 40 ++++++++++++-----
docker/build_tools/Makefile | 51 ++++++++++++++++++----
docker/compute/wrapdocker | 6 +--
docker/config.yml.example | 4 ++
docker/shell/Dockerfile | 19 ++++++++
docker/{compute => shell}/fuse.conf | 0
docker/shell/setup.sh.in | 21 +++++++++
docker/{api => shell}/superuser_token.in | 0
docker/shell/supervisor.conf | 15 +++++++
18 files changed, 187 insertions(+), 51 deletions(-)
delete mode 100644 doc/_includes/_alert_docker.liquid
rename docker/api/{setup-gitolite.sh => setup-gitolite.sh.in} (60%)
create mode 100644 docker/shell/Dockerfile
copy docker/{compute => shell}/fuse.conf (100%)
create mode 100755 docker/shell/setup.sh.in
copy docker/{api => shell}/superuser_token.in (100%)
create mode 100644 docker/shell/supervisor.conf
via 33f52182ea5d0ca602ebe4dff81822ed57a3a847 (commit)
via 94c6d21aed9f33599f7d8eee29481b47b7bb99f7 (commit)
via c58d996cd35e754c56a7c223414f04d0b362c284 (commit)
from ab371a54c3522c06fbc82c9838a24c6f53122a6e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 33f52182ea5d0ca602ebe4dff81822ed57a3a847
Author: Ward Vandewege <ward at curoverse.com>
Date: Sat Jul 19 10:21:14 2014 -0400
* Add shell node to docker installation.
* Remove documentation warning about missing crunch support.
* Various small improvements
refs #3219
diff --git a/doc/_includes/_alert_docker.liquid b/doc/_includes/_alert_docker.liquid
deleted file mode 100644
index be4a8e0..0000000
--- a/doc/_includes/_alert_docker.liquid
+++ /dev/null
@@ -1,4 +0,0 @@
-<div class="alert alert-block alert-info">
- <button type="button" class="close" data-dismiss="alert">×</button>
- <p>The Docker installation is not feature complete. We do not have a Docker container yet for crunch-dispatch and the arvados compute nodes. This means that running pipelines from a Docker-based Arvados install is currently not supported without additional manual configuration. Without that manual configuration, it is possible to use arv-crunch-job to run a 'local' job against your Docker-based Arvados installation. To do this, please refer to the "Debugging a Crunch script":{{site.baseurl}}/user/topics/tutorial-job-debug.html page.</p>
-</div>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 944a695..ad0722a 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -12,8 +12,6 @@ For larger scale installations, a manual installation is more appropriate.
h2. Docker
-{% include 'alert_docker' %}
-
"Installing with Docker":install-docker.html
h2. Manual installation
diff --git a/doc/install/install-docker.html.textile.liquid b/doc/install/install-docker.html.textile.liquid
index ef2b709..cb27c97 100644
--- a/doc/install/install-docker.html.textile.liquid
+++ b/doc/install/install-docker.html.textile.liquid
@@ -4,12 +4,10 @@ navsection: installguide
title: Installing with Docker
...
-{% include 'alert_docker' %}
-
h2. Prerequisites:
# A GNU/Linux (virtual) machine
-# A working Docker installation
+# A working Docker installation (see "Installing Docker":https://docs.docker.com/installation/)
h2. Download the source tree
@@ -34,6 +32,7 @@ parameters:
<pre>
PUBLIC_KEY_PATH
+ARVADOS_USER_NAME
API_HOSTNAME
API_AUTO_ADMIN_USER
</pre>
diff --git a/doc/user/getting_started/check-environment.html.textile.liquid b/doc/user/getting_started/check-environment.html.textile.liquid
index c9d4778..287d708 100644
--- a/doc/user/getting_started/check-environment.html.textile.liquid
+++ b/doc/user/getting_started/check-environment.html.textile.liquid
@@ -37,4 +37,4 @@ However, if you receive the following message:
bc. ARVADOS_API_HOST and ARVADOS_API_TOKEN need to be defined as environment variables
-Then follow the instructions for "getting an API token,":{{site.baseurl}}/user/reference/api-tokens.html and try @arv user current@ again.
+follow the instructions for "getting an API token,":{{site.baseurl}}/user/reference/api-tokens.html and try @arv user current@ again.
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index 0446de2..cba4759 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -43,3 +43,20 @@ grep -q keep_server_1 /tmp/keep_service.list
if [[ "$?" != "0" ]]; then
arv keep_service create --keep-service "$(cat /root/keep_server_1.json)"
fi
+
+# User repository object
+user_uuid=`arv --format=uuid user current`
+repo_uuid=`arv --format=uuid repository create --repository '{"name":"@@ARVADOS_USER_NAME@@","fetch_url":"git at api:@@ARVADOS_USER_NAME@@.git","push_url":"git at api:@@ARVADOS_USER_NAME@@.git"}'`
+echo "User repository uuid is $repo_uuid"
+
+read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
+{
+ "tail_uuid":"$user_uuid",
+ "head_uuid":"$repo_uuid",
+ "link_class":"permission",
+ "name":"can_write"
+}
+EOF
+
+# Shell machine object
+arv virtual_machine create --virtual-machine '{"hostname":"shell"}'
diff --git a/docker/arvdock b/docker/arvdock
index 9336ae1..18472cf 100755
--- a/docker/arvdock
+++ b/docker/arvdock
@@ -19,6 +19,7 @@ function usage {
echo >&2 " -s[port], --sso[=port] SSO server (default port 9901)"
echo >&2 " -a[port], --api[=port] API server (default port 9900)"
echo >&2 " -c[count], --compute[=count] Compute nodes (default starts 2)"
+ echo >&2 " -v, --vm Shell server"
echo >&2 " -n, --nameserver Nameserver"
echo >&2 " -k, --keep Keep servers"
echo >&2 " --ssh Enable SSH access to server containers"
@@ -44,14 +45,14 @@ function start_container {
if [[ "$2" != '' ]]; then
local name="$2"
if [[ "$name" == "api_server" ]]; then
- args="$args --hostname api -P --name $name"
+ args="$args --dns=172.17.42.1 --dns-search=compute.dev.arvados --hostname api -P --name $name"
elif [[ "$name" == "compute" ]]; then
name=$name$COMPUTE_COUNTER
# We need --privileged because we run docker-inside-docker on the compute nodes
- args="$args --hostname compute$COMPUTE_COUNTER -P --privileged --name $name"
+ args="$args --dns=172.17.42.1 --dns-search=compute.dev.arvados --hostname compute$COMPUTE_COUNTER -P --privileged --name $name"
let COMPUTE_COUNTER=$(($COMPUTE_COUNTER + 1))
else
- args="$args --name $name"
+ args="$args --dns=172.17.42.1 --dns-search=dev.arvados --hostname ${name#_server} --name $name"
fi
fi
if [[ "$3" != '' ]]; then
@@ -81,12 +82,12 @@ function start_container {
echo "Starting container:"
#echo " $DOCKER run --dns=127.0.0.1 $args $image"
echo " $DOCKER run $args $image"
- container=`$DOCKER run --dns=172.17.42.1 --dns-search=compute.dev.arvados $args $image`
+ container=`$DOCKER run $args $image`
if [[ "$?" != "0" ]]; then
echo "Unable to start container"
exit 1
fi
- if $ENABLE_SSH
+ if [[ "$name" == "compute" || $ENABLE_SSH ]];
then
ip=$(ip_address $container )
echo
@@ -146,12 +147,13 @@ function do_start {
local start_api=false
local start_compute=false
local start_workbench=false
+ local start_vm=false
local start_nameserver=false
local start_keep=false
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- local TEMP=`getopt -o d::s::a::c::w::nkh \
- --long doc::,sso::,api::,compute::,workbench::,nameserver,keep,help,ssh \
+ local TEMP=`getopt -o d::s::a::c::w::nkvh \
+ --long doc::,sso::,api::,compute::,workbench::,nameserver,keep,vm,help,ssh \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -192,6 +194,10 @@ function do_start {
*) start_workbench=$2; shift 2 ;;
esac
;;
+ -v | --vm)
+ start_vm=true
+ shift
+ ;;
-n | --nameserver)
start_nameserver=true
shift
@@ -222,6 +228,7 @@ function do_start {
$start_api == false &&
$start_compute == false &&
$start_workbench == false &&
+ $start_vm == false &&
$start_nameserver == false &&
$start_keep == false ]]
then
@@ -230,6 +237,7 @@ function do_start {
start_api=9900
start_compute=2
start_workbench=9899
+ start_vm=true
start_nameserver=true
start_keep=true
fi
@@ -303,6 +311,11 @@ function do_start {
start_container "$start_doc:80" "doc_server" '' '' "arvados/doc"
fi
+ if [[ $start_vm != false ]]
+ then
+ start_container "" "shell" '' "api_server:api" "arvados/shell"
+ fi
+
if [[ $start_workbench != false ]]
then
start_container "$start_workbench:80" "workbench_server" '' "api_server:api" "arvados/workbench"
@@ -326,11 +339,12 @@ function do_stop {
local stop_compute=""
local stop_workbench=""
local stop_nameserver=""
+ local stop_vm=""
local stop_keep=""
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- local TEMP=`getopt -o dsacwnkh \
- --long doc,sso,api,compute,workbench,nameserver,keep,help \
+ local TEMP=`getopt -o dsacwnkvh \
+ --long doc,sso,api,compute,workbench,nameserver,keep,vm,help \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -352,7 +366,9 @@ function do_stop {
-w | --workbench)
stop_workbench=workbench_server ; shift ;;
-n | --nameserver )
- stop_keep="skydock skydns" ; shift ;;
+ stop_nameserver="skydock skydns" ; shift ;;
+ -v | --vm )
+ stop_vm="shell" ; shift ;;
-k | --keep )
stop_keep="keep_server_0 keep_server_1" ; shift ;;
--)
@@ -372,6 +388,7 @@ function do_stop {
$stop_api == "" &&
$stop_compute == "" &&
$stop_workbench == "" &&
+ $stop_vm == "" &&
$stop_nameserver == "" &&
$stop_keep == "" ]]
then
@@ -380,11 +397,12 @@ function do_stop {
stop_api=api_server
stop_compute=`$DOCKER ps |grep -P "compute\d+" |grep -v api_server |cut -f1 -d ' '`
stop_workbench=workbench_server
+ stop_vm=shell
stop_nameserver="skydock skydns"
stop_keep="keep_server_0 keep_server_1"
fi
- $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep \
+ $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_vm \
2>/dev/null
}
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 223440e..7b1565a 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -1,17 +1,28 @@
-all: api-image compute-image doc-image workbench-image keep-image sso-image
+all: api-image compute-image doc-image workbench-image keep-image sso-image shell-image
# `make clean' removes the files generated in the build directory
# but does not remove any docker images generated in previous builds
clean:
-rm -rf build
- -rm *-image */generated/*
+ -rm `ls *-image |grep -v debian-arvados-image` */generated/*
- at rmdir */generated
-# `make realclean' will also remove the docker images and force
-# subsequent makes to build the entire chain from the ground up
+DEBIAN_IMAGE := $(shell $(DOCKER) images -q arvados/debian |head -n1)
+
+# `make realclean' will also remove the Arvados docker images (but not the
+# arvados/debian image) and force subsequent makes to build the entire chain
+# from the ground up
realclean: clean
-[ -n "`$(DOCKER) ps -q`" ] && $(DOCKER) stop `$(DOCKER) ps -q`
-$(DOCKER) rm `$(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e doc_server -e workbench_server |cut -f 1 -d' '`
+ -$(DOCKER) rmi `$(DOCKER) images -q arvados/* |grep -v $(DEBIAN_IMAGE)`
+
+# `make mrproper' will remove all Arvados docker images and force
+# subsequent makes to build the entire chain from the ground up
+mrproper: clean
+ -rm debian-arvados-image
+ -[ -n "`$(DOCKER) ps -q`" ] && $(DOCKER) stop `$(DOCKER) ps -q`
+ -$(DOCKER) rm `$(DOCKER) ps -a |grep -e arvados -e api_server -e keep_server -e doc_server -e workbench_server |cut -f 1 -d' '`
-$(DOCKER) rmi `$(DOCKER) images -q arvados/*`
# ============================================================
@@ -32,6 +43,8 @@ JAVA_BWA_SAMTOOLS_DEPS = java-bwa-samtools/Dockerfile
API_DEPS = api/* $(API_GENERATED)
+SHELL_DEPS = shell/* $(SHELL_GENERATED)
+
COMPUTE_DEPS = compute/* $(COMPUTE_GENERATED)
DOC_DEPS = doc/Dockerfile doc/apache2_vhost
@@ -77,6 +90,14 @@ API_GENERATED_IN = \
api/slurm.conf.in \
api/superuser_token.in
+SHELL_GENERATED = \
+ shell/generated/setup.sh \
+ shell/generated/superuser_token
+
+SHELL_GENERATED_IN = \
+ shell/setup.sh.in \
+ shell/superuser_token.in
+
SLURM_GENERATED = \
slurm/generated/slurm.conf
@@ -124,6 +145,9 @@ $(BASE_GENERATED): config.yml $(BUILD)
$(API_GENERATED): config.yml $(API_GENERATED_IN)
$(CONFIG_RB)
+$(SHELL_GENERATED): config.yml $(SHELL_GENERATED_IN)
+ $(CONFIG_RB)
+
$(WORKBENCH_GENERATED): config.yml $(WORKBENCH_GENERATED_IN)
$(CONFIG_RB)
@@ -144,6 +168,7 @@ DOCKER_BUILD = $(DOCKER) build -q --rm=true
# The main Arvados servers: api, doc, workbench, warehouse
api-image: passenger-image $(BUILD) $(API_DEPS)
+ @echo "Building api-image"
mkdir -p api/generated
tar -czf api/generated/api.tar.gz -C build/services api
chmod 755 api/generated/setup.sh
@@ -151,7 +176,11 @@ api-image: passenger-image $(BUILD) $(API_DEPS)
$(DOCKER_BUILD) -t arvados/api api
date >api-image
-slurm-image: base-image $(SLURM_DEPS)
+shell-image: base-image $(BUILD) $(SHELL_DEPS)
+ mkdir -p shell/generated
+ chmod 755 shell/generated/setup.sh
+ $(DOCKER_BUILD) -t arvados/shell shell
+ date >shell-image
compute-image: slurm-image $(BUILD) $(COMPUTE_DEPS)
chmod 755 compute/generated/setup.sh
@@ -164,7 +193,7 @@ doc-image: base-image $(BUILD) $(DOC_DEPS)
$(DOCKER_BUILD) -t arvados/doc doc
date >doc-image
-keep-image: debian-image $(BUILD) $(KEEP_DEPS)
+keep-image: debian-arvados-image $(BUILD) $(KEEP_DEPS)
$(DOCKER_BUILD) -t arvados/keep keep
date >keep-image
@@ -208,10 +237,11 @@ slurm-image: base-image $(SLURM_DEPS)
$(DOCKER_BUILD) -t arvados/slurm slurm
date >slurm-image
-base-image: debian-image $(BASE_DEPS)
+base-image: debian-arvados-image $(BASE_DEPS)
+ @echo "Building base-image"
$(DOCKER_BUILD) -t arvados/base base
date >base-image
-debian-image:
+debian-arvados-image:
./mkimage-debootstrap.sh arvados/debian wheezy ftp://ftp.us.debian.org/debian/
- date >debian-image
+ date >debian-arvados-image
diff --git a/docker/config.yml.example b/docker/config.yml.example
index 515fcbe..30fc1d4 100644
--- a/docker/config.yml.example
+++ b/docker/config.yml.example
@@ -7,6 +7,10 @@
# true when starting the container.
PUBLIC_KEY_PATH:
+# Username for your Arvados user. This will be used as your shell login name
+# as well as the name for your git repository.
+ARVADOS_USER_NAME:
+
# ARVADOS_DOMAIN: the Internet domain of this installation.
# ARVADOS_DNS_SERVER: the authoritative nameserver for ARVADOS_DOMAIN.
ARVADOS_DOMAIN: # e.g. arvados.internal
diff --git a/docker/shell/Dockerfile b/docker/shell/Dockerfile
new file mode 100644
index 0000000..1e1c883
--- /dev/null
+++ b/docker/shell/Dockerfile
@@ -0,0 +1,19 @@
+# Slurm node Docker container.
+
+FROM arvados/base
+MAINTAINER Ward Vandewege <ward at curoverse.com>
+
+RUN apt-get update && apt-get -qqy install supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-fuse cron vim
+
+ADD fuse.conf /etc/fuse.conf
+
+ADD generated/superuser_token /tmp/superuser_token
+
+RUN /usr/local/rvm/bin/rvm-exec default gem install arvados-cli arvados
+
+# Supervisor.
+ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
+ADD generated/setup.sh /usr/local/bin/setup.sh
+
+# Start the supervisor.
+CMD ["/usr/bin/supervisord", "-n"]
diff --git a/docker/shell/fuse.conf b/docker/shell/fuse.conf
new file mode 100644
index 0000000..4ed21ba
--- /dev/null
+++ b/docker/shell/fuse.conf
@@ -0,0 +1,10 @@
+# Set the maximum number of FUSE mounts allowed to non-root users.
+# The default is 1000.
+#
+#mount_max = 1000
+
+# Allow non-root users to specify the 'allow_other' or 'allow_root'
+# mount options.
+#
+user_allow_other
+
diff --git a/docker/shell/setup.sh.in b/docker/shell/setup.sh.in
new file mode 100755
index 0000000..2815201
--- /dev/null
+++ b/docker/shell/setup.sh.in
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+USER_NAME="@@ARVADOS_USER_NAME@@"
+
+useradd $USER_NAME -s /bin/bash
+mkdir /home/$USER_NAME/.ssh -p
+
+cp ~root/.ssh/authorized_keys /home/$USER_NAME/.ssh/authorized_keys
+
+# Install our token
+mkdir -p /home/$USER_NAME/.config/arvados;
+echo "ARVADOS_API_HOST=api" >> /home/$USER_NAME/.config/arvados/settings.conf
+echo "ARVADOS_API_HOST_INSECURE=yes" >> /home/$USER_NAME/.config/arvados/settings.conf
+echo "ARVADOS_API_TOKEN=$(cat /tmp/superuser_token)" >> /home/$USER_NAME/.config/arvados/settings.conf
+chmod 600 /home/$USER_NAME/.config/arvados/settings.conf
+
+chown $USER_NAME:$USER_NAME /home/$USER_NAME -R
+
+rm -f /tmp/superuser_token
+
+
diff --git a/docker/shell/superuser_token.in b/docker/shell/superuser_token.in
new file mode 100644
index 0000000..49bb34e
--- /dev/null
+++ b/docker/shell/superuser_token.in
@@ -0,0 +1 @@
+@@API_SUPERUSER_SECRET@@
diff --git a/docker/shell/supervisor.conf b/docker/shell/supervisor.conf
new file mode 100644
index 0000000..97ad540
--- /dev/null
+++ b/docker/shell/supervisor.conf
@@ -0,0 +1,15 @@
+[program:ssh]
+user=root
+command=/etc/init.d/ssh start
+startsecs=0
+
+[program:cron]
+user=root
+command=/etc/init.d/cron start
+startsecs=0
+
+[program:setup]
+user=root
+command=/usr/local/bin/setup.sh
+startsecs=0
+
commit 94c6d21aed9f33599f7d8eee29481b47b7bb99f7
Author: Ward Vandewege <ward at curoverse.com>
Date: Wed Jul 16 10:23:52 2014 -0400
Whitespace cleanup.
refs #3219
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index c25a0d6..0446de2 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -22,13 +22,13 @@ all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados","fetch_url":"git at api:arvados.git","push_url":"git at api:arvados.git"}'`
echo "Arvados repository uuid is $repo_uuid"
-read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
+read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
{
"tail_uuid":"$all_users_group_uuid",
"head_uuid":"$repo_uuid",
"link_class":"permission",
- "name":"can_read"
-}
+ "name":"can_read"
+}
EOF
# Make sure the necessary keep_service objects exist
diff --git a/docker/api/update-gitolite.rb b/docker/api/update-gitolite.rb
index 1623a8b..779099a 100755
--- a/docker/api/update-gitolite.rb
+++ b/docker/api/update-gitolite.rb
@@ -1,9 +1,5 @@
#!/usr/bin/env ruby
-###################################################################
-# THIS FILE IS MANAGED BY PUPPET -- CHANGES WILL BE OVERWRITTEN #
-###################################################################
-
require 'rubygems'
require 'pp'
require 'arvados'
@@ -143,12 +139,12 @@ begin
Dir.glob(gitolite_admin + '/keydir/arvados/*.pub') do |key_file|
next if key_file =~ /arvados_git_user.pub$/
next if @seen.has_key?(key_file)
- puts "Extra file #{key_file}"
+ puts "Extra file #{key_file}"
@commit = true
Dir.chdir(gitolite_admin)
key_file.gsub!(/^#{gitolite_admin}\//,'')
`git rm #{key_file}`
- end
+ end
if @commit then
message = "#{Time.now().to_s}: update from API"
diff --git a/docker/compute/wrapdocker b/docker/compute/wrapdocker
index e714d5b..14f3a75 100755
--- a/docker/compute/wrapdocker
+++ b/docker/compute/wrapdocker
@@ -7,10 +7,10 @@
CGROUP=/sys/fs/cgroup
: {LOG:=stdio}
-[ -d $CGROUP ] ||
+[ -d $CGROUP ] ||
mkdir $CGROUP
-mountpoint -q $CGROUP ||
+mountpoint -q $CGROUP ||
mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
echo "Could not make a tmpfs mount. Did you use -privileged?"
exit 1
@@ -28,7 +28,7 @@ fi
for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
do
[ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
- mountpoint -q $CGROUP/$SUBSYS ||
+ mountpoint -q $CGROUP/$SUBSYS ||
mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
# The two following sections address a bug which manifests itself
commit c58d996cd35e754c56a7c223414f04d0b362c284
Author: Ward Vandewege <ward at curoverse.com>
Date: Wed Jul 16 10:20:50 2014 -0400
Further changes to make pipeline running work reliably.
refs #3219
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index ad85315..6a70fc3 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -4,14 +4,10 @@ FROM arvados/passenger
MAINTAINER Tim Pierce <twp at curoverse.com>
# Install postgres and apache.
-# Clone a git repository of Arvados source -- not used to build, but
-# will be used by the Commit model and anything else that needs to
-# check a git repo for crunch scripts.
RUN apt-get update && \
apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
- libjson-perl cron && \
- git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
+ libjson-perl cron
ADD munge.key /etc/munge/
RUN chown munge:munge /etc/munge/munge.key && chmod 600 /etc/munge/munge.key
@@ -75,7 +71,7 @@ ADD update-gitolite.rb /usr/local/arvados/
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
ADD ssh.sh /usr/local/bin/ssh.sh
ADD generated/setup.sh /usr/local/bin/setup.sh
-ADD setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
+ADD generated/setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
ADD crunch-dispatch-run.sh /usr/local/bin/crunch-dispatch-run.sh
ADD apache2_foreground.sh /etc/apache2/foreground.sh
diff --git a/docker/api/application.yml.in b/docker/api/application.yml.in
index f832b7f..355c4e5 100644
--- a/docker/api/application.yml.in
+++ b/docker/api/application.yml.in
@@ -51,8 +51,10 @@ production:
# end << '172.16.0.23'
# %>
permit_create_collection_with_unsigned_manifest: true
- git_repositories_dir: /var/cache/git
+ git_repositories_dir: /home/git/repositories
crunch_job_wrapper: :slurm_immediate
+ action_mailer.raise_delivery_errors: false
+ action_mailer.perform_deliveries: false
test:
uuid_prefix: zzzzz
diff --git a/docker/api/setup-gitolite.sh b/docker/api/setup-gitolite.sh.in
similarity index 60%
rename from docker/api/setup-gitolite.sh
rename to docker/api/setup-gitolite.sh.in
index c55fdee..0363063 100755
--- a/docker/api/setup-gitolite.sh
+++ b/docker/api/setup-gitolite.sh.in
@@ -14,6 +14,14 @@ su - git -c "git clone git://github.com/sitaramc/gitolite"
su - git -c "gitolite/install -ln ~/bin"
su - git -c "PATH=/home/git/bin:$PATH gitolite setup -pk ~git/root-authorized_keys.pub"
+# Make sure the repositories are created in such a way that they are readable
+# by the api server
+sed -i 's/0077/0022/g' /home/git/.gitolite.rc
+
+# And make sure that the existing repos are equally readable, or the API server commit model will freak out...
+chmod 755 /home/git/repositories
+chmod +rx /home/git/repositories/*git -R
+
# Now set up the gitolite repo(s) we use
mkdir -p /usr/local/arvados/gitolite-tmp/
# Make ssh store the host key
@@ -50,6 +58,20 @@ git add conf/gitolite.conf
git commit -a -m 'git server setup'
git push
+# Prepopulate the arvados.git repo with our source. Silly, but until we can check out from remote trees,
+# we need this to make the tutorials work.
+su - git -c "git clone --bare git://github.com/curoverse/arvados.git /home/git/repositories/arvados.git"
+
echo "ARVADOS_API_HOST_INSECURE=yes" > /etc/cron.d/gitolite-update
-echo "*/5 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+echo "*/2 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+
+# Create/update the repos now
+. /etc/profile.d/rvm.sh
+export ARVADOS_API_HOST=api
+export ARVADOS_API_HOST_INSECURE=yes
+export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
+/usr/local/arvados/update-gitolite.rb production
+
+echo "PATH=/usr/bin:/bin:/sbin" > /etc/cron.d/arvados-repo-update
+echo "*/5 * * * * git cd ~git/repositories/arvados.git; git fetch https://github.com/curoverse/arvados.git master:master" >> /etc/cron.d/arvados-repo-update
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index f107109..c25a0d6 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -19,7 +19,7 @@ EOF
# Arvados repository object
all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
-repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados"}'`
+repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados","fetch_url":"git at api:arvados.git","push_url":"git at api:arvados.git"}'`
echo "Arvados repository uuid is $repo_uuid"
read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 172da98..223440e 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -61,6 +61,7 @@ API_GENERATED = \
api/generated/omniauth.rb \
api/generated/application.yml \
api/generated/setup.sh \
+ api/generated/setup-gitolite.sh \
api/generated/slurm.conf \
api/generated/superuser_token
@@ -72,6 +73,7 @@ API_GENERATED_IN = \
api/omniauth.rb.in \
api/application.yml.in \
api/setup.sh.in \
+ api/setup-gitolite.sh.in \
api/slurm.conf.in \
api/superuser_token.in
@@ -145,6 +147,7 @@ api-image: passenger-image $(BUILD) $(API_DEPS)
mkdir -p api/generated
tar -czf api/generated/api.tar.gz -C build/services api
chmod 755 api/generated/setup.sh
+ chmod 755 api/generated/setup-gitolite.sh
$(DOCKER_BUILD) -t arvados/api api
date >api-image
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list