[ARVADOS] updated: a8a65836a931811e869a3abba2244cc529ef2dd6
Git user
git at public.curoverse.com
Mon Apr 11 16:20:51 EDT 2016
Summary of changes:
tools/arvbox/bin/arvbox | 9 ++++++++-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 4 ++++
tools/arvbox/lib/arvbox/docker/common.sh | 8 ++++----
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 8 ++++----
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 11 ++++++++---
.../lib/arvbox/docker/service/arv-git-httpd/run-service | 12 ++++++++----
.../arvbox/docker/service/crunch-dispatch-local/run-service | 8 ++++----
tools/arvbox/lib/arvbox/docker/service/keep-web/run-service | 12 ++++++++----
tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service | 12 ++++++++----
tools/arvbox/lib/arvbox/docker/service/sdk/run-service | 4 ++--
10 files changed, 58 insertions(+), 30 deletions(-)
via a8a65836a931811e869a3abba2244cc529ef2dd6 (commit)
from 4ab126d1574b7db2fdb5b0cea253b2df28d7b130 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a8a65836a931811e869a3abba2244cc529ef2dd6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Apr 11 16:19:11 2016 -0400
Separate out Go and Python dependencies into separate directories that don't
get deleted by "reset" because they include code from downloading external
dependencies. Add -data-manager-token to keepstore invocation for datamanager
testing.
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index e21db0d..928adf8 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -47,6 +47,8 @@ PG_DATA="$ARVBOX_DATA/postgres"
VAR_DATA="$ARVBOX_DATA/var"
PASSENGER="$ARVBOX_DATA/passenger"
GEMS="$ARVBOX_DATA/gems"
+PIPCACHE="$ARVBOX_DATA/pip"
+GOSTUFF="$ARVBOX_DATA/gopath"
getip() {
docker inspect $ARVBOX_CONTAINER | grep \"IPAddress\" | head -n1 | tr -d ' ":,\n' | cut -c10-
@@ -149,7 +151,8 @@ run() {
updateconf
wait_for_arvbox
else
- mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS"
+ mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$GOSTUFF"
+
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://github.com/curoverse/arvados.git "$ARVADOS_ROOT"
@@ -173,6 +176,8 @@ run() {
"--volume=$VAR_DATA:/var/lib/arvados:rw" \
"--volume=$PASSENGER:/var/lib/passenger:rw" \
"--volume=$GEMS:/var/lib/gems:rw" \
+ "--volume=$PIPCACHE:/var/lib/pip:rw" \
+ "--volume=$GOSTUFF:/var/lib/gopath:rw" \
arvados/arvbox-dev \
/usr/local/bin/runsvinit -svdir=/etc/test-service
@@ -210,6 +215,8 @@ run() {
"--volume=$VAR_DATA:/var/lib/arvados:rw" \
"--volume=$PASSENGER:/var/lib/passenger:rw" \
"--volume=$GEMS:/var/lib/gems:rw" \
+ "--volume=$PIPCACHE:/var/lib/pip:rw" \
+ "--volume=$GOSTUFF:/var/lib/gopath:rw" \
$PUBLIC \
arvados/arvbox-dev
updateconf
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 1f13415..a04c06d 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -14,4 +14,8 @@ RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/workbench/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/keep-web/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
+RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 4c2de47..0e50a06 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -37,13 +37,13 @@ run_bundler() {
else
frozen=""
fi
- if ! flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
- flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
+ if ! flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
+ flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
fi
}
pip_install() {
- pushd /var/lib/arvados/pip
+ pushd /var/lib/pip
for p in $(ls http*.tar.gz) ; do
if test -f $p ; then
ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
@@ -56,7 +56,7 @@ pip_install() {
done
popd
- if ! pip install --no-index --find-links /var/lib/arvados/pip $1 ; then
+ if ! pip install --no-index --find-links /var/lib/pip $1 ; then
pip install $1
fi
}
diff --git a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
index a3397c1..e447fe7 100755
--- a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
@@ -5,14 +5,14 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/sdk/go/crunchrunner"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/sdk/go/crunchrunner"
install bin/crunchstat bin/crunchrunner /usr/local/bin
export ARVADOS_API_HOST=$localip:${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/keep-setup.sh b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
index b66463f..f77de10 100755
--- a/tools/arvbox/lib/arvbox/docker/keep-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
@@ -6,15 +6,19 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepstore"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keepstore"
install bin/keepstore /usr/local/bin
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
mkdir -p /var/lib/arvados/$1
export ARVADOS_API_HOST=$localip:${services[api]}
@@ -47,5 +51,6 @@ exec /usr/local/bin/keepstore \
-listen=:$2 \
-enforce-permissions=true \
-blob-signing-key-file=/var/lib/arvados/blob_signing_key \
+ -data-manager-token-file=/var/lib/arvados/superuser_token \
-max-buffers=20 \
-volume=/var/lib/arvados/$1
diff --git a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
index 854464e..518fe33 100755
--- a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
@@ -1,19 +1,23 @@
#!/bin/bash
exec 2>&1
-set -eux -o pipefail
+set -ex -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/arv-git-httpd"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/arv-git-httpd"
install bin/arv-git-httpd /usr/local/bin
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
export ARVADOS_API_HOST=$localip:${services[api]}
export ARVADOS_API_HOST_INSECURE=1
export GITOLITE_HTTP_HOME=/var/lib/arvados/git
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
index 211b438..c2d2cb8 100755
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
@@ -5,14 +5,14 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-run"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-dispatch-local"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunch-run"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunch-dispatch-local"
install bin/crunch-run bin/crunch-dispatch-local /usr/local/bin
export ARVADOS_API_HOST=$localip:${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
index a2c6aa1..fe53725 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
@@ -1,19 +1,23 @@
#!/bin/bash
exec 2>&1
-set -eux -o pipefail
+set -ex -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keep-web"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keep-web"
install bin/keep-web /usr/local/bin
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
export ARVADOS_API_HOST=$localip:${services[api]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
index 413a67e..00b2e01 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
@@ -2,19 +2,23 @@
exec 2>&1
sleep 2
-set -eux -o pipefail
+set -ex -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepproxy"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keepproxy"
install bin/keepproxy /usr/local/bin
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
export ARVADOS_API_HOST=$localip:${services[api]}
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 3ee6f2a..29452ab 100755
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -5,10 +5,10 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p ~/.pip /var/lib/arvados/pip
+mkdir -p ~/.pip /var/lib/pip
cat > ~/.pip/pip.conf <<EOF
[global]
-download_cache = /var/lib/arvados/pip
+download_cache = /var/lib/pip
EOF
cd /usr/src/arvados/sdk/cli
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list