[ARVADOS] updated: a8a65836a931811e869a3abba2244cc529ef2dd6

Git user git at public.curoverse.com
Mon Apr 11 16:20:51 EDT 2016


Summary of changes:
 tools/arvbox/bin/arvbox                                      |  9 ++++++++-
 tools/arvbox/lib/arvbox/docker/Dockerfile.demo               |  4 ++++
 tools/arvbox/lib/arvbox/docker/common.sh                     |  8 ++++----
 tools/arvbox/lib/arvbox/docker/crunch-setup.sh               |  8 ++++----
 tools/arvbox/lib/arvbox/docker/keep-setup.sh                 | 11 ++++++++---
 .../lib/arvbox/docker/service/arv-git-httpd/run-service      | 12 ++++++++----
 .../arvbox/docker/service/crunch-dispatch-local/run-service  |  8 ++++----
 tools/arvbox/lib/arvbox/docker/service/keep-web/run-service  | 12 ++++++++----
 tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service | 12 ++++++++----
 tools/arvbox/lib/arvbox/docker/service/sdk/run-service       |  4 ++--
 10 files changed, 58 insertions(+), 30 deletions(-)

       via  a8a65836a931811e869a3abba2244cc529ef2dd6 (commit)
      from  4ab126d1574b7db2fdb5b0cea253b2df28d7b130 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit a8a65836a931811e869a3abba2244cc529ef2dd6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Apr 11 16:19:11 2016 -0400

    Separate out Go and Python dependencies into separate directories that don't
    get deleted by "reset" because they include code from downloading external
    dependencies.  Add -data-manager-token to keepstore invocation for datamanager
    testing.

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index e21db0d..928adf8 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -47,6 +47,8 @@ PG_DATA="$ARVBOX_DATA/postgres"
 VAR_DATA="$ARVBOX_DATA/var"
 PASSENGER="$ARVBOX_DATA/passenger"
 GEMS="$ARVBOX_DATA/gems"
+PIPCACHE="$ARVBOX_DATA/pip"
+GOSTUFF="$ARVBOX_DATA/gopath"
 
 getip() {
     docker inspect $ARVBOX_CONTAINER | grep \"IPAddress\" | head -n1 | tr -d ' ":,\n' | cut -c10-
@@ -149,7 +151,8 @@ run() {
         updateconf
         wait_for_arvbox
     else
-        mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS"
+        mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$GOSTUFF"
+
 
         if ! test -d "$ARVADOS_ROOT" ; then
             git clone https://github.com/curoverse/arvados.git "$ARVADOS_ROOT"
@@ -173,6 +176,8 @@ run() {
                    "--volume=$VAR_DATA:/var/lib/arvados:rw" \
                    "--volume=$PASSENGER:/var/lib/passenger:rw" \
                    "--volume=$GEMS:/var/lib/gems:rw" \
+                   "--volume=$PIPCACHE:/var/lib/pip:rw" \
+                   "--volume=$GOSTUFF:/var/lib/gopath:rw" \
                    arvados/arvbox-dev \
                    /usr/local/bin/runsvinit -svdir=/etc/test-service
 
@@ -210,6 +215,8 @@ run() {
                    "--volume=$VAR_DATA:/var/lib/arvados:rw" \
                    "--volume=$PASSENGER:/var/lib/passenger:rw" \
                    "--volume=$GEMS:/var/lib/gems:rw" \
+                   "--volume=$PIPCACHE:/var/lib/pip:rw" \
+                   "--volume=$GOSTUFF:/var/lib/gopath:rw" \
                    $PUBLIC \
                    arvados/arvbox-dev
             updateconf
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 1f13415..a04c06d 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -14,4 +14,8 @@ RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/workbench/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/keep-web/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
+RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 4c2de47..0e50a06 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -37,13 +37,13 @@ run_bundler() {
     else
         frozen=""
     fi
-    if ! flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
-        flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
+    if ! flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
+        flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
     fi
 }
 
 pip_install() {
-    pushd /var/lib/arvados/pip
+    pushd /var/lib/pip
     for p in $(ls http*.tar.gz) ; do
         if test -f $p ; then
             ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
@@ -56,7 +56,7 @@ pip_install() {
     done
     popd
 
-    if ! pip install --no-index --find-links /var/lib/arvados/pip $1 ; then
+    if ! pip install --no-index --find-links /var/lib/pip $1 ; then
         pip install $1
     fi
 }
diff --git a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
index a3397c1..e447fe7 100755
--- a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
@@ -5,14 +5,14 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/sdk/go/crunchrunner"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/sdk/go/crunchrunner"
 install bin/crunchstat bin/crunchrunner /usr/local/bin
 
 export ARVADOS_API_HOST=$localip:${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/keep-setup.sh b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
index b66463f..f77de10 100755
--- a/tools/arvbox/lib/arvbox/docker/keep-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
@@ -6,15 +6,19 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepstore"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keepstore"
 install bin/keepstore /usr/local/bin
 
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
 mkdir -p /var/lib/arvados/$1
 
 export ARVADOS_API_HOST=$localip:${services[api]}
@@ -47,5 +51,6 @@ exec /usr/local/bin/keepstore \
      -listen=:$2 \
      -enforce-permissions=true \
      -blob-signing-key-file=/var/lib/arvados/blob_signing_key \
+     -data-manager-token-file=/var/lib/arvados/superuser_token \
      -max-buffers=20 \
      -volume=/var/lib/arvados/$1
diff --git a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
index 854464e..518fe33 100755
--- a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
@@ -1,19 +1,23 @@
 #!/bin/bash
 
 exec 2>&1
-set -eux -o pipefail
+set -ex -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/arv-git-httpd"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/arv-git-httpd"
 install bin/arv-git-httpd /usr/local/bin
 
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
 export ARVADOS_API_HOST=$localip:${services[api]}
 export ARVADOS_API_HOST_INSECURE=1
 export GITOLITE_HTTP_HOME=/var/lib/arvados/git
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
index 211b438..c2d2cb8 100755
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
@@ -5,14 +5,14 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-run"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-dispatch-local"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunch-run"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/crunch-dispatch-local"
 install bin/crunch-run bin/crunch-dispatch-local /usr/local/bin
 
 export ARVADOS_API_HOST=$localip:${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
index a2c6aa1..fe53725 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keep-web/run-service
@@ -1,19 +1,23 @@
 #!/bin/bash
 
 exec 2>&1
-set -eux -o pipefail
+set -ex -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keep-web"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keep-web"
 install bin/keep-web /usr/local/bin
 
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
 export ARVADOS_API_HOST=$localip:${services[api]}
 export ARVADOS_API_HOST_INSECURE=1
 export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
index 413a67e..00b2e01 100755
--- a/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
@@ -2,19 +2,23 @@
 
 exec 2>&1
 sleep 2
-set -eux -o pipefail
+set -ex -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/gostuff
-cd /var/lib/arvados/gostuff
+mkdir -p /var/lib/gopath
+cd /var/lib/gopath
 
 export GOPATH=$PWD
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
-flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepproxy"
+flock /var/lib/gopath/gopath.lock go get -t "git.curoverse.com/arvados.git/services/keepproxy"
 install bin/keepproxy /usr/local/bin
 
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
 export ARVADOS_API_HOST=$localip:${services[api]}
 export ARVADOS_API_HOST_INSECURE=1
 export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 3ee6f2a..29452ab 100755
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -5,10 +5,10 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p ~/.pip /var/lib/arvados/pip
+mkdir -p ~/.pip /var/lib/pip
 cat > ~/.pip/pip.conf <<EOF
 [global]
-download_cache = /var/lib/arvados/pip
+download_cache = /var/lib/pip
 EOF
 
 cd /usr/src/arvados/sdk/cli

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list