[ARVADOS] updated: 1.3.0-3182-gb27c53ded
Git user
git at public.arvados.org
Fri Sep 25 00:50:18 UTC 2020
Summary of changes:
tools/arvbox/bin/arvbox | 17 ++++-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 93 +++++++++++++++++++++-----
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 1 +
tools/arvbox/lib/arvbox/docker/common.sh | 2 +
4 files changed, 93 insertions(+), 20 deletions(-)
via b27c53dedf632f614356305bc624befa5477b98e (commit)
from 154bfc562eafc642cc801f25b3c258e3846633ba (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b27c53dedf632f614356305bc624befa5477b98e
Author: Ward Vandewege <ward at curii.com>
Date: Thu Sep 24 20:49:57 2020 -0400
16267: implement review feedback.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 8fd77f1f2..e8e3fa17c 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -363,6 +363,7 @@ stop() {
}
build() {
+ export DOCKER_BUILDKIT=1
if ! test -f "$ARVBOX_DOCKER/Dockerfile.base" ; then
echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
exit 1
@@ -373,12 +374,24 @@ build() {
FORCE=-f
fi
GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD)
- docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
- docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
if test "$1" = localdemo -o "$1" = publicdemo ; then
+ set +e
+ if which greadlink >/dev/null 2>/dev/null ; then
+ ARVADOS_ROOT=$(greadlink -f $(dirname $0)/../../../)
+ else
+ ARVADOS_ROOT=$(readlink -f $(dirname $0)/../../../)
+ fi
+ set -e
+ docker build --build-arg=BUILDTYPE=demo $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVADOS_ROOT"
+ docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
docker tag $FORCE arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest
else
+ if ! test -d "$ARVADOS_ROOT" ; then
+ git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
+ fi
+ docker build --build-arg=BUILDTYPE=dev $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVADOS_ROOT"
+ docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
docker tag $FORCE arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest
fi
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 9031de79b..115754cac 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -1,9 +1,19 @@
+# syntax = docker/dockerfile:experimental
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
-FROM debian:10
+ARG BUILDTYPE
+
+# We're using poor man's conditionals (see
+# https://www.docker.com/blog/advanced-dockerfiles-faster-builds-and-smaller-images-using-buildkit-and-multistage-builds/)
+# here to dtrt in the dev/test scenario and the demo scenario. In the dev/test
+# scenario, we use the docker context (i.e. the copy of Arvados checked out on
+# the host) to build arvados-server. In the demo scenario, we check out a new
+# tree, and use the $arvados_version commit (passed in via an argument).
+###########################################################################################################
+FROM debian:10-slim as dev
ENV DEBIAN_FRONTEND noninteractive
RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list
@@ -12,22 +22,67 @@ RUN apt-get update && \
apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
golang -t buster-backports
-# The arvbox-specific dependencies are
-# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less
RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
- build-essential ca-certificates git libpam0g-dev \
- gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \
- apt-get clean
+ build-essential ca-certificates git libpam0g-dev
ENV GOPATH /var/lib/gopath
+# the --mount option requires the experimental syntax enabled (enables
+# buildkit) on the first line of this file. This Dockerfile must also be built
+# with the DOCKER_BUILDKIT=1 environment variable set.
+RUN --mount=type=bind,target=/usr/src/arvados \
+ cd /usr/src/arvados && \
+ go mod download && \
+ cd cmd/arvados-server && \
+ go install
+
+###########################################################################################################
+FROM debian:10-slim as demo
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list
+
+RUN apt-get update && \
+ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
+ golang -t buster-backports
+
+RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
+ build-essential ca-certificates git libpam0g-dev
+
+ENV GOPATH /var/lib/gopath
+
+ARG arvados_version
+RUN echo arvados_version is git commit $arvados_version
+
RUN cd /usr/src && \
- git clone https://git.arvados.org/arvados.git && \
- cd arvados && \
+ git clone --no-checkout https://git.arvados.org/arvados.git && \
+ git -C arvados checkout ${arvados_version} && \
+ cd /usr/src/arvados && \
go mod download && \
cd cmd/arvados-server && \
- go install && \
- $GOPATH/bin/arvados-server install -type test
+ go install
+
+###########################################################################################################
+FROM ${BUILDTYPE} as base
+
+###########################################################################################################
+FROM debian:10
+ENV DEBIAN_FRONTEND noninteractive
+
+# The arvbox-specific dependencies are
+# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less
+RUN apt-get update && \
+ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
+ gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \
+ apt-get clean
+
+ENV GOPATH /var/lib/gopath
+RUN echo buildtype is $BUILDTYPE
+
+RUN mkdir -p $GOPATH/bin/
+COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server
+RUN $GOPATH/bin/arvados-server --version
+RUN $GOPATH/bin/arvados-server install -type test
RUN /etc/init.d/postgresql start && \
su postgres -c 'dropuser arvados' && \
@@ -41,7 +96,9 @@ VOLUME /var/lib/docker
VOLUME /var/log/nginx
VOLUME /etc/ssl/private
-ADD 8D81803C0EBFCD88.asc /tmp/
+ARG workdir
+
+ADD $workdir/8D81803C0EBFCD88.asc /tmp/
RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \
rm -f /tmp/8D81803C0EBFCD88.asc
@@ -60,16 +117,16 @@ ENV LC_ALL en_US.UTF-8
ARG arvados_version
RUN echo arvados_version is git commit $arvados_version
-ADD fuse.conf /etc/
+COPY $workdir/fuse.conf /etc/
-ADD gitolite.rc \
- keep-setup.sh common.sh createusers.sh \
- logger runsu.sh waitforpostgres.sh \
- yml_override.py api-setup.sh \
- go-setup.sh devenv.sh cluster-config.sh edit_users.py \
+COPY $workdir/gitolite.rc \
+ $workdir/keep-setup.sh $workdir/common.sh $workdir/createusers.sh \
+ $workdir/logger $workdir/runsu.sh $workdir/waitforpostgres.sh \
+ $workdir/yml_override.py $workdir/api-setup.sh \
+ $workdir/go-setup.sh $workdir/devenv.sh $workdir/cluster-config.sh $workdir/edit_users.py \
/usr/local/lib/arvbox/
-ADD runit /etc/runit
+COPY $workdir/runit /etc/runit
# arvbox mounts a docker volume at $ARVADOS_CONTAINER_PATH, make sure that that
# doesn't overlap with the directory where `arvados-server install -type test`
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 192b2a144..777c71356 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -8,6 +8,7 @@ ARG composer_version=arvados-fork
ARG workbench2_version=master
RUN cd /usr/src && \
+ git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
git -C arvados pull && \
git clone --no-checkout https://github.com/arvados/composer.git && \
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 4bfe9dd51..48d356640 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -77,6 +77,8 @@ run_bundler() {
# flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
# fi
# fi
+ # Make sure to put the gem binaries in the right place
+ flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin
if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
fi
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list