[ARVADOS] created: 2.1.0-2065-gafec9ddba
Git user
git at public.arvados.org
Wed Mar 16 13:21:11 UTC 2022
at afec9ddba0b865918815ca3345278decc1fc6697 (commit)
commit afec9ddba0b865918815ca3345278decc1fc6697
Author: Ward Vandewege <ward at curii.com>
Date: Wed Mar 16 08:43:42 2022 -0400
18766: be a bit smarter about the preseeding of the go module cache and
the ruby gems in our docker images used for package building: make sure
to preseed with the currently checked out branch of the source tree,
not 'main'. This avoids potential compatibility issues between the
version of Ruby and certain gems.
The preseeding is worthwhile to speed up repeated package builds. When
testing on an 8 core i7-10510U:
* save ~60s on rails package build if the gems are fresh
* save ~30s on go package build if the go mods are fresh
* total gain for worst case scenario (workbench1 package) is almost 50%
(85 seconds): ~3m down to ~1m35s
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh
index 5ed33dc9f..5f8817f20 100755
--- a/build/package-build-dockerfiles/build-all-build-containers.sh
+++ b/build/package-build-dockerfiles/build-all-build-containers.sh
@@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do
target=${target#./}
echo $target
cd $target
- docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
+ docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
cd ..
done
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 01662d6ac..e44d231ed 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM centos:7 as build_x86_64
# Install go
@@ -64,7 +65,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index edd47cf69..ed0a0cdc1 100644
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
FROM debian:buster as build_x86_64
@@ -51,7 +52,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index cb4c695c4..cfeaf2463 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
FROM debian:bullseye as build_x86_64
@@ -56,7 +57,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index b026fa2a8..9b20b41a4 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM ubuntu:bionic as build_x86_64
# Install go
@@ -50,7 +51,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 1457670ce..f28e6fef1 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM ubuntu:focal as build_x86_64
# Install go
@@ -61,7 +62,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index e06a73297..c1cc2e587 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -195,7 +195,7 @@ fi
echo $TARGET
cd $TARGET
-time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
popd
if test -z "$packages" ; then
@@ -307,16 +307,24 @@ else
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
+ # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
+ # docker run command below.
+ # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
+ cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+ printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
# Build packages.
if docker run \
--rm \
"${docker_volume_args[@]}" \
+ -v $tmpfile:/root/.bundle/config \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
--env "FORCE_BUILD=$FORCE_BUILD" \
--env "ARCH=$ARCH" \
+ --env "MAKE=make --jobs $cores" \
"$IMAGE" $COMMAND
then
echo
@@ -325,6 +333,8 @@ else
FINAL_EXITCODE=$?
echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
fi
+ # Clean up the bundle config file
+ rm -f $tmpfile
fi
if test -n "$package_fails" ; then
commit 221c808a04b490a07123fa44b8630c2f712f5977
Author: Ward Vandewege <ward at curii.com>
Date: Wed Mar 16 08:42:56 2022 -0400
18766: deal with deprecation warnings for our bundler invocation.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index 0257e6524..fa2be6ac7 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -492,7 +492,8 @@ handle_rails_package() {
cd "$srcdir"
mkdir -p tmp
git rev-parse HEAD >git-commit.version
- bundle package --all
+ bundle config set cache_all true
+ bundle package
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
@@ -603,7 +604,8 @@ handle_workbench () {
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
+ bundle config set --local system 'true' >"$STDOUT_IF_DEBUG"
+ bundle install >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list