[ARVADOS] created: 2.1.0-2065-gafec9ddba

Git user git at public.arvados.org
Wed Mar 16 13:21:11 UTC 2022


        at  afec9ddba0b865918815ca3345278decc1fc6697 (commit)


commit afec9ddba0b865918815ca3345278decc1fc6697
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Mar 16 08:43:42 2022 -0400

    18766: be a bit smarter about the preseeding of the go module cache and
    the ruby gems in our docker images used for package building: make sure
    to preseed with the currently checked out branch of the source tree,
    not 'main'. This avoids potential compatibility issues between the
    version of Ruby and certain gems.
    
    The preseeding is worthwhile to speed up repeated package builds. When
    testing on an 8 core i7-10510U:
    
    * save ~60s on rails package build if the gems are fresh
    * save ~30s on go package build if the go mods are fresh
    * total gain for worst case scenario (workbench1 package) is almost 50%
     (85 seconds): ~3m down to ~1m35s
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh
index 5ed33dc9f..5f8817f20 100755
--- a/build/package-build-dockerfiles/build-all-build-containers.sh
+++ b/build/package-build-dockerfiles/build-all-build-containers.sh
@@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do
   target=${target#./}
   echo $target
   cd $target
-  docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
+  docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
   cd ..
 done
 
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 01662d6ac..e44d231ed 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM centos:7 as build_x86_64
 # Install go
@@ -64,7 +65,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index edd47cf69..ed0a0cdc1 100644
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 ## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
 FROM debian:buster as build_x86_64
@@ -51,7 +52,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index cb4c695c4..cfeaf2463 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 ## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
 FROM debian:bullseye as build_x86_64
@@ -56,7 +57,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index b026fa2a8..9b20b41a4 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM ubuntu:bionic as build_x86_64
 # Install go
@@ -50,7 +51,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 1457670ce..f28e6fef1 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM ubuntu:focal as build_x86_64
 # Install go
@@ -61,7 +62,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index e06a73297..c1cc2e587 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -195,7 +195,7 @@ fi
 
 echo $TARGET
 cd $TARGET
-time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
 popd
 
 if test -z "$packages" ; then
@@ -307,16 +307,24 @@ else
     set +e
     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
     set -e
+    # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
+    # docker run command below.
+    # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+    tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
+    cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+    printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
     # Build packages.
     if docker run \
         --rm \
         "${docker_volume_args[@]}" \
+        -v $tmpfile:/root/.bundle/config \
         --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
         --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
         --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
         --env "ONLY_BUILD=$ONLY_BUILD" \
         --env "FORCE_BUILD=$FORCE_BUILD" \
         --env "ARCH=$ARCH" \
+        --env "MAKE=make --jobs $cores" \
         "$IMAGE" $COMMAND
     then
         echo
@@ -325,6 +333,8 @@ else
         FINAL_EXITCODE=$?
         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
     fi
+    # Clean up the bundle config file
+    rm -f $tmpfile
 fi
 
 if test -n "$package_fails" ; then

commit 221c808a04b490a07123fa44b8630c2f712f5977
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Mar 16 08:42:56 2022 -0400

    18766: deal with deprecation warnings for our bundler invocation.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/run-library.sh b/build/run-library.sh
index 0257e6524..fa2be6ac7 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -492,7 +492,8 @@ handle_rails_package() {
         cd "$srcdir"
         mkdir -p tmp
         git rev-parse HEAD >git-commit.version
-        bundle package --all
+        bundle config set cache_all true
+        bundle package
     )
     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
         echo "ERROR: $pkgname package prep failed" >&2
@@ -603,7 +604,8 @@ handle_workbench () {
 
         # We need to bundle to be ready even when we build a package without vendor directory
         # because asset compilation requires it.
-        bundle install --system >"$STDOUT_IF_DEBUG"
+        bundle config set --local system 'true' >"$STDOUT_IF_DEBUG"
+        bundle install >"$STDOUT_IF_DEBUG"
 
         # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
         # and we want that in the package, so it's easier to not exclude the tmp directory

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list