[ARVADOS] updated: 2.1.0-1755-g9e9917f74
Git user
git at public.arvados.org
Thu Dec 23 22:02:22 UTC 2021
Summary of changes:
build/package-build-dockerfiles/Makefile | 16 ++-
.../build-all-build-containers.sh | 2 +-
.../package-build-dockerfiles/debian11/Dockerfile | 54 ++++++----
build/run-build-packages-one-target.sh | 2 +-
build/run-build-packages.sh | 110 +++++++++++----------
build/run-library.sh | 54 ++++++----
6 files changed, 147 insertions(+), 91 deletions(-)
via 9e9917f7423b719d1eaf1ca62b2deade25fea55a (commit)
from 3203561dd0affdcaa1d34ae9b44c07f28201043a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9e9917f7423b719d1eaf1ca62b2deade25fea55a
Author: Ward Vandewege <ward at curii.com>
Date: Thu Dec 23 13:00:06 2021 -0500
17417: add native build support for aarch64 (arm64) for our Debian 11
target. Also speed up the docker image build by telling RVM and
bundler to use all available cores.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 51fee9fce..2592d8686 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0
+SHELL := '/bin/bash'
all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated
centos7/generated: common-generated-all
@@ -24,8 +25,19 @@ ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
-GOTARBALL=go1.17.1.linux-amd64.tar.gz
-NODETARBALL=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_=DOES_NOT_EXIST
+NODETARBALL_=DOES_NOT_EXIST
+GOTARBALL_x86_64=go1.17.1.linux-amd64.tar.gz
+NODETARBALL_x86_64=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_aarch64=go1.17.1.linux-arm64.tar.gz
+NODETARBALL_aarch64=node-v10.23.1-linux-arm64.tar.xz
+
+# Get the bash variable $HOSTTYPE (this requires the SHELL line above)
+HOSTTYPE=$(shell echo $${HOSTTYPE})
+
+GOTARBALL=${GOTARBALL_$(HOSTTYPE)}
+NODETARBALL=${NODETARBALL_$(HOSTTYPE)}
+
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh
index 0d37859ee..5ed33dc9f 100755
--- a/build/package-build-dockerfiles/build-all-build-containers.sh
+++ b/build/package-build-dockerfiles/build-all-build-containers.sh
@@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do
target=${target#./}
echo $target
cd $target
- docker build -t arvados/build:$target .
+ docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
cd ..
done
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index 924c91684..d98bc8b1d 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -2,12 +2,38 @@
#
# SPDX-License-Identifier: AGPL-3.0
+ARG HOSTTYPE
+
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
-FROM debian:bullseye
+FROM debian:bullseye as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM debian:bullseye as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging at arvados.org>
ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
@@ -20,27 +46,21 @@ ADD generated/pkuczynski.asc /tmp/
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- echo "gem: --no-document" >> /etc/gemrc && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
-RUN /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
-# We also need libpam compiled for arm64
-RUN /usr/bin/dpkg --add-architecture arm64
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN echo $MAKE && git clone -b 17417-switch-to-mini_racer --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian11"]
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index d52de98a9..4c55458d4 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -196,7 +196,7 @@ fi
echo $TARGET
cd $TARGET
-time docker build --tag=$IMAGE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
popd
if test -z "$packages" ; then
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 9406be714..0d7f72049 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -374,60 +374,64 @@ if [[ "$?" == "0" ]]; then
fi
# Build the workbench server package
-test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
-if [[ "$?" == "0" ]] ; then
- (
- set -e
-
- # The workbench package has a build-time dependency on the arvados-server
- # package for config manipulation, so install it first.
- cd $WORKSPACE/cmd/arvados-server
- get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
- if [[ ! -e ${arvados_server_pkg_path} ]]; then
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
- fi
- if [[ "$FORMAT" == "deb" ]]; then
- dpkg -i ${arvados_server_pkg_path}
- else
- rpm -i ${arvados_server_pkg_path}
- fi
-
- cd "$WORKSPACE/apps/workbench"
-
- # We need to bundle to be ready even when we build a package without vendor directory
- # because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
-
- # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
- # and we want that in the package, so it's easier to not exclude the tmp directory
- # from the package - empty it instead.
- rm -rf tmp
- mkdir tmp
-
- # Set up an appropriate config.yml
- arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
- mkdir -p /etc/arvados/
- mv /tmp/x /etc/arvados/config.yml
- perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
-
- # Remove generated configuration files so they don't go in the package.
- rm -rf /etc/arvados/
- )
-
- if [[ "$?" != "0" ]]; then
- echo "ERROR: Asset precompilation failed"
- EXITCODE=1
- else
- handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+if [[ "$HOSTTYPE" == "x86_64" ]]; then
+ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+ if [[ "$?" == "0" ]] ; then
+ (
+ set -e
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --system >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up an appropriate config.yml
+ arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
+ mkdir -p /etc/arvados/
+ mv /tmp/x /etc/arvados/config.yml
+ perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+ # Remove generated configuration files so they don't go in the package.
+ rm -rf /etc/arvados/
+ )
+
+ if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+ else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+ fi
fi
+else
+ echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
fi
# clean up temporary GOPATH
diff --git a/build/run-library.sh b/build/run-library.sh
index 33bb09f82..22214ca23 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -137,11 +137,32 @@ package_go_binary() {
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ return 0
+ fi
+ fi
+
+ native_arch="amd64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ fi
+
if [[ -n "$ONLY_ARCH" ]]; then
- package_go_binary_worker "$src_path" "$prog" "$description" "$ONLY_ARCH" "$license_file"
+ if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ONLY_ARCH" ]]; then
+ package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ONLY_ARCH" "$license_file"
+ else
+ echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ONLY_ARCH"
+ fi
else
- for arch in 'amd64' 'arm64'; do
- package_go_binary_worker "$src_path" "$prog" "$description" "$arch" "$license_file"
+ archs=($native_arch)
+ if [[ "$native_arch" == "amd64" ]]; then
+ archs=('amd64' 'arm64')
+ fi
+ for arch in $archs; do
+ package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file"
done
fi
}
@@ -151,17 +172,10 @@ package_go_binary_worker() {
local src_path="$1"; shift
local prog="$1"; shift
local description="$1"; shift
+ local native_arch="${1:-amd64}"; shift
local arch="${1:-amd64}"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
- # arvados-workbench depends on arvados-server at build time, so even when
- # only arvados-workbench is being built, we need to build arvados-server too
- if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
- return 0
- fi
- fi
-
debug_echo "package_go_binary $src_path as $prog"
local basename="${src_path##*/}"
calculate_go_package_version go_package_version $src_path
@@ -173,7 +187,7 @@ package_go_binary_worker() {
fi
echo "BUILDING ${arch}"
- if [[ "$arch" == "arm64" ]]; then
+ if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then
CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
else
GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
@@ -182,7 +196,7 @@ package_go_binary_worker() {
local -a switches=()
binpath=$GOPATH/bin/${basename}
- if [[ "${arch}" != "amd64" ]]; then
+ if [[ "${arch}" != "${native_arch}" ]]; then
switches+=("-a${arch}")
binpath="$GOPATH/bin/linux_${arch}/${basename}"
fi
@@ -313,8 +327,14 @@ get_complete_package_name() {
fi
if [[ "$arch" == "" ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ native_arch="amd64"
+ rpm_native_arch="x86_64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ rpm_native_arch="arm64"
+ fi
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
if [[ "$pkgtype" =~ ^(src)$ ]]; then
rpm_architecture="noarch"
@@ -323,8 +343,8 @@ get_complete_package_name() {
# These python packages have binary components
if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
fi
else
rpm_architecture=$arch
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list