[ARVADOS] updated: 2.1.0-1755-g9e9917f74

Git user git at public.arvados.org
Thu Dec 23 22:02:22 UTC 2021


Summary of changes:
 build/package-build-dockerfiles/Makefile           |  16 ++-
 .../build-all-build-containers.sh                  |   2 +-
 .../package-build-dockerfiles/debian11/Dockerfile  |  54 ++++++----
 build/run-build-packages-one-target.sh             |   2 +-
 build/run-build-packages.sh                        | 110 +++++++++++----------
 build/run-library.sh                               |  54 ++++++----
 6 files changed, 147 insertions(+), 91 deletions(-)

       via  9e9917f7423b719d1eaf1ca62b2deade25fea55a (commit)
      from  3203561dd0affdcaa1d34ae9b44c07f28201043a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 9e9917f7423b719d1eaf1ca62b2deade25fea55a
Author: Ward Vandewege <ward at curii.com>
Date:   Thu Dec 23 13:00:06 2021 -0500

    17417: add native build support for aarch64 (arm64) for our Debian 11
           target. Also speed up the docker image build by telling RVM and
           bundler to use all available cores.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 51fee9fce..2592d8686 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+SHELL := '/bin/bash'
 all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated
 
 centos7/generated: common-generated-all
@@ -24,8 +25,19 @@ ubuntu2004/generated: common-generated-all
 	test -d ubuntu2004/generated || mkdir ubuntu2004/generated
 	cp -f -rlt ubuntu2004/generated common-generated/*
 
-GOTARBALL=go1.17.1.linux-amd64.tar.gz
-NODETARBALL=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_=DOES_NOT_EXIST
+NODETARBALL_=DOES_NOT_EXIST
+GOTARBALL_x86_64=go1.17.1.linux-amd64.tar.gz
+NODETARBALL_x86_64=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_aarch64=go1.17.1.linux-arm64.tar.gz
+NODETARBALL_aarch64=node-v10.23.1-linux-arm64.tar.xz
+
+# Get the bash variable $HOSTTYPE (this requires the SHELL line above)
+HOSTTYPE=$(shell echo $${HOSTTYPE})
+
+GOTARBALL=${GOTARBALL_$(HOSTTYPE)}
+NODETARBALL=${NODETARBALL_$(HOSTTYPE)}
+
 RVMKEY1=mpapis.asc
 RVMKEY2=pkuczynski.asc
 
diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh
index 0d37859ee..5ed33dc9f 100755
--- a/build/package-build-dockerfiles/build-all-build-containers.sh
+++ b/build/package-build-dockerfiles/build-all-build-containers.sh
@@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do
   target=${target#./}
   echo $target
   cd $target
-  docker build -t arvados/build:$target .
+  docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
   cd ..
 done
 
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index 924c91684..d98bc8b1d 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -2,12 +2,38 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+ARG HOSTTYPE
+
 ## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
-FROM debian:bullseye
+FROM debian:bullseye as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM debian:bullseye as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
 MAINTAINER Arvados Package Maintainers <packaging at arvados.org>
 
 ENV DEBIAN_FRONTEND noninteractive
 
+SHELL ["/bin/bash", "-c"]
 # Install dependencies.
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
 
@@ -20,27 +46,21 @@ ADD generated/pkuczynski.asc /tmp/
 RUN gpg --import --no-tty /tmp/mpapis.asc && \
     gpg --import --no-tty /tmp/pkuczynski.asc && \
     curl -L https://get.rvm.io | bash -s stable && \
-    /usr/local/rvm/bin/rvm install 2.5 && \
+    /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
-    echo "gem: --no-document" >> /etc/gemrc && \
+    echo "gem: --no-document" >> ~/.gemrc && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
     /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
-RUN /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
-# We also need libpam compiled for arm64
-RUN /usr/bin/dpkg --add-architecture arm64
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN echo $MAKE && git clone -b 17417-switch-to-mini_racer --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados/services/api && \
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados/apps/workbench && \
+    /usr/local/rvm/bin/rvm-exec default bundle install
 
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian11"]
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index d52de98a9..4c55458d4 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -196,7 +196,7 @@ fi
 
 echo $TARGET
 cd $TARGET
-time docker build --tag=$IMAGE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
 popd
 
 if test -z "$packages" ; then
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 9406be714..0d7f72049 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -374,60 +374,64 @@ if [[ "$?" == "0" ]]; then
 fi
 
 # Build the workbench server package
-test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
-if [[ "$?" == "0" ]] ; then
-  (
-      set -e
-
-      # The workbench package has a build-time dependency on the arvados-server
-      # package for config manipulation, so install it first.
-      cd $WORKSPACE/cmd/arvados-server
-      get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
-      arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
-      if [[ ! -e ${arvados_server_pkg_path} ]]; then
-        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
-      fi
-      if [[ "$FORMAT" == "deb" ]]; then
-        dpkg -i ${arvados_server_pkg_path}
-      else
-        rpm -i ${arvados_server_pkg_path}
-      fi
-
-      cd "$WORKSPACE/apps/workbench"
-
-      # We need to bundle to be ready even when we build a package without vendor directory
-      # because asset compilation requires it.
-      bundle install --system >"$STDOUT_IF_DEBUG"
-
-      # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-      # and we want that in the package, so it's easier to not exclude the tmp directory
-      # from the package - empty it instead.
-      rm -rf tmp
-      mkdir tmp
-
-      # Set up an appropriate config.yml
-      arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
-      mkdir -p /etc/arvados/
-      mv /tmp/x /etc/arvados/config.yml
-      perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
-      ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
-      ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
-
-      # Remove generated configuration files so they don't go in the package.
-      rm -rf /etc/arvados/
-  )
-
-  if [[ "$?" != "0" ]]; then
-    echo "ERROR: Asset precompilation failed"
-    EXITCODE=1
-  else
-    handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
-        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-        --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
-        --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+if [[ "$HOSTTYPE" == "x86_64" ]]; then
+  test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+  if [[ "$?" == "0" ]] ; then
+    (
+        set -e
+
+        # The workbench package has a build-time dependency on the arvados-server
+        # package for config manipulation, so install it first.
+        cd $WORKSPACE/cmd/arvados-server
+        get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+        if [[ ! -e ${arvados_server_pkg_path} ]]; then
+          arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+        fi
+        if [[ "$FORMAT" == "deb" ]]; then
+          dpkg -i ${arvados_server_pkg_path}
+        else
+          rpm -i ${arvados_server_pkg_path}
+        fi
+
+        cd "$WORKSPACE/apps/workbench"
+
+        # We need to bundle to be ready even when we build a package without vendor directory
+        # because asset compilation requires it.
+        bundle install --system >"$STDOUT_IF_DEBUG"
+
+        # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+        # and we want that in the package, so it's easier to not exclude the tmp directory
+        # from the package - empty it instead.
+        rm -rf tmp
+        mkdir tmp
+
+        # Set up an appropriate config.yml
+        arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
+        mkdir -p /etc/arvados/
+        mv /tmp/x /etc/arvados/config.yml
+        perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+        # Remove generated configuration files so they don't go in the package.
+        rm -rf /etc/arvados/
+    )
+
+    if [[ "$?" != "0" ]]; then
+      echo "ERROR: Asset precompilation failed"
+      EXITCODE=1
+    else
+      handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+          "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+          --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+          --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+    fi
   fi
+else
+  echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
 fi
 
 # clean up temporary GOPATH
diff --git a/build/run-library.sh b/build/run-library.sh
index 33bb09f82..22214ca23 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -137,11 +137,32 @@ package_go_binary() {
     local description="$1"; shift
     local license_file="${1:-agpl-3.0.txt}"; shift
 
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+      # arvados-workbench depends on arvados-server at build time, so even when
+      # only arvados-workbench is being built, we need to build arvados-server too
+      if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+        return 0
+      fi
+    fi
+
+    native_arch="amd64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+        native_arch="arm64"
+    fi
+
     if [[ -n "$ONLY_ARCH" ]]; then
-        package_go_binary_worker "$src_path" "$prog" "$description" "$ONLY_ARCH" "$license_file"
+      if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ONLY_ARCH" ]]; then
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ONLY_ARCH" "$license_file"
+      else
+        echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ONLY_ARCH"
+      fi
     else
-      for arch in 'amd64' 'arm64'; do
-        package_go_binary_worker "$src_path" "$prog" "$description" "$arch" "$license_file"
+      archs=($native_arch)
+      if [[ "$native_arch" == "amd64" ]]; then
+        archs=('amd64' 'arm64')
+      fi
+      for arch in $archs; do
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file"
       done
     fi
 }
@@ -151,17 +172,10 @@ package_go_binary_worker() {
     local src_path="$1"; shift
     local prog="$1"; shift
     local description="$1"; shift
+    local native_arch="${1:-amd64}"; shift
     local arch="${1:-amd64}"; shift
     local license_file="${1:-agpl-3.0.txt}"; shift
 
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
-      # arvados-workbench depends on arvados-server at build time, so even when
-      # only arvados-workbench is being built, we need to build arvados-server too
-      if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
-        return 0
-      fi
-    fi
-
     debug_echo "package_go_binary $src_path as $prog"
     local basename="${src_path##*/}"
     calculate_go_package_version go_package_version $src_path
@@ -173,7 +187,7 @@ package_go_binary_worker() {
     fi
 
     echo "BUILDING ${arch}"
-    if [[ "$arch" == "arm64" ]]; then
+    if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then
       CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
     else
       GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
@@ -182,7 +196,7 @@ package_go_binary_worker() {
     local -a switches=()
 
     binpath=$GOPATH/bin/${basename}
-    if [[ "${arch}" != "amd64" ]]; then
+    if [[ "${arch}" != "${native_arch}" ]]; then
       switches+=("-a${arch}")
       binpath="$GOPATH/bin/linux_${arch}/${basename}"
     fi
@@ -313,8 +327,14 @@ get_complete_package_name() {
   fi
 
   if [[ "$arch" == "" ]]; then
-    rpm_architecture="x86_64"
-    deb_architecture="amd64"
+    native_arch="amd64"
+    rpm_native_arch="x86_64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+      native_arch="arm64"
+      rpm_native_arch="arm64"
+    fi
+    rpm_architecture="$rpm_native_arch"
+    deb_architecture="$native_arch"
 
     if [[ "$pkgtype" =~ ^(src)$ ]]; then
       rpm_architecture="noarch"
@@ -323,8 +343,8 @@ get_complete_package_name() {
 
     # These python packages have binary components
     if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
-      rpm_architecture="x86_64"
-      deb_architecture="amd64"
+      rpm_architecture="$rpm_native_arch"
+      deb_architecture="$native_arch"
     fi
   else
     rpm_architecture=$arch

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list