[ARVADOS] updated: 2.1.0-1788-g836a871b3

Git user git at public.arvados.org
Wed Jan 5 18:55:49 UTC 2022


Summary of changes:
 build/run-build-packages-one-target.sh      |  2 +-
 build/run-build-test-packages-one-target.sh | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

  discards  f734ad65ba97e9cb36e01fde8a375c7b637528c6 (commit)
  discards  fe1195005c0e4e86a305f3edd2615e411db13ea1 (commit)
  discards  c29ca1ce0cff520cf85a75be3745f90d168e5971 (commit)
       via  836a871b3a985e74004d709de8ca80db7705180c (commit)
       via  6e14b7d45fb47a654966b528ede41add437215e0 (commit)
       via  7b69c2a7f763190954cb767aa2cd26dc87c727fe (commit)
       via  2d2fcf8e83870114fe6ee20746aac362927d92aa (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (f734ad65ba97e9cb36e01fde8a375c7b637528c6)
            \
             N -- N -- N (836a871b3a985e74004d709de8ca80db7705180c)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 836a871b3a985e74004d709de8ca80db7705180c
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Jan 5 13:54:16 2022 -0500

    17417: add --only-build flag to the
           run-build-test-packages-one-target.sh script.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh
index 1c8296dfc..2e2e3cdb4 100755
--- a/build/run-build-test-packages-one-target.sh
+++ b/build/run-build-test-packages-one-target.sh
@@ -11,6 +11,8 @@ Syntax:
 
 --target <target>
     Distribution to build packages for (default: debian10)
+--only-build <package>
+    Build only a specific package (or $ONLY_BUILD from environment)
 --upload
     If the build and test steps are successful, upload the packages
     to a remote apt repository (default: false)
@@ -44,7 +46,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
 fi
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,debug,upload,rc,target:,build-version: \
+    help,debug,upload,rc,target:,only-build:,build-version: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -68,6 +70,9 @@ while [ $# -gt 0 ]; do
         --target)
             TARGET="$2"; shift
             ;;
+        --only-build)
+            ONLY_BUILD="$2"; shift
+            ;;
         --debug)
             DEBUG=" --debug"
             ;;
@@ -93,6 +98,10 @@ done
 
 build_args+=(--target "$TARGET")
 
+if [[ -n "$ONLY_BUILD" ]]; then
+  build_args+=(--only-build "$ONLY_BUILD")
+fi
+
 exit_cleanly() {
     trap - INT
     report_outcomes

commit 6e14b7d45fb47a654966b528ede41add437215e0
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Jan 5 11:20:23 2022 -0500

    17417: Because of debian bug #983477, libfuse-dev for arm64 does not
           install properly beside the amd64 version. This was fixed in
           debian11. Rather than making run-library.sh even more complicated
           by switching out libfuse-dev before every arm64 cross compiled
           build or by building a custom version of libfuse-dev, I disabled
           cross compilation the older Debian-based distributions we support.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index d7c31411f..efff0acc9 100644
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -12,12 +12,7 @@ ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 # Install nodejs and npm
 ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
 ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
-# On x86, we want some cross-compilation support for arm64
-# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
-ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
-# We also need libpam compiled for arm64
-ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
-ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+# No cross compilation support for debian10 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
 
 FROM debian:buster as build_aarch64
 # Install go
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index e819a28a7..54a6a0ec1 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -17,7 +17,7 @@ ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
 ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
 # We also need libpam compiled for arm64
 ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
-ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64
 
 FROM debian:bullseye as build_aarch64
 # Install go
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index db55a33d4..d37278492 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -11,14 +11,7 @@ ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 # Install nodejs and npm
 ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
 ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
-# On x86, we want some cross-compilation support for arm64
-# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
-ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
-# We also need libpam compiled for arm64, and that requires some sources.list mangling
-ONBUILD RUN /bin/sed -i 's/deb http/deb [ arch=amd64 ] http/' /etc/apt/sources.list
-ONBUILD ADD ports.list /etc/apt/sources.list.d/
-ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
-ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+# No cross compilation support for ubuntu1804 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
 
 FROM ubuntu:bionic as build_aarch64
 # Install go
diff --git a/build/package-build-dockerfiles/ubuntu1804/ports.list b/build/package-build-dockerfiles/ubuntu1804/ports.list
deleted file mode 100644
index 1947c4601..000000000
--- a/build/package-build-dockerfiles/ubuntu1804/ports.list
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe multiverse
-deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe multiverse
-deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted universe multiverse
-deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 438bf878a..58b4bc1ed 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -19,6 +19,10 @@ ONBUILD RUN /bin/sed -i 's/deb http/deb [ arch=amd64 ] http/' /etc/apt/sources.l
 ONBUILD ADD ports.list /etc/apt/sources.list.d/
 ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
 ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+# Note that cross compilation support for ubuntu2004 is disabled in
+# run-library.sh because of
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477. Hopefully
+# ubuntu2204 will have the fix introduced in debian11.
 
 FROM ubuntu:focal as build_aarch64
 # Install go
diff --git a/build/run-library.sh b/build/run-library.sh
index b6bb9ab37..9d0dffc0a 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -182,7 +182,13 @@ package_go_binary() {
     # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
     # but only when building deb packages (centos does not have support for crosscompiling userspace).
     archs=($native_arch)
-    if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+    if [[ "$native_arch" == "amd64" ]] &&
+       [[ "$package_format" == "deb" ]] &&
+       [[ "$TARGET" != "debian10" ]] &&
+       [[ "$TARGET" != "ubuntu1804" ]] &&
+       [[ "$TARGET" != "ubuntu2004" ]]; then
+      # Due to bug https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477 the libfuse-dev package for arm64 does
+      # not install properly side by side with the amd64 version before Debian 11.
       archs+=("arm64")
     fi
     for ta in ${archs[@]}; do

commit 7b69c2a7f763190954cb767aa2cd26dc87c727fe
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Jan 5 10:27:02 2022 -0500

    17417: another optimization: instead of building the
           arvados-python-client sdist regardless of the value of
           $ONLY_BUILD, only do it when it is actually needed.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/run-library.sh b/build/run-library.sh
index 3434f1ae8..b6bb9ab37 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -728,11 +728,17 @@ fpm_build_virtualenv () {
     PYTHON_PKG=$pkg
   fi
 
-  # arvados-python-client sdist should always be built, to be available
-  # for other dependent packages.
-  if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$pkg" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
-    debug_echo -e "Skipping build of $pkg package."
-    return 0
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]]; then
+    # arvados-python-client sdist should always be built if we are building a
+    # python package.
+    if [[ "$ONLY_BUILD" != "python3-arvados-cwl-runner" ]] &&
+       [[ "$ONLY_BUILD" != "python3-arvados-fuse" ]] &&
+       [[ "$ONLY_BUILD" != "python3-crunchstat-summary" ]] &&
+       [[ "$ONLY_BUILD" != "arvados-docker-cleaner" ]] &&
+       [[ "$ONLY_BUILD" != "python3-arvados-user-activity" ]]; then
+      debug_echo -e "Skipping build of $pkg package."
+      return 0
+    fi
   fi
 
   if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then

commit 2d2fcf8e83870114fe6ee20746aac362927d92aa
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Jan 4 20:11:43 2022 -0500

    17417: fix bug to enable arm64 cross-compilation on amd64 for our go
           packages. Also make sure package testing works (at least for
           amd64).
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index f20896383..0c474d00f 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -170,7 +170,7 @@ if [[ -n "$test_packages" ]]; then
     fi
     set -e
     (cd $WORKSPACE/packages/$TARGET
-      dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
+      dpkg-scanpackages --multiversion .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
       apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
     )
   fi
diff --git a/build/run-library.sh b/build/run-library.sh
index ed3211c2a..3434f1ae8 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -183,12 +183,12 @@ package_go_binary() {
     # but only when building deb packages (centos does not have support for crosscompiling userspace).
     archs=($native_arch)
     if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
-      archs=('amd64' 'arm64')
+      archs+=("arm64")
     fi
-    for ta in $archs; do
+    for ta in ${archs[@]}; do
       package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
       retval=$?
-      if [[ "$retval" != 0 ]]; then
+      if [[ $retval -ne 0 ]]; then
         return $retval
       fi
     done
@@ -216,8 +216,8 @@ package_go_binary_worker() {
 
     cd $WORKSPACE/packages/$TARGET
     test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
-    if [[ "$?" != "0" ]]; then
-      return 1
+    if [[ $? -ne 0 ]]; then
+      return 0
     fi
 
     echo "Building $package_format ($target_arch) package for $prog from $src_path"

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list