[ARVADOS] updated: 2.1.0-1782-g8c0013949

Git user git at public.arvados.org
Tue Jan 4 20:23:11 UTC 2022


Summary of changes:
 build/libcloud-pin.sh       | 12 ------------
 build/run-build-packages.sh |  1 -
 build/run-library.sh        |  2 +-
 build/run-tests.sh          |  2 --
 4 files changed, 1 insertion(+), 16 deletions(-)
 delete mode 100644 build/libcloud-pin.sh

  discards  2f8dbfea982f72d8b5bb539486e17f9eb7b6fdc5 (commit)
  discards  8936d9344ee7498abf340a40c0af7880490c4931 (commit)
       via  8c0013949dc735f44f0fa1e064848dd50a7fbf23 (commit)
       via  2d1063022c48e1d1909f1b67e29f9aa6d6c90b41 (commit)
       via  1154089b34acd6e4d1426132456e1cf6bde9a241 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (2f8dbfea982f72d8b5bb539486e17f9eb7b6fdc5)
            \
             N -- N -- N (8c0013949dc735f44f0fa1e064848dd50a7fbf23)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8c0013949dc735f44f0fa1e064848dd50a7fbf23
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Jan 4 14:58:29 2022 -0500

    17417: remove libcloud pin, it is no longer used.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/libcloud-pin.sh b/build/libcloud-pin.sh
deleted file mode 100644
index 9cc2924e1..000000000
--- a/build/libcloud-pin.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-LIBCLOUD_PIN=2.3.1.dev2
-
-using_fork=true
-if [[ $using_fork = true ]]; then
-    LIBCLOUD_PIN_SRC="https://github.com/arvados/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip"
-else
-    LIBCLOUD_PIN_SRC=""
-fi
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index a07355d35..fe177f542 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d318bc60d..29e0d83d0 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
-
 COLUMNS=80
 . `dirname "$(readlink -f "$0")"`/run-library.sh
 

commit 2d1063022c48e1d1909f1b67e29f9aa6d6c90b41
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Jan 4 14:50:27 2022 -0500

    17417: preseed the go module cache in our package build images. This
           makes them (much) bigger, but it really speeds up a quick one-off
           package build.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 9a2ceb065..14a28901c 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -68,7 +68,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
-    /usr/local/rvm/bin/rvm-exec default bundle install
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
 
 # The version of setuptools that comes with CentOS is way too old
 RUN pip3 install 'setuptools<45'
diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index 7f1332714..d7c31411f 100644
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -60,7 +60,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
-    /usr/local/rvm/bin/rvm-exec default bundle install
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
 
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian10"]
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index 02f156785..e819a28a7 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -60,7 +60,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
-    /usr/local/rvm/bin/rvm-exec default bundle install
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
 
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian11"]
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index fd9813883..b1a85d202 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -59,7 +59,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
-    /usr/local/rvm/bin/rvm-exec default bundle install
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
 
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 89cdf199a..1de328029 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -59,7 +59,10 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \
-    /usr/local/rvm/bin/rvm-exec default bundle install
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
+
 
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2004"]
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7aa9f9bd2..a07355d35 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -224,7 +224,6 @@ debug_echo -e "\nGo packages\n"
 
 # Go binaries
 cd $WORKSPACE/packages/$TARGET
-export GOPATH=$(mktemp -d)
 package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
     "Arvados command line tool (beta)"
 package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \

commit 1154089b34acd6e4d1426132456e1cf6bde9a241
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Jan 4 13:55:18 2022 -0500

    17417: more refactoring: reduce use of global variables, move more code
           into wrapper functions in run_library.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/.gitignore b/.gitignore
index 231424acc..07482bde7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ _version.py
 *.log
 arvados-snakeoil-ca.pem
 .vagrant
+packages
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 2e22a9e53..7aa9f9bd2 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -200,31 +200,7 @@ fi
 # Perl packages
 debug_echo -e "\nPerl packages\n"
 
-if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
-  cd "$WORKSPACE/sdk/perl"
-  libarvados_perl_version="$(version_from_git)"
-
-  cd $WORKSPACE/packages/$TARGET
-  test_package_presence libarvados-perl "$libarvados_perl_version"
-
-  if [[ "$?" == "0" ]]; then
-    cd "$WORKSPACE/sdk/perl"
-
-    if [[ -e Makefile ]]; then
-      make realclean >"$STDOUT_IF_DEBUG"
-    fi
-    find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-        -delete
-    rm -rf install
-
-    perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
-        make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
-        fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
-        dir "$(version_from_git)" install/man/=/usr/share/man \
-        "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
-        mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
-  fi
-fi
+handle_libarvados_perl
 
 # Ruby gems
 debug_echo -e "\nRuby gems\n"
@@ -240,38 +216,11 @@ handle_ruby_gem arvados-cli
 cd "$WORKSPACE/services/login-sync"
 handle_ruby_gem arvados-login-sync
 
-# Python packages
-debug_echo -e "\nPython packages\n"
+# arvados-src
+handle_arvados_src
 
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-src" == "$ONLY_BUILD" ]] ; then
-  # arvados-src
-  (
-      cd "$WORKSPACE"
-      COMMIT_HASH=$(format_last_commit_here "%H")
-      arvados_src_version="$(version_from_git)"
-
-      cd $WORKSPACE/packages/$TARGET
-      test_package_presence arvados-src "$arvados_src_version" src ""
-
-      if [[ "$?" == "0" ]]; then
-        cd "$WORKSPACE"
-        SRC_BUILD_DIR=$(mktemp -d)
-        # mktemp creates the directory with 0700 permissions by default
-        chmod 755 $SRC_BUILD_DIR
-        git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
-        cd "$SRC_BUILD_DIR"
-
-        # go into detached-head state
-        git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
-        echo "$COMMIT_HASH" >git-commit.version
-
-        cd $WORKSPACE/packages/$TARGET
-        fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
-
-        rm -rf "$SRC_BUILD_DIR"
-      fi
-  )
-fi
+# Go packages
+debug_echo -e "\nGo packages\n"
 
 # Go binaries
 cd $WORKSPACE/packages/$TARGET
@@ -319,23 +268,26 @@ package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
 package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
     "Arvados PAM authentication module"
 
+# Python packages
+debug_echo -e "\nPython packages\n"
+
 # The Python SDK - Python3 package
-fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
 
 # Arvados cwl runner - Python3 package
-fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
 
 # The FUSE driver - Python3 package
-fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
 
 # The Arvados crunchstat-summary tool
-fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
+fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
 
 # The Docker image cleaner
-fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
+fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
 
 # The Arvados user activity tool
-fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
+fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
 
 # The python->python3 metapackages
 build_metapackage "arvados-fuse" "services/fuse"
@@ -345,94 +297,16 @@ build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
 build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
 build_metapackage "arvados-user-activity" "tools/user-activity"
 
-if [[ -z "$ONLY_BUILD" ]] || [[ "cwltest" == "$ONLY_BUILD" ]] ; then
-  # The cwltest package, which lives out of tree
-  cd "$WORKSPACE"
-  if [[ -e "$WORKSPACE/cwltest" ]]; then
-    rm -rf "$WORKSPACE/cwltest"
-  fi
-  git clone https://github.com/common-workflow-language/cwltest.git
-  # signal to our build script that we want a cwltest executable installed in /usr/bin/
-  mkdir cwltest/bin && touch cwltest/bin/cwltest
-  fpm_build_virtualenv "cwltest" "cwltest" "python3"
-  # The python->python3 metapackage
-  build_metapackage "cwltest" "cwltest"
-  cd "$WORKSPACE"
-  rm -rf "$WORKSPACE/cwltest"
-fi
-
-calculate_go_package_version arvados_server_version cmd/arvados-server
-arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+# The cwltest package, which lives out of tree
+handle_cwltest "$FORMAT" "$ARCH"
 
-# Build the API server package
-test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
-if [[ "$?" == "0" ]]; then
-  handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
-      "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-      --description="Arvados API server - Arvados is a free and open source platform for big data science." \
-      --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
-fi
+# Rails packages
+debug_echo -e "\nRails packages\n"
 
-# Build the workbench server package
-if [[ "$HOSTTYPE" == "x86_64" ]]; then
-  test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
-  if [[ "$?" == "0" ]] ; then
-    (
-        set -e
-
-        # The workbench package has a build-time dependency on the arvados-server
-        # package for config manipulation, so install it first.
-        cd $WORKSPACE/cmd/arvados-server
-        get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
-        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
-        if [[ ! -e ${arvados_server_pkg_path} ]]; then
-          arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
-        fi
-        if [[ "$FORMAT" == "deb" ]]; then
-          dpkg -i ${arvados_server_pkg_path}
-        else
-          rpm -i ${arvados_server_pkg_path}
-        fi
-
-        cd "$WORKSPACE/apps/workbench"
-
-        # We need to bundle to be ready even when we build a package without vendor directory
-        # because asset compilation requires it.
-        bundle install --system >"$STDOUT_IF_DEBUG"
-
-        # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-        # and we want that in the package, so it's easier to not exclude the tmp directory
-        # from the package - empty it instead.
-        rm -rf tmp
-        mkdir tmp
-
-        # Set up an appropriate config.yml
-        arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
-        mkdir -p /etc/arvados/
-        mv /tmp/x /etc/arvados/config.yml
-        perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
-        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
-        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
-
-        # Remove generated configuration files so they don't go in the package.
-        rm -rf /etc/arvados/
-    )
-
-    if [[ "$?" != "0" ]]; then
-      echo "ERROR: Asset precompilation failed"
-      EXITCODE=1
-    else
-      handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
-          "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-          --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
-          --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
-    fi
-  fi
-else
-  echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
-fi
+# The rails api server package
+handle_api_server "$ARCH"
+# The rails workbench package
+handle_workbench "$ARCH"
 
 # clean up temporary GOPATH
 rm -rf "$GOPATH"
diff --git a/build/run-library.sh b/build/run-library.sh
index 92140bbdc..ed3211c2a 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -79,6 +79,25 @@ calculate_python_sdk_cwl_package_versions() {
   cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
 }
 
+# Usage: get_native_arch
+get_native_arch() {
+  # Only amd64 and aarch64 are supported at the moment
+  local native_arch=""
+  case "$HOSTTYPE" in
+    x86_64)
+      native_arch="amd64"
+      ;;
+    aarch64)
+      native_arch="arm64"
+      ;;
+    *)
+      echo "Error: architecture not supported"
+      exit 1
+      ;;
+  esac
+  echo $native_arch
+}
+
 handle_ruby_gem() {
     local gem_name="$1"; shift
     local gem_version="$(nohash_version_from_git)"
@@ -132,51 +151,48 @@ calculate_go_package_version() {
 
 # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
 package_go_binary() {
-    local src_path="$1"; shift
-    local prog="$1"; shift
-    local package_format="$1"; shift
-    local target_arch="$1"; shift
-    local description="$1"; shift
-    local license_file="${1:-agpl-3.0.txt}"; shift
-
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
-      # arvados-workbench depends on arvados-server at build time, so even when
-      # only arvados-workbench is being built, we need to build arvados-server too
-      if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
-        return 0
-      fi
+  local src_path="$1"; shift
+  local prog="$1"; shift
+  local package_format="$1"; shift
+  local target_arch="$1"; shift
+  local description="$1"; shift
+  local license_file="${1:-agpl-3.0.txt}"; shift
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+    # arvados-workbench depends on arvados-server at build time, so even when
+    # only arvados-workbench is being built, we need to build arvados-server too
+    if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+      debug_echo -e "Skipping build of $prog package."
+      return 0
     fi
+  fi
 
-    # Only amd64 and aarch64 are supported
-    native_arch="amd64"
-    if [[ "$HOSTTYPE" == "aarch64" ]]; then
-        native_arch="arm64"
-    fi
+  native_arch=$(get_native_arch)
 
-    if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
-      echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
-      return 1
-    fi
+  if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
+    echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
+    return 1
+  fi
 
-    if [[ -n "$target_arch" ]]; then
-      # A target architecture has been specified
-      package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
-      return $?
-    else
-      # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
-      # but only when building deb packages (centos does not have support for crosscompiling userspace).
-      archs=($native_arch)
-      if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
-        archs=('amd64' 'arm64')
-      fi
-      for ta in $archs; do
-        package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
-        retval=$?
-        if [[ "$retval" != 0 ]]; then
-          return $retval
-        fi
-      done
+  if [[ -n "$target_arch" ]]; then
+    # A target architecture has been specified
+    package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
+    return $?
+  else
+    # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
+    # but only when building deb packages (centos does not have support for crosscompiling userspace).
+    archs=($native_arch)
+    if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+      archs=('amd64' 'arm64')
     fi
+    for ta in $archs; do
+      package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
+      retval=$?
+      if [[ "$retval" != 0 ]]; then
+        return $retval
+      fi
+    done
+  fi
 }
 
 # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
@@ -204,7 +220,7 @@ package_go_binary_worker() {
       return 1
     fi
 
-    echo "Building $pachage_format ($target_arch) package for $prog from $src_path"
+    echo "Building $package_format ($target_arch) package for $prog from $src_path"
     if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
       CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
     else
@@ -241,6 +257,7 @@ package_go_so() {
     local description="$1"; shift
 
     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+      debug_echo -e "Skipping build of $pkg package."
       return 0
     fi
 
@@ -347,10 +364,9 @@ get_complete_package_name() {
   fi
 
   if [[ "$arch" == "" ]]; then
-    native_arch="amd64"
+    native_arch=$(get_native_arch)
     rpm_native_arch="x86_64"
     if [[ "$HOSTTYPE" == "aarch64" ]]; then
-      native_arch="arm64"
       rpm_native_arch="arm64"
     fi
     rpm_architecture="$rpm_native_arch"
@@ -360,12 +376,6 @@ get_complete_package_name() {
       rpm_architecture="noarch"
       deb_architecture="all"
     fi
-
-    # These python packages have binary components
-    if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
-      rpm_architecture="$rpm_native_arch"
-      deb_architecture="$native_arch"
-    fi
   else
     rpm_architecture=$arch
     deb_architecture=$arch
@@ -502,40 +512,247 @@ handle_rails_package() {
     rm -rf "$scripts_dir"
 }
 
+# Usage: handle_api_server [amd64|arm64]
+handle_api_server () {
+  local target_arch="${1:-amd64}"; shift
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then
+    debug_echo -e "Skipping build of arvados-api-server package."
+    return 0
+  fi
+
+  native_arch=$(get_native_arch)
+  if [[ "$target_arch" != "$native_arch" ]]; then
+    echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH"
+    echo
+    exit 1
+  fi
+
+  # Build the API server package
+  test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+  if [[ "$?" == "0" ]]; then
+    calculate_go_package_version arvados_server_version cmd/arvados-server
+    arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+    handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
+        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+        --description="Arvados API server - Arvados is a free and open source platform for big data science." \
+        --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+  fi
+}
+
+# Usage: handle_workbench [amd64|arm64]
+handle_workbench () {
+  local target_arch="${1:-amd64}"; shift
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-workbench" ]] ; then
+    debug_echo -e "Skipping build of arvados-workbench package."
+    return 0
+  fi
+
+  native_arch=$(get_native_arch)
+  if [[ "$target_arch" != "$native_arch" ]]; then
+    echo "Error: no cross compilation support for Rails yet, can not build arvados-workbench for $native_arch"
+    echo
+    exit 1
+  fi
+
+  if [[ "$native_arch" != "amd64" ]]; then
+    echo "Error: building the arvados-workbench package is not yet supported on this architecture ($native_arch)."
+    echo
+    exit 1
+  fi
+
+  # Build the workbench server package
+  test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+  if [[ "$?" == "0" ]] ; then
+    (
+        set -e
+
+        calculate_go_package_version arvados_server_version cmd/arvados-server
+        arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
+        # The workbench package has a build-time dependency on the arvados-server
+        # package for config manipulation, so install it first.
+        cd $WORKSPACE/cmd/arvados-server
+        get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+        if [[ ! -e ${arvados_server_pkg_path} ]]; then
+          arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+        fi
+        if [[ "$FORMAT" == "deb" ]]; then
+          dpkg -i ${arvados_server_pkg_path}
+        else
+          rpm -i ${arvados_server_pkg_path}
+        fi
+
+        cd "$WORKSPACE/apps/workbench"
+
+        # We need to bundle to be ready even when we build a package without vendor directory
+        # because asset compilation requires it.
+        bundle install --system >"$STDOUT_IF_DEBUG"
+
+        # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+        # and we want that in the package, so it's easier to not exclude the tmp directory
+        # from the package - empty it instead.
+        rm -rf tmp
+        mkdir tmp
+
+        # Set up an appropriate config.yml
+        arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
+        mkdir -p /etc/arvados/
+        mv /tmp/x /etc/arvados/config.yml
+        perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+        # Remove generated configuration files so they don't go in the package.
+        rm -rf /etc/arvados/
+    )
+
+    if [[ "$?" != "0" ]]; then
+      echo "ERROR: Asset precompilation failed"
+      EXITCODE=1
+    else
+      handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+          "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+          --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+          --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+    fi
+  fi
+}
+
+# Usage: handle_cwltest [deb|rpm] [amd64|arm64]
+handle_cwltest () {
+  local package_format="$1"; shift
+  local target_arch="${1:-amd64}"; shift
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "python3-cwltest" ]] ; then
+    debug_echo -e "Skipping build of cwltest package."
+    return 0
+  fi
+  cd "$WORKSPACE"
+  if [[ -e "$WORKSPACE/cwltest" ]]; then
+    rm -rf "$WORKSPACE/cwltest"
+  fi
+  git clone https://github.com/common-workflow-language/cwltest.git
+  # signal to our build script that we want a cwltest executable installed in /usr/bin/
+  mkdir cwltest/bin && touch cwltest/bin/cwltest
+  fpm_build_virtualenv "cwltest" "cwltest" "$package_format" "$target_arch"
+  # The python->python3 metapackage
+  build_metapackage "cwltest" "cwltest"
+  cd "$WORKSPACE"
+  rm -rf "$WORKSPACE/cwltest"
+}
+
+# Usage: handle_arvados_src
+handle_arvados_src () {
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
+    debug_echo -e "Skipping build of arvados-src package."
+    return 0
+  fi
+  # arvados-src
+  (
+      cd "$WORKSPACE"
+      COMMIT_HASH=$(format_last_commit_here "%H")
+      arvados_src_version="$(version_from_git)"
+
+      cd $WORKSPACE/packages/$TARGET
+      test_package_presence arvados-src "$arvados_src_version" src ""
+
+      if [[ "$?" == "0" ]]; then
+        cd "$WORKSPACE"
+        SRC_BUILD_DIR=$(mktemp -d)
+        # mktemp creates the directory with 0700 permissions by default
+        chmod 755 $SRC_BUILD_DIR
+        git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+        cd "$SRC_BUILD_DIR"
+
+        # go into detached-head state
+        git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+        echo "$COMMIT_HASH" >git-commit.version
+
+        cd $WORKSPACE/packages/$TARGET
+        fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+        rm -rf "$SRC_BUILD_DIR"
+      fi
+  )
+}
+
+# Usage: handle_libarvados_perl
+handle_libarvados_perl () {
+  if [[ -n "$ONLY_BUILD" ]] || [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then
+    debug_echo -e "Skipping build of libarvados-perl package."
+    return 0
+  fi
+  cd "$WORKSPACE/sdk/perl"
+  libarvados_perl_version="$(version_from_git)"
+
+  cd $WORKSPACE/packages/$TARGET
+  test_package_presence libarvados-perl "$libarvados_perl_version"
+
+  if [[ "$?" == "0" ]]; then
+    cd "$WORKSPACE/sdk/perl"
+
+    if [[ -e Makefile ]]; then
+      make realclean >"$STDOUT_IF_DEBUG"
+    fi
+    find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+        -delete
+    rm -rf install
+
+    perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+        make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+        fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
+        dir "$(version_from_git)" install/man/=/usr/share/man \
+        "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
+        mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  fi
+}
+
 # Build python packages with a virtualenv built-in
+# Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64]
 fpm_build_virtualenv () {
-  PKG=$1
-  shift
-  PKG_DIR=$1
-  shift
-  PACKAGE_TYPE=${1:-python}
-  shift
-  native_arch="amd64"
-  if [[ "$HOSTTYPE" == "aarch64" ]]; then
-    native_arch="arm64"
+  local pkg=$1; shift
+  local pkg_dir=$1; shift
+  local package_format="$1"; shift
+  local target_arch="${1:-amd64}"; shift
+
+  native_arch=$(get_native_arch)
+
+  if [[ "$pkg" != "arvados-docker-cleaner" ]]; then
+    PYTHON_PKG=$PYTHON3_PKG_PREFIX-$pkg
+  else
+    # Exception to our package naming convention
+    PYTHON_PKG=$pkg
   fi
 
-  if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
-      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
-  elif [[ -z "$ARCH" ]]; then
-    for arch in $native_arch; do
-      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
-    done
+  # arvados-python-client sdist should always be built, to be available
+  # for other dependent packages.
+  if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$pkg" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+    debug_echo -e "Skipping build of $pkg package."
+    return 0
+  fi
+
+  if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then
+      fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$target_arch"
+  elif [[ -z "$target_arch" ]]; then
+    fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$native_arch"
   else
-    echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+    echo "Error: no cross compilation support for Python yet, can not build $pkg for $target_arch"
+    return 1
   fi
 }
 
 # Build python packages with a virtualenv built-in
+# Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64|arm64] [amd64|arm64]
 fpm_build_virtualenv_worker () {
-  PKG=$1
-  shift
-  PKG_DIR=$1
-  shift
-  PACKAGE_TYPE=${1:-python}
-  shift
-  arch=${1:-amd64}
-  shift
+  PKG=$1; shift
+  PKG_DIR=$1; shift
+  local package_format="$1"; shift
+  local native_arch="${1:-amd64}"; shift
+  local target_arch=${1:-amd64}; shift
 
   # Set up
   STDOUT_IF_DEBUG=/dev/null
@@ -550,14 +767,9 @@ fpm_build_virtualenv_worker () {
     ARVADOS_BUILDING_ITERATION=1
   fi
 
-  local python=""
-  case "$PACKAGE_TYPE" in
-    python3)
-        python=python3
-        pip=pip3
-        PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
-        ;;
-  esac
+  local python=python3
+  pip=pip3
+  PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
 
   if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
     PYTHON_PKG=$PACKAGE_PREFIX-$PKG
@@ -566,12 +778,6 @@ fpm_build_virtualenv_worker () {
     PYTHON_PKG=$PKG
   fi
 
-  # arvados-python-client sdist should always be built, to be available
-  # for other dependent packages.
-  if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
-    return 0
-  fi
-
   cd $WORKSPACE/$PKG_DIR
 
   rm -rf dist/*
@@ -611,11 +817,11 @@ fpm_build_virtualenv_worker () {
   # We can't do this earlier than here, because we need PYTHON_VERSION...
   # This isn't so bad; the sdist call above is pretty quick compared to
   # the invocation of virtualenv and fpm, below.
-  if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
+  if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$python" "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
     return 0
   fi
 
-  echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
+  echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
 
   # Package the sdist in a virtualenv
   echo "Creating virtualenv..."
@@ -698,10 +904,10 @@ fpm_build_virtualenv_worker () {
   # Finally, generate the package
   echo "Creating package..."
 
-  declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
+  declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$package_format")
 
-  if [[ "${arch}" != "amd64" ]]; then
-    COMMAND_ARR+=("-a${arch}")
+  if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
+    COMMAND_ARR+=("-a$target_arch")
   fi
 
   if [[ "$MAINTAINER" != "" ]]; then
@@ -722,7 +928,7 @@ fpm_build_virtualenv_worker () {
   LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
   COMMAND_ARR+=('--license' "$LICENSE_STRING")
 
-  if [[ "$FORMAT" == "rpm" ]]; then
+  if [[ "$package_format" == "rpm" ]]; then
     # Make sure to conflict with the old rh-python36 packages we used to publish
     COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
   fi
@@ -808,8 +1014,8 @@ fpm_build_virtualenv_worker () {
     echo
     echo -e "\n${COMMAND_ARR[@]}\n"
   else
-    echo `ls *$FORMAT`
-    mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/
+    echo `ls *$package_format`
+    mv $WORKSPACE/$PKG_DIR/dist/*$package_format $WORKSPACE/packages/$TARGET/
   fi
   echo
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list