[ARVADOS] updated: 2.1.0-1782-g8c0013949
Git user
git at public.arvados.org
Tue Jan 4 20:23:11 UTC 2022
Summary of changes:
build/libcloud-pin.sh | 12 ------------
build/run-build-packages.sh | 1 -
build/run-library.sh | 2 +-
build/run-tests.sh | 2 --
4 files changed, 1 insertion(+), 16 deletions(-)
delete mode 100644 build/libcloud-pin.sh
discards 2f8dbfea982f72d8b5bb539486e17f9eb7b6fdc5 (commit)
discards 8936d9344ee7498abf340a40c0af7880490c4931 (commit)
via 8c0013949dc735f44f0fa1e064848dd50a7fbf23 (commit)
via 2d1063022c48e1d1909f1b67e29f9aa6d6c90b41 (commit)
via 1154089b34acd6e4d1426132456e1cf6bde9a241 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (2f8dbfea982f72d8b5bb539486e17f9eb7b6fdc5)
\
N -- N -- N (8c0013949dc735f44f0fa1e064848dd50a7fbf23)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8c0013949dc735f44f0fa1e064848dd50a7fbf23
Author: Ward Vandewege <ward at curii.com>
Date: Tue Jan 4 14:58:29 2022 -0500
17417: remove libcloud pin, it is no longer used.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/libcloud-pin.sh b/build/libcloud-pin.sh
deleted file mode 100644
index 9cc2924e1..000000000
--- a/build/libcloud-pin.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-LIBCLOUD_PIN=2.3.1.dev2
-
-using_fork=true
-if [[ $using_fork = true ]]; then
- LIBCLOUD_PIN_SRC="https://github.com/arvados/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip"
-else
- LIBCLOUD_PIN_SRC=""
-fi
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index a07355d35..fe177f542 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -4,7 +4,6 @@
# SPDX-License-Identifier: AGPL-3.0
. `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados packages
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d318bc60d..29e0d83d0 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -3,8 +3,6 @@
#
# SPDX-License-Identifier: AGPL-3.0
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
-
COLUMNS=80
. `dirname "$(readlink -f "$0")"`/run-library.sh
commit 2d1063022c48e1d1909f1b67e29f9aa6d6c90b41
Author: Ward Vandewege <ward at curii.com>
Date: Tue Jan 4 14:50:27 2022 -0500
17417: preseed the go module cache in our package build images. This
makes them (much) bigger, but it really speeds up a quick one-off
package build.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 9a2ceb065..14a28901c 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -68,7 +68,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
- /usr/local/rvm/bin/rvm-exec default bundle install
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
# The version of setuptools that comes with CentOS is way too old
RUN pip3 install 'setuptools<45'
diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index 7f1332714..d7c31411f 100644
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -60,7 +60,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
- /usr/local/rvm/bin/rvm-exec default bundle install
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian10"]
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index 02f156785..e819a28a7 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -60,7 +60,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
- /usr/local/rvm/bin/rvm-exec default bundle install
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian11"]
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index fd9813883..b1a85d202 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -59,7 +59,9 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
- /usr/local/rvm/bin/rvm-exec default bundle install
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 89cdf199a..1de328029 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -59,7 +59,10 @@ RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
- /usr/local/rvm/bin/rvm-exec default bundle install
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2004"]
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7aa9f9bd2..a07355d35 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -224,7 +224,6 @@ debug_echo -e "\nGo packages\n"
# Go binaries
cd $WORKSPACE/packages/$TARGET
-export GOPATH=$(mktemp -d)
package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
"Arvados command line tool (beta)"
package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \
commit 1154089b34acd6e4d1426132456e1cf6bde9a241
Author: Ward Vandewege <ward at curii.com>
Date: Tue Jan 4 13:55:18 2022 -0500
17417: more refactoring: reduce use of global variables, move more code
into wrapper functions in run_library.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/.gitignore b/.gitignore
index 231424acc..07482bde7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ _version.py
*.log
arvados-snakeoil-ca.pem
.vagrant
+packages
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 2e22a9e53..7aa9f9bd2 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -200,31 +200,7 @@ fi
# Perl packages
debug_echo -e "\nPerl packages\n"
-if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
- cd "$WORKSPACE/sdk/perl"
- libarvados_perl_version="$(version_from_git)"
-
- cd $WORKSPACE/packages/$TARGET
- test_package_presence libarvados-perl "$libarvados_perl_version"
-
- if [[ "$?" == "0" ]]; then
- cd "$WORKSPACE/sdk/perl"
-
- if [[ -e Makefile ]]; then
- make realclean >"$STDOUT_IF_DEBUG"
- fi
- find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
- -delete
- rm -rf install
-
- perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
- make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
- fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
- dir "$(version_from_git)" install/man/=/usr/share/man \
- "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
- mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
- fi
-fi
+handle_libarvados_perl
# Ruby gems
debug_echo -e "\nRuby gems\n"
@@ -240,38 +216,11 @@ handle_ruby_gem arvados-cli
cd "$WORKSPACE/services/login-sync"
handle_ruby_gem arvados-login-sync
-# Python packages
-debug_echo -e "\nPython packages\n"
+# arvados-src
+handle_arvados_src
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-src" == "$ONLY_BUILD" ]] ; then
- # arvados-src
- (
- cd "$WORKSPACE"
- COMMIT_HASH=$(format_last_commit_here "%H")
- arvados_src_version="$(version_from_git)"
-
- cd $WORKSPACE/packages/$TARGET
- test_package_presence arvados-src "$arvados_src_version" src ""
-
- if [[ "$?" == "0" ]]; then
- cd "$WORKSPACE"
- SRC_BUILD_DIR=$(mktemp -d)
- # mktemp creates the directory with 0700 permissions by default
- chmod 755 $SRC_BUILD_DIR
- git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
- cd "$SRC_BUILD_DIR"
-
- # go into detached-head state
- git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
- echo "$COMMIT_HASH" >git-commit.version
-
- cd $WORKSPACE/packages/$TARGET
- fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
-
- rm -rf "$SRC_BUILD_DIR"
- fi
- )
-fi
+# Go packages
+debug_echo -e "\nGo packages\n"
# Go binaries
cd $WORKSPACE/packages/$TARGET
@@ -319,23 +268,26 @@ package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
"Arvados PAM authentication module"
+# Python packages
+debug_echo -e "\nPython packages\n"
+
# The Python SDK - Python3 package
-fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
# Arvados cwl runner - Python3 package
-fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
# The FUSE driver - Python3 package
-fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
# The Arvados crunchstat-summary tool
-fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
+fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
# The Docker image cleaner
-fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
+fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
# The Arvados user activity tool
-fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
+fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
# The python->python3 metapackages
build_metapackage "arvados-fuse" "services/fuse"
@@ -345,94 +297,16 @@ build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
build_metapackage "arvados-user-activity" "tools/user-activity"
-if [[ -z "$ONLY_BUILD" ]] || [[ "cwltest" == "$ONLY_BUILD" ]] ; then
- # The cwltest package, which lives out of tree
- cd "$WORKSPACE"
- if [[ -e "$WORKSPACE/cwltest" ]]; then
- rm -rf "$WORKSPACE/cwltest"
- fi
- git clone https://github.com/common-workflow-language/cwltest.git
- # signal to our build script that we want a cwltest executable installed in /usr/bin/
- mkdir cwltest/bin && touch cwltest/bin/cwltest
- fpm_build_virtualenv "cwltest" "cwltest" "python3"
- # The python->python3 metapackage
- build_metapackage "cwltest" "cwltest"
- cd "$WORKSPACE"
- rm -rf "$WORKSPACE/cwltest"
-fi
-
-calculate_go_package_version arvados_server_version cmd/arvados-server
-arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+# The cwltest package, which lives out of tree
+handle_cwltest "$FORMAT" "$ARCH"
-# Build the API server package
-test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
-if [[ "$?" == "0" ]]; then
- handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
-fi
+# Rails packages
+debug_echo -e "\nRails packages\n"
-# Build the workbench server package
-if [[ "$HOSTTYPE" == "x86_64" ]]; then
- test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
- if [[ "$?" == "0" ]] ; then
- (
- set -e
-
- # The workbench package has a build-time dependency on the arvados-server
- # package for config manipulation, so install it first.
- cd $WORKSPACE/cmd/arvados-server
- get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
- if [[ ! -e ${arvados_server_pkg_path} ]]; then
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
- fi
- if [[ "$FORMAT" == "deb" ]]; then
- dpkg -i ${arvados_server_pkg_path}
- else
- rpm -i ${arvados_server_pkg_path}
- fi
-
- cd "$WORKSPACE/apps/workbench"
-
- # We need to bundle to be ready even when we build a package without vendor directory
- # because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
-
- # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
- # and we want that in the package, so it's easier to not exclude the tmp directory
- # from the package - empty it instead.
- rm -rf tmp
- mkdir tmp
-
- # Set up an appropriate config.yml
- arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
- mkdir -p /etc/arvados/
- mv /tmp/x /etc/arvados/config.yml
- perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
-
- # Remove generated configuration files so they don't go in the package.
- rm -rf /etc/arvados/
- )
-
- if [[ "$?" != "0" ]]; then
- echo "ERROR: Asset precompilation failed"
- EXITCODE=1
- else
- handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
- fi
- fi
-else
- echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
-fi
+# The rails api server package
+handle_api_server "$ARCH"
+# The rails workbench package
+handle_workbench "$ARCH"
# clean up temporary GOPATH
rm -rf "$GOPATH"
diff --git a/build/run-library.sh b/build/run-library.sh
index 92140bbdc..ed3211c2a 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -79,6 +79,25 @@ calculate_python_sdk_cwl_package_versions() {
cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
+# Usage: get_native_arch
+get_native_arch() {
+ # Only amd64 and aarch64 are supported at the moment
+ local native_arch=""
+ case "$HOSTTYPE" in
+ x86_64)
+ native_arch="amd64"
+ ;;
+ aarch64)
+ native_arch="arm64"
+ ;;
+ *)
+ echo "Error: architecture not supported"
+ exit 1
+ ;;
+ esac
+ echo $native_arch
+}
+
handle_ruby_gem() {
local gem_name="$1"; shift
local gem_version="$(nohash_version_from_git)"
@@ -132,51 +151,48 @@ calculate_go_package_version() {
# Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
package_go_binary() {
- local src_path="$1"; shift
- local prog="$1"; shift
- local package_format="$1"; shift
- local target_arch="$1"; shift
- local description="$1"; shift
- local license_file="${1:-agpl-3.0.txt}"; shift
-
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
- # arvados-workbench depends on arvados-server at build time, so even when
- # only arvados-workbench is being built, we need to build arvados-server too
- if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
- return 0
- fi
+ local src_path="$1"; shift
+ local prog="$1"; shift
+ local package_format="$1"; shift
+ local target_arch="$1"; shift
+ local description="$1"; shift
+ local license_file="${1:-agpl-3.0.txt}"; shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ debug_echo -e "Skipping build of $prog package."
+ return 0
fi
+ fi
- # Only amd64 and aarch64 are supported
- native_arch="amd64"
- if [[ "$HOSTTYPE" == "aarch64" ]]; then
- native_arch="arm64"
- fi
+ native_arch=$(get_native_arch)
- if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
- echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
- return 1
- fi
+ if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
+ echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
+ return 1
+ fi
- if [[ -n "$target_arch" ]]; then
- # A target architecture has been specified
- package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
- return $?
- else
- # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
- # but only when building deb packages (centos does not have support for crosscompiling userspace).
- archs=($native_arch)
- if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
- archs=('amd64' 'arm64')
- fi
- for ta in $archs; do
- package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
- retval=$?
- if [[ "$retval" != 0 ]]; then
- return $retval
- fi
- done
+ if [[ -n "$target_arch" ]]; then
+ # A target architecture has been specified
+ package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
+ return $?
+ else
+ # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
+ # but only when building deb packages (centos does not have support for crosscompiling userspace).
+ archs=($native_arch)
+ if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+ archs=('amd64' 'arm64')
fi
+ for ta in $archs; do
+ package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
+ retval=$?
+ if [[ "$retval" != 0 ]]; then
+ return $retval
+ fi
+ done
+ fi
}
# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
@@ -204,7 +220,7 @@ package_go_binary_worker() {
return 1
fi
- echo "Building $pachage_format ($target_arch) package for $prog from $src_path"
+ echo "Building $package_format ($target_arch) package for $prog from $src_path"
if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
else
@@ -241,6 +257,7 @@ package_go_so() {
local description="$1"; shift
if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+ debug_echo -e "Skipping build of $pkg package."
return 0
fi
@@ -347,10 +364,9 @@ get_complete_package_name() {
fi
if [[ "$arch" == "" ]]; then
- native_arch="amd64"
+ native_arch=$(get_native_arch)
rpm_native_arch="x86_64"
if [[ "$HOSTTYPE" == "aarch64" ]]; then
- native_arch="arm64"
rpm_native_arch="arm64"
fi
rpm_architecture="$rpm_native_arch"
@@ -360,12 +376,6 @@ get_complete_package_name() {
rpm_architecture="noarch"
deb_architecture="all"
fi
-
- # These python packages have binary components
- if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="$rpm_native_arch"
- deb_architecture="$native_arch"
- fi
else
rpm_architecture=$arch
deb_architecture=$arch
@@ -502,40 +512,247 @@ handle_rails_package() {
rm -rf "$scripts_dir"
}
+# Usage: handle_api_server [amd64|arm64]
+handle_api_server () {
+ local target_arch="${1:-amd64}"; shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then
+ debug_echo -e "Skipping build of arvados-api-server package."
+ return 0
+ fi
+
+ native_arch=$(get_native_arch)
+ if [[ "$target_arch" != "$native_arch" ]]; then
+ echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH"
+ echo
+ exit 1
+ fi
+
+ # Build the API server package
+ test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+ if [[ "$?" == "0" ]]; then
+ calculate_go_package_version arvados_server_version cmd/arvados-server
+ arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+ handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados API server - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+ fi
+}
+
+# Usage: handle_workbench [amd64|arm64]
+handle_workbench () {
+ local target_arch="${1:-amd64}"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-workbench" ]] ; then
+ debug_echo -e "Skipping build of arvados-workbench package."
+ return 0
+ fi
+
+ native_arch=$(get_native_arch)
+ if [[ "$target_arch" != "$native_arch" ]]; then
+ echo "Error: no cross compilation support for Rails yet, can not build arvados-workbench for $native_arch"
+ echo
+ exit 1
+ fi
+
+ if [[ "$native_arch" != "amd64" ]]; then
+ echo "Error: building the arvados-workbench package is not yet supported on this architecture ($native_arch)."
+ echo
+ exit 1
+ fi
+
+ # Build the workbench server package
+ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+ if [[ "$?" == "0" ]] ; then
+ (
+ set -e
+
+ calculate_go_package_version arvados_server_version cmd/arvados-server
+ arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --system >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up an appropriate config.yml
+ arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
+ mkdir -p /etc/arvados/
+ mv /tmp/x /etc/arvados/config.yml
+ perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+ # Remove generated configuration files so they don't go in the package.
+ rm -rf /etc/arvados/
+ )
+
+ if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+ else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+ fi
+ fi
+}
+
+# Usage: handle_cwltest [deb|rpm] [amd64|arm64]
+handle_cwltest () {
+ local package_format="$1"; shift
+ local target_arch="${1:-amd64}"; shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "python3-cwltest" ]] ; then
+ debug_echo -e "Skipping build of cwltest package."
+ return 0
+ fi
+ cd "$WORKSPACE"
+ if [[ -e "$WORKSPACE/cwltest" ]]; then
+ rm -rf "$WORKSPACE/cwltest"
+ fi
+ git clone https://github.com/common-workflow-language/cwltest.git
+ # signal to our build script that we want a cwltest executable installed in /usr/bin/
+ mkdir cwltest/bin && touch cwltest/bin/cwltest
+ fpm_build_virtualenv "cwltest" "cwltest" "$package_format" "$target_arch"
+ # The python->python3 metapackage
+ build_metapackage "cwltest" "cwltest"
+ cd "$WORKSPACE"
+ rm -rf "$WORKSPACE/cwltest"
+}
+
+# Usage: handle_arvados_src
+handle_arvados_src () {
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
+ debug_echo -e "Skipping build of arvados-src package."
+ return 0
+ fi
+ # arvados-src
+ (
+ cd "$WORKSPACE"
+ COMMIT_HASH=$(format_last_commit_here "%H")
+ arvados_src_version="$(version_from_git)"
+
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence arvados-src "$arvados_src_version" src ""
+
+ if [[ "$?" == "0" ]]; then
+ cd "$WORKSPACE"
+ SRC_BUILD_DIR=$(mktemp -d)
+ # mktemp creates the directory with 0700 permissions by default
+ chmod 755 $SRC_BUILD_DIR
+ git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+ cd "$SRC_BUILD_DIR"
+
+ # go into detached-head state
+ git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+ echo "$COMMIT_HASH" >git-commit.version
+
+ cd $WORKSPACE/packages/$TARGET
+ fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+ rm -rf "$SRC_BUILD_DIR"
+ fi
+ )
+}
+
+# Usage: handle_libarvados_perl
+handle_libarvados_perl () {
+ if [[ -n "$ONLY_BUILD" ]] || [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then
+ debug_echo -e "Skipping build of libarvados-perl package."
+ return 0
+ fi
+ cd "$WORKSPACE/sdk/perl"
+ libarvados_perl_version="$(version_from_git)"
+
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence libarvados-perl "$libarvados_perl_version"
+
+ if [[ "$?" == "0" ]]; then
+ cd "$WORKSPACE/sdk/perl"
+
+ if [[ -e Makefile ]]; then
+ make realclean >"$STDOUT_IF_DEBUG"
+ fi
+ find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+ -delete
+ rm -rf install
+
+ perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+ fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
+ dir "$(version_from_git)" install/man/=/usr/share/man \
+ "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
+ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+ fi
+}
+
# Build python packages with a virtualenv built-in
+# Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64]
fpm_build_virtualenv () {
- PKG=$1
- shift
- PKG_DIR=$1
- shift
- PACKAGE_TYPE=${1:-python}
- shift
- native_arch="amd64"
- if [[ "$HOSTTYPE" == "aarch64" ]]; then
- native_arch="arm64"
+ local pkg=$1; shift
+ local pkg_dir=$1; shift
+ local package_format="$1"; shift
+ local target_arch="${1:-amd64}"; shift
+
+ native_arch=$(get_native_arch)
+
+ if [[ "$pkg" != "arvados-docker-cleaner" ]]; then
+ PYTHON_PKG=$PYTHON3_PKG_PREFIX-$pkg
+ else
+ # Exception to our package naming convention
+ PYTHON_PKG=$pkg
fi
- if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
- fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
- elif [[ -z "$ARCH" ]]; then
- for arch in $native_arch; do
- fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
- done
+ # arvados-python-client sdist should always be built, to be available
+ # for other dependent packages.
+ if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$pkg" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+ debug_echo -e "Skipping build of $pkg package."
+ return 0
+ fi
+
+ if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then
+ fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$target_arch"
+ elif [[ -z "$target_arch" ]]; then
+ fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$native_arch"
else
- echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+ echo "Error: no cross compilation support for Python yet, can not build $pkg for $target_arch"
+ return 1
fi
}
# Build python packages with a virtualenv built-in
+# Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64|arm64] [amd64|arm64]
fpm_build_virtualenv_worker () {
- PKG=$1
- shift
- PKG_DIR=$1
- shift
- PACKAGE_TYPE=${1:-python}
- shift
- arch=${1:-amd64}
- shift
+ PKG=$1; shift
+ PKG_DIR=$1; shift
+ local package_format="$1"; shift
+ local native_arch="${1:-amd64}"; shift
+ local target_arch=${1:-amd64}; shift
# Set up
STDOUT_IF_DEBUG=/dev/null
@@ -550,14 +767,9 @@ fpm_build_virtualenv_worker () {
ARVADOS_BUILDING_ITERATION=1
fi
- local python=""
- case "$PACKAGE_TYPE" in
- python3)
- python=python3
- pip=pip3
- PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
- ;;
- esac
+ local python=python3
+ pip=pip3
+ PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
PYTHON_PKG=$PACKAGE_PREFIX-$PKG
@@ -566,12 +778,6 @@ fpm_build_virtualenv_worker () {
PYTHON_PKG=$PKG
fi
- # arvados-python-client sdist should always be built, to be available
- # for other dependent packages.
- if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
- return 0
- fi
-
cd $WORKSPACE/$PKG_DIR
rm -rf dist/*
@@ -611,11 +817,11 @@ fpm_build_virtualenv_worker () {
# We can't do this earlier than here, because we need PYTHON_VERSION...
# This isn't so bad; the sdist call above is pretty quick compared to
# the invocation of virtualenv and fpm, below.
- if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
+ if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$python" "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
return 0
fi
- echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
+ echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
# Package the sdist in a virtualenv
echo "Creating virtualenv..."
@@ -698,10 +904,10 @@ fpm_build_virtualenv_worker () {
# Finally, generate the package
echo "Creating package..."
- declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
+ declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$package_format")
- if [[ "${arch}" != "amd64" ]]; then
- COMMAND_ARR+=("-a${arch}")
+ if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
+ COMMAND_ARR+=("-a$target_arch")
fi
if [[ "$MAINTAINER" != "" ]]; then
@@ -722,7 +928,7 @@ fpm_build_virtualenv_worker () {
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
- if [[ "$FORMAT" == "rpm" ]]; then
+ if [[ "$package_format" == "rpm" ]]; then
# Make sure to conflict with the old rh-python36 packages we used to publish
COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
fi
@@ -808,8 +1014,8 @@ fpm_build_virtualenv_worker () {
echo
echo -e "\n${COMMAND_ARR[@]}\n"
else
- echo `ls *$FORMAT`
- mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/
+ echo `ls *$package_format`
+ mv $WORKSPACE/$PKG_DIR/dist/*$package_format $WORKSPACE/packages/$TARGET/
fi
echo
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list