[arvados] updated: 2.7.0-5681-ga3b72ab765
git repository hosting
git at public.arvados.org
Sat Jan 6 21:11:24 UTC 2024
Summary of changes:
build/run-build-packages.sh | 20 ++++++++++++++++----
build/run-library.sh | 41 ++++++++++++++++++-----------------------
2 files changed, 34 insertions(+), 27 deletions(-)
via a3b72ab765012aea0926571d97ebd31ddbf9ea68 (commit)
from fc30a5602e315037ee79daebd7b93afaff96ac36 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a3b72ab765012aea0926571d97ebd31ddbf9ea68
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Jan 6 16:00:25 2024 -0500
20846: Use Python build variables more consistently
Rather than have the virtualenv process use its own python command,
ensure PYTHON3_EXECUTABLE points to a specific minor version, and use it
throughout the entire build process.
The changes to the virtualenv manipulation with sed are meant to clean
things up with fewer commands and tighter regexps, but not make any
functional change beyond referring to a specific minor Python version.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index e1363955a2..df7031fca5 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -106,8 +106,19 @@ if [[ "$DEBUG" != 0 ]]; then
DASHQ_UNLESS_DEBUG=
fi
-
-PYTHON3_EXECUTABLE=python3
+# The next section defines a bunch of constants used to build distro packages
+# for our Python tools. Because those packages include C extensions, they need
+# to depend on and refer to a specific minor version of Python 3. The logic
+# below should Just Work for most cases, but you can override variables for a
+# specific distro if you need to to do something weird.
+# * PYTHON3_VERSION: The major+minor version of Python we build against
+# (e.g., "3.11")
+# * PYTHON3_EXECUTABLE: The command to run that version of Python,
+# either a full path or something in $PATH (e.g., "python3.11")
+# * PYTHON3_PACKAGE: The name of the distro package that provides
+# $PYTHON3_EXECUTABLE. Our Python packages will all depend on this.
+# * PYTHON3_PKG_PREFIX: The prefix used in the names of all of our Python
+# packages. This should match distro convention.
PYTHON3_PKG_PREFIX=python3
case "$TARGET" in
centos*|rocky*)
@@ -121,13 +132,14 @@ case "$TARGET" in
exit 1
;;
esac
-: "${PYTHON3_VERSION:=$("$PYTHON3_EXECUTABLE" -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')}"
+: "${PYTHON3_VERSION:=$("${PYTHON3_EXECUTABLE:-python3}" -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')}"
+: "${PYTHON3_EXECUTABLE:=python$PYTHON3_VERSION}"
case "$FORMAT" in
deb)
: "${PYTHON3_PACKAGE:=python$PYTHON3_VERSION}"
;;
rpm)
- : "${PYTHON3_PACKAGE:=$(rpm -qf "$(command -v "python$PYTHON3_VERSION")" --queryformat '%{NAME}\n')}"
+ : "${PYTHON3_PACKAGE:=$(rpm -qf "$(command -v "$PYTHON3_EXECUTABLE")" --queryformat '%{NAME}\n')}"
;;
esac
diff --git a/build/run-library.sh b/build/run-library.sh
index 883219d67c..bb224a7172 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -694,9 +694,7 @@ fpm_build_virtualenv_worker () {
ARVADOS_BUILDING_ITERATION=1
fi
- local python="python$PYTHON3_VERSION"
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
-
if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
PYTHON_PKG=$PACKAGE_PREFIX-$PKG
else
@@ -707,14 +705,14 @@ fpm_build_virtualenv_worker () {
cd $WORKSPACE/$PKG_DIR
rm -rf dist/*
- local venv_dir="dist/build/usr/share/$python/dist/$PYTHON_PKG"
+ local venv_dir="dist/build/usr/share/python$PYTHON3_VERSION/dist/$PYTHON_PKG"
echo "Creating virtualenv..."
- if ! "$python" -m venv "$venv_dir"; then
- printf "Error, unable to run\n %s -m venv %s\n" "$python" "$venv_dir"
+ if ! "$PYTHON3_EXECUTABLE" -m venv "$venv_dir"; then
+ printf "Error, unable to run\n %s -m venv %s\n" "$PYTHON3_EXECUTABLE" "$venv_dir"
exit 1
fi
- local venv_py="$venv_dir/bin/$python"
+ local venv_py="$venv_dir/bin/python$PYTHON3_VERSION"
if ! "$venv_py" -m pip install --upgrade $DASHQ_UNLESS_DEBUG $CACHE_FLAG pip setuptools wheel; then
printf "Error, unable to upgrade pip, setuptools, and wheel with
%s -m pip install --upgrade $DASHQ_UNLESS_DEBUG $CACHE_FLAG pip setuptools wheel
@@ -747,7 +745,7 @@ fpm_build_virtualenv_worker () {
# See if we actually need to build this package; does it exist already?
# We can't do this earlier than here, because we need PYTHON_VERSION.
- if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$python" "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
+ if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" python3 "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
return 0
fi
@@ -767,19 +765,16 @@ fpm_build_virtualenv_worker () {
# scripts too. This is a functional replacement of the 237 line
# virtualenv_tools.py script that doesn't work in python3 without serious
# patching, minus the parts we don't need (modifying pyc files, etc).
+ local sys_venv_dir="${venv_dir#dist/build/}"
+ local sys_venv_py="$sys_venv_dir/bin/python$PYTHON3_VERSION"
for binfile in `ls bin/`; do
- if ! file --mime bin/$binfile |grep -q binary; then
- # Not a binary file
- if [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
- # these 'activate' scripts need special treatment
- sed -i "s/VIRTUAL_ENV=\".*\"/VIRTUAL_ENV=\"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
- sed -i "s/VIRTUAL_ENV \".*\"/VIRTUAL_ENV \"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
- else
- if grep -q -E '^#!.*/bin/python\d?' bin/$binfile; then
- # Replace shebang line
- sed -i "1 s/^.*$/#!\/usr\/share\/$python\/dist\/$PYTHON_PKG\/bin\/python/" bin/$binfile
- fi
- fi
+ if file --mime "bin/$binfile" | grep -q binary; then
+ : # Nothing to do for binary files
+ elif [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
+ sed -ri "s at VIRTUAL_ENV(=| )\".*\"@VIRTUAL_ENV\\1\"/$sys_venv_dir\"@" "bin/$binfile"
+ else
+ # Replace shebang line
+ sed -ri "1 s@^#\![^[:space:]]+/bin/python[0-9.]*@#\!/$sys_venv_py@" "bin/$binfile"
fi
done
@@ -873,7 +868,7 @@ fpm_build_virtualenv_worker () {
# make sure the systemd service file ends up in the right place
# used by arvados-docker-cleaner
if [[ -e "${systemd_unit}" ]]; then
- COMMAND_ARR+=("usr/share/$python/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
+ COMMAND_ARR+=("usr/share/python$PYTHON3_VERSION/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
fi
COMMAND_ARR+=("${fpm_args[@]}")
@@ -886,13 +881,13 @@ fpm_build_virtualenv_worker () {
# because those are the ones we rewrote the shebang line of, above.
if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
- COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/$binary=/usr/bin/")
+ COMMAND_ARR+=("$sys_venv_dir/bin/$binary=/usr/bin/")
done
fi
# the python3-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
+ if [[ -e "$WORKSPACE/$PKG_DIR/$venv_dir/bin/cwltool" ]]; then
+ COMMAND_ARR+=("$sys_venv_dir/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list