[arvados] created: 2.7.0-6264-gaca95ae873
git repository hosting
git at public.arvados.org
Sun Mar 24 02:30:09 UTC 2024
at aca95ae8737d90adcee57929caa4239d2a7e1f66 (commit)
commit aca95ae8737d90adcee57929caa4239d2a7e1f66
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 23 14:16:35 2024 -0400
21601: Build Python distro packages from wheels
The immediate problem this solves is that, by building and installing
from a repository of wheels, Python packages can find their
interdependencies without any special logic in the build process.
Other benefits:
* Eliminates some redundant work. We don't have to build the Python SDK
from source multiple times. We can use the published cwltest wheel
instead of building our own.
* Prepares the code for PEP 517 compliance. We only invoke setup.py to
build packages that have not been updated yet. We introspect packages
from their wheels, so we no longer have to introspect the source to
build distro packages.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/pypkg_info.py b/build/pypkg_info.py
new file mode 100644
index 0000000000..45f8d16eab
--- /dev/null
+++ b/build/pypkg_info.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+"""pypkg_info.py - Introspect installed Python packages
+
+This tool can read metadata about any Python package installed in the current
+environment and report it out in various formats. We use this mainly to pass
+information through when building distribution packages.
+"""
+
+import argparse
+import enum
+import importlib.metadata
+import os
+import sys
+
+from pathlib import PurePath
+
+class RawFormat:
+ def format_metadata(self, key, value):
+ return value
+
+ def format_path(self, path):
+ return str(path)
+
+
+class FPMFormat(RawFormat):
+ PYTHON_METADATA_MAP = {
+ 'summary': 'description',
+ }
+
+ def format_metadata(self, key, value):
+ key = key.lower()
+ key = self.PYTHON_METADATA_MAP.get(key, key)
+ return f'--{key}={value}'
+
+
+class Formats(enum.Enum):
+ RAW = RawFormat
+ FPM = FPMFormat
+
+ @classmethod
+ def from_arg(cls, arg):
+ try:
+ return cls[arg.upper()]
+ except KeyError:
+ raise ValueError(f"unknown format {arg!r}") from None
+
+
+def report_binfiles(args):
+ bin_names = [
+ PurePath('bin', path.name)
+ for pkg_name in args.package_names
+ for path in importlib.metadata.distribution(pkg_name).files
+ if path.parts[-3:-1] == ('..', 'bin')
+ ]
+ fmt = args.format.value().format_path
+ return (fmt(path) for path in bin_names)
+
+def report_metadata(args):
+ dist = importlib.metadata.distribution(args.package_name)
+ fmt = args.format.value().format_metadata
+ for key in args.metadata_key:
+ yield fmt(key, dist.metadata.get(key, ''))
+
+def unescape_str(arg):
+ arg = arg.replace('\'', '\\\'')
+ return eval(f"'''{arg}'''", {})
+
+def parse_arguments(arglist=None):
+ parser = argparse.ArgumentParser()
+ parser.set_defaults(action=None)
+ format_names = ', '.join(fmt.name.lower() for fmt in Formats)
+ parser.add_argument(
+ '--format', '-f',
+ choices=list(Formats),
+ default=Formats.RAW,
+ type=Formats.from_arg,
+ help=f"Output format. Choices are: {format_names}",
+ )
+ parser.add_argument(
+ '--delimiter', '-d',
+ default='\n',
+ type=unescape_str,
+ help="Line ending. Python backslash escapes are supported. Default newline.",
+ )
+ subparsers = parser.add_subparsers()
+
+ binfiles = subparsers.add_parser('binfiles')
+ binfiles.set_defaults(action=report_binfiles)
+ binfiles.add_argument(
+ 'package_names',
+ nargs=argparse.ONE_OR_MORE,
+ )
+
+ metadata = subparsers.add_parser('metadata')
+ metadata.set_defaults(action=report_metadata)
+ metadata.add_argument(
+ 'package_name',
+ )
+ metadata.add_argument(
+ 'metadata_key',
+ nargs=argparse.ONE_OR_MORE,
+ )
+
+ args = parser.parse_args()
+ if args.action is None:
+ parser.error("subcommand is required")
+ return args
+
+def main(arglist=None):
+ args = parse_arguments(arglist)
+ try:
+ for line in args.action(args):
+ print(line, end=args.delimiter)
+ except importlib.metadata.PackageNotFoundError as error:
+ print(f"error: package not found: {error.args[0]}", file=sys.stderr)
+ return os.EX_NOTFOUND
+ else:
+ return os.EX_OK
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 77ce054318..ada3bf8b6c 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -278,27 +278,17 @@ package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
# Python packages
debug_echo -e "\nPython packages\n"
-# The Python SDK - Python3 package
+# Before a Python package can be built, its dependencies must already be built.
+# This list is ordered accordingly.
+setup_build_virtualenv
+fpm_build_virtualenv cwltest "==2.3.20230108193615" "$FORMAT" "$ARCH"
fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
-
-# Arvados cwl runner - Python3 package
-fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
-
-# The FUSE driver - Python3 package
-fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
-
-# The Arvados crunchstat-summary tool
fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
-
-# The Docker image cleaner
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
-
-# The Arvados user activity tool
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
-# The cwltest package, which lives out of tree
-handle_cwltest "$FORMAT" "$ARCH"
-
# Workbench2
package_workbench2
diff --git a/build/run-library.sh b/build/run-library.sh
index a395db8b77..b15446ec53 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -121,8 +121,8 @@ package_workbench2() {
local src=services/workbench2
local dst=/var/www/arvados-workbench2/workbench2
local description="Arvados Workbench 2"
- local version="$(version_from_git)"
cd "$WORKSPACE/$src"
+ local version="$(version_from_git)"
rm -rf ./build
NODE_ENV=production yarn install
VERSION="$version" BUILD_NUMBER="$(default_iteration "$pkgname" "$version" yarn)" GIT_COMMIT="$(git rev-parse HEAD | head -c9)" yarn build
@@ -566,34 +566,6 @@ handle_api_server () {
fi
}
-# Usage: handle_cwltest [deb|rpm] [amd64|arm64]
-handle_cwltest () {
- local package_format="$1"; shift
- local target_arch="${1:-amd64}"; shift
-
- if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "python3-cwltest" ]] ; then
- debug_echo -e "Skipping build of cwltest package."
- return 0
- fi
- cd "$WORKSPACE"
- if [[ -e "$WORKSPACE/cwltest" ]]; then
- rm -rf "$WORKSPACE/cwltest"
- fi
- git clone https://github.com/common-workflow-language/cwltest.git
-
- # The subsequent release of cwltest confirms that files exist on disk, since
- # our files are in Keep, all the tests fail.
- # We should add [optional] Arvados support to cwltest so it can access
- # Keep but for the time being just package the last working version.
- (cd cwltest && git checkout 2.3.20230108193615)
-
- # signal to our build script that we want a cwltest executable installed in /usr/bin/
- mkdir cwltest/bin && touch cwltest/bin/cwltest
- fpm_build_virtualenv "cwltest" "cwltest" "$package_format" "$target_arch"
- cd "$WORKSPACE"
- rm -rf "$WORKSPACE/cwltest"
-}
-
# Usage: handle_arvados_src
handle_arvados_src () {
if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
@@ -629,6 +601,13 @@ handle_arvados_src () {
)
}
+setup_build_virtualenv() {
+ PYTHON_BUILDROOT="$(mktemp --directory --tmpdir pybuild.XXXXXXXX)"
+ "$PYTHON3_EXECUTABLE" -m venv "$PYTHON_BUILDROOT/venv"
+ "$PYTHON_BUILDROOT/venv/bin/pip" install --upgrade build piprepo setuptools wheel
+ mkdir "$PYTHON_BUILDROOT/wheelhouse"
+}
+
# Build python packages with a virtualenv built-in
# Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64]
fpm_build_virtualenv () {
@@ -638,27 +617,6 @@ fpm_build_virtualenv () {
local target_arch="${1:-amd64}"; shift
native_arch=$(get_native_arch)
-
- if [[ "$pkg" != "arvados-docker-cleaner" ]]; then
- PYTHON_PKG=$PYTHON3_PKG_PREFIX-$pkg
- else
- # Exception to our package naming convention
- PYTHON_PKG=$pkg
- fi
-
- if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]]; then
- # arvados-python-client sdist should always be built if we are building a
- # python package.
- if [[ "$ONLY_BUILD" != "python3-arvados-cwl-runner" ]] &&
- [[ "$ONLY_BUILD" != "python3-arvados-fuse" ]] &&
- [[ "$ONLY_BUILD" != "python3-crunchstat-summary" ]] &&
- [[ "$ONLY_BUILD" != "arvados-docker-cleaner" ]] &&
- [[ "$ONLY_BUILD" != "python3-arvados-user-activity" ]]; then
- debug_echo -e "Skipping build of $pkg package."
- return 0
- fi
- fi
-
if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then
fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$target_arch"
elif [[ -z "$target_arch" ]]; then
@@ -699,91 +657,100 @@ fpm_build_virtualenv_worker () {
PYTHON_PKG=$PKG
fi
- cd $WORKSPACE/$PKG_DIR
+ # We must always add a wheel to our repository, even if we're not building
+ # this distro package, because it might be a dependency for a later
+ # package we do build.
+ if [[ "$PKG_DIR" =~ ^.=[0-9]+\. ]]; then
+ # Not source to build, but a version to download.
+ # The rest of the function expects a filesystem path, so set one afterwards.
+ "$PYTHON_BUILDROOT/venv/bin/pip" download --dest="$PYTHON_BUILDROOT/wheelhouse" "$PKG$PKG_DIR" \
+ && PKG_DIR="$PYTHON_BUILDROOT/nonexistent"
+ else
+ # Make PKG_DIR absolute.
+ PKG_DIR="$(env -C "$WORKSPACE" readlink -e "$PKG_DIR")"
+ if [[ -e "$PKG_DIR/pyproject.toml" ]]; then
+ "$PYTHON_BUILDROOT/venv/bin/python" -m build --outdir="$PYTHON_BUILDROOT/wheelhouse" "$PKG_DIR"
+ else
+ env -C "$PKG_DIR" "$PYTHON_BUILDROOT/venv/bin/python" setup.py bdist_wheel --dist-dir="$PYTHON_BUILDROOT/wheelhouse"
+ fi
+ fi
+ if [[ $? -ne 0 ]]; then
+ printf "Error, unable to download/build wheel for %s @ %s" "$PKG" "$PKG_DIR"
+ exit 1
+ elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
+ printf "Error, unable to update local wheel repository"
+ exit 1
+ fi
- rm -rf dist/*
- local venv_dir="dist/build/usr/lib/$PYTHON_PKG"
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ local venv_dir="$PYTHON_BUILDROOT/$PYTHON_PKG"
echo "Creating virtualenv..."
if ! "$PYTHON3_EXECUTABLE" -m venv "$venv_dir"; then
printf "Error, unable to run\n %s -m venv %s\n" "$PYTHON3_EXECUTABLE" "$venv_dir"
exit 1
fi
- local venv_py="$venv_dir/bin/python$PYTHON3_VERSION"
- if ! "$venv_py" -m pip install --upgrade $DASHQ_UNLESS_DEBUG $CACHE_FLAG pip setuptools wheel; then
- printf "Error, unable to upgrade pip, setuptools, and wheel with
- %s -m pip install --upgrade $DASHQ_UNLESS_DEBUG $CACHE_FLAG pip setuptools wheel
-" "$venv_py"
+ local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
+ if [[ -z "$pip_wheel" ]]; then
+ printf "Error, unable to find built wheel for $PKG"
exit 1
- fi
-
- # filter a useless warning (when building the cwltest package) from the stderr output
- if ! "$venv_py" setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
- echo "Error, unable to run $venv_py setup.py sdist for $PKG"
+ elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
+ printf "Error, unable to run
+ %s/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url=file://%s %s
+" "$venv_dir" "$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"
exit 1
fi
- if [[ "arvados-python-client" == "$PKG" ]]; then
- PYSDK_PATH="-f $(pwd)/dist/"
- fi
-
- if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
- return 0
- fi
-
- # Determine the package version from the generated sdist archive
- if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
- UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
- PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
- else
- PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
- UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
- fi
+ # Determine the package version from the wheel
+ PYTHON_VERSION="$("$venv_dir/bin/python" "$WORKSPACE/build/pypkg_info.py" metadata "$PKG" Version)"
+ UNFILTERED_PYTHON_VERSION="$(echo "$PYTHON_VERSION" | sed 's/\.dev/~dev/; s/\([0-9]\)rc/\1~rc/')"
# See if we actually need to build this package; does it exist already?
# We can't do this earlier than here, because we need PYTHON_VERSION.
if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" python3 "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
return 0
fi
-
echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
- local sdist_path="$(ls dist/*.tar.gz)"
- if ! "$venv_py" -m pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PYSDK_PATH "$sdist_path"; then
- printf "Error, unable to run
- %s -m pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG %s %s
-" "$venv_py" "$PYSDK_PATH" "$sdist_path"
- exit 1
- fi
-
- pushd "$venv_dir" >$STDOUT_IF_DEBUG
-
# Replace the shebang lines in all python scripts, and handle the activate
# scripts too. This is a functional replacement of the 237 line
# virtualenv_tools.py script that doesn't work in python3 without serious
# patching, minus the parts we don't need (modifying pyc files, etc).
- local sys_venv_dir="${venv_dir#dist/build/}"
+ local sys_venv_dir="/usr/lib/$PYTHON_PKG"
local sys_venv_py="$sys_venv_dir/bin/python$PYTHON3_VERSION"
- for binfile in `ls bin/`; do
- if file --mime "bin/$binfile" | grep -q binary; then
+ find "$venv_dir/bin" -type f | while read binfile; do
+ if file --mime "$binfile" | grep -q binary; then
: # Nothing to do for binary files
- elif [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
- sed -ri "s at VIRTUAL_ENV(=| )\".*\"@VIRTUAL_ENV\\1\"/$sys_venv_dir\"@" "bin/$binfile"
+ elif [[ "$binfile" =~ /activate(.csh|.fish|)$ ]]; then
+ sed -ri "s at VIRTUAL_ENV(=| )\".*\"@VIRTUAL_ENV\\1\"$sys_venv_dir\"@" "$binfile"
else
# Replace shebang line
- sed -ri "1 s@^#\![^[:space:]]+/bin/python[0-9.]*@#\!/$sys_venv_py@" "bin/$binfile"
+ sed -ri "1 s@^#\![^[:space:]]+/bin/python[0-9.]*@#\!$sys_venv_py@" "$binfile"
fi
done
- popd >$STDOUT_IF_DEBUG
- cd dist
-
- find build -iname '*.py[co]' -delete
-
- # Finally, generate the package
- echo "Creating package..."
-
- declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$package_format")
+ # Using `env -C` sets the directory where the package is built.
+ # Using `fpm --chdir` sets the root directory for source arguments.
+ declare -a COMMAND_ARR=(
+ env -C "$PYTHON_BUILDROOT" fpm
+ --chdir="$venv_dir"
+ --name="$PYTHON_PKG"
+ --version="$UNFILTERED_PYTHON_VERSION"
+ --input-type=dir
+ --output-type="$package_format"
+ --depends="$PYTHON3_PACKAGE"
+ --iteration="$ARVADOS_BUILDING_ITERATION"
+ --replaces="python-$PKG"
+ --url="https://arvados.org"
+ )
+ # Append fpm flags corresponding to Python package metadata.
+ readarray -d "" -O "${#COMMAND_ARR[@]}" -t COMMAND_ARR < \
+ <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" \
+ --delimiter=\\0 --format=fpm \
+ metadata "$PKG" License Summary)
if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
COMMAND_ARR+=("-a$target_arch")
@@ -797,32 +764,16 @@ fpm_build_virtualenv_worker () {
COMMAND_ARR+=('--vendor' "$VENDOR")
fi
- COMMAND_ARR+=('--url' 'https://arvados.org')
-
- # Get description
- DESCRIPTION=`grep '\sdescription' $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
- COMMAND_ARR+=('--description' "$DESCRIPTION")
-
- # Get license string
- LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
- COMMAND_ARR+=('--license' "$LICENSE_STRING")
-
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g))
- COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
- COMMAND_ARR+=('-n' "$PYTHON_PKG")
- COMMAND_ARR+=('-C' "build")
-
- systemd_unit="$WORKSPACE/$PKG_DIR/$PKG.service"
+ systemd_unit="$PKG_DIR/$PKG.service"
if [[ -e "${systemd_unit}" ]]; then
COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
fi
- COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
case "$package_format" in
deb)
COMMAND_ARR+=(
@@ -845,7 +796,7 @@ fpm_build_virtualenv_worker () {
declare -a fpm_args=()
declare -a fpm_depends=()
- fpminfo="$WORKSPACE/$PKG_DIR/fpm-info.sh"
+ fpminfo="$PKG_DIR/fpm-info.sh"
if [[ -e "$fpminfo" ]]; then
echo "Loading fpm overrides from $fpminfo"
if ! source "$fpminfo"; then
@@ -858,37 +809,24 @@ fpm_build_virtualenv_worker () {
COMMAND_ARR+=('--depends' "$i")
done
- for i in "${fpm_depends[@]}"; do
- COMMAND_ARR+=('--replaces' "python-$PKG")
- done
-
# make sure the systemd service file ends up in the right place
# used by arvados-docker-cleaner
if [[ -e "${systemd_unit}" ]]; then
- COMMAND_ARR+=("$sys_venv_dir/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
+ COMMAND_ARR+=("share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
fi
COMMAND_ARR+=("${fpm_args[@]}")
- # Make sure to install all our package binaries in /usr/bin. We have to
- # walk $WORKSPACE/$PKG_DIR/bin rather than $venv_dir/bin to get the list
- # because the latter also includes scripts installed by all the
- # dependencies in the virtualenv, which may conflict with other
- # packages. We have to take the copies of our binaries from the latter
- # directory, though, because those are the ones we rewrote the shebang
- # line of, above.
- if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
- for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
- COMMAND_ARR+=("$sys_venv_dir/bin/$binary=/usr/bin/")
- done
- fi
+ while read -d "" binpath; do
+ COMMAND_ARR+=("$binpath=/usr/$binpath")
+ done < <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" --delimiter=\\0 binfiles "$PKG")
# the python3-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/$venv_dir/bin/cwltool" ]]; then
- COMMAND_ARR+=("$sys_venv_dir/bin/cwltool=/usr/bin/")
+ if [[ "$PKG" == arvados-cwl-runner ]]; then
+ COMMAND_ARR+=("bin/cwltool=/usr/bin/cwltool")
fi
- COMMAND_ARR+=(".")
+ COMMAND_ARR+=(".=$sys_venv_dir")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
@@ -901,8 +839,8 @@ fpm_build_virtualenv_worker () {
echo
echo -e "\n${COMMAND_ARR[@]}\n"
else
- echo `ls *$package_format`
- mv $WORKSPACE/$PKG_DIR/dist/*$package_format $WORKSPACE/packages/$TARGET/
+ ls "$PYTHON_BUILDROOT"/*."$package_format"
+ mv "$PYTHON_BUILDROOT"/*."$package_format" "$WORKSPACE/packages/$TARGET/"
fi
echo
}
commit 64f53eb394384ba13e2e33f4794516577e4db528
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 23 14:17:47 2024 -0400
21601: Remove PySDK from `install deps`
It's already installed by `install env`. 🤷
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index c425a48a94..ca7c793a49 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1058,7 +1058,6 @@ install_deps() {
# Install parts needed by test suites
do_install env
do_install cmd/arvados-server go
- do_install sdk/python pip "${VENV3DIR}/bin/"
do_install tools/crunchstat-summary pip "${VENV3DIR}/bin/"
do_install sdk/ruby-google-api-client
do_install sdk/ruby
commit 55703cfd8ee34f76a230ff256f51367f4fe3f77e
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Mar 18 23:04:59 2024 -0400
21601: Install wheel in run-tests.sh VENV3DIR
We do this basically everywhere else in our build infrastructure, so
doing it here helps avoid discrepancies between those environments.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 1f28915a29..c425a48a94 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -636,13 +636,15 @@ install_env() {
setup_virtualenv "$VENV3DIR"
. "$VENV3DIR/bin/activate"
+ # wheel modernizes the venv (as of early 2024) and makes it more closely
+ # match our package build environment.
# PyYAML is a test requirement used by run_test_server.py and needed for
# other, non-Python tests.
# pdoc is needed to build PySDK documentation.
# We run `setup.py build` first to generate _version.py.
- env -C "$WORKSPACE/sdk/python" python3 setup.py build \
- && python3 -m pip install "$WORKSPACE/sdk/python" \
- && python3 -m pip install PyYAML pdoc \
+ pip install PyYAML pdoc wheel \
+ && env -C "$WORKSPACE/sdk/python" python3 setup.py build \
+ && pip install "$WORKSPACE/sdk/python" \
|| fatal "installing Python SDK and related dependencies failed"
}
commit 81b42d35b22babd40a068a33f40d9be2c94eb5e5
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:23:44 2024 -0400
21601: Specify Python interdependencies with ~=
This has the same rationale as using <= before, but it's stricter. It
should prevent pip from using release versions to satisfy development
dependencies in the future, and help root out bugs in our build
processes.
DRY up this logic in arvados_version.py.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 043b52cb81..551bd964b1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -9,16 +9,9 @@ import sys
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_cwl")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados-cwl-runner',
version=version,
@@ -36,10 +29,9 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
+ *arvados_version.iter_dependencies(version),
'cwltool==3.1.20230601100705',
'schema-salad==8.4.20230601112322',
- 'arvados-python-client{}'.format(pysdk_dep),
- 'crunchstat-summary{}'.format(pysdk_dep),
'ciso8601 >= 2.0.0',
'networkx < 2.6',
'msgpack==1.0.3',
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 00429de844..e13e51609a 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -12,12 +12,10 @@ from pathlib import Path
from setuptools import setup, find_packages
from setuptools.command import build_py
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados")
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
class BuildPython(build_py.build_py):
"""Extend setuptools `build_py` to generate API documentation
@@ -111,6 +109,7 @@ setup(name='arvados-python-client',
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
+ *arvados_version.iter_dependencies(version),
'ciso8601 >=2.0.0',
'future',
'google-api-core <2.11.0', # 2.11.0rc1 is incompatible with google-auth<2
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 565fdcd1ea..9c69879b45 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -10,12 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name="arvados-docker-cleaner",
version=version,
@@ -33,6 +31,7 @@ setup(name="arvados-docker-cleaner",
('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt', 'arvados-docker-cleaner.service']),
],
install_requires=[
+ *arvados_version.iter_dependencies(version),
'docker>=6.1.0',
'setuptools',
],
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 80a26980f1..07c9dcf5e1 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -10,17 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_fuse")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados_fuse',
version=version,
@@ -39,7 +32,7 @@ setup(name='arvados_fuse',
('share/doc/arvados_fuse', ['agpl-3.0.txt', 'README.rst']),
],
install_requires=[
- 'arvados-python-client{}'.format(pysdk_dep),
+ *arvados_version.iter_dependencies(version),
'llfuse >= 1.3.6',
'future',
'python-daemon',
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 4bb9df2ba9..24a6bf5e4f 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -10,17 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='crunchstat_summary',
version=version,
@@ -39,7 +32,7 @@ setup(name='crunchstat_summary',
('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client{}'.format(pysdk_dep),
+ *arvados_version.iter_dependencies(version),
],
python_requires="~=3.8",
test_suite='tests',
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index b594f88a7c..794b6afe42 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -12,6 +12,7 @@
# it reads _version.py and generates dependencies from it.
import os
+import re
import runpy
import subprocess
import sys
@@ -123,6 +124,22 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ # A packaged development release should be installed with other
+ # development packages built from the same source, but those
+ # dependencies may have earlier "dev" versions (read: less recent
+ # Git commit timestamps). This compatible version dependency
+ # expresses that as closely as possible. Allowing versions
+ # compatible with .dev0 allows any development release.
+ # Regular expression borrowed partially from
+ # <https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers-regex>
+ dep_ver, match_count = re.subn(r'\.dev(0|[1-9][0-9]*)$', '.dev0', version, 1)
+ dep_op = '~=' if match_count else '=='
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield f'{dep_pkg}{dep_op}{dep_ver}'
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
index 4b7ec16b93..8611fa47a1 100755
--- a/tools/user-activity/setup.py
+++ b/tools/user-activity/setup.py
@@ -10,11 +10,9 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados-user-activity',
version=version,
@@ -31,7 +29,7 @@ setup(name='arvados-user-activity',
('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client >= 2.2.0.dev20201118185221',
+ *arvados_version.iter_dependencies(version),
],
python_requires="~=3.8",
zip_safe=True,
commit a683ec930d6658145b2e96131288f4ef3000a7d5
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:10:15 2024 -0400
21601: Move --short-tests-only check to arvados_version
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index e3d66aa472..00429de844 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -17,11 +17,7 @@ README = os.path.join(SETUP_DIR, 'README.rst')
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "arvados")
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
class BuildPython(build_py.build_py):
"""Extend setuptools `build_py` to generate API documentation
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 2b386c70b4..565fdcd1ea 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -15,11 +15,7 @@ README = os.path.join(SETUP_DIR, 'README.rst')
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name="arvados-docker-cleaner",
version=version,
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index b04829652e..80a26980f1 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -20,11 +20,7 @@ if os.environ.get('ARVADOS_BUILDING_VERSION', False):
else:
# On dev releases, arvados-python-client may have a different timestamp
pysdk_dep = "<={}".format(version)
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name='arvados_fuse',
version=version,
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 98be9f2702..4bb9df2ba9 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -20,11 +20,7 @@ if os.environ.get('ARVADOS_BUILDING_VERSION', False):
else:
# On dev releases, arvados-python-client may have a different timestamp
pysdk_dep = "<={}".format(version)
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name='crunchstat_summary',
version=version,
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
commit 7548c375d046ce564630de403f8707baab547e1a
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:02:59 2024 -0400
21601: Make arvados_version.py more declarative
The main goal of this change is to introduce the metadata maps near the
top of the file, which we will use to build additional
functionality. The rest of the changes are just modernization or
clean-up based on that.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index a78dbfcf2b..020ec38738 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -1,63 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../tools/crunchstat-summary")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
-
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
-
- return read_version(setup_dir, module)
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
- print(get_version(SETUP_DIR, "arvados_cwl"))
+ print(get_version())
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 092131d930..020ec38738 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -1,61 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
-
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
-
- return read_version(setup_dir, module)
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
- print(get_version(SETUP_DIR, "arvados"))
+ print(get_version())
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 38e6f564e7..020ec38738 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -1,57 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list