[arvados] created: 2.7.0-5817-g7bfde74c81
git repository hosting
git at public.arvados.org
Fri Jan 12 20:47:24 UTC 2024
at 7bfde74c810b997a300b42a007b096a30233d8a1 (commit)
commit 7bfde74c810b997a300b42a007b096a30233d8a1
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Jan 12 14:30:45 2024 -0500
21230: Fix binstubs generation in arvbox
At some point, bundle started installing binstubs in `bin/` by
default. Various pieces of arvbox assume they go in `binstubs/` and
break when they don't. Explicitly pass options to get the old behavior
where needed.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 218c6361ba..3e00b0ea20 100755
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -9,12 +9,10 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
cd /usr/src/arvados/sdk/ruby
-run_bundler
-bundler_binstubs
+run_bundler --binstubs=binstubs
cd /usr/src/arvados/sdk/cli
-run_bundler
-bundler_binstubs
+run_bundler --binstubs=binstubs
python_srcdir="$(mktemp --directory --tmpdir pysrc.XXXXXXXX)"
trap 'rm -rf "$python_srcdir"' EXIT INT TERM QUIT
diff --git a/tools/arvbox/lib/arvbox/docker/service/vm/run-service b/tools/arvbox/lib/arvbox/docker/service/vm/run-service
index 5e952b8d5c..2079bb1d0b 100755
--- a/tools/arvbox/lib/arvbox/docker/service/vm/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/vm/run-service
@@ -16,8 +16,7 @@ if test "$1" != "--only-deps" ; then
fi
cd /usr/src/arvados/services/login-sync
-run_bundler
-bundler_binstubs
+run_bundler --binstubs=binstubs
if test "$1" = "--only-deps" ; then
exit
commit 946fb6cd2a97b715b285e326c244debacbec05a2
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Jan 12 10:52:00 2024 -0500
21230: Use virtualenv in arvbox
This is preparation to let us build arvbox on Debian 12, which has
adopted PEP 668 and doesn't permit installing packages outside a
virtualenv anymore.
This does move pdoc installation out of the doc service and into the
Dockerfile. The doc code currently doesn't have any way to version this
dependency, and it rarely changes in development, so this seems fine as
a code simplification.
All the code that tries to "pip install from cache only, then use the
network" is gone because as best I can tell it's already non-functional:
the pip cache isn't structured in the way that it assumes
anymore. Generally pip looks at what's installed in the virtualenv and
only installs what it *must* to satisfy dependencies, so this seems
fine.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 778b752fd4..6a03b303f7 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -79,10 +79,10 @@ FROM debian:11
ENV DEBIAN_FRONTEND noninteractive
# The arvbox-specific dependencies are
-# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less
+# gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less
RUN apt-get update && \
apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
- gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less vim-tiny && \
+ gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less vim-tiny && \
apt-get clean
ENV GOPATH /var/lib/gopath
@@ -93,6 +93,18 @@ COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server
RUN $GOPATH/bin/arvados-server --version
RUN $GOPATH/bin/arvados-server install -type test
+# Set up a virtualenv for all Python tools in arvbox.
+# This is used mainly by the `sdk` service, but `doc` and internal scripts
+# also rely on it.
+# 1. Install wheel just to modernize the virtualenv.
+# 2. Install setuptools as an sdk build dependency; PyYAML for all tests
+# and yml_override;py; and pdoc for the doc service.
+# Everything else is installed by the sdk service on boot.
+RUN python3 -m venv /opt/arvados-py \
+ && /opt/arvados-py/bin/pip install --no-cache-dir wheel \
+ && /opt/arvados-py/bin/pip install --no-cache-dir setuptools PyYAML pdoc \
+ && ln -s /opt/arvados-py/bin/pdoc /usr/local/bin/
+
RUN /etc/init.d/postgresql start && \
su postgres -c 'dropuser arvados' && \
su postgres -c 'createuser -s arvbox' && \
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 9c5df83c0e..81516ef08a 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -6,7 +6,7 @@ export RUBY_VERSION=3.2.2
export BUNDLER_VERSION=2.4.22
export DEBIAN_FRONTEND=noninteractive
-export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
@@ -100,24 +100,3 @@ bundler_binstubs() {
fi
flock $GEMLOCK $BUNDLER binstubs --all
}
-
-PYCMD=""
-pip_install() {
- pushd /var/lib/pip
- for p in $(ls http*.tar.gz) $(ls http*.tar.bz2) $(ls http*.whl) $(ls http*.zip) ; do
- if test -f $p ; then
- ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
- fi
- done
- popd
-
- if [ "$PYCMD" = "python3" ]; then
- if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
- pip3 install --prefix /usr/local $1
- fi
- else
- if ! pip install --no-index --find-links /var/lib/pip $1 ; then
- pip install $1
- fi
- fi
-}
diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh
index 4cafd8c09c..9224b80f52 100755
--- a/tools/arvbox/lib/arvbox/docker/createusers.sh
+++ b/tools/arvbox/lib/arvbox/docker/createusers.sh
@@ -14,6 +14,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
mkdir -p $ARVADOS_CONTAINER_PATH/git \
/var/lib/passenger /var/lib/gopath \
/var/lib/pip /var/lib/npm
+ /opt/arvados-py/bin/pip config --site set global.cache-dir /var/lib/pip
if test -z "$ARVBOX_HOME" ; then
ARVBOX_HOME=$ARVADOS_CONTAINER_PATH
@@ -31,7 +32,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
useradd --groups docker crunch
if [[ "$1" != --no-chown ]] ; then
- chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH \
+ chown arvbox:arvbox -R /usr/local /opt/arvados-py $ARVADOS_CONTAINER_PATH \
/var/lib/passenger /var/lib/postgresql \
/var/lib/nginx /var/log/nginx /etc/ssl/private \
/var/lib/gopath /var/lib/pip /var/lib/npm
@@ -43,7 +44,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
-export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
diff --git a/tools/arvbox/lib/arvbox/docker/edit_users.py b/tools/arvbox/lib/arvbox/docker/edit_users.py
index ab046b11d4..cb44b984b7 100755
--- a/tools/arvbox/lib/arvbox/docker/edit_users.py
+++ b/tools/arvbox/lib/arvbox/docker/edit_users.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/opt/arvados-py/bin/python3
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
diff --git a/tools/arvbox/lib/arvbox/docker/service/doc/run-service b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
index c40e1175ed..8d3fdc8365 100755
--- a/tools/arvbox/lib/arvbox/docker/service/doc/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
@@ -18,13 +18,10 @@ cd /usr/src/arvados/doc
run_bundler --without=development
# Generating the Python and R docs is expensive, so for development if the file
-# "no-sdk" exists then skip the Python and R stuff.
+# "no-sdk" exists then skip installing R stuff.
if [[ ! -f /usr/src/arvados/doc/no-sdk ]] ; then
cd /usr/src/arvados/sdk/R
R --quiet --vanilla --file=install_deps.R
-
- export PYCMD=python3
- pip_install pdoc
fi
if test "$1" = "--only-deps" ; then
diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 5bff561052..218c6361ba 100755
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -8,12 +8,6 @@ set -eux -o pipefail
. /usr/local/lib/arvbox/common.sh
-mkdir -p ~/.pip /var/lib/pip
-cat > ~/.pip/pip.conf <<EOF
-[global]
-download_cache = /var/lib/pip
-EOF
-
cd /usr/src/arvados/sdk/ruby
run_bundler
bundler_binstubs
@@ -22,18 +16,10 @@ cd /usr/src/arvados/sdk/cli
run_bundler
bundler_binstubs
-export PYCMD=python3
-
-pip_install wheel
-
-cd /usr/src/arvados/sdk/python
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/services/fuse
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/sdk/cwl
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
+python_srcdir="$(mktemp --directory --tmpdir pysrc.XXXXXXXX)"
+trap 'rm -rf "$python_srcdir"' EXIT INT TERM QUIT
+for subdir in sdk/python services/fuse sdk/cwl; do
+ env -C "/usr/src/arvados/$subdir" \
+ /opt/arvados-py/bin/python3 setup.py sdist --dist-dir="$python_srcdir"
+done
+/opt/arvados-py/bin/pip install "$python_srcdir"/*
diff --git a/tools/arvbox/lib/arvbox/docker/yml_override.py b/tools/arvbox/lib/arvbox/docker/yml_override.py
index 7f35ac1d68..deea83f909 100755
--- a/tools/arvbox/lib/arvbox/docker/yml_override.py
+++ b/tools/arvbox/lib/arvbox/docker/yml_override.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/opt/arvados-py/bin/python3
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
commit 11cc387401be06df6d4711d7888aba8549620b98
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Jan 12 09:39:41 2024 -0500
21230: Remove pip install from get-package-version
Reasons to remove this: it can't work on Debian 12+, and the code has
obviously gotten stale with its original rationale ('setuptools<45' is
definitely not "latest setuptools").
Nobody can tell me what context this runs in. I'm going to hope that it
runs in some context where setuptools is already available. If it turns
out I'm wrong, then hopefully that will break the build in a way that
lets me assess our options.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/get-package-version.sh b/build/get-package-version.sh
index 5147f7bba6..390b5dd828 100755
--- a/build/get-package-version.sh
+++ b/build/get-package-version.sh
@@ -44,12 +44,6 @@ elif [[ "$TYPE_LANG" == "python3" ]]; then
rm -rf dist/*
- # Get the latest setuptools
- if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
- echo "Error, unable to upgrade setuptools with"
- echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
- exit 1
- fi
# filter a useless warning (when building the cwltest package) from the stderr output
if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching' |grep -v 'for version number calculation'); then
echo "Error, unable to run python3 setup.py sdist for $SRC_PATH"
commit d498f264665a3986fb02e91199ea47a7493069c8
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Jan 12 09:31:27 2024 -0500
21230: Clean up unnecessary Python packages from package build Dockerfiles
* python3-pip: `ensurepip` is standard as of Python 3.4+.
* python3-setuptools: We want to install this in our build
virtualenv anyway.
* python3-virtualenv: python3-venv is sufficient.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index 1427c64fa5..9e2b3a381f 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -46,7 +46,7 @@ ENV DEBIAN_FRONTEND noninteractive
SHELL ["/bin/bash", "-c"]
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
# Install RVM
ADD generated/mpapis.asc /tmp/
diff --git a/build/package-build-dockerfiles/debian12/Dockerfile b/build/package-build-dockerfiles/debian12/Dockerfile
index 859c1a8597..fa1d095e79 100644
--- a/build/package-build-dockerfiles/debian12/Dockerfile
+++ b/build/package-build-dockerfiles/debian12/Dockerfile
@@ -44,7 +44,7 @@ ENV DEBIAN_FRONTEND noninteractive
SHELL ["/bin/bash", "-c"]
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-virtualenv python3-dev libpam-dev equivs
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
# Install RVM
ADD generated/mpapis.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index ac23462289..deaac7c0ab 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -50,7 +50,7 @@ ENV DEBIAN_FRONTEND noninteractive
SHELL ["/bin/bash", "-c"]
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
# Install RVM
ADD generated/mpapis.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu2204/Dockerfile b/build/package-build-dockerfiles/ubuntu2204/Dockerfile
index ce96710ead..79664fea6b 100644
--- a/build/package-build-dockerfiles/ubuntu2204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2204/Dockerfile
@@ -44,7 +44,7 @@ ENV DEBIAN_FRONTEND noninteractive
SHELL ["/bin/bash", "-c"]
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
# Install RVM
ADD generated/mpapis.asc /tmp/
commit 5b0b92405919dd8f7bf10e0a2068d5f1e1816d84
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Jan 12 09:26:34 2024 -0500
21230: Remove `pip install virtualenv` from package build Dockerfiles
Installing the python3-venv package is sufficient.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index be818587b3..1427c64fa5 100644
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -48,9 +48,6 @@ SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index a21f8a2ab9..ac23462289 100644
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -52,9 +52,6 @@ SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu2204/Dockerfile b/build/package-build-dockerfiles/ubuntu2204/Dockerfile
index 533cdd8641..ce96710ead 100644
--- a/build/package-build-dockerfiles/ubuntu2204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2204/Dockerfile
@@ -46,9 +46,6 @@ SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list