[arvados] updated: 2.7.0-5807-gacf1fa4559
git repository hosting
git at public.arvados.org
Wed Jan 10 18:49:40 UTC 2024
Summary of changes:
build/build-dev-docker-jobs-image.sh | 72 ++++++++----------------------------
sdk/dev-jobs.dockerfile | 45 +++++-----------------
2 files changed, 24 insertions(+), 93 deletions(-)
via acf1fa4559c007f127a12c4da3ae644a5ad3a6ac (commit)
via 2c540bf7fb83471751e90fcb44cfc04dc3ed333f (commit)
via 22af992e198a2acfa276050bf7b0325029f67000 (commit)
from 2f0d2cee42749bb86af303b38438dd5e3b236d60 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit acf1fa4559c007f127a12c4da3ae644a5ad3a6ac
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Jan 10 13:43:20 2024 -0500
21367/21230: Set up dev-jobs with virtualenv
The functional goal here is to do all pip installs inside the Dockerfile
inside a virtualenv to avoid the global pip install issues.
Changes that fall out of that:
* Installing python3-venv is all we need to go back to the -slim Debian
image.
* Stop installing other Python packages we'll just install inside the
virtualenv anyway.
* Rather than having separate arguments and code blocks for every
package we might want to install, just set up a dedicated build
context for the Docker image with all the packages we want to install,
and install them all unconditionally during the image build. This is
much less code and hopefully easier to follow.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index dfa437fdfc..a0ea05383a 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -17,7 +17,6 @@ WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
CWL_UTILS=path (optional) Path to cwl-utils git repository.
-PYCMD=pythonexec (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
EOF
@@ -28,64 +27,31 @@ if [[ -z "$WORKSPACE" ]] ; then
echo "Using WORKSPACE $WORKSPACE"
fi
-cd "$WORKSPACE"
-
-py=python3
-pipcmd=pip
-if [[ -n "$PYCMD" ]] ; then
- py="$PYCMD"
-fi
-if [[ $py = python3 ]] ; then
- pipcmd=pip3
-fi
-
-(cd sdk/python && python3 setup.py sdist)
-sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
-
-(cd sdk/cwl && python3 setup.py sdist)
-runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
-
-rm -rf sdk/cwl/salad_dist
-mkdir -p sdk/cwl/salad_dist
-if [[ -n "$SALAD" ]] ; then
- (cd "$SALAD" && python3 setup.py sdist)
- salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
- cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
-fi
-
-rm -rf sdk/cwl/cwltool_dist
-mkdir -p sdk/cwl/cwltool_dist
-if [[ -n "$CWLTOOL" ]] ; then
- (cd "$CWLTOOL" && python3 setup.py sdist)
- cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
- cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
-fi
-
-rm -rf sdk/cwl/cwlutils_dist
-mkdir -p sdk/cwl/cwlutils_dist
-if [[ -n "$CWL_UTILS" ]] ; then
- (cd "$CWL_UTILS" && python3 setup.py sdist)
- cwlutils=$(cd "$CWL_UTILS/dist" && ls -t cwl-utils-*.tar.gz | head -n1)
- cp "$CWL_UTILS/dist/$cwlutils" $WORKSPACE/sdk/cwl/cwlutils_dist
-fi
+context_dir="$(mktemp --directory --tmpdir dev-jobs.XXXXXXXX)"
+trap 'rm -rf "$context_dir"' EXIT INT TERM QUIT
+ts_path="$context_dir/.timestamp"
+
+for src_dir in "$WORKSPACE/sdk/python" "${CWLTOOL:-}" "${CWL_UTILS:-}" "${SALAD:-}" "$WORKSPACE/sdk/cwl"; do
+ if [[ -z "$src_dir" ]]; then
+ continue
+ fi
+ touch "$ts_path"
+ env -C "$src_dir" python3 setup.py sdist
+ find "$src_dir/dist/" -maxdepth 1 -type f -cnewer "$ts_path" -print0 \
+ | xargs -0 cp --target="$context_dir/"
+done
+rm "$ts_path"
+cd "$WORKSPACE"
. build/run-library.sh
-
# This defines python_sdk_version and cwl_runner_version with python-style
# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
set -x
docker build --no-cache \
- --build-arg sdk=$sdk \
- --build-arg runner=$runner \
- --build-arg salad=$salad \
- --build-arg cwltool=$cwltool \
- --build-arg pythoncmd=$py \
- --build-arg pipcmd=$pipcmd \
- --build-arg cwlutils=$cwlutils \
-f "$WORKSPACE/sdk/dev-jobs.dockerfile" \
-t arvados/jobs:$cwl_runner_version \
- "$WORKSPACE/sdk"
+ "$context_dir"
arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 656572eb4f..016163307f 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -9,47 +9,20 @@
# version.
#
# Use arvados/build/build-dev-docker-jobs-image.sh to build.
-#
-# (This dockerfile file must be located in the arvados/sdk/ directory because
-# of the docker build root.)
-FROM debian:bullseye
+FROM debian:bullseye-slim
MAINTAINER Arvados Package Maintainers <packaging at arvados.org>
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG pythoncmd=python3
-ARG pipcmd=pip3
-
-RUN apt-get update -q && apt-get install -qy --no-install-recommends \
- git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
- libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential ${pythoncmd}-setuptools
-
-ARG sdk
-ARG runner
-ARG salad
-ARG cwlutils
-ARG cwltool
-
-ADD python/dist/$sdk /tmp/
-ADD cwl/salad_dist/$salad /tmp/
-ADD cwl/cwltool_dist/$cwltool /tmp/
-ADD cwl/cwlutils_dist/$cwlutils /tmp/
-ADD cwl/dist/$runner /tmp/
-
-RUN $pipcmd install wheel
-RUN cd /tmp/arvados-python-client-* && $pipcmd install .
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwl-utils-* ; then cd /tmp/cwl-utils-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
-RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
+RUN DEBIAN_FRONTEND=noninteractive apt-get update -q && apt-get install -qy --no-install-recommends \
+ git python3-dev python3-venv libcurl4-gnutls-dev libgnutls28-dev nodejs build-essential
-# Sometimes Python dependencies install successfully but don't
-# actually work. So run arvados-cwl-runner here to catch fun
-# dependency errors like pkg_resources.DistributionNotFound.
-RUN arvados-cwl-runner --version
+ADD * /usr/local/src/
+RUN python3 -m venv /opt/arvados-py
+ENV PATH=/opt/arvados-py/bin:/usr/local/bin:/usr/bin:/bin
+RUN python3 -m pip install --no-cache-dir wheel
+# Run a-c-r afterward to check for a successful install.
+RUN python3 -m pip install --no-cache-dir /usr/local/src/* && arvados-cwl-runner --version
-# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
--gecos 'Crunch execution user' crunch && \
/usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job
commit 2c540bf7fb83471751e90fcb44cfc04dc3ed333f
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Jan 10 13:33:36 2024 -0500
21367: Remove environment check
You only need for this for `arv-keepdocker` at the end of the process,
which can read from settings.conf just fine too. Even if it fails, the
developer will have the command handy to fix their environment and try
that again easily.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index d88489d465..dfa437fdfc 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -28,13 +28,6 @@ if [[ -z "$WORKSPACE" ]] ; then
echo "Using WORKSPACE $WORKSPACE"
fi
-if [[ -z "$ARVADOS_API_HOST" || -z "$ARVADOS_API_TOKEN" ]] ; then
- echo "$helpmessage"
- echo
- echo "Must set ARVADOS_API_HOST and ARVADOS_API_TOKEN"
- exit 1
-fi
-
cd "$WORKSPACE"
py=python3
commit 22af992e198a2acfa276050bf7b0325029f67000
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Jan 10 13:33:13 2024 -0500
21367: Remove debug print
Doubly redundant since we're using `set -x` here anyway.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index bf1ab34189..d88489d465 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -95,5 +95,4 @@ docker build --no-cache \
-t arvados/jobs:$cwl_runner_version \
"$WORKSPACE/sdk"
-echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list