[ARVADOS] created: 1.2.0-486-g0886f4750
Git user
git at public.curoverse.com
Thu Dec 20 11:53:20 EST 2018
at 0886f4750b498851c10af40e227a953dc5108816 (commit)
commit 0886f4750b498851c10af40e227a953dc5108816
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Thu Dec 20 11:52:01 2018 -0500
9945: checkpoint for run-library, this commit is very messy and will need
squashed into another one.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index b595cc8a0..e47b2df85 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -251,7 +251,7 @@ test_package_presence() {
rpm_architecture=$arch
deb_architecture=$arch
fi
-
+#FIXME this doesn't work because the PYTHON_PREFIX is not included
if [[ "$FORMAT" == "deb" ]]; then
local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
else
@@ -281,8 +281,9 @@ test_package_presence() {
echo "Package $complete_pkgname exists, not rebuilding!"
curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
echo "Package $complete_pkgname exists, not rebuilding!"
+ mv "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" "$WORKSPACE/packages/$TARGET/"
return 1
else
echo "Package $complete_pkgname not found, building"
@@ -359,6 +360,342 @@ handle_rails_package() {
rm -rf "$scripts_dir"
}
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv () {
+ PKG=$1
+ shift
+ PKG_DIR=$1
+ shift
+ PACKAGE_TYPE=${1:-python}
+ shift
+
+ # Set up
+ STDOUT_IF_DEBUG=/dev/null
+ STDERR_IF_DEBUG=/dev/null
+ DASHQ_UNLESS_DEBUG=-q
+ if [[ "$DEBUG" != "0" ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+ fi
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
+ local python=""
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ python=python2.7
+ PACKAGE_PREFIX=$PYTHON2_PKG_PREFIX
+ ;;
+ python3)
+ PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
+ python=python3
+ ;;
+ esac
+
+ if [[ "$PKG" != "libpam-arvados" ]] &&
+ [[ "$PKG" != "arvados-node-manager" ]] &&
+ [[ "$PKG" != "arvados-docker-cleaner" ]]; then
+ PYTHON_PKG=$PACKAGE_PREFIX-$PKG
+ else
+ # Exception to our package naming convention
+ PYTHON_PKG=$PKG
+ fi
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ echo "Building $FORMAT package for $PKG from $PKG_DIR"
+ cd $WORKSPACE/$PKG_DIR
+
+ # Create an sdist
+ echo "Creating sdist..."
+
+ rm -rf dist/*
+ $python setup.py $DASHQ_UNLESS_DEBUG sdist
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run python setup.py sdist for $PKG"
+ exit 1
+ fi
+
+ # Determine the package version from the generated sdist archive
+ PACKAGE_PATH=`(cd dist; ls *tar.gz)`
+ PYTHON_VERSION=${PACKAGE_PATH#$PKG-}
+
+ # For historical reasons, arvados-fuse is called arvados_fuse in python land
+ # Same with crunchstat-summary.
+ # We should fix this, but for now let's just make this script dtrt.
+ PKG_ALTERNATE=${PKG//-/_}
+ PYTHON_VERSION=${PYTHON_VERSION#$PKG_ALTERNATE-}
+
+ # An even more complicated exception
+ if [[ "$PKG" == "libpam-arvados" ]]; then
+ PYTHON_VERSION=${PYTHON_VERSION#arvados-pam-}
+ fi
+
+ if [[ "$PACKAGE_PATH" == "$PYTHON_VERSION" ]]; then
+ echo "Error, unable to determine python package version for $PKG from $PACKAGE_PATH"
+ exit 1
+ fi
+ PYTHON_VERSION=${PYTHON_VERSION%.tar.gz}
+
+ # See if we actually need to build this package; does it exist already?
+ # We can't do this earlier than here, because we need PYTHON_VERSION...
+ # This isn't so bad; the sdist call above is pretty quick compared to
+ # the invocation of virtualenv and fpm, below.
+ test_package_presence "$PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION amd64
+ if [[ "$?" != "0" ]]; then
+ return 0
+ fi
+
+ # Package the sdist in a virtualenv
+ echo "Creating virtualenv..."
+
+ cd dist
+
+ rm -rf build
+ rm -f $PYTHON_PKG*deb
+
+# if [[ "$python" == "python2.7" ]]; then
+# mkdir -p build/usr/share/$python/dist/$PYTHON_PKG
+ virtualenv_command="virtualenv --python `which $python` $DASHQ_UNLESS_DEBUG build/usr/share/$python/dist/$PYTHON_PKG"
+# else
+# # depends on python3-venv on debian/ubuntu
+# virtualenv_command="python3 $DASHQ_UNLESS_DEBUG -m venv build/usr/share/$python/dist/$PYTHON_PKG"
+# fi
+
+ $virtualenv_command
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run"
+ echo " $virtualenv_command"
+ exit 1
+ fi
+
+ if [[ "$TARGET" != "centos7" ]] || [[ "$python" != "python3" ]]; then
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip
+ else
+ # on centos7, we're stuck on python3.3, which is no longer supported by the latest pip
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG pip==10.0.1
+ # we also have to tweak ruamel.yaml because 0.15.77 doesn't support python 3.3 (and 0.15.54 doesn't either)
+set -x
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip download $DASHQ_UNLESS_DEBUG $CACHE_FLAG ruamel.yaml==0.15.77 --no-binary :all:
+ tar xzf ruamel.yaml-0.15.77.tar.gz
+ cd ruamel.yaml-0.15.77
+ sed -i "s/supported=.(2, 7), (3, 4).,/supported=[(2, 7), (3, 3)],/" __init__.py
+ cd ..
+ tar czf ruamel.yaml-0.15.77.tar.gz ruamel.yaml-0.15.77
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip download $DASHQ_UNLESS_DEBUG $CACHE_FLAG ./ruamel.yaml-0.15.77.tar.gz
+# rm -rf ruamel.*
+#FIXME this needs work
+set +x
+ fi
+# build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG virtualenv-tools
+
+ if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+ else
+ # centos7 needs these special tweaks to install python-arvados-fuse
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
+ PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+ fi
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run"
+ echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+ exit 1
+ fi
+
+ cd build/usr/share/$python/dist/$PYTHON_PKG/
+
+ # Replace the shebang lines in all python scripts, and handle the activate
+ # scripts too This is a functional replacement of the 237 line
+ # virtualenv_tools.py script that doesn't work in python3 without serious
+ # patching, minus the parts we don't need (modifying pyc files, etc).
+ for binfile in `ls bin/`; do
+ file --mime bin/$binfile |grep -q binary
+ if [[ "$?" != "0" ]]; then
+ # Not a binary file
+ if [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
+ # these 'activate' scripts need special treatment
+ sed -i "s/VIRTUAL_ENV=\".*\"/VIRTUAL_ENV=\"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
+ sed -i "s/VIRTUAL_ENV \".*\"/VIRTUAL_ENV \"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
+ else
+ grep -q -E '^#!.*/bin/python\d?' bin/$binfile
+ if [[ "$?" == "0" ]]; then
+ # Replace shebang line
+ sed -i "1 s/^.*$/#!\/usr\/share\/$python\/dist\/$PYTHON_PKG\/bin\/python/" bin/$binfile
+ fi
+ fi
+ fi
+ done
+
+# sed -i "s/'\/bin\/python'/\('\/bin\/python','\/bin\/python2','\/bin\/python3'\)/g" lib/python*/site-packages/virtualenv_tools.py
+#
+# # make virtualenv_tools python3 compatible. Sigh.
+# sed -i "s/from types import CodeType/from types import CodeType\nfrom io import open\nfrom builtins import str/" lib/python*/site-packages/virtualenv_tools.py
+# sed -i "s/print \(.*\)$/print\(\1\)/" lib/python*/site-packages/virtualenv_tools.py
+# sed -i "s/\\\)/\\\\/" lib/python*/site-packages/virtualenv_tools.py
+# sed -i "s/py_ver$/py_ver)/" lib/python*/site-packages/virtualenv_tools.py
+# sed -i "s/with open(script_filename)/with open(script_filename, encoding='utf-8')/" lib/python*/site-packages/virtualenv_tools.py
+# sed -i "s/f.writelines(lines)/f.writelines(str(lines))/" lib/python*/site-packages/virtualenv_tools.py
+#
+# # make the virtualenv-tools code less ridiculous (trying to edit directories and
+# # binary files, seriously?) so that it works in python3
+# cat > virtualenv_tools.fix <<EOF
+# # mimick file(1) behavior
+# textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
+# is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
+# if os.path.isdir(script_filename):
+# return
+# if is_binary_string(open(script_filename, 'rb').read(1024)):
+# return
+#EOF
+# sed -i "/\"\"\"Updates shebang lines for actual scripts.\"\"\"/r virtualenv_tools.fix" lib/python*/site-packages/virtualenv_tools.py
+#
+## ./bin/virtualenv-tools --update-path /usr/share/$python/dist/$PYTHON_PKG >$STDOUT_IF_DEBUG
+#
+# if [[ "$?" != "0" ]]; then
+# echo "Error, unable to run"
+# echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/virtualenv-tools --update-path /usr/share/$python/dist/$PYTHON_PKG"
+# exit 1
+# fi
+
+ cd - >$STDOUT_IF_DEBUG
+
+ find build -iname *.pyc -exec rm {} \;
+ find build -iname *.pyo -exec rm {} \;
+
+ # Finally, generate the package
+ echo "Creating package..."
+
+ declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
+
+ if [[ "$MAINTAINER" != "" ]]; then
+ COMMAND_ARR+=('--maintainer' "$MAINTAINER")
+ fi
+
+ if [[ "$VENDOR" != "" ]]; then
+ COMMAND_ARR+=('--vendor' "$VENDOR")
+ fi
+
+ COMMAND_ARR+=('--url' 'https://arvados.org')
+
+ # Get description
+ DESCRIPTION=`grep '\sdescription' $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
+ COMMAND_ARR+=('--description' "$DESCRIPTION")
+
+ # Get license string
+ LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
+ COMMAND_ARR+=('--license' "$LICENSE_STRING")
+
+ # 12271 - As FPM-generated packages don't include scripts by default, the
+ # packages cleanup on upgrade depends on files being listed on the %files
+ # section in the generated SPEC files. To remove DIRECTORIES, they need to
+ # be listed in that sectiontoo, so we need to add this parameter to properly
+ # remove lingering dirs. But this only works for python2: if used on
+ # python33, it includes dirs like /opt/rh/python33 that belong to
+ # other packages.
+ if [[ "$FORMAT" == "rpm" ]] && [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--rpm-auto-add-directories')
+ fi
+
+ if [[ "$PKG" == "arvados-python-client" ]]; then
+ if [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
+ else
+ COMMAND_ARR+=('--conflicts' "$PYTHON2_PKG_PREFIX-$PKG")
+ fi
+ fi
+
+ if [[ "${DEBUG:-0}" != "0" ]]; then
+ COMMAND_ARR+=('--verbose' '--log' 'info')
+ fi
+
+ COMMAND_ARR+=('-v' "$PYTHON_VERSION")
+ COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
+ COMMAND_ARR+=('-n' "$PYTHON_PKG")
+ COMMAND_ARR+=('--before-remove' "$WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh")
+ COMMAND_ARR+=('--after-install' "$WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh")
+ COMMAND_ARR+=('-C' "build")
+
+ if [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--depends' "$PYTHON2_PACKAGE")
+ else
+ COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
+ fi
+
+ # avoid warning
+ COMMAND_ARR+=('--deb-no-default-config-files')
+
+ # Append --depends X and other arguments specified by fpm-info.sh in
+ # the package source dir. These are added last so they can override
+ # the arguments added by this script.
+ declare -a fpm_args=()
+ declare -a fpm_depends=()
+
+ fpminfo="$WORKSPACE/$PKG_DIR/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
+ exit 1
+ fi
+ fi
+
+ if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
+ FPM_BINARIES=""
+ for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
+ FPM_BINARIES+="'$binary' "
+ done
+ fi
+
+ # create a custom version of the package scripts, with fpm_binaries populated
+ cp -f $WORKSPACE/build/python-package-scripts/before-remove.sh $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ cp -f $WORKSPACE/build/python-package-scripts/after-install.sh $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+ sed -i s/%FPM_BINARIES/"$FPM_BINARIES"/g $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ sed -i s/%FPM_BINARIES/"$FPM_BINARIES"/g $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+
+ # Make sure the package scripts know the correct path where its files are installed
+ sed -i s/%PYTHON/$python/g $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ sed -i s/%PYTHON/$python/g $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--depends' "$i")
+ done
+
+ COMMAND_ARR+=("${fpm_args[@]}")
+
+ COMMAND_ARR+=(".")
+
+#echo -e "\n${COMMAND_ARR[@]}\n"
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+
+# fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+
+ # if something went wrong and debug is off, print out the fpm command that errored
+ if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
+ echo "fpm returned an error execurin the command:"
+ echo
+ echo -e "\n${COMMAND_ARR[@]}\n"
+ else
+ echo `ls *$FORMAT`
+ mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/
+ fi
+ echo
+
+ # clean up
+ rm -f $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ rm -f $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+}
+
# Build packages for everything
fpm_build () {
# The package source. Depending on the source type, this can be a
commit 0ddb70d7350801a56c1754f02975c95ee2687c1a
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Thu Dec 20 11:51:39 2018 -0500
9945: first set of changes.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index ba616eef1..20a887127 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -6,7 +6,7 @@ FROM centos:7
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies.
-RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git python-virtualenv
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 1244e3f55..134cece79 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 42094d53b..9674c0cf2 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -9,7 +9,10 @@ MAINTAINER Nico Cesar <nico at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
+
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv virtualenv-tools
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index a6bf65bc1..798680e48 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index 17bf89f3b..b9e966c74 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index c66de4690..3891df393 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install RVM
ADD generated/rvm.asc /tmp/
diff --git a/build/python-package-scripts/after-install.sh b/build/python-package-scripts/after-install.sh
new file mode 100755
index 000000000..8851f28ec
--- /dev/null
+++ b/build/python-package-scripts/after-install.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+# Detect rpm-based systems: the exit code of the following command is zero
+# on rpm-based systems
+if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then
+ # Red Hat ("%{...}" is interpolated at package build time)
+ pkg="%{name}"
+ pkgtype=rpm
+ prefix="${RPM_INSTALL_PREFIX}"
+else
+ # Debian
+ script="$(basename "${0}")"
+ pkg="${script%.postinst}"
+ pkgtype=deb
+ prefix=/usr
+fi
+
+# populated from the build script
+# dash only supports one array, $@
+if [ "%FPM_BINARIES" != "" ]; then
+ set %FPM_BINARIES
+fi
+
+# Install symlinks to the binary/binaries
+if [ "$pkg" != "" ]; then
+
+ if [ "%FPM_BINARIES" != "" ]; then
+ # read from $@
+ for binary do
+ if [ -e /usr/bin/$binary ]; then
+ rm -f /usr/bin/$binary
+ fi
+ ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/$binary
+ done
+ fi
+
+ # special case for arvados-cwl-runner
+ if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then
+ if [ -e /usr/bin/cwl-runner ]; then
+ rm -f /usr/bin/cwl-runner
+ fi
+ ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/cwl-runner
+ fi
+fi
+
diff --git a/build/python-package-scripts/before-remove.sh b/build/python-package-scripts/before-remove.sh
new file mode 100755
index 000000000..2ea15ee70
--- /dev/null
+++ b/build/python-package-scripts/before-remove.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+# Detect rpm-based systems: the exit code of the following command is zero
+# on rpm-based systems
+if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then
+ # Red Hat ("%{...}" is interpolated at package build time)
+ pkg="%{name}"
+ pkgtype=rpm
+ prefix="${RPM_INSTALL_PREFIX}"
+else
+ # Debian
+ script="$(basename "${0}")"
+ pkg="${script%.prerm}"
+ pkgtype=deb
+ prefix=/usr
+fi
+
+# populated from the build script
+# dash only supports one array, $@
+if [ "%FPM_BINARIES" != "" ]; then
+ set %FPM_BINARIES
+fi
+
+if [ "$pkg" != "" ]; then
+ # Remove the binary python files so the package manager doesn't throw warnings
+ # on removing the package.
+ find /usr/share/%PYTHON/dist/$pkg -iname *.pyc -exec rm {} \; || true
+ find /usr/share/%PYTHON/dist/$pkg -iname *.pyo -exec rm {} \; || true
+
+ if [ "%FPM_BINARIES" != "" ]; then
+ # read from $@
+ for binary do
+ if [ -L /usr/bin/$binary ]; then
+ # Remove the symlinks we installed
+ rm -f /usr/bin/$binary
+ fi
+ done
+ fi
+
+ if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then
+ if [ -L /usr/bin/cwl-runner ]; then
+ rm -f /usr/bin/cwl-runner
+ fi
+ fi
+fi
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 73e2e6286..efc20b234 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -348,38 +348,12 @@ for i in ${deps[@]}; do
fi
done
-
# The Python SDK
-# Please resist the temptation to add --no-python-fix-name to the fpm call here
-# (which would remove the python- prefix from the package name), because this
-# package is a dependency of arvados-fuse, and fpm can not omit the python-
-# prefix from only one of the dependencies of a package... Maybe I could
-# whip up a patch and send it upstream, but that will be for another day. Ward,
-# 2014-05-15
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/python/build"
-arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)}
-test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
-if [[ "$?" == "0" ]]; then
+fpm_build_virtualenv "arvados-python-client" "sdk/python"
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
- fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
-fi
-
-# cwl-runner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/cwl/build"
-arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)}
-declare -a iterargs=()
-if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
- arvados_cwl_runner_iteration=4
- iterargs+=(--iteration $arvados_cwl_runner_iteration)
-else
- arvados_cwl_runner_iteration=
-fi
-test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
-if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" "${iterargs[@]}"
-fi
+# Arvadow cwl runner
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
# schema_salad. This is a python dependency of arvados-cwl-runner,
# but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -421,15 +395,7 @@ if [[ $TARGET =~ debian|ubuntu ]]; then
fi
# The FUSE driver
-# Please see comment about --no-python-fix-name above; we stay consistent and do
-# not omit the python- prefix first.
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/fuse/build"
-arvados_fuse_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)}
-test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
-if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
-fi
+fpm_build_virtualenv "arvados-fuse" "services/fuse"
# The node manager
cd $WORKSPACE/packages/$TARGET
diff --git a/services/dockercleaner/README.rst b/services/dockercleaner/README.rst
new file mode 100644
index 000000000..dd2b7e9cf
--- /dev/null
+++ b/services/dockercleaner/README.rst
@@ -0,0 +1,5 @@
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: Apache-2.0
+
+Arvados Docker Cleaner.
diff --git a/services/dockercleaner/bin/arvados-docker-cleaner b/services/dockercleaner/bin/arvados-docker-cleaner
new file mode 100755
index 000000000..c00593fbf
--- /dev/null
+++ b/services/dockercleaner/bin/arvados-docker-cleaner
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+from __future__ import absolute_import, print_function
+
+from arvados_docker.cleaner import main
+main()
diff --git a/tools/crunchstat-summary/README.rst b/tools/crunchstat-summary/README.rst
new file mode 100644
index 000000000..fa8e1bd3d
--- /dev/null
+++ b/tools/crunchstat-summary/README.rst
@@ -0,0 +1,5 @@
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: Apache-2.0
+
+Arvados Crunchstat Summary.
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 04adba849..642428181 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -23,7 +23,7 @@ if '--short-tests-only' in sys.argv:
setup(name='crunchstat_summary',
version=version,
- description='read crunch log files and summarize resource usage',
+ description='Arvados crunchstat-summary reads crunch log files and summarizes resource usage',
author='Arvados',
author_email='info at arvados.org',
url="https://arvados.org",
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list