[ARVADOS] updated: 1.2.0-488-g5a09ff5d4
Git user
git at public.curoverse.com
Mon Jan 21 21:21:40 EST 2019
Summary of changes:
build/build.list | 57 -------
build/package-build-dockerfiles/Makefile | 12 +-
build/package-build-dockerfiles/centos7/Dockerfile | 12 +-
build/package-build-dockerfiles/debian8/Dockerfile | 9 +-
build/package-build-dockerfiles/debian9/Dockerfile | 8 +-
.../ubuntu1404/Dockerfile | 11 +-
.../ubuntu1604/Dockerfile | 9 +-
.../ubuntu1804/Dockerfile | 9 +-
build/package-test-dockerfiles/Makefile | 12 +-
build/package-test-dockerfiles/centos7/Dockerfile | 13 +-
build/package-test-dockerfiles/debian8/Dockerfile | 6 +-
build/package-test-dockerfiles/debian9/Dockerfile | 6 +-
.../package-test-dockerfiles/ubuntu1404/Dockerfile | 6 +-
.../package-test-dockerfiles/ubuntu1604/Dockerfile | 6 +-
.../package-test-dockerfiles/ubuntu1804/Dockerfile | 6 +-
build/package-testing/deb-common-test-packages.sh | 29 +++-
build/package-testing/rpm-common-test-packages.sh | 23 ++-
.../test-package-arvados-node-manager.sh | 13 +-
...t-package-python27-python-arvados-cwl-runner.sh | 11 +-
.../test-package-python27-python-arvados-fuse.sh | 9 +-
...ackage-python27-python-arvados-python-client.sh | 9 +-
build/run-build-packages.sh | 184 +--------------------
build/run-library.sh | 127 ++++----------
{backports/python-ciso8601 => sdk/cwl}/fpm-info.sh | 6 +-
{backports/deb-fuse => sdk/python}/fpm-info.sh | 4 +-
services/fuse/fpm-info.sh | 9 +
.../workbench => services/nodemanager}/fpm-info.sh | 5 +-
.../crunchstat-summary}/fpm-info.sh | 5 +-
28 files changed, 216 insertions(+), 400 deletions(-)
delete mode 100644 build/build.list
copy {backports/python-ciso8601 => sdk/cwl}/fpm-info.sh (60%)
copy {backports/deb-fuse => sdk/python}/fpm-info.sh (68%)
copy {apps/workbench => services/nodemanager}/fpm-info.sh (65%)
copy {apps/workbench => tools/crunchstat-summary}/fpm-info.sh (65%)
discards 0886f4750b498851c10af40e227a953dc5108816 (commit)
via 5a09ff5d42b0f8b71ca6775813e0844067363e12 (commit)
via 167b927cd6bc0cd4ff5de2a523959eec71d8a17d (commit)
via 55558d2bbe2af6e285cc1ad458fce58ebe452b24 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (0886f4750b498851c10af40e227a953dc5108816)
\
N -- N -- N (5a09ff5d42b0f8b71ca6775813e0844067363e12)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5a09ff5d42b0f8b71ca6775813e0844067363e12
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Mon Jan 21 20:46:50 2019 -0500
9945: use virtualenv for our python packages
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/build/build.list b/build/build.list
deleted file mode 100644
index 502460bc3..000000000
--- a/build/build.list
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
-debian8,debian9,centos7|python-gflags|2.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|google-api-python-client|1.6.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
-debian8,debian9,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rsa|3.4.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|uritemplate|3.0.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|httplib2|0.9.2|3|python|all
-debian8,debian9,centos7,ubuntu1404,ubuntu1604|ws4py|0.4.2|2|python|all
-debian8,debian9,centos7|pykka|1.2.1|2|python|all
-debian8,debian9,ubuntu1404,centos7|six|1.10.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|ciso8601|1.0.6|3|python|amd64
-debian8,debian9,centos7|pycrypto|2.6.1|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804|backports.ssl_match_hostname|3.5.0.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|llfuse|1.2|3|python|amd64
-debian8,debian9,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pyyaml|3.12|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rdflib|4.2.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|mistune|0.8.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|typing|3.6.4|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|avro|1.8.1|2|python|all
-debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|cachecontrol|0.11.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pathlib2|2.3.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|scandir|1.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|python3|all
-debian8,debian9,centos7|six|1.10.0|2|python3|all
-debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
-debian8,ubuntu1404,centos7|requests|2.6.1|2|python|all
-centos7|contextlib2|0.5.4|2|python|all
-centos7|isodate|0.5.4|2|python|all
-centos7|python-daemon|2.1.2|1|python|all
-centos7|pbr|0.11.1|2|python|all
-centos7|pyparsing|2.1.10|2|python|all
-centos7|keepalive|0.5|2|python|all
-centos7|networkx|1.11|0|python|all
-centos7|psutil|5.0.1|0|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.15.77|1|python|amd64|--python-setup-py-arguments --single-version-externally-managed --depends 'python-ruamel.ordereddict >= 0.4.9'
-all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
-all|junit-xml|1.8|3|python|all
-all|rdflib-jsonld|0.4.0|2|python|all
-all|futures|3.0.5|2|python|all
-all|future|0.16.0|2|python|all
-all|future|0.16.0|2|python3|all
-all|mypy-extensions|0.3.0|1|python|all
-all|prov|1.5.1|0|python|all
-all|bagit|1.6.4|0|python|all
-all|typing-extensions|3.6.5|0|python|all
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 23400cf48..9bba1f161 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -29,8 +29,8 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
# Need to "touch" RPM database to workaround bug in interaction between
# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install python33
-RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
+RUN touch /var/lib/rpm/* && yum -q -y install rh-python35
+RUN scl enable rh-python35 "easy_install-3.5 pip" && easy_install-2.7 pip
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
@@ -38,4 +38,4 @@ RUN pip install --upgrade setuptools
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
-CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
+CMD ["scl", "enable", "rh-python35", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 493aef7be..1e6c62ce7 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 98da1c97a..fcaf0bc0a 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install virtualenv
-RUN /usr/bin/pip install virtualenv virtualenv-tools
+RUN /usr/bin/pip install virtualenv
# Install RVM
ADD generated/mpapis.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index 8b0d9bc65..a88435a5b 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -8,7 +8,10 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3.4-venv python3.4-dev
+
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
# Install RVM
ADD generated/mpapis.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index de15c9ecd..be3911eff 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index 0dc431373..8995d14be 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
# Install RVM
ADD generated/mpapis.asc /tmp/
ADD generated/pkuczynski.asc /tmp/
diff --git a/build/package-testing/deb-common-test-packages.sh b/build/package-testing/deb-common-test-packages.sh
index b5325224e..77017ba97 100755
--- a/build/package-testing/deb-common-test-packages.sh
+++ b/build/package-testing/deb-common-test-packages.sh
@@ -2,9 +2,19 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
-
set -eu
+# Set up
+DEBUG=${ARVADOS_DEBUG:-0}
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQQ_UNLESS_DEBUG=-qq
+if [[ "$DEBUG" != "0" ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQQ_UNLESS_DEBUG=
+fi
+
# Multiple .deb based distros symlink to this script, so extract the target
# from the invocation path.
target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
@@ -13,8 +23,9 @@ export ARV_PACKAGES_DIR="/arvados/packages/$target"
dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
-apt-get -qq update
-apt-get --assume-yes --allow-unauthenticated install "$1"
+apt-get $DASHQQ_UNLESS_DEBUG update
+
+apt-get $DASHQQ_UNLESS_DEBUG -y --allow-unauthenticated install "$1" >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
@@ -35,12 +46,14 @@ fi
dpkg-deb -x $debpkg .
-while read so && [ -n "$so" ]; do
- echo
- echo "== Packages dependencies for $so =="
- ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
-done <<EOF
+if [[ "$DEBUG" != "0" ]]; then
+ while read so && [ -n "$so" ]; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+ done <<EOF
$(find -name '*.so')
EOF
+fi
exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/build/package-testing/rpm-common-test-packages.sh b/build/package-testing/rpm-common-test-packages.sh
index 268611cb7..12450dd4f 100755
--- a/build/package-testing/rpm-common-test-packages.sh
+++ b/build/package-testing/rpm-common-test-packages.sh
@@ -5,6 +5,15 @@
set -eu
+# Set up
+DEBUG=${ARVADOS_DEBUG:-0}
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+if [[ "$DEBUG" != "0" ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+fi
+
target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b')
yum -q clean all
@@ -14,7 +23,7 @@ export ARV_PACKAGES_DIR="/arvados/packages/$target"
rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
-yum install --assumeyes $1
+yum install --assumeyes -e 0 $1
rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
@@ -41,10 +50,12 @@ cd /tmp/opts
rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
-find -name '*.so' | while read so; do
- echo -e "\n== Packages dependencies for $so =="
- ldd "$so" \
- | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
-done
+if [[ "$DEBUG" != "0" ]]; then
+ find -name '*.so' | while read so; do
+ echo -e "\n== Packages dependencies for $so =="
+ ldd "$so" \
+ | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+ done
+fi
exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/build/package-testing/test-package-arvados-node-manager.sh b/build/package-testing/test-package-arvados-node-manager.sh
index c699fb45b..e2ada8ed6 100755
--- a/build/package-testing/test-package-arvados-node-manager.sh
+++ b/build/package-testing/test-package-arvados-node-manager.sh
@@ -3,9 +3,20 @@
#
# SPDX-License-Identifier: AGPL-3.0
-exec python <<EOF
+set -e
+arvados-node-manager --version
+set +e
+
+PYTHON=`ls /usr/share/python2.7/dist/arvados-node-manager/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+ set -e
+ exec $PYTHON <<EOF
import libcloud.compute.types
import libcloud.compute.providers
libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
print "Successfully imported compatible libcloud library"
EOF
+else
+ exit 1
+fi
diff --git a/build/package-testing/test-package-python27-python-arvados-cwl-runner.sh b/build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
index e499238d8..02da021af 100755
--- a/build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
+++ b/build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
@@ -4,10 +4,17 @@
# SPDX-License-Identifier: AGPL-3.0
set -e
-
arvados-cwl-runner --version
+set +e
+
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-cwl-runner/bin/python2.7`
-exec python <<EOF
+if [ "$PYTHON" != "" ]; then
+ set -e
+ exec $PYTHON <<EOF
import arvados_cwl
print "arvados-cwl-runner version", arvados_cwl.__version__
EOF
+else
+ exit 1
+fi
diff --git a/build/package-testing/test-package-python27-python-arvados-fuse.sh b/build/package-testing/test-package-python27-python-arvados-fuse.sh
index 152d1eb69..8d21951ff 100755
--- a/build/package-testing/test-package-python27-python-arvados-fuse.sh
+++ b/build/package-testing/test-package-python27-python-arvados-fuse.sh
@@ -3,7 +3,14 @@
#
# SPDX-License-Identifier: AGPL-3.0
-exec python <<EOF
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-fuse/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+ set -e
+ exec $PYTHON <<EOF
import arvados_fuse
print "Successfully imported arvados_fuse"
EOF
+else
+ exit 1
+fi
diff --git a/build/package-testing/test-package-python27-python-arvados-python-client.sh b/build/package-testing/test-package-python27-python-arvados-python-client.sh
index 2603e034a..3e737ef2b 100755
--- a/build/package-testing/test-package-python27-python-arvados-python-client.sh
+++ b/build/package-testing/test-package-python27-python-arvados-python-client.sh
@@ -3,7 +3,14 @@
#
# SPDX-License-Identifier: AGPL-3.0
-exec python2.7 <<EOF
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-python-client/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+ set -e
+ exec $PYTHON <<EOF
import arvados
print "Successfully imported arvados"
EOF
+else
+ exit 1
+fi
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index efc20b234..04b4a7956 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -117,7 +117,7 @@ case "$TARGET" in
PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
- PYTHON3_PREFIX=/opt/rh/python33/root/usr
+ PYTHON3_PREFIX=/opt/rh/rh-python35/root/usr
PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
export PYCURL_SSL_LIBRARY=nss
;;
@@ -327,27 +327,6 @@ package_go_binary tools/keep-rsync keep-rsync \
package_go_binary tools/keep-exercise keep-exercise \
"Performance testing tool for Arvados Keep"
-
-# we need explicit debian_revision values in the dependencies for ruamel.yaml, because we have a package iteration
-# greater than zero. So we parse setup.py, get the ruamel.yaml dependencies, tell fpm not to automatically include
-# them in the package being built, and re-add them manually with an appropriate debian_revision value.
-# See #14552 for the reason for this (nasty) workaround. We use ${ruamel_depends[@]} in a few places further down
-# in this script.
-# Ward, 2018-11-28
-IFS=', ' read -r -a deps <<< `grep ruamel.yaml $WORKSPACE/sdk/python/setup.py |cut -f 3 -dl |sed -e "s/'//g"`
-declare -a ruamel_depends=()
-for i in ${deps[@]}; do
- i=`echo "$i" | sed -e 's!\([0-9]\)! \1!'`
- if [[ $i =~ .*\>.* ]]; then
- ruamel_depends+=(--depends "python-ruamel.yaml $i-1")
- elif [[ $i =~ .*\<.* ]]; then
- ruamel_depends+=(--depends "python-ruamel.yaml $i-9")
- else
- echo "Encountered ruamel dependency that I can't parse. Aborting..."
- exit 1
- fi
-done
-
# The Python SDK
fpm_build_virtualenv "arvados-python-client" "sdk/python"
fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
@@ -355,34 +334,6 @@ fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
# Arvadow cwl runner
fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
-# schema_salad. This is a python dependency of arvados-cwl-runner,
-# but we can't use the usual PYTHONPACKAGES way to build this package due to the
-# intricacies of how version numbers get generated in setup.py: we need a specific version,
-# e.g. 1.7.20160316203940. If we don't explicitly list that version with the -v
-# argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get
-# a package with version 1.7. That's because our gittagger hack is not being
-# picked up by self.distribution.get_version(), which is called from
-# https://github.com/jordansissel/fpm/blob/master/lib/fpm/package/pyfpm/get_metadata.py
-# by means of this command:
-#
-# python2.7 setup.py --command-packages=pyfpm get_metadata --output=metadata.json
-#
-# So we build this thing separately.
-#
-# Ward, 2016-03-17
-saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python 2
-if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
-fi
-
-# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python 3
-if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion --iteration 3 --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
-fi
-
# The PAM module
if [[ $TARGET =~ debian|ubuntu ]]; then
cd $WORKSPACE/packages/$TARGET
@@ -398,34 +349,13 @@ fi
fpm_build_virtualenv "arvados-fuse" "services/fuse"
# The node manager
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/nodemanager/build"
-nodemanager_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-1}"
-test_package_presence arvados-node-manager "$nodemanager_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration "$iteration"
-fi
+fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
# The Docker image cleaner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/dockercleaner/build"
-dockercleaner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-4}"
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
-fi
+fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
# The Arvados crunchstat-summary tool
-cd $WORKSPACE/packages/$TARGET
-crunchstat_summary_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-2}"
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
- rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
-fi
+fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary"
# Forked libcloud
if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
@@ -452,112 +382,6 @@ then
rm -rf $LIBCLOUD_DIR
fi
-# Python 2 dependencies
-declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
-# Add --no-use-wheel if this pip knows it.
-pip install --no-use-wheel >/dev/null 2>&1
-case "$?" in
- 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
- 1) ;;
- 2) ;;
- *) echo "WARNING: 'pip install --no-use-wheel' test returned unknown exit code $?" ;;
-esac
-
-while read -r line || [[ -n "$line" ]]; do
-# echo "Text read from file: $line"
- if [[ "$line" =~ ^# ]]; then
- continue
- fi
- IFS='|'; arr=($line); unset IFS
-
- dist=${arr[0]}
-
- IFS=',';dists=($dist); unset IFS
-
- MATCH=0
- for d in "${dists[@]}"; do
- if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
- MATCH=1
- fi
- done
-
- if [[ "$MATCH" != "1" ]]; then
- continue
- fi
- name=${arr[1]}
- version=${arr[2]}
- iteration=${arr[3]}
- pkgtype=${arr[4]}
- arch=${arr[5]}
- extra=${arr[6]}
- declare -a 'extra_arr=('"$extra"')'
-
- if [[ "$FORMAT" == "rpm" ]]; then
- if [[ "$arch" == "all" ]]; then
- arch="noarch"
- fi
- if [[ "$arch" == "amd64" ]]; then
- arch="x86_64"
- fi
- fi
-
- if [[ "$pkgtype" == "python" ]]; then
- outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
- else
- outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
- fi
-
- if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
- continue
- fi
-
- case "$name" in
- httplib2|google-api-python-client)
- test_package_presence $outname $version $pkgtype $iteration $arch
- if [[ "$?" == "0" ]]; then
- # Work around 0640 permissions on some package files.
- # See #7591 and #7991.
- pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
- set -e
- cd "$pyfpm_workdir"
- PIP_VERSION=`python$PYTHON2_VERSION -c "import pip; print(pip.__version__)" |cut -f1 -d.`
- if (( $PIP_VERSION < 8 )); then
- pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
- else
- pip download --no-deps --no-binary :all: "$name==$version"
- fi
- # Sometimes pip gives us a tarball, sometimes a zip file...
- DOWNLOADED=`ls $name-*`
- [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
- [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
- cd "$name"-*/
- "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
- chmod -R go+rX .
- set +e
- fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
- # The upload step uses the package timestamp to determine
- # if it is new. --no-clobber plays nice with that.
- mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
- )
- if [ 0 != "$?" ]; then
- echo "ERROR: $name build process failed"
- EXITCODE=1
- fi
- if [ -n "$pyfpm_workdir" ]; then
- rm -rf "$pyfpm_workdir"
- fi
- fi
- ;;
- *)
- test_package_presence $outname $version $pkgtype $iteration $arch
- if [[ "$?" == "0" ]]; then
- fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
- fi
- ;;
- esac
-
-done <`dirname "$(readlink -f "$0")"`"/build.list"
-
# Build the API server package
test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
if [[ "$?" == "0" ]]; then
diff --git a/build/run-library.sh b/build/run-library.sh
index f079be24d..16cb0bc1e 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -623,8 +623,7 @@ fpm_build_virtualenv () {
FPM_RESULTS=$("${COMMAND_ARR[@]}")
FPM_EXIT_CODE=$?
-# FIXME
-# fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+ fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
# if something went wrong and debug is off, print out the fpm command that errored
if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
diff --git a/sdk/cwl/fpm-info.sh b/sdk/cwl/fpm-info.sh
new file mode 100644
index 000000000..5bfee91af
--- /dev/null
+++ b/sdk/cwl/fpm-info.sh
@@ -0,0 +1,12 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+case "$TARGET" in
+ debian8)
+ fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
+ ;;
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
diff --git a/sdk/python/fpm-info.sh b/sdk/python/fpm-info.sh
new file mode 100644
index 000000000..7a89cf03a
--- /dev/null
+++ b/sdk/python/fpm-info.sh
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+case "$TARGET" in
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
diff --git a/services/fuse/fpm-info.sh b/services/fuse/fpm-info.sh
index 6cf69fb78..f789abe69 100644
--- a/services/fuse/fpm-info.sh
+++ b/services/fuse/fpm-info.sh
@@ -3,3 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0
fpm_depends+=(fuse)
+
+case "$TARGET" in
+ centos*)
+ fpm_depends+=(fuse-libs)
+ ;;
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
diff --git a/services/fuse/fpm-info.sh b/services/nodemanager/fpm-info.sh
similarity index 50%
copy from services/fuse/fpm-info.sh
copy to services/nodemanager/fpm-info.sh
index 6cf69fb78..0abc6a08e 100644
--- a/services/fuse/fpm-info.sh
+++ b/services/nodemanager/fpm-info.sh
@@ -2,4 +2,8 @@
#
# SPDX-License-Identifier: AGPL-3.0
-fpm_depends+=(fuse)
+case "$TARGET" in
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
diff --git a/services/fuse/fpm-info.sh b/tools/crunchstat-summary/fpm-info.sh
similarity index 50%
copy from services/fuse/fpm-info.sh
copy to tools/crunchstat-summary/fpm-info.sh
index 6cf69fb78..0abc6a08e 100644
--- a/services/fuse/fpm-info.sh
+++ b/tools/crunchstat-summary/fpm-info.sh
@@ -2,4 +2,8 @@
#
# SPDX-License-Identifier: AGPL-3.0
-fpm_depends+=(fuse)
+case "$TARGET" in
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
commit 167b927cd6bc0cd4ff5de2a523959eec71d8a17d
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Thu Dec 20 11:52:01 2018 -0500
9945: run-library changes
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index b595cc8a0..f079be24d 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -281,7 +281,7 @@ test_package_presence() {
echo "Package $complete_pkgname exists, not rebuilding!"
curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
echo "Package $complete_pkgname exists, not rebuilding!"
return 1
else
@@ -359,6 +359,289 @@ handle_rails_package() {
rm -rf "$scripts_dir"
}
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv () {
+ PKG=$1
+ shift
+ PKG_DIR=$1
+ shift
+ PACKAGE_TYPE=${1:-python}
+ shift
+
+ # Set up
+ STDOUT_IF_DEBUG=/dev/null
+ STDERR_IF_DEBUG=/dev/null
+ DASHQ_UNLESS_DEBUG=-q
+ if [[ "$DEBUG" != "0" ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+ fi
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
+ local python=""
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ python=python2.7
+ PACKAGE_PREFIX=$PYTHON2_PKG_PREFIX
+ ;;
+ python3)
+ PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
+ python=python3
+ ;;
+ esac
+
+ if [[ "$PKG" != "libpam-arvados" ]] &&
+ [[ "$PKG" != "arvados-node-manager" ]] &&
+ [[ "$PKG" != "arvados-docker-cleaner" ]]; then
+ PYTHON_PKG=$PACKAGE_PREFIX-$PKG
+ else
+ # Exception to our package naming convention
+ PYTHON_PKG=$PKG
+ fi
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ echo "Building $FORMAT package for $PKG from $PKG_DIR"
+ cd $WORKSPACE/$PKG_DIR
+
+ # Create an sdist
+ echo "Creating sdist..."
+
+ rm -rf dist/*
+ $python setup.py $DASHQ_UNLESS_DEBUG sdist
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run python setup.py sdist for $PKG"
+ exit 1
+ fi
+
+ # Determine the package version from the generated sdist archive
+ PACKAGE_PATH=`(cd dist; ls *tar.gz)`
+ PYTHON_VERSION=${PACKAGE_PATH#$PKG-}
+
+ # For historical reasons, arvados-fuse is called arvados_fuse in python land
+ # Same with crunchstat-summary.
+ # We should fix this, but for now let's just make this script dtrt.
+ PKG_ALTERNATE=${PKG//-/_}
+ PYTHON_VERSION=${PYTHON_VERSION#$PKG_ALTERNATE-}
+
+ # An even more complicated exception
+ if [[ "$PKG" == "libpam-arvados" ]]; then
+ PYTHON_VERSION=${PYTHON_VERSION#arvados-pam-}
+ fi
+
+ if [[ "$PACKAGE_PATH" == "$PYTHON_VERSION" ]]; then
+ echo "Error, unable to determine python package version for $PKG from $PACKAGE_PATH"
+ exit 1
+ fi
+ PYTHON_VERSION=${PYTHON_VERSION%.tar.gz}
+
+ # See if we actually need to build this package; does it exist already?
+ # We can't do this earlier than here, because we need PYTHON_VERSION...
+ # This isn't so bad; the sdist call above is pretty quick compared to
+ # the invocation of virtualenv and fpm, below.
+ test_package_presence "$PYTHON_PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION amd64
+ if [[ "$?" != "0" ]]; then
+ echo "exists"
+ return 0
+ fi
+
+ # Package the sdist in a virtualenv
+ echo "Creating virtualenv..."
+
+ cd dist
+
+ rm -rf build
+ rm -f $PYTHON_PKG*deb
+
+ virtualenv_command="virtualenv --python `which $python` $DASHQ_UNLESS_DEBUG build/usr/share/$python/dist/$PYTHON_PKG"
+
+ $virtualenv_command
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run"
+ echo " $virtualenv_command"
+ exit 1
+ fi
+
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel
+
+ if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+ else
+ # centos7 needs these special tweaks to install python-arvados-fuse
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
+ PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+ fi
+
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to run"
+ echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+ exit 1
+ fi
+
+ cd build/usr/share/$python/dist/$PYTHON_PKG/
+
+ # Replace the shebang lines in all python scripts, and handle the activate
+ # scripts too This is a functional replacement of the 237 line
+ # virtualenv_tools.py script that doesn't work in python3 without serious
+ # patching, minus the parts we don't need (modifying pyc files, etc).
+ for binfile in `ls bin/`; do
+ file --mime bin/$binfile |grep -q binary
+ if [[ "$?" != "0" ]]; then
+ # Not a binary file
+ if [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
+ # these 'activate' scripts need special treatment
+ sed -i "s/VIRTUAL_ENV=\".*\"/VIRTUAL_ENV=\"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
+ sed -i "s/VIRTUAL_ENV \".*\"/VIRTUAL_ENV \"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
+ else
+ grep -q -E '^#!.*/bin/python\d?' bin/$binfile
+ if [[ "$?" == "0" ]]; then
+ # Replace shebang line
+ sed -i "1 s/^.*$/#!\/usr\/share\/$python\/dist\/$PYTHON_PKG\/bin\/python/" bin/$binfile
+ fi
+ fi
+ fi
+ done
+
+ cd - >$STDOUT_IF_DEBUG
+
+ find build -iname *.pyc -exec rm {} \;
+ find build -iname *.pyo -exec rm {} \;
+
+ # Finally, generate the package
+ echo "Creating package..."
+
+ declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
+
+ if [[ "$MAINTAINER" != "" ]]; then
+ COMMAND_ARR+=('--maintainer' "$MAINTAINER")
+ fi
+
+ if [[ "$VENDOR" != "" ]]; then
+ COMMAND_ARR+=('--vendor' "$VENDOR")
+ fi
+
+ COMMAND_ARR+=('--url' 'https://arvados.org')
+
+ # Get description
+ DESCRIPTION=`grep '\sdescription' $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
+ COMMAND_ARR+=('--description' "$DESCRIPTION")
+
+ # Get license string
+ LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
+ COMMAND_ARR+=('--license' "$LICENSE_STRING")
+
+ # 12271 - As FPM-generated packages don't include scripts by default, the
+ # packages cleanup on upgrade depends on files being listed on the %files
+ # section in the generated SPEC files. To remove DIRECTORIES, they need to
+ # be listed in that sectiontoo, so we need to add this parameter to properly
+ # remove lingering dirs. But this only works for python2: if used on
+ # python33, it includes dirs like /opt/rh/python33 that belong to
+ # other packages.
+ if [[ "$FORMAT" == "rpm" ]] && [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--rpm-auto-add-directories')
+ fi
+
+ if [[ "$PKG" == "arvados-python-client" ]]; then
+ if [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
+ else
+ COMMAND_ARR+=('--conflicts' "$PYTHON2_PKG_PREFIX-$PKG")
+ fi
+ fi
+
+ if [[ "${DEBUG:-0}" != "0" ]]; then
+ COMMAND_ARR+=('--verbose' '--log' 'info')
+ fi
+
+ COMMAND_ARR+=('-v' "$PYTHON_VERSION")
+ COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
+ COMMAND_ARR+=('-n' "$PYTHON_PKG")
+ COMMAND_ARR+=('--before-remove' "$WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh")
+ COMMAND_ARR+=('--after-install' "$WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh")
+ COMMAND_ARR+=('-C' "build")
+
+ if [[ "$python" == "python2.7" ]]; then
+ COMMAND_ARR+=('--depends' "$PYTHON2_PACKAGE")
+ else
+ COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
+ fi
+
+ # avoid warning
+ COMMAND_ARR+=('--deb-no-default-config-files')
+
+ # Append --depends X and other arguments specified by fpm-info.sh in
+ # the package source dir. These are added last so they can override
+ # the arguments added by this script.
+ declare -a fpm_args=()
+ declare -a fpm_depends=()
+
+ fpminfo="$WORKSPACE/$PKG_DIR/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ if [[ "$?" != "0" ]]; then
+ echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
+ exit 1
+ fi
+ fi
+
+ if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
+ FPM_BINARIES=""
+ for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
+ FPM_BINARIES+="'$binary' "
+ done
+ fi
+
+ # create a custom version of the package scripts, with fpm_binaries populated
+ cp -f $WORKSPACE/build/python-package-scripts/before-remove.sh $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ cp -f $WORKSPACE/build/python-package-scripts/after-install.sh $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+ sed -i s/%FPM_BINARIES/"$FPM_BINARIES"/g $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ sed -i s/%FPM_BINARIES/"$FPM_BINARIES"/g $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+
+ # Make sure the package scripts know the correct path where its files are installed
+ sed -i s/%PYTHON/$python/g $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ sed -i s/%PYTHON/$python/g $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--depends' "$i")
+ done
+
+ COMMAND_ARR+=("${fpm_args[@]}")
+
+ COMMAND_ARR+=(".")
+
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+
+# FIXME
+# fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+
+ # if something went wrong and debug is off, print out the fpm command that errored
+ if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
+ echo "fpm returned an error execurin the command:"
+ echo
+ echo -e "\n${COMMAND_ARR[@]}\n"
+ else
+ echo `ls *$FORMAT`
+ mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/
+ fi
+ echo
+
+ # clean up
+ rm -f $WORKSPACE/build/python-package-scripts/before-remove-$PKG.tmp.sh
+ rm -f $WORKSPACE/build/python-package-scripts/after-install-$PKG.tmp.sh
+}
+
# Build packages for everything
fpm_build () {
# The package source. Depending on the source type, this can be a
commit 55558d2bbe2af6e285cc1ad458fce58ebe452b24
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Sun Jan 20 13:12:45 2019 -0500
9945: there is a new rvm gpg key
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index cdd44d9c5..523205021 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -30,9 +30,10 @@ ubuntu1804/generated: common-generated-all
GOTARBALL=go1.10.1.linux-amd64.tar.gz
NODETARBALL=node-v6.11.2-linux-x64.tar.xz
-RVMKEY=rvm.asc
+RVMKEY1=mpapis.asc
+RVMKEY2=pkuczynski.asc
-common-generated-all: common-generated/$(GOTARBALL) common-generated/$(NODETARBALL) common-generated/$(RVMKEY)
+common-generated-all: common-generated/$(GOTARBALL) common-generated/$(NODETARBALL) common-generated/$(RVMKEY1) common-generated/$(RVMKEY2)
common-generated/$(GOTARBALL): common-generated
wget -cqO common-generated/$(GOTARBALL) http://storage.googleapis.com/golang/$(GOTARBALL)
@@ -40,8 +41,11 @@ common-generated/$(GOTARBALL): common-generated
common-generated/$(NODETARBALL): common-generated
wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v6.11.2/$(NODETARBALL)
-common-generated/$(RVMKEY): common-generated
- wget -cqO common-generated/$(RVMKEY) https://rvm.io/mpapis.asc
+common-generated/$(RVMKEY1): common-generated
+ wget -cqO common-generated/$(RVMKEY1) https://rvm.io/mpapis.asc
+
+common-generated/$(RVMKEY2): common-generated
+ wget -cqO common-generated/$(RVMKEY2) https://rvm.io/pkuczynski.asc
common-generated:
mkdir common-generated
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 20a887127..23400cf48 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -9,8 +9,10 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git python-virtualenv
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 134cece79..493aef7be 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -11,8 +11,10 @@ ENV DEBIAN_FRONTEND noninteractive
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 9674c0cf2..98da1c97a 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -15,8 +15,10 @@ RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev pyth
RUN /usr/bin/pip install virtualenv virtualenv-tools
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --no-tty --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index 798680e48..8b0d9bc65 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -11,8 +11,10 @@ ENV DEBIAN_FRONTEND noninteractive
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index b9e966c74..de15c9ecd 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -11,8 +11,10 @@ ENV DEBIAN_FRONTEND noninteractive
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index 3891df393..0dc431373 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -11,8 +11,10 @@ ENV DEBIAN_FRONTEND noninteractive
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile
index f2fd49b10..c6d5a15fb 100644
--- a/build/package-test-dockerfiles/Makefile
+++ b/build/package-test-dockerfiles/Makefile
@@ -28,12 +28,16 @@ ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -rlt ubuntu1804/generated common-generated/*
-RVMKEY=rvm.asc
+RVMKEY1=mpapis.asc
+RVMKEY2=pkuczynski.asc
-common-generated-all: common-generated/$(RVMKEY)
+common-generated-all: common-generated/$(RVMKEY1) common-generated/$(RVMKEY2)
-common-generated/$(RVMKEY): common-generated
- wget -cqO common-generated/$(RVMKEY) https://rvm.io/mpapis.asc
+common-generated/$(RVMKEY1): common-generated
+ wget -cqO common-generated/$(RVMKEY1) https://rvm.io/mpapis.asc
+
+common-generated/$(RVMKEY2): common-generated
+ wget -cqO common-generated/$(RVMKEY2) https://rvm.io/pkuczynski.asc
common-generated:
mkdir common-generated
diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile
index 36be0ba32..7a0cb9bd5 100644
--- a/build/package-test-dockerfiles/centos7/Dockerfile
+++ b/build/package-test-dockerfiles/centos7/Dockerfile
@@ -9,13 +9,16 @@ MAINTAINER Ward Vandewege <wvandewege at veritasgenetics.com>
RUN yum -q -y install scl-utils centos-release-scl which tar
# Install RVM
-ADD generated/rvm.asc /tmp/
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
RUN touch /var/lib/rpm/* && \
- gpg --import /tmp/rvm.asc && \
+ gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundle && \
- /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+ /usr/local/rvm/bin/rvm alias create default ruby-2.3
+# && \
+# /usr/local/rvm/bin/rvm-exec default gem install bundle && \
+# /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
diff --git a/build/package-test-dockerfiles/debian8/Dockerfile b/build/package-test-dockerfiles/debian8/Dockerfile
index fdefadea5..2168f725a 100644
--- a/build/package-test-dockerfiles/debian8/Dockerfile
+++ b/build/package-test-dockerfiles/debian8/Dockerfile
@@ -12,8 +12,10 @@ RUN apt-get update && \
apt-get -y install --no-install-recommends curl ca-certificates
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3
diff --git a/build/package-test-dockerfiles/debian9/Dockerfile b/build/package-test-dockerfiles/debian9/Dockerfile
index c682ccc7c..9c46ef601 100644
--- a/build/package-test-dockerfiles/debian9/Dockerfile
+++ b/build/package-test-dockerfiles/debian9/Dockerfile
@@ -12,8 +12,10 @@ RUN apt-get update && \
apt-get -y install --no-install-recommends curl ca-certificates gpg procps
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --no-tty --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3
diff --git a/build/package-test-dockerfiles/ubuntu1404/Dockerfile b/build/package-test-dockerfiles/ubuntu1404/Dockerfile
index 5f5b1d881..c05dbee0f 100644
--- a/build/package-test-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-test-dockerfiles/ubuntu1404/Dockerfile
@@ -12,8 +12,10 @@ RUN apt-get update && \
apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3
diff --git a/build/package-test-dockerfiles/ubuntu1604/Dockerfile b/build/package-test-dockerfiles/ubuntu1604/Dockerfile
index 1f65c7a47..615ab1c00 100644
--- a/build/package-test-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-test-dockerfiles/ubuntu1604/Dockerfile
@@ -12,8 +12,10 @@ RUN apt-get update && \
apt-get -y install --no-install-recommends curl ca-certificates
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3
diff --git a/build/package-test-dockerfiles/ubuntu1804/Dockerfile b/build/package-test-dockerfiles/ubuntu1804/Dockerfile
index 9d326c729..d530d22a2 100644
--- a/build/package-test-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-test-dockerfiles/ubuntu1804/Dockerfile
@@ -12,8 +12,10 @@ RUN apt-get update && \
apt-get -y install --no-install-recommends curl ca-certificates gnupg2
# Install RVM
-ADD generated/rvm.asc /tmp/
-RUN gpg --import /tmp/rvm.asc && \
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.3
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list