[ARVADOS] created: 4aea0dbc8b9b4a5f898ea206d337ad88e6ef7268
Git user
git at public.curoverse.com
Fri May 27 20:04:50 EDT 2016
at 4aea0dbc8b9b4a5f898ea206d337ad88e6ef7268 (commit)
commit 4aea0dbc8b9b4a5f898ea206d337ad88e6ef7268
Author: Brett Smith <brett at curoverse.com>
Date: Fri May 27 18:34:56 2016 -0400
9309: Add packages and tests for CentOS 7.
diff --git a/backports/python-apache-libcloud/fpm-info.sh b/backports/python-apache-libcloud/fpm-info.sh
new file mode 100644
index 0000000..0333bdf
--- /dev/null
+++ b/backports/python-apache-libcloud/fpm-info.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+case $TARGET in
+ centos7)
+ # fpm incorrectly transforms the dependency name in this case.
+ fpm_depends+=(python-backports-ssl_match_hostname)
+ fpm_args+=(--python-disable-dependency backports.ssl-match-hostname)
+ ;;
+esac
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 9216f82..ad081e9 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -1,9 +1,13 @@
-all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
+all: centos6/generated centos7/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
centos6/generated: common-generated-all
test -d centos6/generated || mkdir centos6/generated
cp -rlt centos6/generated common-generated/*
+centos7/generated: common-generated-all
+ test -d centos7/generated || mkdir centos7/generated
+ cp -rlt centos7/generated common-generated/*
+
debian7/generated: common-generated-all
test -d debian7/generated || mkdir debian7/generated
cp -rlt debian7/generated common-generated/*
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
new file mode 100644
index 0000000..a37c97f
--- /dev/null
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -0,0 +1,25 @@
+FROM centos:7
+MAINTAINER Brett Smith <brett at curoverse.com>
+
+# Install build dependencies provided in base distribution
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python33
+RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
+
+ENV WORKSPACE /arvados
+CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile
new file mode 100644
index 0000000..6bc40bf
--- /dev/null
+++ b/build/package-test-dockerfiles/centos7/Dockerfile
@@ -0,0 +1,15 @@
+FROM centos:7
+MAINTAINER Brett Smith <brett at curoverse.com>
+
+RUN yum -q -y install scl-utils centos-release-scl which tar
+
+# Install RVM
+RUN touch /var/lib/rpm/* && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundle && \
+ /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
diff --git a/build/package-test-dockerfiles/centos7/localrepo.repo b/build/package-test-dockerfiles/centos7/localrepo.repo
new file mode 100644
index 0000000..ebb8765
--- /dev/null
+++ b/build/package-test-dockerfiles/centos7/localrepo.repo
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos7
+gpgcheck=0
+enabled=1
diff --git a/build/package-testing/rpm-common-test-packages.sh b/build/package-testing/rpm-common-test-packages.sh
new file mode 100755
index 0000000..4d0c32b
--- /dev/null
+++ b/build/package-testing/rpm-common-test-packages.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eu
+
+target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b')
+
+yum -q clean all
+touch /var/lib/rpm/*
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
+
+yum install --assumeyes $1
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
+
+diff "$ARV_PACKAGES_DIR/$1".{before,after} >"$ARV_PACKAGES_DIR/$1.diff" || true
+
+# Enable any Software Collections that the package depended on.
+if [[ -d /opt/rh ]]; then
+ # We have to stage the list to a file, because `ls | while read` would
+ # make a subshell, causing the `source` lines to have no effect.
+ scl_list=$(mktemp)
+ ls /opt/rh >"$scl_list"
+
+ # SCL scripts aren't designed to run with -eu.
+ set +eu
+ while read scl; do
+ source scl_source enable "$scl"
+ done <"$scl_list"
+ set -eu
+ rm "$scl_list"
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
+
+find -name '*.so' | while read so; do
+ echo -e "\n== Packages dependencies for $so =="
+ ldd "$so" \
+ | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+done
+
+exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/build/package-testing/test-package-arvados-api-server.sh b/build/package-testing/test-package-arvados-api-server.sh
index e975448..66d5969 100755
--- a/build/package-testing/test-package-arvados-api-server.sh
+++ b/build/package-testing/test-package-arvados-api-server.sh
@@ -7,7 +7,7 @@ case "$TARGET" in
apt-get install -y nginx
dpkg-reconfigure arvados-api-server
;;
- centos6)
+ centos*)
yum install --assumeyes httpd
yum reinstall --assumeyes arvados-api-server
;;
diff --git a/build/package-testing/test-package-arvados-sso-server.sh b/build/package-testing/test-package-arvados-sso-server.sh
index c1a377e..ab0b225 100755
--- a/build/package-testing/test-package-arvados-sso-server.sh
+++ b/build/package-testing/test-package-arvados-sso-server.sh
@@ -18,7 +18,7 @@ case "$TARGET" in
debian*|ubuntu*)
FORMAT=deb
;;
- centos6)
+ centos*)
FORMAT=rpm
;;
*)
@@ -75,12 +75,9 @@ fi
if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then
# We need to set up our database configuration now.
if [[ "$FORMAT" == "rpm" ]]; then
- # postgres packaging on CentOS6 is kind of primitive, needs an initdb
service postgresql initdb
- if [ "$TARGET" = "centos6" ]; then
- sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
- sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
- fi
+ sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
+ sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
fi
service postgresql start
diff --git a/build/package-testing/test-package-arvados-workbench.sh b/build/package-testing/test-package-arvados-workbench.sh
index 1be4dea..5deb1a0 100755
--- a/build/package-testing/test-package-arvados-workbench.sh
+++ b/build/package-testing/test-package-arvados-workbench.sh
@@ -7,7 +7,7 @@ case "$TARGET" in
apt-get install -y nginx
dpkg-reconfigure arvados-workbench
;;
- centos6)
+ centos*)
yum install --assumeyes httpd
yum reinstall --assumeyes arvados-workbench
;;
diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh
deleted file mode 100755
index 4e05364..0000000
--- a/build/package-testing/test-packages-centos6.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-yum -q clean all
-touch /var/lib/rpm/*
-
-export ARV_PACKAGES_DIR=/arvados/packages/centos6
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
-
-yum install --assumeyes $1
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
-
-set +e
-diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff"
-set -e
-
-SCL=""
-if scl enable python27 true 2>/dev/null ; then
- SCL="scl enable python27"
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
-
-shared=$(find -name '*.so')
-if test -n "$shared" ; then
- for so in $shared ; do
- echo
- echo "== Packages dependencies for $so =="
- $SCL ldd "$so" \
- | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
- done
-fi
-
-if test -n "$SCL" ; then
- exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'"
-else
- exec /jenkins/package-testing/common-test-packages.sh "$1"
-fi
diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh
new file mode 120000
index 0000000..64ef604
--- /dev/null
+++ b/build/package-testing/test-packages-centos6.sh
@@ -0,0 +1 @@
+rpm-common-test-packages.sh
\ No newline at end of file
diff --git a/build/package-testing/test-packages-centos7.sh b/build/package-testing/test-packages-centos7.sh
new file mode 120000
index 0000000..64ef604
--- /dev/null
+++ b/build/package-testing/test-packages-centos7.sh
@@ -0,0 +1 @@
+rpm-common-test-packages.sh
\ No newline at end of file
diff --git a/build/run-build-packages-sso.sh b/build/run-build-packages-sso.sh
index cc673a6..b5dcdfc 100755
--- a/build/run-build-packages-sso.sh
+++ b/build/run-build-packages-sso.sh
@@ -80,7 +80,7 @@ case "$TARGET" in
ubuntu1404)
FORMAT=deb
;;
- centos6)
+ centos6|centos7)
FORMAT=rpm
;;
*)
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 0312bc3..ccd9d40 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -154,6 +154,22 @@ case "$TARGET" in
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
export PYCURL_SSL_LIBRARY=nss
;;
+ centos7)
+ FORMAT=rpm
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka pyexecjs jsonschema \
+ ciso8601 pycrypto 'pycurl<7.21.5' \
+ python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
+ 'rdflib>=4.2.0' shellescape mistune typing avro \
+ isodate pyparsing sparqlwrapper html5lib keepalive)
+ PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+ export PYCURL_SSL_LIBRARY=nss
+ ;;
*)
echo -e "$0: Unknown target '$TARGET'.\n" >&2
exit 1
@@ -431,7 +447,7 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover
# Ward, 2016-03-17
fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-schema-salad" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.11.20160506154702 schema_salad
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
+# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
# Ward, 2016-05-26
fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t deb --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --verbose --log info --iteration 1 --python-bin python2.7 --python-easyinstall easy_install-2.7 --python-package-name-prefix python --depends python2.7 --python-setup-py-arguments "--single-version-externally-managed" --debug ruamel.yaml
commit ca2b0c524f73ad1dcbad724b695ea59f1b00645e
Author: Brett Smith <brett at curoverse.com>
Date: Fri May 27 17:50:40 2016 -0400
9309: Look for fpm-info in backports/$PACKAGE_NAME.
This lets us define additional fpm flags when we build a non-dir
package from a source directory.
diff --git a/build/run-library.sh b/build/run-library.sh
index 8d97ada..0867978 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -178,8 +178,8 @@ fpm_build () {
# pip).
PACKAGE=$1
shift
- # The name of the package to build. Defaults to $PACKAGE.
- PACKAGE_NAME=${1:-$PACKAGE}
+ # The name of the package to build.
+ PACKAGE_NAME=$1
shift
# Optional: the vendor of the package. Should be "Curoverse, Inc." for
# packages of our own software. Passed to fpm --vendor.
@@ -229,7 +229,7 @@ fpm_build () {
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
+ if [[ -n "$PACKAGE_NAME" ]]; then
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fi
@@ -251,19 +251,21 @@ fpm_build () {
declare -a build_depends=()
declare -a fpm_depends=()
declare -a fpm_exclude=()
- declare -a fpm_dirs=(
+ local fpm_dir
+ if [[ dir = "$PACKAGE_TYPE" ]]; then
# source dir part of 'dir' package ("/source=/dest" => "/source"):
- "${PACKAGE%%=/*}"
+ fpm_dir="${PACKAGE%%=/*}"
+ elif [[ -n "$PACKAGE_NAME" ]]; then
+ fpm_dir="${WORKSPACE}/backports/${PACKAGE_NAME}"
+ else
# backports ("llfuse==0.41.1" => "backports/python-llfuse")
- "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
- for pkgdir in "${fpm_dirs[@]}"; do
- fpminfo="$pkgdir/fpm-info.sh"
- if [[ -e "$fpminfo" ]]; then
- debug_echo "Loading fpm overrides from $fpminfo"
- source "$fpminfo"
- break
- fi
- done
+ fpm_dir="${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}"
+ fi
+ fpminfo="${fpm_dir}/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ debug_echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ fi
for pkg in "${build_depends[@]}"; do
if [[ $TARGET =~ debian|ubuntu ]]; then
pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list