[ARVADOS-DEV] created: a23620386db119bf670221368016484a5a2f1f93
git at public.curoverse.com
git at public.curoverse.com
Fri Oct 16 14:31:12 EDT 2015
at a23620386db119bf670221368016484a5a2f1f93 (commit)
commit a23620386db119bf670221368016484a5a2f1f93
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Oct 16 14:29:57 2015 -0400
7370: run-build-packages-all-targets hands off to run-build-packages-one-target, suppports --test-packages option.
diff --git a/jenkins/run-build-packages-all-targets.sh b/jenkins/run-build-packages-all-targets.sh
index 8c889a0..19befb3 100755
--- a/jenkins/run-build-packages-all-targets.sh
+++ b/jenkins/run-build-packages-all-targets.sh
@@ -10,6 +10,8 @@ Options:
--command
Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install tests
--debug
Output debug information (default: false)
@@ -36,7 +38,7 @@ fi
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,command: \
+ help,test-packages,debug,command: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -44,6 +46,7 @@ fi
COMMAND=
DEBUG=
+TEST_PACKAGES=
eval set -- "$PARSEDOPTS"
while [ $# -gt 0 ]; do
@@ -54,11 +57,14 @@ while [ $# -gt 0 ]; do
exit 1
;;
--debug)
- DEBUG=" --debug"
+ DEBUG="--debug"
;;
--command)
COMMAND="$2"; shift
;;
+ --test-packages)
+ TEST_PACKAGES="--test-packages"
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
@@ -69,36 +75,6 @@ while [ $# -gt 0 ]; do
shift
done
-if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND$DEBUG"
-fi
-
-FINAL_EXITCODE=0
-JENKINS_DIR=$(dirname "$(readlink -e "$0")")
-
-run_docker() {
- local tag=$1; shift
- if [[ "$COMMAND" != "" ]]; then
- COMMAND="$COMMAND --target $tag"
- fi
- if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
- --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
- # Success - nothing more to do.
- true
- else
- FINAL_EXITCODE=$?
- echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
- fi
-}
-
-# In case it's needed, build the containers. This costs just a few
-# seconds when the containers already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
-time ./build-all-build-containers.sh
-
for dockerfile_path in $(find -name Dockerfile); do
- run_docker "$(basename $(dirname "$dockerfile_path"))"
+ ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES
done
-
-exit $FINAL_EXITCODE
commit ad5f9554d13754e16def528a24a2669f7cbd4e5d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 15 16:40:54 2015 -0400
7370: Fix centos6 package builder container.
diff --git a/jenkins/dockerfiles/centos6/Dockerfile b/jenkins/dockerfiles/centos6/Dockerfile
index d0daef6..0a733e3 100644
--- a/jenkins/dockerfiles/centos6/Dockerfile
+++ b/jenkins/dockerfiles/centos6/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.1 && \
/usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm
+ /usr/local/rvm/bin/rvm-exec default gem install bundle fpm
# Download and install Software Collections for newer requirements
ADD https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm /opt/
commit 72c6e7bcd898caa3a2ed718e66b71568fe66bc7d
Merge: d20e333 b67e599
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 15 16:24:07 2015 -0400
Merge branch 'master' into 7370-package-install-testing
Conflicts:
jenkins/run-build-packages-one-target.sh
diff --cc jenkins/run-build-packages-one-target.sh
index 3e0e136,2d8d8df..5b8f2ce
--- a/jenkins/run-build-packages-one-target.sh
+++ b/jenkins/run-build-packages-one-target.sh
@@@ -10,8 -10,8 +10,10 @@@ Syntax
Distribution to build packages for (default: debian7)
--command
Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install test script "test-packages-$target.sh"
+ --debug
+ Output debug information (default: false)
WORKSPACE=path Path to the Arvados source tree to build packages from
@@@ -34,7 -34,7 +36,7 @@@ if ! [[ -d "$WORKSPACE" ]]; the
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,target:,command: \
- help,debug,target:,command: \
++ help,debug,test-packages,target:,command: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@@ -72,48 -73,34 +78,48 @@@ don
set -e
-if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
+if [[ -n "$test_packages" ]]; then
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+ createrepo $WORKSPACE/packages/$TARGET
+ fi
+
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+ (cd $WORKSPACE/packages/$TARGET
+ dpkg-scanpackages . /dev/null | gzip -c > Packages.gz
+ )
+ fi
+
+ COMMAND="/jenkins/test-packages-$TARGET.sh"
+ IMAGE="arvados/package-test:$TARGET"
+else
+ IMAGE="arvados/build:$TARGET"
+ if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
++ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
+ fi
fi
-FINAL_EXITCODE=0
JENKINS_DIR=$(dirname "$(readlink -e "$0")")
-run_docker() {
- local tag=$1; shift
- if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
- --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
- # Success - nothing more to do.
- true
- else
- FINAL_EXITCODE=$?
- echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
- fi
-}
+if [[ -n "$test_packages" ]]; then
+ pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+ pushd "$JENKINS_DIR/dockerfiles"
+fi
-# In case it's needed, build the container. This costs just a few
-# seconds when the container already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
echo $TARGET
cd $TARGET
-time docker build -t arvados/build:$TARGET .
-cd ..
+time docker build --tag=$IMAGE .
+popd
-run_docker $TARGET
+FINAL_EXITCODE=0
+
+if docker run --rm -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
+ --env ARVADOS_DEBUG=1 "$IMAGE" $COMMAND; then
+ # Success - nothing more to do.
+ true
+else
+ FINAL_EXITCODE=$?
+ echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
+fi
exit $FINAL_EXITCODE
commit d20e33394de2a85f17399e0e4d9d7d2e52c94178
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 15 16:21:56 2015 -0400
7370: Refactoring way the package test is set up and executed.
diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index f40ba7b..d9d0301 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -1,29 +1,9 @@
-#!/bin/sh
+#!/bin/bash
-if test "$1" = --run-test ; then
+# Multiple .deb based distros symlink to this script, so extract the target
+# from the invocation path.
+target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
- if test -z "$WORKSPACE" ; then
- echo "Must set WORKSPACE"
- exit 1
- fi
-
- self=$(readlink -f $0)
- base=$(dirname $self)
-
- cd $WORKSPACE/packages/$2
- dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
-
- exec docker run \
- --rm \
- --volume=$WORKSPACE/packages/$2:/mnt \
- --volume=$self:/root/run-test.sh \
- --volume=$base/common-test-packages.sh:/root/common-test.sh \
- --workdir=/mnt \
- $3 \
- /root/run-test.sh
-fi
-
-echo "deb file:///mnt /" >>/etc/apt/sources.list
apt-get -qq update
if ! apt-get -qq --assume-yes --force-yes install \
python-arvados-python-client python-arvados-fuse arvados-node-manager
@@ -34,7 +14,7 @@ fi
mkdir -p /tmp/opts
cd /tmp/opts
-for r in /mnt/python-*amd64.deb ; do
+for r in /arvados/packages/$target/python-*amd64.deb ; do
dpkg-deb -x $r .
done
@@ -44,4 +24,4 @@ for so in $(find . -name "*.so") ; do
ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
done
-exec /root/common-test.sh
+exec /jenkins/common-test-packages.sh
diff --git a/jenkins/dockerfiles/centos6/Dockerfile b/jenkins/dockerfiles/centos6/Dockerfile
index ca974df..d0daef6 100644
--- a/jenkins/dockerfiles/centos6/Dockerfile
+++ b/jenkins/dockerfiles/centos6/Dockerfile
@@ -19,7 +19,10 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
ADD https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm /opt/
RUN yum -q -y install /opt/rhscl-*.rpm
-RUN yum -q -y install python27 rh-python34
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python27 rh-python34
ENV WORKSPACE /arvados
CMD ["scl", "enable", "rh-python34", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]
diff --git a/jenkins/package-test-dockerfiles/centos6/Dockerfile b/jenkins/package-test-dockerfiles/centos6/Dockerfile
new file mode 100644
index 0000000..5e354e2
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/Dockerfile
@@ -0,0 +1,9 @@
+FROM centos:6
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN yum -q install --assumeyes scl-utils && \
+ curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm && \
+ yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm && \
+ yum -q install --assumeyes python27
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/centos6/localrepo.repo b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
new file mode 100644
index 0000000..ac6b898
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos6
+gpgcheck=0
+enabled=1
diff --git a/jenkins/package-test-dockerfiles/debian7/Dockerfile b/jenkins/package-test-dockerfiles/debian7/Dockerfile
new file mode 100644
index 0000000..fb789c6
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian7/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:7
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list
diff --git a/jenkins/package-test-dockerfiles/debian8/Dockerfile b/jenkins/package-test-dockerfiles/debian8/Dockerfile
new file mode 100644
index 0000000..fbcd8f4
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian8/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:8
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
new file mode 100644
index 0000000..4b6e936
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:precise
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
new file mode 100644
index 0000000..4f49d92
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:trusty
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/run-build-packages-one-target.sh b/jenkins/run-build-packages-one-target.sh
index 5451a4e..3e0e136 100755
--- a/jenkins/run-build-packages-one-target.sh
+++ b/jenkins/run-build-packages-one-target.sh
@@ -10,6 +10,8 @@ Syntax:
Distribution to build packages for (default: debian7)
--command
Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install test script "test-packages-$target.sh"
WORKSPACE=path Path to the Arvados source tree to build packages from
@@ -32,7 +34,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,target:,command: \
+ help,test-packages,target:,command: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -55,6 +57,9 @@ while [ $# -gt 0 ]; do
--command)
COMMAND="$2"; shift
;;
+ --test-packages)
+ test_packages=1
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
@@ -67,34 +72,48 @@ done
set -e
-if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+if [[ -n "$test_packages" ]]; then
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+ createrepo $WORKSPACE/packages/$TARGET
+ fi
+
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+ (cd $WORKSPACE/packages/$TARGET
+ dpkg-scanpackages . /dev/null | gzip -c > Packages.gz
+ )
+ fi
+
+ COMMAND="/jenkins/test-packages-$TARGET.sh"
+ IMAGE="arvados/package-test:$TARGET"
+else
+ IMAGE="arvados/build:$TARGET"
+ if [[ "$COMMAND" != "" ]]; then
+ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+ fi
fi
-FINAL_EXITCODE=0
JENKINS_DIR=$(dirname "$(readlink -e "$0")")
-run_docker() {
- local tag=$1; shift
- if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
- --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
- # Success - nothing more to do.
- true
- else
- FINAL_EXITCODE=$?
- echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
- fi
-}
+if [[ -n "$test_packages" ]]; then
+ pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+ pushd "$JENKINS_DIR/dockerfiles"
+fi
-# In case it's needed, build the container. This costs just a few
-# seconds when the container already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
echo $TARGET
cd $TARGET
-time docker build -t arvados/build:$TARGET .
-cd ..
+time docker build --tag=$IMAGE .
+popd
-run_docker $TARGET
+FINAL_EXITCODE=0
+
+if docker run --rm -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
+ --env ARVADOS_DEBUG=1 "$IMAGE" $COMMAND; then
+ # Success - nothing more to do.
+ true
+else
+ FINAL_EXITCODE=$?
+ echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
+fi
exit $FINAL_EXITCODE
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 555b9f1..c8a62b5 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -1,62 +1,29 @@
-#!/bin/sh
+#!/bin/bash
-if test "$1" = --run-test ; then
-
- if test -z "$WORKSPACE" ; then
- echo "Must set WORKSPACE"
+if ! python --version |& grep "Python 2.7" >/dev/null ; then
+ # Re-execute in the software collections Python 2.7 environment.
+ exec scl enable python27 $0
+else
+ yum -q clean all
+ yum -q update
+ if ! yum -q install --assumeyes \
+ python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
+ then
exit 1
fi
- self=$(readlink -f $0)
- base=$(dirname $self)
+ mkdir -p /tmp/opts
+ cd /tmp/opts
- createrepo $WORKSPACE/packages/centos6
+ for r in /arvados/packages/centos6/python27-python-*x86_64.rpm ; do
+ rpm2cpio $r | cpio -idm
+ done
- exec docker run \
- --rm \
- --volume=$WORKSPACE/packages/centos6:/mnt \
- --volume=$(readlink -f $0):/root/run-test.sh \
- --volume=$base/common-test-packages.sh:/root/common-test.sh \
- --workdir=/mnt \
- centos:6 \
- /root/run-test.sh --install-scl
-fi
+ for so in $(find . -name "*.so") ; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+ done
-if test "$1" = --install-scl ; then
- yum -q install --assumeyes scl-utils
- curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
- yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
- yum -q install --assumeyes python27
- exec scl enable python27 $0
+ exec /jenkins/common-test-packages.sh
fi
-
-cat >/etc/yum.repos.d/localrepo.repo <<EOF
-[localrepo]
-name=Arvados Test
-baseurl=file:///mnt
-gpgcheck=0
-enabled=1
-EOF
-
-yum -q clean all
-yum -q update
-if ! yum -q install --assumeyes \
- python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
-then
- exit 1
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-for r in /mnt/python27-python-*x86_64.rpm ; do
- rpm2cpio $r | cpio -idm
-done
-
-for so in $(find . -name "*.so") ; do
- echo
- echo "== Packages dependencies for $so =="
- ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
-done
-
-exec /root/common-test.sh
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
deleted file mode 100755
index c5cbfe6..0000000
--- a/jenkins/test-packages-debian7.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-debian7.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
deleted file mode 100755
index 5502f46..0000000
--- a/jenkins/test-packages-debian8.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-debian8.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
deleted file mode 100755
index 00727a0..0000000
--- a/jenkins/test-packages-ubuntu1204.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
deleted file mode 100755
index bfc2c85..0000000
--- a/jenkins/test-packages-ubuntu1404.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1404 ubuntu:trusty
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
commit 39520ae3caa8227e58471b8783106a3c2dba1304
Merge: 823a16a 2d11eaa
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 15 10:40:17 2015 -0400
Merge branch 'master' into 7370-package-install-testing
commit 823a16ae78acdadd17d6f563dfafe4678c364906
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Aug 31 15:23:00 2015 -0400
Tests compatible libcloud package is installed.
diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
index 52bbba0..c9d77c8 100755
--- a/jenkins/common-test-packages.sh
+++ b/jenkins/common-test-packages.sh
@@ -15,6 +15,11 @@ if ! python <<EOF
import arvados
import arvados_fuse
print "Successly imported arvados and arvados_fuse"
+
+import libcloud.compute.types
+import libcloud.compute.providers
+libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
+print "Successly imported compatible libcloud library"
EOF
then
FAIL=1
diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index d781c8b..f40ba7b 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -25,7 +25,9 @@ fi
echo "deb file:///mnt /" >>/etc/apt/sources.list
apt-get -qq update
-if ! apt-get -qq --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+if ! apt-get -qq --assume-yes --force-yes install \
+ python-arvados-python-client python-arvados-fuse arvados-node-manager
+then
exit 1
fi
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 580bb27..555b9f1 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -40,7 +40,9 @@ EOF
yum -q clean all
yum -q update
-if ! yum -q install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+if ! yum -q install --assumeyes \
+ python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
+then
exit 1
fi
commit 21a3d01379891f2670991e4d24804e1dc87a1ab1
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Aug 31 11:18:05 2015 -0400
Log package dependencies for each shared object. Add -q so make the logs a bit
less chatty.
diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
index 869f054..52bbba0 100755
--- a/jenkins/common-test-packages.sh
+++ b/jenkins/common-test-packages.sh
@@ -2,6 +2,7 @@
FAIL=0
+echo
for so in $(find . -name "*.so") ; do
if ldd $so | grep "not found" ; then
echo "^^^ Missing while scanning $so ^^^"
@@ -9,6 +10,7 @@ for so in $(find . -name "*.so") ; do
fi
done
+echo
if ! python <<EOF
import arvados
import arvados_fuse
diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index fbebe24..d781c8b 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -24,8 +24,8 @@ if test "$1" = --run-test ; then
fi
echo "deb file:///mnt /" >>/etc/apt/sources.list
-apt-get update
-if ! apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+apt-get -qq update
+if ! apt-get -qq --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
exit 1
fi
@@ -36,4 +36,10 @@ for r in /mnt/python-*amd64.deb ; do
dpkg-deb -x $r .
done
+for so in $(find . -name "*.so") ; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+done
+
exec /root/common-test.sh
diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
index dce7362..baca7b7 100755
--- a/jenkins/run-test-packages.sh
+++ b/jenkins/run-test-packages.sh
@@ -7,11 +7,17 @@ fi
FAIL=0
+ERRORS=""
+
for pkg in ./test-packages-*.sh ; do
+ echo
+ echo "== Running $pkg =="
+ echo
if ! $pkg --run-test ; then
FAIL=1
- echo "$pkg has install errors"
+ ERRORS="$ERRORS\n$pkg has install errors"
fi
done
+echo $ERRORS
exit $FAIL
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index ea3ed09..580bb27 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -23,10 +23,10 @@ if test "$1" = --run-test ; then
fi
if test "$1" = --install-scl ; then
- yum install --assumeyes scl-utils
+ yum -q install --assumeyes scl-utils
curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
- yum install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
- yum install --assumeyes python27
+ yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
+ yum -q install --assumeyes python27
exec scl enable python27 $0
fi
@@ -38,9 +38,9 @@ gpgcheck=0
enabled=1
EOF
-yum clean all
-yum update
-if ! yum install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+yum -q clean all
+yum -q update
+if ! yum -q install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
exit 1
fi
@@ -51,4 +51,10 @@ for r in /mnt/python27-python-*x86_64.rpm ; do
rpm2cpio $r | cpio -idm
done
+for so in $(find . -name "*.so") ; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+done
+
exec /root/common-test.sh
commit 8713e250470011db40d0cc9f1a02afbf98d2853c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Sat Aug 29 15:15:55 2015 -0400
Sets up local repositories for both centos (yum) and debian (apt), installs
from there as if an end user would.
diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
new file mode 100755
index 0000000..869f054
--- /dev/null
+++ b/jenkins/common-test-packages.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+FAIL=0
+
+for so in $(find . -name "*.so") ; do
+ if ldd $so | grep "not found" ; then
+ echo "^^^ Missing while scanning $so ^^^"
+ FAIL=1
+ fi
+done
+
+if ! python <<EOF
+import arvados
+import arvados_fuse
+print "Successly imported arvados and arvados_fuse"
+EOF
+then
+ FAIL=1
+fi
+
+exit $FAIL
diff --git a/jenkins/test-packages-deb-common.sh b/jenkins/deb-common-test-packages.sh
similarity index 57%
rename from jenkins/test-packages-deb-common.sh
rename to jenkins/deb-common-test-packages.sh
index d2c586c..fbebe24 100755
--- a/jenkins/test-packages-deb-common.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -8,13 +8,16 @@ if test "$1" = --run-test ; then
fi
self=$(readlink -f $0)
- cd $WORKSPACE/packages/debian7
+ base=$(dirname $self)
+
+ cd $WORKSPACE/packages/$2
dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
exec docker run \
--rm \
--volume=$WORKSPACE/packages/$2:/mnt \
--volume=$self:/root/run-test.sh \
+ --volume=$base/common-test-packages.sh:/root/common-test.sh \
--workdir=/mnt \
$3 \
/root/run-test.sh
@@ -22,9 +25,15 @@ fi
echo "deb file:///mnt /" >>/etc/apt/sources.list
apt-get update
-apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse
+if ! apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+ exit 1
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+for r in /mnt/python-*amd64.deb ; do
+ dpkg-deb -x $r .
+done
-python <<EOF
-import arvados
-import arvados_fuse
-EOF
+exec /root/common-test.sh
diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
index c4f5bec..dce7362 100755
--- a/jenkins/run-test-packages.sh
+++ b/jenkins/run-test-packages.sh
@@ -5,6 +5,13 @@ if test -z "$WORKSPACE" ; then
exit 1
fi
-for pkg in test-packages-*.sh ; do
- $pkg --run-test
+FAIL=0
+
+for pkg in ./test-packages-*.sh ; do
+ if ! $pkg --run-test ; then
+ FAIL=1
+ echo "$pkg has install errors"
+ fi
done
+
+exit $FAIL
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 6dd87e9..ea3ed09 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -7,10 +7,16 @@ if test "$1" = --run-test ; then
exit 1
fi
+ self=$(readlink -f $0)
+ base=$(dirname $self)
+
+ createrepo $WORKSPACE/packages/centos6
+
exec docker run \
--rm \
--volume=$WORKSPACE/packages/centos6:/mnt \
--volume=$(readlink -f $0):/root/run-test.sh \
+ --volume=$base/common-test-packages.sh:/root/common-test.sh \
--workdir=/mnt \
centos:6 \
/root/run-test.sh --install-scl
@@ -24,7 +30,19 @@ if test "$1" = --install-scl ; then
exec scl enable python27 $0
fi
-yum install --assumeyes python27-python*.rpm
+cat >/etc/yum.repos.d/localrepo.repo <<EOF
+[localrepo]
+name=Arvados Test
+baseurl=file:///mnt
+gpgcheck=0
+enabled=1
+EOF
+
+yum clean all
+yum update
+if ! yum install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+ exit 1
+fi
mkdir -p /tmp/opts
cd /tmp/opts
@@ -33,18 +51,4 @@ for r in /mnt/python27-python-*x86_64.rpm ; do
rpm2cpio $r | cpio -idm
done
-FAIL=0
-
-for so in $(find . -name "*.so") ; do
- if ldd $so | grep "not found" ; then
- echo "^^^ Missing while scanning $so ^^^"
- FAIL=1
- fi
-done
-
-python <<EOF
-import arvados
-import arvados_fuse
-EOF
-
-exit $FAIL
+exec /root/common-test.sh
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
index c48761c..c5cbfe6 100755
--- a/jenkins/test-packages-debian7.sh
+++ b/jenkins/test-packages-debian7.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-exec ./test-packages-deb-common.sh --run-test debian7 debian:wheezy
+exec ./deb-common-test-packages.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
index 6e42806..5502f46 100755
--- a/jenkins/test-packages-debian8.sh
+++ b/jenkins/test-packages-debian8.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-exec ./test-packages-deb-common.sh --run-test debian8 debian:jessie
+exec ./deb-common-test-packages.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
index a8af4d8..00727a0 100755
--- a/jenkins/test-packages-ubuntu1204.sh
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-exec ./test-packages-deb-common.sh --run-test ubuntu1204 ubuntu:precise
+exec ./deb-common-test-packages.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
index a70a1f6..bfc2c85 100755
--- a/jenkins/test-packages-ubuntu1404.sh
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-exec ./test-packages-deb-common.sh --run-test ubuntu1404 ubuntu:trusty
+exec ./deb-common-test-packages.sh --run-test ubuntu1404 ubuntu:trusty
commit 77f198433d31d492d86c38359940acd7f5012cf8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 28 17:18:20 2015 -0400
Script to automatically test python package installation on clean Docker
images.
diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
new file mode 100755
index 0000000..c4f5bec
--- /dev/null
+++ b/jenkins/run-test-packages.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if test -z "$WORKSPACE" ; then
+ echo "Must set WORKSPACE"
+ exit 1
+fi
+
+for pkg in test-packages-*.sh ; do
+ $pkg --run-test
+done
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
new file mode 100755
index 0000000..6dd87e9
--- /dev/null
+++ b/jenkins/test-packages-centos6.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+if test "$1" = --run-test ; then
+
+ if test -z "$WORKSPACE" ; then
+ echo "Must set WORKSPACE"
+ exit 1
+ fi
+
+ exec docker run \
+ --rm \
+ --volume=$WORKSPACE/packages/centos6:/mnt \
+ --volume=$(readlink -f $0):/root/run-test.sh \
+ --workdir=/mnt \
+ centos:6 \
+ /root/run-test.sh --install-scl
+fi
+
+if test "$1" = --install-scl ; then
+ yum install --assumeyes scl-utils
+ curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
+ yum install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
+ yum install --assumeyes python27
+ exec scl enable python27 $0
+fi
+
+yum install --assumeyes python27-python*.rpm
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+for r in /mnt/python27-python-*x86_64.rpm ; do
+ rpm2cpio $r | cpio -idm
+done
+
+FAIL=0
+
+for so in $(find . -name "*.so") ; do
+ if ldd $so | grep "not found" ; then
+ echo "^^^ Missing while scanning $so ^^^"
+ FAIL=1
+ fi
+done
+
+python <<EOF
+import arvados
+import arvados_fuse
+EOF
+
+exit $FAIL
diff --git a/jenkins/test-packages-deb-common.sh b/jenkins/test-packages-deb-common.sh
new file mode 100755
index 0000000..d2c586c
--- /dev/null
+++ b/jenkins/test-packages-deb-common.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+if test "$1" = --run-test ; then
+
+ if test -z "$WORKSPACE" ; then
+ echo "Must set WORKSPACE"
+ exit 1
+ fi
+
+ self=$(readlink -f $0)
+ cd $WORKSPACE/packages/debian7
+ dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
+
+ exec docker run \
+ --rm \
+ --volume=$WORKSPACE/packages/$2:/mnt \
+ --volume=$self:/root/run-test.sh \
+ --workdir=/mnt \
+ $3 \
+ /root/run-test.sh
+fi
+
+echo "deb file:///mnt /" >>/etc/apt/sources.list
+apt-get update
+apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse
+
+python <<EOF
+import arvados
+import arvados_fuse
+EOF
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
new file mode 100755
index 0000000..c48761c
--- /dev/null
+++ b/jenkins/test-packages-debian7.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
new file mode 100755
index 0000000..6e42806
--- /dev/null
+++ b/jenkins/test-packages-debian8.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
new file mode 100755
index 0000000..a8af4d8
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
new file mode 100755
index 0000000..a70a1f6
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test ubuntu1404 ubuntu:trusty
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list