[ARVADOS-DEV] created: a23620386db119bf670221368016484a5a2f1f93

git at public.curoverse.com git at public.curoverse.com
Fri Oct 16 14:30:17 EDT 2015


        at  a23620386db119bf670221368016484a5a2f1f93 (commit)


commit a23620386db119bf670221368016484a5a2f1f93
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Oct 16 14:29:57 2015 -0400

    7370: run-build-packages-all-targets hands off to run-build-packages-one-target, suppports --test-packages option.

diff --git a/jenkins/run-build-packages-all-targets.sh b/jenkins/run-build-packages-all-targets.sh
index 8c889a0..19befb3 100755
--- a/jenkins/run-build-packages-all-targets.sh
+++ b/jenkins/run-build-packages-all-targets.sh
@@ -10,6 +10,8 @@ Options:
 
 --command
     Build command to execute (default: use built-in Docker image command)
+--test-packages
+    Run package install tests
 --debug
     Output debug information (default: false)
 
@@ -36,7 +38,7 @@ fi
 set -e
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,debug,command: \
+    help,test-packages,debug,command: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -44,6 +46,7 @@ fi
 
 COMMAND=
 DEBUG=
+TEST_PACKAGES=
 
 eval set -- "$PARSEDOPTS"
 while [ $# -gt 0 ]; do
@@ -54,11 +57,14 @@ while [ $# -gt 0 ]; do
             exit 1
             ;;
         --debug)
-            DEBUG=" --debug"
+            DEBUG="--debug"
             ;;
         --command)
             COMMAND="$2"; shift
             ;;
+        --test-packages)
+            TEST_PACKAGES="--test-packages"
+            ;;
         --)
             if [ $# -gt 1 ]; then
                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
@@ -69,36 +75,6 @@ while [ $# -gt 0 ]; do
     shift
 done
 
-if [[ "$COMMAND" != "" ]]; then
-  COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND$DEBUG"
-fi
-
-FINAL_EXITCODE=0
-JENKINS_DIR=$(dirname "$(readlink -e "$0")")
-
-run_docker() {
-    local tag=$1; shift
-    if [[ "$COMMAND" != "" ]]; then
-      COMMAND="$COMMAND --target $tag"
-    fi
-    if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
-          --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
-        # Success - nothing more to do.
-        true
-    else
-        FINAL_EXITCODE=$?
-        echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
-    fi
-}
-
-# In case it's needed, build the containers. This costs just a few
-# seconds when the containers already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
-time ./build-all-build-containers.sh
-
 for dockerfile_path in $(find -name Dockerfile); do
-    run_docker "$(basename $(dirname "$dockerfile_path"))"
+    ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES
 done
-
-exit $FINAL_EXITCODE

commit ad5f9554d13754e16def528a24a2669f7cbd4e5d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Oct 15 16:40:54 2015 -0400

    7370: Fix centos6 package builder container.

diff --git a/jenkins/dockerfiles/centos6/Dockerfile b/jenkins/dockerfiles/centos6/Dockerfile
index d0daef6..0a733e3 100644
--- a/jenkins/dockerfiles/centos6/Dockerfile
+++ b/jenkins/dockerfiles/centos6/Dockerfile
@@ -13,7 +13,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.1 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
-    /usr/local/rvm/bin/rvm-exec default gem install fpm
+    /usr/local/rvm/bin/rvm-exec default gem install bundle fpm
 
 # Download and install Software Collections for newer requirements
 ADD https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm /opt/

commit 72c6e7bcd898caa3a2ed718e66b71568fe66bc7d
Merge: d20e333 b67e599
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Oct 15 16:24:07 2015 -0400

    Merge branch 'master' into 7370-package-install-testing
    
    Conflicts:
    	jenkins/run-build-packages-one-target.sh

diff --cc jenkins/run-build-packages-one-target.sh
index 3e0e136,2d8d8df..5b8f2ce
--- a/jenkins/run-build-packages-one-target.sh
+++ b/jenkins/run-build-packages-one-target.sh
@@@ -10,8 -10,8 +10,10 @@@ Syntax
      Distribution to build packages for (default: debian7)
  --command
      Build command to execute (default: use built-in Docker image command)
 +--test-packages
 +    Run package install test script "test-packages-$target.sh"
+ --debug
+     Output debug information (default: false)
  
  WORKSPACE=path         Path to the Arvados source tree to build packages from
  
@@@ -34,7 -34,7 +36,7 @@@ if ! [[ -d "$WORKSPACE" ]]; the
  fi
  
  PARSEDOPTS=$(getopt --name "$0" --longoptions \
-     help,test-packages,target:,command: \
 -    help,debug,target:,command: \
++    help,debug,test-packages,target:,command: \
      -- "" "$@")
  if [ $? -ne 0 ]; then
      exit 1
@@@ -72,48 -73,34 +78,48 @@@ don
  
  set -e
  
 -if [[ "$COMMAND" != "" ]]; then
 -  COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 +if [[ -n "$test_packages" ]]; then
 +    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
 +        createrepo $WORKSPACE/packages/$TARGET
 +    fi
 +
 +    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
 +        (cd $WORKSPACE/packages/$TARGET
 +         dpkg-scanpackages . /dev/null | gzip -c > Packages.gz
 +        )
 +    fi
 +
 +    COMMAND="/jenkins/test-packages-$TARGET.sh"
 +    IMAGE="arvados/package-test:$TARGET"
 +else
 +    IMAGE="arvados/build:$TARGET"
 +    if [[ "$COMMAND" != "" ]]; then
-         COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
++        COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 +    fi
  fi
  
 -FINAL_EXITCODE=0
  JENKINS_DIR=$(dirname "$(readlink -e "$0")")
  
 -run_docker() {
 -    local tag=$1; shift
 -    if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
 -          --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
 -        # Success - nothing more to do.
 -        true
 -    else
 -        FINAL_EXITCODE=$?
 -        echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
 -    fi
 -}
 +if [[ -n "$test_packages" ]]; then
 +    pushd "$JENKINS_DIR/package-test-dockerfiles"
 +else
 +    pushd "$JENKINS_DIR/dockerfiles"
 +fi
  
 -# In case it's needed, build the container. This costs just a few
 -# seconds when the container already exist, so it's not a big deal to
 -# do it on each run.
 -cd "$JENKINS_DIR/dockerfiles"
  echo $TARGET
  cd $TARGET
 -time docker build -t arvados/build:$TARGET .
 -cd ..
 +time docker build --tag=$IMAGE .
 +popd
  
 -run_docker $TARGET
 +FINAL_EXITCODE=0
 +
 +if docker run --rm -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
 +          --env ARVADOS_DEBUG=1 "$IMAGE" $COMMAND; then
 +    # Success - nothing more to do.
 +    true
 +else
 +    FINAL_EXITCODE=$?
 +    echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
 +fi
  
  exit $FINAL_EXITCODE

commit d20e33394de2a85f17399e0e4d9d7d2e52c94178
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Oct 15 16:21:56 2015 -0400

    7370: Refactoring way the package test is set up and executed.

diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index f40ba7b..d9d0301 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -1,29 +1,9 @@
-#!/bin/sh
+#!/bin/bash
 
-if test "$1" = --run-test ; then
+# Multiple .deb based distros symlink to this script, so extract the target
+# from the invocation path.
+target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
 
-    if test -z "$WORKSPACE" ; then
-        echo "Must set WORKSPACE"
-        exit 1
-    fi
-
-    self=$(readlink -f $0)
-    base=$(dirname $self)
-
-    cd $WORKSPACE/packages/$2
-    dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
-
-    exec docker run \
-         --rm \
-         --volume=$WORKSPACE/packages/$2:/mnt \
-         --volume=$self:/root/run-test.sh \
-         --volume=$base/common-test-packages.sh:/root/common-test.sh \
-         --workdir=/mnt \
-         $3 \
-         /root/run-test.sh
-fi
-
-echo "deb file:///mnt /" >>/etc/apt/sources.list
 apt-get -qq update
 if ! apt-get -qq --assume-yes --force-yes install \
      python-arvados-python-client python-arvados-fuse arvados-node-manager
@@ -34,7 +14,7 @@ fi
 mkdir -p /tmp/opts
 cd /tmp/opts
 
-for r in /mnt/python-*amd64.deb ; do
+for r in /arvados/packages/$target/python-*amd64.deb ; do
     dpkg-deb -x $r .
 done
 
@@ -44,4 +24,4 @@ for so in $(find . -name "*.so") ; do
     ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
 done
 
-exec /root/common-test.sh
+exec /jenkins/common-test-packages.sh
diff --git a/jenkins/dockerfiles/centos6/Dockerfile b/jenkins/dockerfiles/centos6/Dockerfile
index ca974df..d0daef6 100644
--- a/jenkins/dockerfiles/centos6/Dockerfile
+++ b/jenkins/dockerfiles/centos6/Dockerfile
@@ -19,7 +19,10 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
 ADD https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm /opt/
 
 RUN yum -q -y install /opt/rhscl-*.rpm
-RUN yum -q -y install python27 rh-python34
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python27 rh-python34
 
 ENV WORKSPACE /arvados
 CMD ["scl", "enable", "rh-python34", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]
diff --git a/jenkins/package-test-dockerfiles/centos6/Dockerfile b/jenkins/package-test-dockerfiles/centos6/Dockerfile
new file mode 100644
index 0000000..5e354e2
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/Dockerfile
@@ -0,0 +1,9 @@
+FROM centos:6
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN yum -q install --assumeyes scl-utils && \
+    curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm && \
+    yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm && \
+    yum -q install --assumeyes python27
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/centos6/localrepo.repo b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
new file mode 100644
index 0000000..ac6b898
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos6
+gpgcheck=0
+enabled=1
diff --git a/jenkins/package-test-dockerfiles/debian7/Dockerfile b/jenkins/package-test-dockerfiles/debian7/Dockerfile
new file mode 100644
index 0000000..fb789c6
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian7/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:7
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list
diff --git a/jenkins/package-test-dockerfiles/debian8/Dockerfile b/jenkins/package-test-dockerfiles/debian8/Dockerfile
new file mode 100644
index 0000000..fbcd8f4
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian8/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:8
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
new file mode 100644
index 0000000..4b6e936
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:precise
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
new file mode 100644
index 0000000..4f49d92
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:trusty
+MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/run-build-packages-one-target.sh b/jenkins/run-build-packages-one-target.sh
index 5451a4e..3e0e136 100755
--- a/jenkins/run-build-packages-one-target.sh
+++ b/jenkins/run-build-packages-one-target.sh
@@ -10,6 +10,8 @@ Syntax:
     Distribution to build packages for (default: debian7)
 --command
     Build command to execute (default: use built-in Docker image command)
+--test-packages
+    Run package install test script "test-packages-$target.sh"
 
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 
@@ -32,7 +34,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
 fi
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,target:,command: \
+    help,test-packages,target:,command: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -55,6 +57,9 @@ while [ $# -gt 0 ]; do
         --command)
             COMMAND="$2"; shift
             ;;
+        --test-packages)
+            test_packages=1
+            ;;
         --)
             if [ $# -gt 1 ]; then
                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
@@ -67,34 +72,48 @@ done
 
 set -e
 
-if [[ "$COMMAND" != "" ]]; then
-  COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+if [[ -n "$test_packages" ]]; then
+    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+        createrepo $WORKSPACE/packages/$TARGET
+    fi
+
+    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+        (cd $WORKSPACE/packages/$TARGET
+         dpkg-scanpackages . /dev/null | gzip -c > Packages.gz
+        )
+    fi
+
+    COMMAND="/jenkins/test-packages-$TARGET.sh"
+    IMAGE="arvados/package-test:$TARGET"
+else
+    IMAGE="arvados/build:$TARGET"
+    if [[ "$COMMAND" != "" ]]; then
+        COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+    fi
 fi
 
-FINAL_EXITCODE=0
 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 
-run_docker() {
-    local tag=$1; shift
-    if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
-          --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
-        # Success - nothing more to do.
-        true
-    else
-        FINAL_EXITCODE=$?
-        echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
-    fi
-}
+if [[ -n "$test_packages" ]]; then
+    pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+    pushd "$JENKINS_DIR/dockerfiles"
+fi
 
-# In case it's needed, build the container. This costs just a few
-# seconds when the container already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
 echo $TARGET
 cd $TARGET
-time docker build -t arvados/build:$TARGET .
-cd ..
+time docker build --tag=$IMAGE .
+popd
 
-run_docker $TARGET
+FINAL_EXITCODE=0
+
+if docker run --rm -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
+          --env ARVADOS_DEBUG=1 "$IMAGE" $COMMAND; then
+    # Success - nothing more to do.
+    true
+else
+    FINAL_EXITCODE=$?
+    echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
+fi
 
 exit $FINAL_EXITCODE
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 555b9f1..c8a62b5 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -1,62 +1,29 @@
-#!/bin/sh
+#!/bin/bash
 
-if test "$1" = --run-test ; then
-
-    if test -z "$WORKSPACE" ; then
-        echo "Must set WORKSPACE"
+if ! python --version |& grep "Python 2.7" >/dev/null ; then
+    # Re-execute in the software collections Python 2.7 environment.
+    exec scl enable python27 $0
+else
+    yum -q clean all
+    yum -q update
+    if ! yum -q install --assumeyes \
+         python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
+    then
         exit 1
     fi
 
-    self=$(readlink -f $0)
-    base=$(dirname $self)
+    mkdir -p /tmp/opts
+    cd /tmp/opts
 
-    createrepo $WORKSPACE/packages/centos6
+    for r in /arvados/packages/centos6/python27-python-*x86_64.rpm ; do
+        rpm2cpio $r | cpio -idm
+    done
 
-    exec docker run \
-         --rm \
-         --volume=$WORKSPACE/packages/centos6:/mnt \
-         --volume=$(readlink -f $0):/root/run-test.sh \
-         --volume=$base/common-test-packages.sh:/root/common-test.sh \
-         --workdir=/mnt \
-         centos:6 \
-         /root/run-test.sh --install-scl
-fi
+    for so in $(find . -name "*.so") ; do
+        echo
+        echo "== Packages dependencies for $so =="
+        ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+    done
 
-if test "$1" = --install-scl ; then
-    yum -q install --assumeyes scl-utils
-    curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum -q install --assumeyes python27
-    exec scl enable python27 $0
+    exec /jenkins/common-test-packages.sh
 fi
-
-cat >/etc/yum.repos.d/localrepo.repo <<EOF
-[localrepo]
-name=Arvados Test
-baseurl=file:///mnt
-gpgcheck=0
-enabled=1
-EOF
-
-yum -q clean all
-yum -q update
-if ! yum -q install --assumeyes \
-     python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
-then
-    exit 1
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-for r in /mnt/python27-python-*x86_64.rpm ; do
-    rpm2cpio $r | cpio -idm
-done
-
-for so in $(find . -name "*.so") ; do
-    echo
-    echo "== Packages dependencies for $so =="
-    ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
-done
-
-exec /root/common-test.sh
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
deleted file mode 100755
index c5cbfe6..0000000
--- a/jenkins/test-packages-debian7.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-debian7.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
deleted file mode 100755
index 5502f46..0000000
--- a/jenkins/test-packages-debian8.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-debian8.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
deleted file mode 100755
index 00727a0..0000000
--- a/jenkins/test-packages-ubuntu1204.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
deleted file mode 100755
index bfc2c85..0000000
--- a/jenkins/test-packages-ubuntu1404.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1404 ubuntu:trusty
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
new file mode 120000
index 0000000..54ce94c
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file

commit 39520ae3caa8227e58471b8783106a3c2dba1304
Merge: 823a16a 2d11eaa
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Oct 15 10:40:17 2015 -0400

    Merge branch 'master' into 7370-package-install-testing


commit 823a16ae78acdadd17d6f563dfafe4678c364906
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Aug 31 15:23:00 2015 -0400

    Tests compatible libcloud package is installed.

diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
index 52bbba0..c9d77c8 100755
--- a/jenkins/common-test-packages.sh
+++ b/jenkins/common-test-packages.sh
@@ -15,6 +15,11 @@ if ! python <<EOF
 import arvados
 import arvados_fuse
 print "Successly imported arvados and arvados_fuse"
+
+import libcloud.compute.types
+import libcloud.compute.providers
+libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
+print "Successly imported compatible libcloud library"
 EOF
 then
     FAIL=1
diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index d781c8b..f40ba7b 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -25,7 +25,9 @@ fi
 
 echo "deb file:///mnt /" >>/etc/apt/sources.list
 apt-get -qq update
-if ! apt-get -qq --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+if ! apt-get -qq --assume-yes --force-yes install \
+     python-arvados-python-client python-arvados-fuse arvados-node-manager
+then
     exit 1
 fi
 
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 580bb27..555b9f1 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -40,7 +40,9 @@ EOF
 
 yum -q clean all
 yum -q update
-if ! yum -q install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+if ! yum -q install --assumeyes \
+     python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
+then
     exit 1
 fi
 

commit 21a3d01379891f2670991e4d24804e1dc87a1ab1
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Aug 31 11:18:05 2015 -0400

    Log package dependencies for each shared object.  Add -q so make the logs a bit
    less chatty.

diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
index 869f054..52bbba0 100755
--- a/jenkins/common-test-packages.sh
+++ b/jenkins/common-test-packages.sh
@@ -2,6 +2,7 @@
 
 FAIL=0
 
+echo
 for so in $(find . -name "*.so") ; do
     if ldd $so | grep "not found" ; then
         echo "^^^ Missing while scanning $so ^^^"
@@ -9,6 +10,7 @@ for so in $(find . -name "*.so") ; do
     fi
 done
 
+echo
 if ! python <<EOF
 import arvados
 import arvados_fuse
diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index fbebe24..d781c8b 100755
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -24,8 +24,8 @@ if test "$1" = --run-test ; then
 fi
 
 echo "deb file:///mnt /" >>/etc/apt/sources.list
-apt-get update
-if ! apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+apt-get -qq update
+if ! apt-get -qq --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
     exit 1
 fi
 
@@ -36,4 +36,10 @@ for r in /mnt/python-*amd64.deb ; do
     dpkg-deb -x $r .
 done
 
+for so in $(find . -name "*.so") ; do
+    echo
+    echo "== Packages dependencies for $so =="
+    ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+done
+
 exec /root/common-test.sh
diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
index dce7362..baca7b7 100755
--- a/jenkins/run-test-packages.sh
+++ b/jenkins/run-test-packages.sh
@@ -7,11 +7,17 @@ fi
 
 FAIL=0
 
+ERRORS=""
+
 for pkg in ./test-packages-*.sh ; do
+    echo
+    echo "== Running $pkg =="
+    echo
     if ! $pkg --run-test ; then
         FAIL=1
-        echo "$pkg has install errors"
+        ERRORS="$ERRORS\n$pkg has install errors"
     fi
 done
 
+echo $ERRORS
 exit $FAIL
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index ea3ed09..580bb27 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -23,10 +23,10 @@ if test "$1" = --run-test ; then
 fi
 
 if test "$1" = --install-scl ; then
-    yum install --assumeyes scl-utils
+    yum -q install --assumeyes scl-utils
     curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum install --assumeyes python27
+    yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
+    yum -q install --assumeyes python27
     exec scl enable python27 $0
 fi
 
@@ -38,9 +38,9 @@ gpgcheck=0
 enabled=1
 EOF
 
-yum clean all
-yum update
-if ! yum install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+yum -q clean all
+yum -q update
+if ! yum -q install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
     exit 1
 fi
 
@@ -51,4 +51,10 @@ for r in /mnt/python27-python-*x86_64.rpm ; do
     rpm2cpio $r | cpio -idm
 done
 
+for so in $(find . -name "*.so") ; do
+    echo
+    echo "== Packages dependencies for $so =="
+    ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+done
+
 exec /root/common-test.sh

commit 8713e250470011db40d0cc9f1a02afbf98d2853c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Sat Aug 29 15:15:55 2015 -0400

    Sets up local repositories for both centos (yum) and debian (apt), installs
    from there as if an end user would.

diff --git a/jenkins/common-test-packages.sh b/jenkins/common-test-packages.sh
new file mode 100755
index 0000000..869f054
--- /dev/null
+++ b/jenkins/common-test-packages.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+FAIL=0
+
+for so in $(find . -name "*.so") ; do
+    if ldd $so | grep "not found" ; then
+        echo "^^^ Missing while scanning $so ^^^"
+        FAIL=1
+    fi
+done
+
+if ! python <<EOF
+import arvados
+import arvados_fuse
+print "Successly imported arvados and arvados_fuse"
+EOF
+then
+    FAIL=1
+fi
+
+exit $FAIL
diff --git a/jenkins/test-packages-deb-common.sh b/jenkins/deb-common-test-packages.sh
similarity index 57%
rename from jenkins/test-packages-deb-common.sh
rename to jenkins/deb-common-test-packages.sh
index d2c586c..fbebe24 100755
--- a/jenkins/test-packages-deb-common.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -8,13 +8,16 @@ if test "$1" = --run-test ; then
     fi
 
     self=$(readlink -f $0)
-    cd $WORKSPACE/packages/debian7
+    base=$(dirname $self)
+
+    cd $WORKSPACE/packages/$2
     dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
 
     exec docker run \
          --rm \
          --volume=$WORKSPACE/packages/$2:/mnt \
          --volume=$self:/root/run-test.sh \
+         --volume=$base/common-test-packages.sh:/root/common-test.sh \
          --workdir=/mnt \
          $3 \
          /root/run-test.sh
@@ -22,9 +25,15 @@ fi
 
 echo "deb file:///mnt /" >>/etc/apt/sources.list
 apt-get update
-apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse
+if ! apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse ; then
+    exit 1
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+for r in /mnt/python-*amd64.deb ; do
+    dpkg-deb -x $r .
+done
 
-python <<EOF
-import arvados
-import arvados_fuse
-EOF
+exec /root/common-test.sh
diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
index c4f5bec..dce7362 100755
--- a/jenkins/run-test-packages.sh
+++ b/jenkins/run-test-packages.sh
@@ -5,6 +5,13 @@ if test -z "$WORKSPACE" ; then
     exit 1
 fi
 
-for pkg in test-packages-*.sh ; do
-    $pkg --run-test
+FAIL=0
+
+for pkg in ./test-packages-*.sh ; do
+    if ! $pkg --run-test ; then
+        FAIL=1
+        echo "$pkg has install errors"
+    fi
 done
+
+exit $FAIL
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 6dd87e9..ea3ed09 100755
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -7,10 +7,16 @@ if test "$1" = --run-test ; then
         exit 1
     fi
 
+    self=$(readlink -f $0)
+    base=$(dirname $self)
+
+    createrepo $WORKSPACE/packages/centos6
+
     exec docker run \
          --rm \
          --volume=$WORKSPACE/packages/centos6:/mnt \
          --volume=$(readlink -f $0):/root/run-test.sh \
+         --volume=$base/common-test-packages.sh:/root/common-test.sh \
          --workdir=/mnt \
          centos:6 \
          /root/run-test.sh --install-scl
@@ -24,7 +30,19 @@ if test "$1" = --install-scl ; then
     exec scl enable python27 $0
 fi
 
-yum install --assumeyes python27-python*.rpm
+cat >/etc/yum.repos.d/localrepo.repo <<EOF
+[localrepo]
+name=Arvados Test
+baseurl=file:///mnt
+gpgcheck=0
+enabled=1
+EOF
+
+yum clean all
+yum update
+if ! yum install --assumeyes python27-python-arvados-python-client python27-python-arvados-fuse ; then
+    exit 1
+fi
 
 mkdir -p /tmp/opts
 cd /tmp/opts
@@ -33,18 +51,4 @@ for r in /mnt/python27-python-*x86_64.rpm ; do
     rpm2cpio $r | cpio -idm
 done
 
-FAIL=0
-
-for so in $(find . -name "*.so") ; do
-    if ldd $so | grep "not found" ; then
-        echo "^^^ Missing while scanning $so ^^^"
-        FAIL=1
-    fi
-done
-
-python <<EOF
-import arvados
-import arvados_fuse
-EOF
-
-exit $FAIL
+exec /root/common-test.sh
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
index c48761c..c5cbfe6 100755
--- a/jenkins/test-packages-debian7.sh
+++ b/jenkins/test-packages-debian7.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec ./test-packages-deb-common.sh --run-test debian7 debian:wheezy
+exec ./deb-common-test-packages.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
index 6e42806..5502f46 100755
--- a/jenkins/test-packages-debian8.sh
+++ b/jenkins/test-packages-debian8.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec ./test-packages-deb-common.sh --run-test debian8 debian:jessie
+exec ./deb-common-test-packages.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
index a8af4d8..00727a0 100755
--- a/jenkins/test-packages-ubuntu1204.sh
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec ./test-packages-deb-common.sh --run-test ubuntu1204 ubuntu:precise
+exec ./deb-common-test-packages.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
index a70a1f6..bfc2c85 100755
--- a/jenkins/test-packages-ubuntu1404.sh
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec ./test-packages-deb-common.sh --run-test ubuntu1404 ubuntu:trusty
+exec ./deb-common-test-packages.sh --run-test ubuntu1404 ubuntu:trusty

commit 77f198433d31d492d86c38359940acd7f5012cf8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Aug 28 17:18:20 2015 -0400

    Script to automatically test python package installation on clean Docker
    images.

diff --git a/jenkins/run-test-packages.sh b/jenkins/run-test-packages.sh
new file mode 100755
index 0000000..c4f5bec
--- /dev/null
+++ b/jenkins/run-test-packages.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if test -z "$WORKSPACE" ; then
+    echo "Must set WORKSPACE"
+    exit 1
+fi
+
+for pkg in test-packages-*.sh ; do
+    $pkg --run-test
+done
diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
new file mode 100755
index 0000000..6dd87e9
--- /dev/null
+++ b/jenkins/test-packages-centos6.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+if test "$1" = --run-test ; then
+
+    if test -z "$WORKSPACE" ; then
+        echo "Must set WORKSPACE"
+        exit 1
+    fi
+
+    exec docker run \
+         --rm \
+         --volume=$WORKSPACE/packages/centos6:/mnt \
+         --volume=$(readlink -f $0):/root/run-test.sh \
+         --workdir=/mnt \
+         centos:6 \
+         /root/run-test.sh --install-scl
+fi
+
+if test "$1" = --install-scl ; then
+    yum install --assumeyes scl-utils
+    curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
+    yum install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
+    yum install --assumeyes python27
+    exec scl enable python27 $0
+fi
+
+yum install --assumeyes python27-python*.rpm
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+for r in /mnt/python27-python-*x86_64.rpm ; do
+    rpm2cpio $r | cpio -idm
+done
+
+FAIL=0
+
+for so in $(find . -name "*.so") ; do
+    if ldd $so | grep "not found" ; then
+        echo "^^^ Missing while scanning $so ^^^"
+        FAIL=1
+    fi
+done
+
+python <<EOF
+import arvados
+import arvados_fuse
+EOF
+
+exit $FAIL
diff --git a/jenkins/test-packages-deb-common.sh b/jenkins/test-packages-deb-common.sh
new file mode 100755
index 0000000..d2c586c
--- /dev/null
+++ b/jenkins/test-packages-deb-common.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+if test "$1" = --run-test ; then
+
+    if test -z "$WORKSPACE" ; then
+        echo "Must set WORKSPACE"
+        exit 1
+    fi
+
+    self=$(readlink -f $0)
+    cd $WORKSPACE/packages/debian7
+    dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
+
+    exec docker run \
+         --rm \
+         --volume=$WORKSPACE/packages/$2:/mnt \
+         --volume=$self:/root/run-test.sh \
+         --workdir=/mnt \
+         $3 \
+         /root/run-test.sh
+fi
+
+echo "deb file:///mnt /" >>/etc/apt/sources.list
+apt-get update
+apt-get --assume-yes --force-yes install python-arvados-python-client python-arvados-fuse
+
+python <<EOF
+import arvados
+import arvados_fuse
+EOF
diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
new file mode 100755
index 0000000..c48761c
--- /dev/null
+++ b/jenkins/test-packages-debian7.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test debian7 debian:wheezy
diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
new file mode 100755
index 0000000..6e42806
--- /dev/null
+++ b/jenkins/test-packages-debian8.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test debian8 debian:jessie
diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
new file mode 100755
index 0000000..a8af4d8
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test ubuntu1204 ubuntu:precise
diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
new file mode 100755
index 0000000..a70a1f6
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec ./test-packages-deb-common.sh --run-test ubuntu1404 ubuntu:trusty

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list