[ARVADOS-DEV] updated: 6c6c1c28e120e21ca469c3e411f2f56475af6753
Git user
git at public.curoverse.com
Mon May 16 21:00:23 EDT 2016
Summary of changes:
jenkins/create-plot-data-from-log.sh | 59 --
jenkins/package-build-dockerfiles/.gitignore | 2 -
jenkins/package-build-dockerfiles/Makefile | 29 -
jenkins/package-build-dockerfiles/README | 13 -
.../build-all-build-containers.sh | 16 -
.../package-build-dockerfiles/centos6/Dockerfile | 31 -
.../package-build-dockerfiles/debian7/Dockerfile | 19 -
.../package-build-dockerfiles/debian8/Dockerfile | 19 -
.../ubuntu1204/Dockerfile | 19 -
.../ubuntu1404/Dockerfile | 19 -
.../package-test-dockerfiles/centos6/Dockerfile | 20 -
.../centos6/localrepo.repo | 5 -
.../package-test-dockerfiles/debian7/Dockerfile | 14 -
.../package-test-dockerfiles/debian8/Dockerfile | 14 -
.../package-test-dockerfiles/ubuntu1204/Dockerfile | 14 -
.../package-test-dockerfiles/ubuntu1404/Dockerfile | 14 -
jenkins/package-testing/common-test-packages.sh | 28 -
.../package-testing/deb-common-test-packages.sh | 37 -
.../test-package-arvados-api-server.sh | 20 -
.../test-package-arvados-node-manager.sh | 7 -
.../test-package-arvados-sso-server.sh | 172 -----
.../test-package-arvados-workbench.sh | 20 -
.../test-package-python27-python-arvados-fuse.sh | 6 -
...ackage-python27-python-arvados-python-client.sh | 6 -
jenkins/package-testing/test-packages-centos6.sh | 44 --
jenkins/package-testing/test-packages-debian7.sh | 1 -
jenkins/package-testing/test-packages-debian8.sh | 1 -
.../package-testing/test-packages-ubuntu1204.sh | 1 -
.../package-testing/test-packages-ubuntu1404.sh | 1 -
jenkins/rails-package-scripts/README.md | 14 -
.../rails-package-scripts/arvados-api-server.sh | 32 -
.../rails-package-scripts/arvados-sso-server.sh | 9 -
jenkins/rails-package-scripts/arvados-workbench.sh | 7 -
jenkins/rails-package-scripts/postinst.sh | 251 -------
jenkins/rails-package-scripts/postrm.sh | 23 -
jenkins/rails-package-scripts/prerm.sh | 22 -
jenkins/rails-package-scripts/step2.sh | 28 -
jenkins/run-build-docker-images.sh | 167 -----
jenkins/run-build-docker-jobs-image.sh | 164 ----
jenkins/run-build-packages-all-targets.sh | 98 ---
jenkins/run-build-packages-one-target.sh | 203 -----
jenkins/run-build-packages-sso.sh | 161 ----
jenkins/run-build-packages.sh | 543 --------------
jenkins/run-library.sh | 344 ---------
jenkins/run-tests.sh | 831 ---------------------
jenkins/run_upload_packages.py | 6 +-
46 files changed, 3 insertions(+), 3551 deletions(-)
delete mode 100755 jenkins/create-plot-data-from-log.sh
delete mode 100644 jenkins/package-build-dockerfiles/.gitignore
delete mode 100644 jenkins/package-build-dockerfiles/Makefile
delete mode 100644 jenkins/package-build-dockerfiles/README
delete mode 100755 jenkins/package-build-dockerfiles/build-all-build-containers.sh
delete mode 100644 jenkins/package-build-dockerfiles/centos6/Dockerfile
delete mode 100644 jenkins/package-build-dockerfiles/debian7/Dockerfile
delete mode 100644 jenkins/package-build-dockerfiles/debian8/Dockerfile
delete mode 100644 jenkins/package-build-dockerfiles/ubuntu1204/Dockerfile
delete mode 100644 jenkins/package-build-dockerfiles/ubuntu1404/Dockerfile
delete mode 100644 jenkins/package-test-dockerfiles/centos6/Dockerfile
delete mode 100644 jenkins/package-test-dockerfiles/centos6/localrepo.repo
delete mode 100644 jenkins/package-test-dockerfiles/debian7/Dockerfile
delete mode 100644 jenkins/package-test-dockerfiles/debian8/Dockerfile
delete mode 100644 jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
delete mode 100644 jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
delete mode 100755 jenkins/package-testing/common-test-packages.sh
delete mode 100755 jenkins/package-testing/deb-common-test-packages.sh
delete mode 100755 jenkins/package-testing/test-package-arvados-api-server.sh
delete mode 100755 jenkins/package-testing/test-package-arvados-node-manager.sh
delete mode 100755 jenkins/package-testing/test-package-arvados-sso-server.sh
delete mode 100755 jenkins/package-testing/test-package-arvados-workbench.sh
delete mode 100755 jenkins/package-testing/test-package-python27-python-arvados-fuse.sh
delete mode 100755 jenkins/package-testing/test-package-python27-python-arvados-python-client.sh
delete mode 100755 jenkins/package-testing/test-packages-centos6.sh
delete mode 120000 jenkins/package-testing/test-packages-debian7.sh
delete mode 120000 jenkins/package-testing/test-packages-debian8.sh
delete mode 120000 jenkins/package-testing/test-packages-ubuntu1204.sh
delete mode 120000 jenkins/package-testing/test-packages-ubuntu1404.sh
delete mode 100644 jenkins/rails-package-scripts/README.md
delete mode 100644 jenkins/rails-package-scripts/arvados-api-server.sh
delete mode 100644 jenkins/rails-package-scripts/arvados-sso-server.sh
delete mode 100644 jenkins/rails-package-scripts/arvados-workbench.sh
delete mode 100644 jenkins/rails-package-scripts/postinst.sh
delete mode 100644 jenkins/rails-package-scripts/postrm.sh
delete mode 100644 jenkins/rails-package-scripts/prerm.sh
delete mode 100644 jenkins/rails-package-scripts/step2.sh
delete mode 100755 jenkins/run-build-docker-images.sh
delete mode 100755 jenkins/run-build-docker-jobs-image.sh
delete mode 100755 jenkins/run-build-packages-all-targets.sh
delete mode 100755 jenkins/run-build-packages-one-target.sh
delete mode 100755 jenkins/run-build-packages-sso.sh
delete mode 100755 jenkins/run-build-packages.sh
delete mode 100755 jenkins/run-library.sh
delete mode 100755 jenkins/run-tests.sh
via 6c6c1c28e120e21ca469c3e411f2f56475af6753 (commit)
via 2fecb120e5b9be7a85f209e90114731f3e95f709 (commit)
from c441e6c17584f7d4e8e4b858c3e14cdd6c46c4bf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6c6c1c28e120e21ca469c3e411f2f56475af6753
Author: Ward Vandewege <ward at curoverse.com>
Date: Mon May 16 21:00:08 2016 -0400
Remove a bunch of files that were moved to the arvados repository.
No issue #
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
deleted file mode 100755
index ce3bfed..0000000
--- a/jenkins/create-plot-data-from-log.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-
-build=$1
-file=$2
-outputdir=$3
-
-usage() {
- echo "./$0 build_number file_to_parse output_dir"
- echo "this script will use the build output to generate *csv and *txt"
- echo "for jenkins plugin plot https://github.com/jenkinsci/plot-plugin/"
-}
-
-if [ $# -ne 3 ]
-then
- usage
- exit 1
-fi
-
-if [ ! -e $file ]
-then
- usage
- echo "$file doesn't exist! exiting"
- exit 2
-fi
-if [ ! -w $outputdir ]
-then
- usage
- echo "$outputdir isn't writeable! exiting"
- exit 3
-fi
-
-#------------------------------
-## MAXLINE is the amount of lines that will read after the pattern
-## is match (the logfile could be hundred thousands lines long).
-## 1000 should be safe enough to capture all the output of the individual test
-MAXLINES=1000
-
-## TODO: check $build and $file make sense
-
-for test in \
- test_Create_and_show_large_collection_with_manifest_text_of_20000000 \
- test_Create,_show,_and_update_description_for_large_collection_with_manifest_text_of_100000 \
- test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
-do
- cleaned_test=$(echo $test | tr -d ",.:;/")
- (zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail -n +1 | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$cleaned_test-$build.txt
- result=$?
- if [ $result -eq 0 ]
- then
- echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
- echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
- echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
- #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
- else
- echo "$test was't found on $file"
- cleaned_test=$(echo $test | tr -d ",.:;/")
- > $outputdir/$cleaned_test.csv
- fi
-done
diff --git a/jenkins/package-build-dockerfiles/.gitignore b/jenkins/package-build-dockerfiles/.gitignore
deleted file mode 100644
index ceee9fa..0000000
--- a/jenkins/package-build-dockerfiles/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*/generated
-common-generated/
diff --git a/jenkins/package-build-dockerfiles/Makefile b/jenkins/package-build-dockerfiles/Makefile
deleted file mode 100644
index 9216f82..0000000
--- a/jenkins/package-build-dockerfiles/Makefile
+++ /dev/null
@@ -1,29 +0,0 @@
-all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
-
-centos6/generated: common-generated-all
- test -d centos6/generated || mkdir centos6/generated
- cp -rlt centos6/generated common-generated/*
-
-debian7/generated: common-generated-all
- test -d debian7/generated || mkdir debian7/generated
- cp -rlt debian7/generated common-generated/*
-
-debian8/generated: common-generated-all
- test -d debian8/generated || mkdir debian8/generated
- cp -rlt debian8/generated common-generated/*
-
-ubuntu1204/generated: common-generated-all
- test -d ubuntu1204/generated || mkdir ubuntu1204/generated
- cp -rlt ubuntu1204/generated common-generated/*
-
-ubuntu1404/generated: common-generated-all
- test -d ubuntu1404/generated || mkdir ubuntu1404/generated
- cp -rlt ubuntu1404/generated common-generated/*
-
-common-generated-all: common-generated/golang-amd64.tar.gz
-
-common-generated/golang-amd64.tar.gz: common-generated
- wget -cqO common-generated/golang-amd64.tar.gz http://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
-
-common-generated:
- mkdir common-generated
diff --git a/jenkins/package-build-dockerfiles/README b/jenkins/package-build-dockerfiles/README
deleted file mode 100644
index 0dfab94..0000000
--- a/jenkins/package-build-dockerfiles/README
+++ /dev/null
@@ -1,13 +0,0 @@
-==================
-DOCKER IMAGE BUILD
-==================
-
-1. `make`
-2. `cd DISTRO`
-3. `docker build -t arvados/build:DISTRO .`
-
-==============
-BUILD PACKAGES
-==============
-
-`docker run -v /path/to/your/arvados-dev/jenkins:/jenkins -v /path/to/your/arvados:/arvados arvados/build:DISTRO`
diff --git a/jenkins/package-build-dockerfiles/build-all-build-containers.sh b/jenkins/package-build-dockerfiles/build-all-build-containers.sh
deleted file mode 100755
index 34ffcce..0000000
--- a/jenkins/package-build-dockerfiles/build-all-build-containers.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-make
-
-for target in `find -maxdepth 1 -type d |grep -v generated`; do
- if [[ "$target" == "." ]]; then
- continue
- fi
- target=${target#./}
- echo $target
- cd $target
- docker build -t arvados/build:$target .
- cd ..
-done
-
-
diff --git a/jenkins/package-build-dockerfiles/centos6/Dockerfile b/jenkins/package-build-dockerfiles/centos6/Dockerfile
deleted file mode 100644
index cfd94c8..0000000
--- a/jenkins/package-build-dockerfiles/centos6/Dockerfile
+++ /dev/null
@@ -1,31 +0,0 @@
-FROM centos:6
-MAINTAINER Brett Smith <brett at curoverse.com>
-
-# Install build dependencies provided in base distribution
-RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar scl-utils centos-release-SCL postgresql-devel
-
-# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
-
-# Need to "touch" RPM database to workaround bug in interaction between
-# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install python27 python33
-RUN scl enable python33 "easy_install-3.3 pip" && scl enable python27 "easy_install-2.7 pip"
-
-RUN cd /tmp && \
- curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
- rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
- sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
-
-RUN touch /var/lib/rpm/* && yum install --assumeyes git
-
-ENV WORKSPACE /arvados
-CMD ["scl", "enable", "python33", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]
diff --git a/jenkins/package-build-dockerfiles/debian7/Dockerfile b/jenkins/package-build-dockerfiles/debian7/Dockerfile
deleted file mode 100644
index 0d04590..0000000
--- a/jenkins/package-build-dockerfiles/debian7/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM debian:wheezy
-MAINTAINER Ward Vandewege <ward at curoverse.com>
-
-# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libpq-dev python-pip
-
-# Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
-
-# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian7"]
diff --git a/jenkins/package-build-dockerfiles/debian8/Dockerfile b/jenkins/package-build-dockerfiles/debian8/Dockerfile
deleted file mode 100644
index fcd390f..0000000
--- a/jenkins/package-build-dockerfiles/debian8/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM debian:jessie
-MAINTAINER Ward Vandewege <ward at curoverse.com>
-
-# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip
-
-# Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
-
-# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
diff --git a/jenkins/package-build-dockerfiles/ubuntu1204/Dockerfile b/jenkins/package-build-dockerfiles/ubuntu1204/Dockerfile
deleted file mode 100644
index 158053c..0000000
--- a/jenkins/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM ubuntu:precise
-MAINTAINER Ward Vandewege <ward at curoverse.com>
-
-# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential
-
-# Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
-
-# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1204"]
diff --git a/jenkins/package-build-dockerfiles/ubuntu1404/Dockerfile b/jenkins/package-build-dockerfiles/ubuntu1404/Dockerfile
deleted file mode 100644
index 0b8ee7a..0000000
--- a/jenkins/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM ubuntu:trusty
-MAINTAINER Brett Smith <brett at curoverse.com>
-
-# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip
-
-# Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
-
-# Install golang binary
-ADD generated/golang-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
diff --git a/jenkins/package-test-dockerfiles/centos6/Dockerfile b/jenkins/package-test-dockerfiles/centos6/Dockerfile
deleted file mode 100644
index 69927a1..0000000
--- a/jenkins/package-test-dockerfiles/centos6/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM centos:6
-MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
-
-RUN yum -q install --assumeyes scl-utils centos-release-SCL \
- which tar
-
-# Install RVM
-RUN touch /var/lib/rpm/* && \
- gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundle fpm
-
-RUN cd /tmp && \
- curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
- rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
- sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
-
-COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/centos6/localrepo.repo b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
deleted file mode 100644
index ac6b898..0000000
--- a/jenkins/package-test-dockerfiles/centos6/localrepo.repo
+++ /dev/null
@@ -1,5 +0,0 @@
-[localrepo]
-name=Arvados Test
-baseurl=file:///arvados/packages/centos6
-gpgcheck=0
-enabled=1
diff --git a/jenkins/package-test-dockerfiles/debian7/Dockerfile b/jenkins/package-test-dockerfiles/debian7/Dockerfile
deleted file mode 100644
index c9a2fdc..0000000
--- a/jenkins/package-test-dockerfiles/debian7/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM debian:7
-MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
-
-# Install RVM
-RUN apt-get update && apt-get -y install curl procps && \
- gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list
diff --git a/jenkins/package-test-dockerfiles/debian8/Dockerfile b/jenkins/package-test-dockerfiles/debian8/Dockerfile
deleted file mode 100644
index cde1847..0000000
--- a/jenkins/package-test-dockerfiles/debian8/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM debian:8
-MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
-
-# Install RVM
-RUN apt-get update && apt-get -y install curl && \
- gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
diff --git a/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
deleted file mode 100644
index 0cb77c8..0000000
--- a/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM ubuntu:precise
-MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
-
-# Install RVM
-RUN apt-get update && apt-get -y install curl && \
- gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
deleted file mode 100644
index 6c4d0e9..0000000
--- a/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM ubuntu:trusty
-MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
-
-# Install RVM
-RUN apt-get update && apt-get -y install curl && \
- gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.1 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.1
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file
diff --git a/jenkins/package-testing/common-test-packages.sh b/jenkins/package-testing/common-test-packages.sh
deleted file mode 100755
index 2dc67ab..0000000
--- a/jenkins/package-testing/common-test-packages.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-FAIL=0
-
-echo
-
-while read so && [ -n "$so" ]; do
- if ldd "$so" | grep "not found" ; then
- echo "^^^ Missing while scanning $so ^^^"
- FAIL=1
- fi
-done <<EOF
-$(find -name '*.so')
-EOF
-
-if test -x "/jenkins/package-testing/test-package-$1.sh" ; then
- if ! "/jenkins/package-testing/test-package-$1.sh" ; then
- FAIL=1
- fi
-fi
-
-if test $FAIL = 0 ; then
- echo "Package $1 passed"
-fi
-
-exit $FAIL
diff --git a/jenkins/package-testing/deb-common-test-packages.sh b/jenkins/package-testing/deb-common-test-packages.sh
deleted file mode 100755
index 5f32a60..0000000
--- a/jenkins/package-testing/deb-common-test-packages.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# Multiple .deb based distros symlink to this script, so extract the target
-# from the invocation path.
-target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
-
-export ARV_PACKAGES_DIR="/arvados/packages/$target"
-
-dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
-
-apt-get -qq update
-apt-get --assume-yes --force-yes install "$1"
-
-dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
-
-set +e
-diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" > "$ARV_PACKAGES_DIR/$1.diff"
-set -e
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-export ARV_PACKAGES_DIR="/arvados/packages/$target"
-
-dpkg-deb -x $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) .
-
-while read so && [ -n "$so" ]; do
- echo
- echo "== Packages dependencies for $so =="
- ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
-done <<EOF
-$(find -name '*.so')
-EOF
-
-exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/jenkins/package-testing/test-package-arvados-api-server.sh b/jenkins/package-testing/test-package-arvados-api-server.sh
deleted file mode 100755
index e975448..0000000
--- a/jenkins/package-testing/test-package-arvados-api-server.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-set -e
-cd /var/www/arvados-api/current/
-
-case "$TARGET" in
- debian*|ubuntu*)
- apt-get install -y nginx
- dpkg-reconfigure arvados-api-server
- ;;
- centos6)
- yum install --assumeyes httpd
- yum reinstall --assumeyes arvados-api-server
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-api-server.gems"
diff --git a/jenkins/package-testing/test-package-arvados-node-manager.sh b/jenkins/package-testing/test-package-arvados-node-manager.sh
deleted file mode 100755
index 2f416d1..0000000
--- a/jenkins/package-testing/test-package-arvados-node-manager.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-exec python <<EOF
-import libcloud.compute.types
-import libcloud.compute.providers
-libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
-print "Successfully imported compatible libcloud library"
-EOF
diff --git a/jenkins/package-testing/test-package-arvados-sso-server.sh b/jenkins/package-testing/test-package-arvados-sso-server.sh
deleted file mode 100755
index c1a377e..0000000
--- a/jenkins/package-testing/test-package-arvados-sso-server.sh
+++ /dev/null
@@ -1,172 +0,0 @@
-#!/bin/bash
-
-set -e
-
-EXITCODE=0
-DEBUG=${ARVADOS_DEBUG:-0}
-
-STDOUT_IF_DEBUG=/dev/null
-STDERR_IF_DEBUG=/dev/null
-DASHQ_UNLESS_DEBUG=-q
-if [[ "$DEBUG" != 0 ]]; then
- STDOUT_IF_DEBUG=/dev/stdout
- STDERR_IF_DEBUG=/dev/stderr
- DASHQ_UNLESS_DEBUG=
-fi
-
-case "$TARGET" in
- debian*|ubuntu*)
- FORMAT=deb
- ;;
- centos6)
- FORMAT=rpm
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: $WORKSPACE is not a directory"
- echo >&2
- exit 1
-fi
-
-title () {
- txt="********** $1 **********"
- printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
-}
-
-checkexit() {
- if [[ "$1" != "0" ]]; then
- title "!!!!!! $2 FAILED !!!!!!"
- fi
-}
-
-
-# Find the SSO server package
-
-cd "$WORKSPACE"
-
-if [[ ! -d "/var/www/arvados-sso" ]]; then
- echo "/var/www/arvados-sso should exist"
- exit 1
-fi
-
-if [[ ! -e "/etc/arvados/sso/application.yml" ]]; then
- mkdir -p /etc/arvados/sso/
- RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
- cp config/application.yml.example /etc/arvados/sso/application.yml
- sed -i -e 's/uuid_prefix: ~/uuid_prefix: zzzzz/' /etc/arvados/sso/application.yml
- sed -i -e "s/secret_token: ~/secret_token: $RANDOM_PASSWORD/" /etc/arvados/sso/application.yml
-fi
-
-if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then
- # We need to set up our database configuration now.
- if [[ "$FORMAT" == "rpm" ]]; then
- # postgres packaging on CentOS6 is kind of primitive, needs an initdb
- service postgresql initdb
- if [ "$TARGET" = "centos6" ]; then
- sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
- sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
- fi
- fi
- service postgresql start
-
- RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
- cat >/etc/arvados/sso/database.yml <<EOF
-production:
- adapter: postgresql
- encoding: utf8
- database: sso_provider_production
- username: sso_provider_user
- password: $RANDOM_PASSWORD
- host: localhost
-EOF
-
- su postgres -c "psql -c \"CREATE USER sso_provider_user WITH PASSWORD '$RANDOM_PASSWORD'\""
- su postgres -c "createdb sso_provider_production -O sso_provider_user"
-fi
-
-if [[ "$FORMAT" == "deb" ]]; then
- # Test 2: the package should reconfigure cleanly
- dpkg-reconfigure arvados-sso-server || EXITCODE=3
-
- cd /var/www/arvados-sso/current/
- /usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-sso-server.gems"
-
- # Test 3: the package should remove cleanly
- apt-get remove arvados-sso-server --yes || EXITCODE=3
-
- checkexit $EXITCODE "apt-get remove arvados-sso-server --yes"
-
- # Test 4: the package configuration should remove cleanly
- dpkg --purge arvados-sso-server || EXITCODE=4
-
- checkexit $EXITCODE "dpkg --purge arvados-sso-server"
-
- if [[ -e "/var/www/arvados-sso" ]]; then
- EXITCODE=4
- fi
-
- checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
-
- # Test 5: the package should remove cleanly with --purge
-
- apt-get remove arvados-sso-server --purge --yes || EXITCODE=5
-
- checkexit $EXITCODE "apt-get remove arvados-sso-server --purge --yes"
-
- if [[ -e "/var/www/arvados-sso" ]]; then
- EXITCODE=5
- fi
-
- checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
-
-elif [[ "$FORMAT" == "rpm" ]]; then
-
- # Set up Nginx first
- # (courtesy of https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/el6/install_passenger.html)
- yum install -q -y epel-release pygpgme curl
- curl --fail -sSLo /etc/yum.repos.d/passenger.repo https://oss-binaries.phusionpassenger.com/yum/definitions/el-passenger.repo
- yum install -q -y nginx passenger
- sed -i -e 's/^# passenger/passenger/' /etc/nginx/conf.d/passenger.conf
- # Done setting up Nginx
-
- # Test 2: the package should reinstall cleanly
- yum --assumeyes reinstall arvados-sso-server || EXITCODE=3
-
- cd /var/www/arvados-sso/current/
- /usr/local/rvm/bin/rvm-exec default bundle list >$ARV_PACKAGES_DIR/arvados-sso-server.gems
-
- # Test 3: the package should remove cleanly
- yum -q -y remove arvados-sso-server || EXITCODE=3
-
- checkexit $EXITCODE "yum -q -y remove arvados-sso-server"
-
- if [[ -e "/var/www/arvados-sso" ]]; then
- EXITCODE=3
- fi
-
- checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
-
-fi
-
-if [[ "$EXITCODE" == "0" ]]; then
- echo "Testing complete, no errors!"
-else
- echo "Errors while testing!"
-fi
-
-exit $EXITCODE
diff --git a/jenkins/package-testing/test-package-arvados-workbench.sh b/jenkins/package-testing/test-package-arvados-workbench.sh
deleted file mode 100755
index 1be4dea..0000000
--- a/jenkins/package-testing/test-package-arvados-workbench.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-set -e
-cd /var/www/arvados-workbench/current/
-
-case "$TARGET" in
- debian*|ubuntu*)
- apt-get install -y nginx
- dpkg-reconfigure arvados-workbench
- ;;
- centos6)
- yum install --assumeyes httpd
- yum reinstall --assumeyes arvados-workbench
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-workbench.gems"
diff --git a/jenkins/package-testing/test-package-python27-python-arvados-fuse.sh b/jenkins/package-testing/test-package-python27-python-arvados-fuse.sh
deleted file mode 100755
index 1654be9..0000000
--- a/jenkins/package-testing/test-package-python27-python-arvados-fuse.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-exec python <<EOF
-import arvados_fuse
-print "Successfully imported arvados_fuse"
-EOF
diff --git a/jenkins/package-testing/test-package-python27-python-arvados-python-client.sh b/jenkins/package-testing/test-package-python27-python-arvados-python-client.sh
deleted file mode 100755
index 0772fbf..0000000
--- a/jenkins/package-testing/test-package-python27-python-arvados-python-client.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-exec python <<EOF
-import arvados
-print "Successfully imported arvados"
-EOF
diff --git a/jenkins/package-testing/test-packages-centos6.sh b/jenkins/package-testing/test-packages-centos6.sh
deleted file mode 100755
index 4e05364..0000000
--- a/jenkins/package-testing/test-packages-centos6.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-yum -q clean all
-touch /var/lib/rpm/*
-
-export ARV_PACKAGES_DIR=/arvados/packages/centos6
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
-
-yum install --assumeyes $1
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
-
-set +e
-diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff"
-set -e
-
-SCL=""
-if scl enable python27 true 2>/dev/null ; then
- SCL="scl enable python27"
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
-
-shared=$(find -name '*.so')
-if test -n "$shared" ; then
- for so in $shared ; do
- echo
- echo "== Packages dependencies for $so =="
- $SCL ldd "$so" \
- | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
- done
-fi
-
-if test -n "$SCL" ; then
- exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'"
-else
- exec /jenkins/package-testing/common-test-packages.sh "$1"
-fi
diff --git a/jenkins/package-testing/test-packages-debian7.sh b/jenkins/package-testing/test-packages-debian7.sh
deleted file mode 120000
index 54ce94c..0000000
--- a/jenkins/package-testing/test-packages-debian7.sh
+++ /dev/null
@@ -1 +0,0 @@
-deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/package-testing/test-packages-debian8.sh b/jenkins/package-testing/test-packages-debian8.sh
deleted file mode 120000
index 54ce94c..0000000
--- a/jenkins/package-testing/test-packages-debian8.sh
+++ /dev/null
@@ -1 +0,0 @@
-deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/package-testing/test-packages-ubuntu1204.sh b/jenkins/package-testing/test-packages-ubuntu1204.sh
deleted file mode 120000
index 54ce94c..0000000
--- a/jenkins/package-testing/test-packages-ubuntu1204.sh
+++ /dev/null
@@ -1 +0,0 @@
-deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/package-testing/test-packages-ubuntu1404.sh b/jenkins/package-testing/test-packages-ubuntu1404.sh
deleted file mode 120000
index 54ce94c..0000000
--- a/jenkins/package-testing/test-packages-ubuntu1404.sh
+++ /dev/null
@@ -1 +0,0 @@
-deb-common-test-packages.sh
\ No newline at end of file
diff --git a/jenkins/rails-package-scripts/README.md b/jenkins/rails-package-scripts/README.md
deleted file mode 100644
index 3a93c31..0000000
--- a/jenkins/rails-package-scripts/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-When run-build-packages.sh builds a Rails package, it generates the package's pre/post-inst/rm scripts by concatenating:
-
-1. package_name.sh, which defines variables about where package files live and some human-readable names about them.
-2. step2.sh, which uses those to define some utility variables and set defaults for things that aren't set.
-3. stepname.sh, like postinst.sh, prerm.sh, etc., which uses all this information to do the actual work.
-
-Since our build process is a tower of shell scripts, concatenating files seemed like the least worst option to share code between these files and packages. More advanced code generation would've been too much trouble to integrate into our build process at this time. Trying to inject portions of files into other files seemed error-prone and likely to introduce bugs to the end result.
-
-postinst.sh lets the early parts define a few hooks to control behavior:
-
-* After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles. By default this is a noop function (in step2.sh). API server defines this to set up the old omniauth.rb conffile.
-* Before it restarts nginx, it calls setup_before_nginx_restart. By default this is a noop function (in step2.sh). API server defines this to set up the internal git repository, if necessary.
-* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. SSO server uses db:schema:load. Workbench doesn't set this, which causes the postinst to skip all database work.
-* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task. SSO clears this flag (it doesn't have that task code).
diff --git a/jenkins/rails-package-scripts/arvados-api-server.sh b/jenkins/rails-package-scripts/arvados-api-server.sh
deleted file mode 100644
index c2b99f0..0000000
--- a/jenkins/rails-package-scripts/arvados-api-server.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-# This file declares variables common to all scripts for one Rails package.
-
-PACKAGE_NAME=arvados-api-server
-INSTALL_PATH=/var/www/arvados-api
-CONFIG_PATH=/etc/arvados/api
-DOC_URL="http://doc.arvados.org/install/install-api-server.html#configure"
-
-RAILSPKG_DATABASE_LOAD_TASK=db:structure:load
-setup_extra_conffiles() {
- setup_conffile initializers/omniauth.rb
-}
-
-setup_before_nginx_restart() {
- # initialize git_internal_dir
- # usually /var/lib/arvados/internal.git (set in application.default.yml )
- if [ "$APPLICATION_READY" = "1" ]; then
- GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:check 2>&1 | grep git_internal_dir | awk '{ print $2 }')
- if [ ! -e "$GIT_INTERNAL_DIR" ]; then
- run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
- mkdir -p "$GIT_INTERNAL_DIR"
- run_and_report "Initializing git_internal_dir '$GIT_INTERNAL_DIR'" \
- git init --quiet --bare $GIT_INTERNAL_DIR
- else
- echo "Initializing git_internal_dir $GIT_INTERNAL_DIR: directory exists, skipped."
- fi
- run_and_report "Making sure '$GIT_INTERNAL_DIR' has the right permission" \
- chown -R "$WWW_OWNER:" "$GIT_INTERNAL_DIR"
- else
- echo "Initializing git_internal_dir... skipped."
- fi
-}
diff --git a/jenkins/rails-package-scripts/arvados-sso-server.sh b/jenkins/rails-package-scripts/arvados-sso-server.sh
deleted file mode 100644
index 10b2ee2..0000000
--- a/jenkins/rails-package-scripts/arvados-sso-server.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-# This file declares variables common to all scripts for one Rails package.
-
-PACKAGE_NAME=arvados-sso-server
-INSTALL_PATH=/var/www/arvados-sso
-CONFIG_PATH=/etc/arvados/sso
-DOC_URL="http://doc.arvados.org/install/install-sso.html#configure"
-RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
-RAILSPKG_SUPPORTS_CONFIG_CHECK=0
diff --git a/jenkins/rails-package-scripts/arvados-workbench.sh b/jenkins/rails-package-scripts/arvados-workbench.sh
deleted file mode 100644
index f2b8a56..0000000
--- a/jenkins/rails-package-scripts/arvados-workbench.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-# This file declares variables common to all scripts for one Rails package.
-
-PACKAGE_NAME=arvados-workbench
-INSTALL_PATH=/var/www/arvados-workbench
-CONFIG_PATH=/etc/arvados/workbench
-DOC_URL="http://doc.arvados.org/install/install-workbench-app.html#configure"
diff --git a/jenkins/rails-package-scripts/postinst.sh b/jenkins/rails-package-scripts/postinst.sh
deleted file mode 100644
index 6fac26b..0000000
--- a/jenkins/rails-package-scripts/postinst.sh
+++ /dev/null
@@ -1,251 +0,0 @@
-#!/bin/sh
-# This code runs after package variable definitions and step2.sh.
-
-set -e
-
-DATABASE_READY=1
-APPLICATION_READY=1
-
-if [ -s "$HOME/.rvm/scripts/rvm" ] || [ -s "/usr/local/rvm/scripts/rvm" ]; then
- COMMAND_PREFIX="/usr/local/rvm/bin/rvm-exec default"
-else
- COMMAND_PREFIX=
-fi
-
-report_not_ready() {
- local ready_flag="$1"; shift
- local config_file="$1"; shift
- if [ "1" != "$ready_flag" ]; then cat >&2 <<EOF
-
-PLEASE NOTE:
-
-The $PACKAGE_NAME package was not configured completely because
-$config_file needs some tweaking.
-Please refer to the documentation at
-<$DOC_URL> for more details.
-
-When $(basename "$config_file") has been modified,
-reconfigure or reinstall this package.
-
-EOF
- fi
-}
-
-report_web_service_warning() {
- local warning="$1"; shift
- cat >&2 <<EOF
-
-WARNING: $warning.
-
-To override, set the WEB_SERVICE environment variable to the name of the service
-hosting the Rails server.
-
-For Debian-based systems, then reconfigure this package with dpkg-reconfigure.
-
-For RPM-based systems, then reinstall this package.
-
-EOF
-}
-
-run_and_report() {
- # Usage: run_and_report ACTION_MSG CMD
- # This is the usual wrapper that prints ACTION_MSG, runs CMD, then writes
- # a message about whether CMD succeeded or failed. Returns the exit code
- # of CMD.
- local action_message="$1"; shift
- local retcode=0
- echo -n "$action_message..."
- if "$@"; then
- echo " done."
- else
- retcode=$?
- echo " failed."
- fi
- return $retcode
-}
-
-setup_confdirs() {
- for confdir in "$@"; do
- if [ ! -d "$confdir" ]; then
- install -d -g "$WWW_OWNER" -m 0750 "$confdir"
- fi
- done
-}
-
-setup_conffile() {
- # Usage: setup_conffile CONFFILE_PATH [SOURCE_PATH]
- # Both paths are relative to RELEASE_CONFIG_PATH.
- # This function will try to safely ensure that a symbolic link for
- # the configuration file points from RELEASE_CONFIG_PATH to CONFIG_PATH.
- # If SOURCE_PATH is given, this function will try to install that file as
- # the configuration file in CONFIG_PATH, and return 1 if the file in
- # CONFIG_PATH is unmodified from the source.
- local conffile_relpath="$1"; shift
- local conffile_source="$1"
- local release_conffile="$RELEASE_CONFIG_PATH/$conffile_relpath"
- local etc_conffile="$CONFIG_PATH/$(basename "$conffile_relpath")"
-
- # Note that -h can return true and -e will return false simultaneously
- # when the target is a dangling symlink. We're okay with that outcome,
- # so check -h first.
- if [ ! -h "$release_conffile" ]; then
- if [ ! -e "$release_conffile" ]; then
- ln -s "$etc_conffile" "$release_conffile"
- # If there's a config file in /var/www identical to the one in /etc,
- # overwrite it with a symlink after porting its permissions.
- elif cmp --quiet "$release_conffile" "$etc_conffile"; then
- local ownership="$(stat -c "%u:%g" "$release_conffile")"
- local owning_group="${ownership#*:}"
- if [ 0 != "$owning_group" ]; then
- chgrp "$owning_group" "$CONFIG_PATH" /etc/arvados
- fi
- chown "$ownership" "$etc_conffile"
- chmod --reference="$release_conffile" "$etc_conffile"
- ln --force -s "$etc_conffile" "$release_conffile"
- fi
- fi
-
- if [ -n "$conffile_source" ]; then
- if [ ! -e "$etc_conffile" ]; then
- install -g "$WWW_OWNER" -m 0640 \
- "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"
- return 1
- # Even if $etc_conffile already existed, it might be unmodified from
- # the source. This is especially likely when a user installs, updates
- # database.yml, then reconfigures before they update application.yml.
- # Use cmp to be sure whether $etc_conffile is modified.
- elif cmp --quiet "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"; then
- return 1
- fi
- fi
-}
-
-prepare_database() {
- DB_MIGRATE_STATUS=`$COMMAND_PREFIX bundle exec rake db:migrate:status 2>&1 || true`
- if echo $DB_MIGRATE_STATUS | grep -qF 'Schema migrations table does not exist yet.'; then
- # The database exists, but the migrations table doesn't.
- run_and_report "Setting up database" $COMMAND_PREFIX bundle exec \
- rake "$RAILSPKG_DATABASE_LOAD_TASK" db:seed
- elif echo $DB_MIGRATE_STATUS | grep -q '^database: '; then
- run_and_report "Running db:migrate" \
- $COMMAND_PREFIX bundle exec rake db:migrate
- elif echo $DB_MIGRATE_STATUS | grep -q 'database .* does not exist'; then
- if ! run_and_report "Running db:setup" \
- $COMMAND_PREFIX bundle exec rake db:setup 2>/dev/null; then
- echo "Warning: unable to set up database." >&2
- DATABASE_READY=0
- fi
- else
- echo "Warning: Database is not ready to set up. Skipping database setup." >&2
- DATABASE_READY=0
- fi
-}
-
-configure_version() {
- WEB_SERVICE=${WEB_SERVICE:-$(service --status-all 2>/dev/null \
- | grep -Eo '\bnginx|httpd[^[:space:]]*' || true)}
- if [ -z "$WEB_SERVICE" ]; then
- report_web_service_warning "Web service (Nginx or Apache) not found"
- elif [ "$WEB_SERVICE" != "$(echo "$WEB_SERVICE" | head -n 1)" ]; then
- WEB_SERVICE=$(echo "$WEB_SERVICE" | head -n 1)
- report_web_service_warning \
- "Multiple web services found. Choosing the first one ($WEB_SERVICE)"
- fi
-
- if [ -e /etc/redhat-release ]; then
- # Recognize any service that starts with "nginx"; e.g., nginx16.
- if [ "$WEB_SERVICE" != "${WEB_SERVICE#nginx}" ]; then
- WWW_OWNER=nginx
- else
- WWW_OWNER=apache
- fi
- else
- # Assume we're on a Debian-based system for now.
- # Both Apache and Nginx run as www-data by default.
- WWW_OWNER=www-data
- fi
-
- echo
- echo "Assumption: $WEB_SERVICE is configured to serve Rails from"
- echo " $RELEASE_PATH"
- echo "Assumption: $WEB_SERVICE and passenger run as $WWW_OWNER"
- echo
-
- echo -n "Creating symlinks to configuration in $CONFIG_PATH ..."
- setup_confdirs /etc/arvados "$CONFIG_PATH"
- setup_conffile environments/production.rb environments/production.rb.example \
- || true
- setup_conffile application.yml application.yml.example || APPLICATION_READY=0
- if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
- setup_conffile database.yml database.yml.example || DATABASE_READY=0
- fi
- setup_extra_conffiles
- echo "... done."
-
- # Before we do anything else, make sure some directories and files are in place
- if [ ! -e $SHARED_PATH/log ]; then mkdir -p $SHARED_PATH/log; fi
- if [ ! -e $RELEASE_PATH/tmp ]; then mkdir -p $RELEASE_PATH/tmp; fi
- if [ ! -e $RELEASE_PATH/log ]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi
- if [ ! -e $SHARED_PATH/log/production.log ]; then touch $SHARED_PATH/log/production.log; fi
-
- cd "$RELEASE_PATH"
- export RAILS_ENV=production
-
- if ! $COMMAND_PREFIX bundle --version >/dev/null; then
- run_and_report "Installing bundle" $COMMAND_PREFIX gem install bundle
- fi
-
- run_and_report "Running bundle install" \
- $COMMAND_PREFIX bundle install --path $SHARED_PATH/vendor_bundle --local --quiet
-
- echo -n "Ensuring directory and file permissions ..."
- # Ensure correct ownership of a few files
- chown "$WWW_OWNER:" $RELEASE_PATH/config/environment.rb
- chown "$WWW_OWNER:" $RELEASE_PATH/config.ru
- chown "$WWW_OWNER:" $RELEASE_PATH/Gemfile.lock
- chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
- chown -R "$WWW_OWNER:" $SHARED_PATH/log
- case "$RAILSPKG_DATABASE_LOAD_TASK" in
- db:schema:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/schema.rb ;;
- db:structure:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/structure.sql ;;
- esac
- chmod 644 $SHARED_PATH/log/*
- chmod -R 2775 $RELEASE_PATH/tmp
- echo "... done."
-
- if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
- prepare_database
- fi
-
- if [ 11 = "$RAILSPKG_SUPPORTS_CONFIG_CHECK$APPLICATION_READY" ]; then
- run_and_report "Checking application.yml for completeness" \
- $COMMAND_PREFIX bundle exec rake config:check || APPLICATION_READY=0
- fi
-
- # precompile assets; thankfully this does not take long
- if [ "$APPLICATION_READY" = "1" ]; then
- run_and_report "Precompiling assets" \
- $COMMAND_PREFIX bundle exec rake assets:precompile -q -s 2>/dev/null \
- || APPLICATION_READY=0
- else
- echo "Precompiling assets... skipped."
- fi
- chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
-
- setup_before_nginx_restart
-
- if [ ! -z "$WEB_SERVICE" ]; then
- service "$WEB_SERVICE" restart
- fi
-}
-
-if [ "$1" = configure ]; then
- # This is a debian-based system
- configure_version
-elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
- # This is an rpm-based system
- configure_version
-fi
-
-report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
-report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
diff --git a/jenkins/rails-package-scripts/postrm.sh b/jenkins/rails-package-scripts/postrm.sh
deleted file mode 100644
index 2d63f0b..0000000
--- a/jenkins/rails-package-scripts/postrm.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh
-# This code runs after package variable definitions and step2.sh.
-
-set -e
-
-purge () {
- rm -rf $SHARED_PATH/vendor_bundle
- rm -rf $SHARED_PATH/log
- rm -rf $CONFIG_PATH
- rmdir $SHARED_PATH || true
- rmdir $INSTALL_PATH || true
-}
-
-if [ "$1" = 'purge' ]; then
- # This is a debian-based system and purge was requested
- purge
-elif [ "$1" = "0" ]; then
- # This is an rpm-based system, no guarantees are made, always purge
- # Apparently yum doesn't actually remember what it installed.
- # Clean those files up here, then purge.
- rm -rf $RELEASE_PATH
- purge
-fi
diff --git a/jenkins/rails-package-scripts/prerm.sh b/jenkins/rails-package-scripts/prerm.sh
deleted file mode 100644
index 4ef5904..0000000
--- a/jenkins/rails-package-scripts/prerm.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-# This code runs after package variable definitions and step2.sh.
-
-remove () {
- rm -f $RELEASE_PATH/config/database.yml
- rm -f $RELEASE_PATH/config/environments/production.rb
- rm -f $RELEASE_PATH/config/application.yml
- # Old API server configuration file.
- rm -f $RELEASE_PATH/config/initializers/omniauth.rb
- rm -rf $RELEASE_PATH/public/assets/
- rm -rf $RELEASE_PATH/tmp
- rm -rf $RELEASE_PATH/.bundle
- rm -rf $RELEASE_PATH/log
-}
-
-if [ "$1" = 'remove' ]; then
- # This is a debian-based system and removal was requested
- remove
-elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
- # This is an rpm-based system
- remove
-fi
diff --git a/jenkins/rails-package-scripts/step2.sh b/jenkins/rails-package-scripts/step2.sh
deleted file mode 100644
index 816b906..0000000
--- a/jenkins/rails-package-scripts/step2.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/sh
-# This code runs after package variable definitions, before the actual
-# pre/post package work, to set some variable and function defaults.
-
-if [ -z "$INSTALL_PATH" ]; then
- cat >&2 <<EOF
-
-PACKAGE BUILD ERROR: $0 is missing package metadata.
-
-This package is buggy. Please mail <support at curoverse.com> to let
-us know the name and version number of the package you tried to
-install, and we'll get it fixed.
-
-EOF
- exit 3
-fi
-
-RELEASE_PATH=$INSTALL_PATH/current
-RELEASE_CONFIG_PATH=$RELEASE_PATH/config
-SHARED_PATH=$INSTALL_PATH/shared
-
-RAILSPKG_SUPPORTS_CONFIG_CHECK=${RAILSPKG_SUPPORTS_CONFIG_CHECK:-1}
-if ! type setup_extra_conffiles >/dev/null 2>&1; then
- setup_extra_conffiles() { return; }
-fi
-if ! type setup_before_nginx_restart >/dev/null 2>&1; then
- setup_before_nginx_restart() { return; }
-fi
diff --git a/jenkins/run-build-docker-images.sh b/jenkins/run-build-docker-images.sh
deleted file mode 100755
index 0a5841d..0000000
--- a/jenkins/run-build-docker-images.sh
+++ /dev/null
@@ -1,167 +0,0 @@
-#!/bin/bash
-
-function usage {
- echo >&2
- echo >&2 "usage: $0 [options]"
- echo >&2
- echo >&2 "$0 options:"
- echo >&2 " -t, --tags [csv_tags] comma separated tags"
- echo >&2 " -u, --upload Upload the images (docker push)"
- echo >&2 " -h, --help Display this help and exit"
- echo >&2
- echo >&2 " If no options are given, just builds the images."
-}
-
-upload=false
-
-# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hut: \
- --long help,upload,tags: \
- -n "$0" -- "$@"`
-
-if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
-# Note the quotes around `$TEMP': they are essential!
-eval set -- "$TEMP"
-
-while [ $# -ge 1 ]
-do
- case $1 in
- -u | --upload)
- upload=true
- shift
- ;;
- -t | --tags)
- case "$2" in
- "")
- echo "ERROR: --tags needs a parameter";
- usage;
- exit 1
- ;;
- *)
- tags=$2;
- shift 2
- ;;
- esac
- ;;
- --)
- shift
- break
- ;;
- *)
- usage
- exit 1
- ;;
- esac
-done
-
-
-EXITCODE=0
-
-COLUMNS=80
-
-title () {
- printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
-}
-
-docker_push () {
- if [[ ! -z "$tags" ]]
- then
- for tag in $( echo $tags|tr "," " " )
- do
- $DOCKER tag $1 $1:$tag
- done
- fi
-
- # Sometimes docker push fails; retry it a few times if necessary.
- for i in `seq 1 5`; do
- $DOCKER push $*
- ECODE=$?
- if [[ "$ECODE" == "0" ]]; then
- break
- fi
- done
-
- if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker push $* failed !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
- fi
-}
-
-timer_reset() {
- t0=$SECONDS
-}
-
-timer() {
- echo -n "$(($SECONDS - $t0))s"
-}
-
-# Sanity check
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-echo $WORKSPACE
-
-# find the docker binary
-DOCKER=`which docker.io`
-
-if [[ "$DOCKER" == "" ]]; then
- DOCKER=`which docker`
-fi
-
-if [[ "$DOCKER" == "" ]]; then
- title "Error: you need to have docker installed. Could not find the docker executable."
- exit 1
-fi
-
-# DOCKER
-title "Starting docker build"
-
-timer_reset
-
-# clean up the docker build environment
-cd "$WORKSPACE"
-
-tools/arvbox/bin/arvbox build dev
-ECODE=$?
-
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker BUILD FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
-fi
-
-tools/arvbox/bin/arvbox build localdemo
-
-ECODE=$?
-
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker BUILD FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
-fi
-
-title "docker build complete (`timer`)"
-
-title "uploading images"
-
-timer_reset
-
-if [[ "$ECODE" != "0" ]]; then
- title "upload arvados images SKIPPED because build failed"
-else
- if [[ $upload == true ]]; then
- ## 20150526 nico -- *sometimes* dockerhub needs re-login
- ## even though credentials are already in .dockercfg
- docker login -u arvados
-
- docker_push arvados/arvbox-dev
- docker_push arvados/arvbox-demo
- title "upload arvados images complete (`timer`)"
- else
- title "upload arvados images SKIPPED because no --upload option set"
- fi
-fi
-
-exit $EXITCODE
diff --git a/jenkins/run-build-docker-jobs-image.sh b/jenkins/run-build-docker-jobs-image.sh
deleted file mode 100755
index fcf849b..0000000
--- a/jenkins/run-build-docker-jobs-image.sh
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/bin/bash
-
-function usage {
- echo >&2
- echo >&2 "usage: $0 [options]"
- echo >&2
- echo >&2 "$0 options:"
- echo >&2 " -t, --tags [csv_tags] comma separated tags"
- echo >&2 " -u, --upload Upload the images (docker push)"
- echo >&2 " -h, --help Display this help and exit"
- echo >&2
- echo >&2 " If no options are given, just builds the images."
-}
-
-upload=false
-
-# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hut: \
- --long help,upload,tags: \
- -n "$0" -- "$@"`
-
-if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
-# Note the quotes around `$TEMP': they are essential!
-eval set -- "$TEMP"
-
-while [ $# -ge 1 ]
-do
- case $1 in
- -u | --upload)
- upload=true
- shift
- ;;
- -t | --tags)
- case "$2" in
- "")
- echo "ERROR: --tags needs a parameter";
- usage;
- exit 1
- ;;
- *)
- tags=$2;
- shift 2
- ;;
- esac
- ;;
- --)
- shift
- break
- ;;
- *)
- usage
- exit 1
- ;;
- esac
-done
-
-
-EXITCODE=0
-
-COLUMNS=80
-
-title () {
- printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
-}
-
-docker_push () {
- if [[ ! -z "$tags" ]]
- then
- for tag in $( echo $tags|tr "," " " )
- do
- $DOCKER tag -f $1 $1:$tag
- done
- fi
-
- # Sometimes docker push fails; retry it a few times if necessary.
- for i in `seq 1 5`; do
- $DOCKER push $*
- ECODE=$?
- if [[ "$ECODE" == "0" ]]; then
- break
- fi
- done
-
- if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker push $* failed !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
- fi
-}
-
-timer_reset() {
- t0=$SECONDS
-}
-
-timer() {
- echo -n "$(($SECONDS - $t0))s"
-}
-
-# Sanity check
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-echo $WORKSPACE
-
-# find the docker binary
-DOCKER=`which docker.io`
-
-if [[ "$DOCKER" == "" ]]; then
- DOCKER=`which docker`
-fi
-
-if [[ "$DOCKER" == "" ]]; then
- title "Error: you need to have docker installed. Could not find the docker executable."
- exit 1
-fi
-
-# DOCKER
-title "Starting docker build"
-
-timer_reset
-
-# clean up the docker build environment
-cd "$WORKSPACE"
-cd docker
-rm -f jobs-image
-rm -f config.yml
-
-# Get test config.yml file
-cp $HOME/docker/config.yml .
-
-./build.sh jobs-image
-
-ECODE=$?
-
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker BUILD FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
-fi
-
-title "docker build complete (`timer`)"
-
-title "uploading images"
-
-timer_reset
-
-if [[ "$ECODE" != "0" ]]; then
- title "upload arvados images SKIPPED because build failed"
-else
- if [[ $upload == true ]]; then
- ## 20150526 nico -- *sometimes* dockerhub needs re-login
- ## even though credentials are already in .dockercfg
- docker login -u arvados
-
- docker_push arvados/jobs
- title "upload arvados images complete (`timer`)"
- else
- title "upload arvados images SKIPPED because no --upload option set"
- fi
-fi
-
-exit $EXITCODE
diff --git a/jenkins/run-build-packages-all-targets.sh b/jenkins/run-build-packages-all-targets.sh
deleted file mode 100755
index f1a1e1c..0000000
--- a/jenkins/run-build-packages-all-targets.sh
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/bin/bash
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Orchestrate run-build-packages.sh for every target
-
-Syntax:
- WORKSPACE=/path/to/arvados $(basename $0) [options]
-
-Options:
-
---command
- Build command to execute (default: use built-in Docker image command)
---test-packages
- Run package install tests
---debug
- Output debug information (default: false)
-
-WORKSPACE=path Path to the Arvados source tree to build packages from
-
-EOF
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: $WORKSPACE is not a directory"
- echo >&2
- exit 1
-fi
-
-set -e
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
- -- "" "$@")
-if [ $? -ne 0 ]; then
- exit 1
-fi
-
-COMMAND=
-DEBUG=
-TEST_PACKAGES=
-ONLY_TEST=
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
- case "$1" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --debug)
- DEBUG="--debug"
- ;;
- --command)
- COMMAND="$2"; shift
- ;;
- --test-packages)
- TEST_PACKAGES="--test-packages"
- ;;
- --only-test)
- ONLY_TEST="$1 $2"; shift
- ;;
- --)
- if [ $# -gt 1 ]; then
- echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-cd $(dirname $0)
-
-FINAL_EXITCODE=0
-
-for dockerfile_path in $(find -name Dockerfile); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
- true
- else
- FINAL_EXITCODE=$?
- fi
-done
-
-if test $FINAL_EXITCODE != 0 ; then
- echo "Build packages failed with code $FINAL_EXITCODE" >&2
-fi
-
-exit $FINAL_EXITCODE
diff --git a/jenkins/run-build-packages-one-target.sh b/jenkins/run-build-packages-one-target.sh
deleted file mode 100755
index c5e0a89..0000000
--- a/jenkins/run-build-packages-one-target.sh
+++ /dev/null
@@ -1,203 +0,0 @@
-#!/bin/bash
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Orchestrate run-build-packages.sh for one target
-
-Syntax:
- WORKSPACE=/path/to/arvados $(basename $0) [options]
-
---target <target>
- Distribution to build packages for (default: debian7)
---command
- Build command to execute (default: use built-in Docker image command)
---test-packages
- Run package install test script "test-packages-$target.sh"
---debug
- Output debug information (default: false)
---only-test
- Test only a specific package
-
-WORKSPACE=path Path to the Arvados source tree to build packages from
-
-EOF
-
-set -e
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: $WORKSPACE is not a directory"
- echo >&2
- exit 1
-fi
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test: \
- -- "" "$@")
-if [ $? -ne 0 ]; then
- exit 1
-fi
-
-TARGET=debian7
-COMMAND=
-DEBUG=
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
- case "$1" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --target)
- TARGET="$2"; shift
- ;;
- --only-test)
- packages="$2"; shift
- ;;
- --debug)
- DEBUG=" --debug"
- ;;
- --command)
- COMMAND="$2"; shift
- ;;
- --test-packages)
- test_packages=1
- ;;
- --)
- if [ $# -gt 1 ]; then
- echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-set -e
-
-if [[ -n "$test_packages" ]]; then
- if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
- createrepo $WORKSPACE/packages/$TARGET
- fi
-
- if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
- (cd $WORKSPACE/packages/$TARGET
- dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz
- )
- fi
-
- COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
- IMAGE="arvados/package-test:$TARGET"
-else
- IMAGE="arvados/build:$TARGET"
- if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
- fi
-fi
-
-JENKINS_DIR=$(dirname "$(readlink -e "$0")")
-
-if [[ -n "$test_packages" ]]; then
- pushd "$JENKINS_DIR/package-test-dockerfiles"
-else
- pushd "$JENKINS_DIR/package-build-dockerfiles"
- make "$TARGET/generated"
-fi
-
-echo $TARGET
-cd $TARGET
-time docker build --tag=$IMAGE .
-popd
-
-if test -z "$packages" ; then
- packages="arvados-api-server
- arvados-data-manager
- arvados-docker-cleaner
- arvados-git-httpd
- arvados-node-manager
- arvados-src
- arvados-workbench
- crunchstat
- keepproxy
- keep-rsync
- keepstore
- keep-web
- libarvados-perl"
-
- case "$TARGET" in
- centos6)
- packages="$packages python27-python-arvados-fuse
- python27-python-arvados-python-client"
- ;;
- *)
- packages="$packages python-arvados-fuse
- python-arvados-python-client"
- ;;
- esac
-fi
-
-FINAL_EXITCODE=0
-
-package_fails=""
-
-mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
-mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
-
-docker_volume_args=(
- -v "$JENKINS_DIR:/jenkins"
- -v "$WORKSPACE:/arvados"
- -v /arvados/services/api/vendor/bundle
- -v /arvados/apps/workbench/vendor/bundle
- -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
- -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
-)
-
-if [[ -n "$test_packages" ]]; then
- for p in $packages ; do
- echo
- echo "START: $p test on $IMAGE" >&2
- if docker run --rm \
- "${docker_volume_args[@]}" \
- --env ARVADOS_DEBUG=1 \
- --env "TARGET=$TARGET" \
- --env "WORKSPACE=/arvados" \
- "$IMAGE" $COMMAND $p
- then
- echo "OK: $p test on $IMAGE succeeded" >&2
- else
- FINAL_EXITCODE=$?
- package_fails="$package_fails $p"
- echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
- fi
- done
-else
- echo
- echo "START: build packages on $IMAGE" >&2
- if docker run --rm \
- "${docker_volume_args[@]}" \
- --env ARVADOS_DEBUG=1 \
- "$IMAGE" $COMMAND
- then
- echo
- echo "OK: build packages on $IMAGE succeeded" >&2
- else
- FINAL_EXITCODE=$?
- echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
- fi
-fi
-
-if test -n "$package_fails" ; then
- echo "Failed package tests:$package_fails" >&2
-fi
-
-exit $FINAL_EXITCODE
diff --git a/jenkins/run-build-packages-sso.sh b/jenkins/run-build-packages-sso.sh
deleted file mode 100755
index cc673a6..0000000
--- a/jenkins/run-build-packages-sso.sh
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/bin/bash
-
-JENKINS_DIR=$(dirname $(readlink -e "$0"))
-. "$JENKINS_DIR/run-library.sh"
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Build Arvados SSO server package
-
-Syntax:
- WORKSPACE=/path/to/arvados-sso $(basename $0) [options]
-
-Options:
-
---debug
- Output debug information (default: false)
---target
- Distribution to build packages for (default: debian7)
-
-WORKSPACE=path Path to the Arvados SSO source tree to build packages from
-
-EOF
-
-EXITCODE=0
-DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian7
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,build-bundle-packages,debug,target: \
- -- "" "$@")
-if [ $? -ne 0 ]; then
- exit 1
-fi
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
- case "$1" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --target)
- TARGET="$2"; shift
- ;;
- --debug)
- DEBUG=1
- ;;
- --test-packages)
- test_packages=1
- ;;
- --)
- if [ $# -gt 1 ]; then
- echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-STDOUT_IF_DEBUG=/dev/null
-STDERR_IF_DEBUG=/dev/null
-DASHQ_UNLESS_DEBUG=-q
-if [[ "$DEBUG" != 0 ]]; then
- STDOUT_IF_DEBUG=/dev/stdout
- STDERR_IF_DEBUG=/dev/stderr
- DASHQ_UNLESS_DEBUG=
-fi
-
-case "$TARGET" in
- debian7)
- FORMAT=deb
- ;;
- debian8)
- FORMAT=deb
- ;;
- ubuntu1204)
- FORMAT=deb
- ;;
- ubuntu1404)
- FORMAT=deb
- ;;
- centos6)
- FORMAT=rpm
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: $WORKSPACE is not a directory"
- echo >&2
- exit 1
-fi
-
-# Test for fpm
-fpm --version >/dev/null 2>&1
-
-if [[ "$?" != 0 ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: fpm not found"
- echo >&2
- exit 1
-fi
-
-RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
-RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
-if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
- # error; for some reason, the path is not accessible
- # to the script (e.g. permissions re-evaled after suid)
- exit 1 # fail
-fi
-
-debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
-debug_echo "Workspace is $WORKSPACE"
-
-if [[ -f /etc/profile.d/rvm.sh ]]; then
- source /etc/profile.d/rvm.sh
- GEM="rvm-exec default gem"
-else
- GEM=gem
-fi
-
-# Make all files world-readable -- jenkins runs with umask 027, and has checked
-# out our git tree here
-chmod o+r "$WORKSPACE" -R
-
-# More cleanup - make sure all executables that we'll package are 755
-# No executables in the sso server package
-#find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
-
-# Now fix our umask to something better suited to building and publishing
-# gems and packages
-umask 0022
-
-debug_echo "umask is" `umask`
-
-if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
- mkdir -p "$WORKSPACE/packages/$TARGET"
-fi
-
-# Build the SSO server package
-handle_rails_package arvados-sso-server "$WORKSPACE" \
- "$WORKSPACE/LICENCE" --url="https://arvados.org" \
- --description="Arvados SSO server - Arvados is a free and open source platform for big data science." \
- --license="Expat license"
-
-exit $EXITCODE
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
deleted file mode 100755
index 6970929..0000000
--- a/jenkins/run-build-packages.sh
+++ /dev/null
@@ -1,543 +0,0 @@
-#!/bin/bash
-
-. `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Build Arvados packages
-
-Syntax:
- WORKSPACE=/path/to/arvados $(basename $0) [options]
-
-Options:
-
---build-bundle-packages (default: false)
- Build api server and workbench packages with vendor/bundle included
---debug
- Output debug information (default: false)
---target
- Distribution to build packages for (default: debian7)
---command
- Build command to execute (defaults to the run command defined in the
- Docker image)
-
-WORKSPACE=path Path to the Arvados source tree to build packages from
-
-EOF
-
-EXITCODE=0
-DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian7
-COMMAND=
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,build-bundle-packages,debug,target: \
- -- "" "$@")
-if [ $? -ne 0 ]; then
- exit 1
-fi
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
- case "$1" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --target)
- TARGET="$2"; shift
- ;;
- --debug)
- DEBUG=1
- ;;
- --command)
- COMMAND="$2"; shift
- ;;
- --)
- if [ $# -gt 1 ]; then
- echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-if [[ "$COMMAND" != "" ]]; then
- COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
-fi
-
-STDOUT_IF_DEBUG=/dev/null
-STDERR_IF_DEBUG=/dev/null
-DASHQ_UNLESS_DEBUG=-q
-if [[ "$DEBUG" != 0 ]]; then
- STDOUT_IF_DEBUG=/dev/stdout
- STDERR_IF_DEBUG=/dev/stderr
- DASHQ_UNLESS_DEBUG=
-fi
-
-declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
-
-PYTHON2_VERSION=2.7
-PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
-
-case "$TARGET" in
- debian7)
- FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
- PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
- 'pycurl<7.21.5' contextlib2)
- PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
- ;;
- debian8)
- FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
- PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
- 'pycurl<7.21.5')
- PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
- ;;
- ubuntu1204)
- FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
- PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
- contextlib2 \
- 'pycurl<7.21.5')
- PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
- ;;
- ubuntu1404)
- FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
- PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
- google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
- rsa 'pycurl<7.21.5' backports.ssl_match_hostname)
- PYTHON3_BACKPORTS=(docker-py requests websocket-client)
- ;;
- centos6)
- FORMAT=rpm
- PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
- PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
- PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
- PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
- PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
- python-daemon lockfile llfuse==0.41.1 'pbr<1.0')
- PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
- export PYCURL_SSL_LIBRARY=nss
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-# Test for fpm
-fpm --version >/dev/null 2>&1
-
-if [[ "$?" != 0 ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: fpm not found"
- echo >&2
- exit 1
-fi
-
-EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
-EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
-
-RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
-RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
-if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
- # error; for some reason, the path is not accessible
- # to the script (e.g. permissions re-evaled after suid)
- exit 1 # fail
-fi
-
-debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
-debug_echo "Workspace is $WORKSPACE"
-
-if [[ -f /etc/profile.d/rvm.sh ]]; then
- source /etc/profile.d/rvm.sh
- GEM="rvm-exec default gem"
-else
- GEM=gem
-fi
-
-# Make all files world-readable -- jenkins runs with umask 027, and has checked
-# out our git tree here
-chmod o+r "$WORKSPACE" -R
-
-# More cleanup - make sure all executables that we'll package are 755
-find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
-
-# Now fix our umask to something better suited to building and publishing
-# gems and packages
-umask 0022
-
-debug_echo "umask is" `umask`
-
-if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
- mkdir -p $WORKSPACE/packages/$TARGET
-fi
-
-# Perl packages
-debug_echo -e "\nPerl packages\n"
-
-cd "$WORKSPACE/sdk/perl"
-
-if [[ -e Makefile ]]; then
- make realclean >"$STDOUT_IF_DEBUG"
-fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
- -delete
-rm -rf install
-
-perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
- make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
- fpm_build install/lib/=/usr/share libarvados-perl \
- "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
- "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
- mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
-
-# Ruby gems
-debug_echo -e "\nRuby gems\n"
-
-FPM_GEM_PREFIX=$($GEM environment gemdir)
-
-cd "$WORKSPACE/sdk/ruby"
-handle_ruby_gem arvados
-
-cd "$WORKSPACE/sdk/cli"
-handle_ruby_gem arvados-cli
-
-cd "$WORKSPACE/services/login-sync"
-handle_ruby_gem arvados-login-sync
-
-# Python packages
-debug_echo -e "\nPython packages\n"
-
-cd "$WORKSPACE/sdk/pam"
-handle_python_package
-
-cd "$WORKSPACE/sdk/python"
-handle_python_package
-
-cd "$WORKSPACE/sdk/cwl"
-handle_python_package
-
-cd "$WORKSPACE/services/fuse"
-handle_python_package
-
-cd "$WORKSPACE/services/nodemanager"
-handle_python_package
-
-# arvados-src
-(
- set -e
-
- cd "$WORKSPACE"
- COMMIT_HASH=$(format_last_commit_here "%H")
-
- SRC_BUILD_DIR=$(mktemp -d)
- # mktemp creates the directory with 0700 permissions by default
- chmod 755 $SRC_BUILD_DIR
- git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
- cd "$SRC_BUILD_DIR"
-
- # go into detached-head state
- git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
- echo "$COMMIT_HASH" >git-commit.version
-
- cd "$SRC_BUILD_DIR"
- PKG_VERSION=$(version_from_git)
- cd $WORKSPACE/packages/$TARGET
- fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
-
- rm -rf "$SRC_BUILD_DIR"
-)
-
-# On older platforms we need to publish a backport of libfuse >=2.9.2,
-# and we need to build and install it here in order to even build an
-# llfuse package.
-cd $WORKSPACE/packages/$TARGET
-if [[ $TARGET =~ ubuntu1204 ]]; then
- # port libfuse 2.9.2 to Ubuntu 12.04
- LIBFUSE_DIR=$(mktemp -d)
- (
- cd $LIBFUSE_DIR
- # download fuse 2.9.2 ubuntu 14.04 source package
- file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
- file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
- file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
-
- # install dpkg-source and dpkg-buildpackage commands
- apt-get install -y --no-install-recommends dpkg-dev
-
- # extract source and apply patches
- dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
- rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
-
- # add new version to changelog
- cd fuse-2.9.2
- (
- echo "fuse (2.9.2-5) precise; urgency=low"
- echo
- echo " * Backported from trusty-security to precise"
- echo
- echo " -- Joshua Randall <jcrandall at alum.mit.edu> Thu, 4 Feb 2016 11:31:00 -0000"
- echo
- cat debian/changelog
- ) > debian/changelog.new
- mv debian/changelog.new debian/changelog
-
- # install build-deps and build
- apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
- dpkg-buildpackage -rfakeroot -b
- )
- fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
- dpkg -i \
- "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
- "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
- "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
- apt-get -y --no-install-recommends -f install
- rm -rf $LIBFUSE_DIR
-elif [[ $TARGET =~ centos6 ]]; then
- # port fuse 2.9.2 to centos 6
- # install tools to build rpm from source
- yum install -y rpm-build redhat-rpm-config
- LIBFUSE_DIR=$(mktemp -d)
- (
- cd "$LIBFUSE_DIR"
- # download fuse 2.9.2 centos 7 source rpm
- file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
- (
- # modify source rpm spec to remove conflict on filesystem version
- mkdir -p /root/rpmbuild/SOURCES
- cd /root/rpmbuild/SOURCES
- rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
- perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
- )
- # build rpms from source
- rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
- rm -f fuse-2.9.2-6.el7.src.rpm
- # move built RPMs to LIBFUSE_DIR
- mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- rm -rf /root/rpmbuild
- )
- fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
- fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
- yum install -y \
- "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
- "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
- "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
-fi
-
-# Go binaries
-cd $WORKSPACE/packages/$TARGET
-export GOPATH=$(mktemp -d)
-package_go_binary services/keepstore keepstore \
- "Keep storage daemon, accessible to clients on the LAN"
-package_go_binary services/keepproxy keepproxy \
- "Make a Keep cluster accessible to clients that are not on the LAN"
-package_go_binary services/keep-web keep-web \
- "Static web hosting service for user data stored in Arvados Keep"
-package_go_binary services/datamanager arvados-data-manager \
- "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
-package_go_binary services/arv-git-httpd arvados-git-httpd \
- "Provide authenticated http access to Arvados-hosted git repositories"
-package_go_binary services/crunchstat crunchstat \
- "Gather cpu/memory/network statistics of running Crunch jobs"
-package_go_binary tools/keep-rsync keep-rsync \
- "Copy all data from one set of Keep servers to another"
-
-# The Python SDK
-# Please resist the temptation to add --no-python-fix-name to the fpm call here
-# (which would remove the python- prefix from the package name), because this
-# package is a dependency of arvados-fuse, and fpm can not omit the python-
-# prefix from only one of the dependencies of a package... Maybe I could
-# whip up a patch and send it upstream, but that will be for another day. Ward,
-# 2014-05-15
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
-
-# cwl-runner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
-
-# The PAM module
-if [[ $TARGET =~ debian|ubuntu ]]; then
- cd $WORKSPACE/packages/$TARGET
- rm -rf "$WORKSPACE/sdk/pam/build"
- fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
-fi
-
-# The FUSE driver
-# Please see comment about --no-python-fix-name above; we stay consistent and do
-# not omit the python- prefix first.
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
-
-# The node manager
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
-
-# The Docker image cleaner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
-
-# Forked libcloud
-LIBCLOUD_DIR=$(mktemp -d)
-(
- cd $LIBCLOUD_DIR
- git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
- git checkout apache-libcloud-$LIBCLOUD_PIN
- # libcloud is absurdly noisy without -q, so force -q here
- OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
- DASHQ_UNLESS_DEBUG=-q
- handle_python_package
- DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
-)
-fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
-rm -rf $LIBCLOUD_DIR
-
-# Python 2 dependencies
-declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
-# Add --no-use-wheel if this pip knows it.
-pip wheel --help >/dev/null 2>&1
-case "$?" in
- 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
- 2) ;;
- *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
-esac
-
-for deppkg in "${PYTHON_BACKPORTS[@]}"; do
- outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
- case "$deppkg" in
- httplib2|google-api-python-client)
- # Work around 0640 permissions on some package files.
- # See #7591 and #7991.
- pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
- set -e
- cd "$pyfpm_workdir"
- pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
- tar -xf "$deppkg"-*.tar*
- cd "$deppkg"-*/
- "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
- chmod -R go+rX .
- set +e
- # --iteration 2 provides an upgrade for previously built
- # buggy packages.
- fpm_build . "$outname" "" python "" --iteration 2
- # The upload step uses the package timestamp to determine
- # whether it's new. --no-clobber plays nice with that.
- mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
- )
- if [ 0 != "$?" ]; then
- echo "ERROR: $deppkg build process failed"
- EXITCODE=1
- fi
- if [ -n "$pyfpm_workdir" ]; then
- rm -rf "$pyfpm_workdir"
- fi
- ;;
- *)
- fpm_build "$deppkg" "$outname"
- ;;
- esac
-done
-
-# Python 3 dependencies
-for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
- outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
- # The empty string is the vendor argument: these aren't Curoverse software.
- fpm_build "$deppkg" "$outname" "" python3
-done
-
-# Build the API server package
-handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
-
-# Build the workbench server package
-(
- set -e
- cd "$WORKSPACE/apps/workbench"
-
- # We need to bundle to be ready even when we build a package without vendor directory
- # because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
-
- # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
- # and we want that in the package, so it's easier to not exclude the tmp directory
- # from the package - empty it instead.
- rm -rf tmp
- mkdir tmp
-
- # Set up application.yml and production.rb so that asset precompilation works
- \cp config/application.yml.example config/application.yml -f
- \cp config/environments/production.rb.example config/environments/production.rb -f
- sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
-
- RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
-
- # Remove generated configuration files so they don't go in the package.
- rm config/application.yml config/environments/production.rb
-)
-
-if [[ "$?" != "0" ]]; then
- echo "ERROR: Asset precompilation failed"
- EXITCODE=1
-else
- handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
-fi
-
-# clean up temporary GOPATH
-rm -rf "$GOPATH"
-
-exit $EXITCODE
diff --git a/jenkins/run-library.sh b/jenkins/run-library.sh
deleted file mode 100755
index bc76dd6..0000000
--- a/jenkins/run-library.sh
+++ /dev/null
@@ -1,344 +0,0 @@
-#!/bin/bash
-
-# A library of functions shared by the various scripts in this directory.
-
-# This is the timestamp about when we merged changed to include licenses
-# with Arvados packages. We use it as a heuristic to add revisions for
-# older packages.
-LICENSE_PACKAGE_TS=20151208015500
-
-debug_echo () {
- echo "$@" >"$STDOUT_IF_DEBUG"
-}
-
-find_easy_install() {
- for version_suffix in "$@"; do
- if "easy_install$version_suffix" --version >/dev/null 2>&1; then
- echo "easy_install$version_suffix"
- return 0
- fi
- done
- cat >&2 <<EOF
-$helpmessage
-
-Error: easy_install$1 (from Python setuptools module) not found
-
-EOF
- exit 1
-}
-
-format_last_commit_here() {
- local format="$1"; shift
- TZ=UTC git log -n1 --first-parent "--format=format:$format" .
-}
-
-version_from_git() {
- # Generates a version number from the git log for the current working
- # directory, and writes it to stdout.
- local git_ts git_hash
- declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
-}
-
-nohash_version_from_git() {
- version_from_git | cut -d. -f1-3
-}
-
-timestamp_from_git() {
- format_last_commit_here "%ct"
-}
-
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
-}
-
-handle_ruby_gem() {
- local gem_name="$1"; shift
- local gem_version="$(nohash_version_from_git)"
- local gem_src_dir="$(pwd)"
-
- if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
- find -maxdepth 1 -name "${gem_name}-*.gem" -delete
-
- # -q appears to be broken in gem version 2.2.2
- $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
- fi
-}
-
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
-package_go_binary() {
- local src_path="$1"; shift
- local prog="$1"; shift
- local description="$1"; shift
- local license_file="${1:-agpl-3.0.txt}"; shift
-
- debug_echo "package_go_binary $src_path as $prog"
-
- local basename="${src_path##*/}"
-
- mkdir -p "$GOPATH/src/git.curoverse.com"
- ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
-
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
- local version="$(version_from_git)"
- local timestamp="$(timestamp_from_git)"
-
- # If the command imports anything from the Arvados SDK, bump the
- # version number and build a new package whenever the SDK changes.
- if grep -qr git.curoverse.com/arvados .; then
- cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
- if [[ $(timestamp_from_git) -gt "$timestamp" ]]; then
- version=$(version_from_git)
- fi
- fi
-
- cd $WORKSPACE/packages/$TARGET
- go get "git.curoverse.com/arvados.git/$src_path"
- fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file"
-}
-
-default_iteration() {
- local package_name="$1"; shift
- local package_version="$1"; shift
- local iteration=1
- if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
- [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
- iteration=2
- fi
- echo $iteration
-}
-
-_build_rails_package_scripts() {
- local pkgname="$1"; shift
- local destdir="$1"; shift
- local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
- for scriptname in postinst prerm postrm; do
- cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
- >"$destdir/$scriptname" || return $?
- done
-}
-
-handle_rails_package() {
- local pkgname="$1"; shift
- local srcdir="$1"; shift
- local license_path="$1"; shift
- local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
- local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
- set -e
- _build_rails_package_scripts "$pkgname" "$scripts_dir"
- cd "$srcdir"
- mkdir -p tmp
- version_from_git >"$version_file"
- git rev-parse HEAD >git-commit.version
- bundle package --all
- )
- if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
- echo "ERROR: $pkgname package prep failed" >&2
- rm -rf "$scripts_dir" "$version_file"
- EXITCODE=1
- return 1
- fi
- local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
- "$(cat "$version_file")")
- local license_arg="$license_path=$railsdir/$(basename "$license_path")"
- # --iteration=5 accommodates the package script bugfixes #8371 and #8413.
- local -a switches=(--iteration=5
- --after-install "$scripts_dir/postinst"
- --before-remove "$scripts_dir/prerm"
- --after-remove "$scripts_dir/postrm")
- # For some reason fpm excludes need to not start with /.
- local exclude_root="${railsdir#/}"
- # .git and packages are for the SSO server, which is built from its
- # repository root.
- local -a exclude_list=(.git packages tmp log coverage Capfile\* \
- config/deploy\* config/application.yml)
- # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
- if [[ "$pkgname" != "arvados-workbench" ]]; then
- exclude_list+=('config/database.yml')
- fi
- for exclude in ${exclude_list[@]}; do
- switches+=(-x "$exclude_root/$exclude")
- done
- fpm_build "${pos_args[@]}" "${switches[@]}" \
- -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
- rm -rf "$scripts_dir" "$version_file"
-}
-
-# Build packages for everything
-fpm_build () {
- # The package source. Depending on the source type, this can be a
- # path, or the name of the package in an upstream repository (e.g.,
- # pip).
- PACKAGE=$1
- shift
- # The name of the package to build. Defaults to $PACKAGE.
- PACKAGE_NAME=${1:-$PACKAGE}
- shift
- # Optional: the vendor of the package. Should be "Curoverse, Inc." for
- # packages of our own software. Passed to fpm --vendor.
- VENDOR=$1
- shift
- # The type of source package. Passed to fpm -s. Default "python".
- PACKAGE_TYPE=${1:-python}
- shift
- # Optional: the package version number. Passed to fpm -v.
- VERSION=$1
- shift
-
- case "$PACKAGE_TYPE" in
- python)
- # All Arvados Python2 packages depend on Python 2.7.
- # Make sure we build with that for consistency.
- set -- "$@" --python-bin python2.7 \
- --python-easyinstall "$EASY_INSTALL2" \
- --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
- --depends "$PYTHON2_PACKAGE"
- ;;
- python3)
- # fpm does not actually support a python3 package type. Instead
- # we recognize it as a convenience shortcut to add several
- # necessary arguments to fpm's command line later, after we're
- # done handling positional arguments.
- PACKAGE_TYPE=python
- set -- "$@" --python-bin python3 \
- --python-easyinstall "$EASY_INSTALL3" \
- --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
- --depends "$PYTHON3_PACKAGE"
- ;;
- esac
-
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
- if [ python = "$PACKAGE_TYPE" ]; then
- COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
- if [ deb = "$FORMAT" ]; then
- # Dependencies are built from setup.py. Since setup.py will never
- # refer to Debian package iterations, it doesn't make sense to
- # enforce those in the .deb dependencies.
- COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
- fi
- fi
-
- if [[ "${DEBUG:-0}" != "0" ]]; then
- COMMAND_ARR+=('--verbose' '--log' 'info')
- fi
-
- if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
- COMMAND_ARR+=('-n' "$PACKAGE_NAME")
- fi
-
- if [[ "$VENDOR" != "" ]]; then
- COMMAND_ARR+=('--vendor' "$VENDOR")
- fi
-
- if [[ "$VERSION" != "" ]]; then
- COMMAND_ARR+=('-v' "$VERSION")
- fi
- # We can always add an --iteration here. If another one is specified in $@,
- # that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")")
-
- # Append --depends X and other arguments specified by fpm-info.sh in
- # the package source dir. These are added last so they can override
- # the arguments added by this script.
- declare -a fpm_args=()
- declare -a build_depends=()
- declare -a fpm_depends=()
- declare -a fpm_exclude=()
- declare -a fpm_dirs=(
- # source dir part of 'dir' package ("/source=/dest" => "/source"):
- "${PACKAGE%%=/*}"
- # backports ("llfuse==0.41.1" => "backports/python-llfuse")
- "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
- for pkgdir in "${fpm_dirs[@]}"; do
- fpminfo="$pkgdir/fpm-info.sh"
- if [[ -e "$fpminfo" ]]; then
- debug_echo "Loading fpm overrides from $fpminfo"
- source "$fpminfo"
- break
- fi
- done
- for pkg in "${build_depends[@]}"; do
- if [[ $TARGET =~ debian|ubuntu ]]; then
- pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
- if [[ -e $pkg_deb ]]; then
- echo "Installing build_dep $pkg from $pkg_deb"
- dpkg -i "$pkg_deb"
- else
- echo "Attemping to install build_dep $pkg using apt-get"
- apt-get install -y "$pkg"
- fi
- apt-get -y -f install
- else
- pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
- if [[ -e $pkg_rpm ]]; then
- echo "Installing build_dep $pkg from $pkg_rpm"
- rpm -i "$pkg_rpm"
- else
- echo "Attemping to install build_dep $pkg"
- rpm -i "$pkg"
- fi
- fi
- done
- for i in "${fpm_depends[@]}"; do
- COMMAND_ARR+=('--depends' "$i")
- done
- for i in "${fpm_exclude[@]}"; do
- COMMAND_ARR+=('--exclude' "$i")
- done
-
- # Append remaining function arguments directly to fpm's command line.
- for i; do
- COMMAND_ARR+=("$i")
- done
-
- COMMAND_ARR+=("${fpm_args[@]}")
-
- COMMAND_ARR+=("$PACKAGE")
-
- debug_echo -e "\n${COMMAND_ARR[@]}\n"
-
- FPM_RESULTS=$("${COMMAND_ARR[@]}")
- FPM_EXIT_CODE=$?
-
- fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
-}
-
-# verify build results
-fpm_verify () {
- FPM_EXIT_CODE=$1
- shift
- FPM_RESULTS=$@
-
- FPM_PACKAGE_NAME=''
- if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
- FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
- fi
-
- if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
- EXITCODE=1
- echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
- echo
- echo $FPM_RESULTS
- echo
- elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
- echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
- elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
- echo "Error building package for $1:\n $FPM_RESULTS"
- fi
-}
-
-install_package() {
- PACKAGES=$@
- if [[ "$FORMAT" == "deb" ]]; then
- $SUDO apt-get install $PACKAGES --yes
- elif [[ "$FORMAT" == "rpm" ]]; then
- $SUDO yum -q -y install $PACKAGES
- fi
-}
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
deleted file mode 100755
index 41b237b..0000000
--- a/jenkins/run-tests.sh
+++ /dev/null
@@ -1,831 +0,0 @@
-#!/bin/bash
-
-. `dirname "$(readlink -f "$0")"`/libcloud-pin
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Install and test Arvados components.
-
-Exit non-zero if any tests fail.
-
-Syntax:
- $(basename $0) WORKSPACE=/path/to/arvados [options]
-
-Options:
-
---skip FOO Do not test the FOO component.
---only FOO Do not test anything except the FOO component.
---temp DIR Install components and dependencies under DIR instead of
- making a new temporary directory. Implies --leave-temp.
---leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
- Instead, show the path to give as --temp to reuse them in
- subsequent invocations.
---skip-install Do not run any install steps. Just run tests.
- You should provide GOPATH, GEMHOME, and VENVDIR options
- from a previous invocation if you use this option.
---only-install Run specific install step
-WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with api server config files to copy into source tree.
- (If none given, leave config files alone in source tree.)
-services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
- Restrict apiserver tests to the given file
-sdk/python_test="--test-suite test.test_keep_locator"
- Restrict Python SDK tests to the given class
-apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
- Restrict Workbench tests to the given file
-services/arv-git-httpd_test="-check.vv"
- Show all log messages, even when tests pass (also works
- with services/keepstore_test etc.)
-ARVADOS_DEBUG=1
- Print more debug messages
-envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
- *_test, and other examples shown above.
-
-Assuming --skip-install is not given, all components are installed
-into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
-test suites depend on other components being installed, and installing
-everything tends to be quicker than debugging dependencies.
-
-As a special concession to the current CI server config, CONFIGSRC
-defaults to $HOME/arvados-api-server if that directory exists.
-
-More information and background:
-
-https://arvados.org/projects/arvados/wiki/Running_tests
-
-Available tests:
-
-apps/workbench
-apps/workbench_benchmark
-apps/workbench_profile
-doc
-services/api
-services/arv-git-httpd
-services/crunchstat
-services/dockercleaner
-services/fuse
-services/keep-web
-services/keepproxy
-services/keepstore
-services/login-sync
-services/nodemanager
-services/crunch-dispatch-local
-sdk/cli
-sdk/pam
-sdk/python
-sdk/ruby
-sdk/go/arvadosclient
-sdk/go/keepclient
-sdk/go/manifest
-sdk/go/blockdigest
-sdk/go/streamer
-sdk/go/crunchrunner
-sdk/cwl
-tools/crunchstat-summary
-tools/keep-rsync
-
-EOF
-
-# First make sure to remove any ARVADOS_ variables from the calling
-# environment that could interfere with the tests.
-unset $(env | cut -d= -f1 | grep \^ARVADOS_)
-
-# Reset other variables that could affect our [tests'] behavior by
-# accident.
-GITDIR=
-GOPATH=
-VENVDIR=
-VENV3DIR=
-PYTHONPATH=
-GEMHOME=
-PERLINSTALLBASE=
-
-COLUMNS=80
-
-skip_install=
-temp=
-temp_preserve=
-
-clear_temp() {
- if [[ -z "$temp" ]]; then
- # we didn't even get as far as making a temp dir
- :
- elif [[ -z "$temp_preserve" ]]; then
- rm -rf "$temp"
- else
- echo "Leaving behind temp dirs in $temp"
- fi
-}
-
-fatal() {
- clear_temp
- echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
- exit 1
-}
-
-report_outcomes() {
- for x in "${successes[@]}"
- do
- echo "Pass: $x"
- done
-
- if [[ ${#failures[@]} == 0 ]]
- then
- echo "All test suites passed."
- else
- echo "Failures (${#failures[@]}):"
- for x in "${failures[@]}"
- do
- echo "Fail: $x"
- done
- fi
-}
-
-exit_cleanly() {
- trap - INT
- create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
- rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
- stop_services
- rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
- report_outcomes
- clear_temp
- exit ${#failures}
-}
-
-sanity_checks() {
- ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
- || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
- echo Checking dependencies:
- echo -n 'virtualenv: '
- virtualenv --version \
- || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
- echo -n 'go: '
- go version \
- || fatal "No go binary. See http://golang.org/doc/install"
- echo -n 'gcc: '
- gcc --version | egrep ^gcc \
- || fatal "No gcc. Try: apt-get install build-essential"
- echo -n 'fuse.h: '
- find /usr/include -wholename '*fuse/fuse.h' \
- || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
- echo -n 'pyconfig.h: '
- find /usr/include -name pyconfig.h | egrep --max-count=1 . \
- || fatal "No pyconfig.h. Try: apt-get install python-dev"
- echo -n 'nginx: '
- PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
- || fatal "No nginx. Try: apt-get install nginx"
- echo -n 'perl: '
- perl -v | grep version \
- || fatal "No perl. Try: apt-get install perl"
- for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
- echo -n "perl $mod: "
- perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
- || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
- done
- echo -n 'gitolite: '
- which gitolite \
- || fatal "No gitolite. Try: apt-get install gitolite3"
-}
-
-rotate_logfile() {
- # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
- # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
- if [[ -f "$1/$2" ]]; then
- THEDATE=`date +%Y%m%d%H%M%S`
- mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
- gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
- fi
-}
-
-declare -a failures
-declare -A skip
-declare -A testargs
-skip[apps/workbench_profile]=1
-
-while [[ -n "$1" ]]
-do
- arg="$1"; shift
- case "$arg" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --skip)
- skipwhat="$1"; shift
- skip[$skipwhat]=1
- ;;
- --only)
- only="$1"; skip[$1]=""; shift
- ;;
- --skip-install)
- skip_install=1
- ;;
- --only-install)
- skip_install=1
- only_install="$1"; shift
- ;;
- --temp)
- temp="$1"; shift
- temp_preserve=1
- ;;
- --leave-temp)
- temp_preserve=1
- ;;
- --retry)
- retry=1
- ;;
- *_test=*)
- suite="${arg%%_test=*}"
- args="${arg#*=}"
- testargs["$suite"]="$args"
- ;;
- *=*)
- eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
- ;;
- *)
- echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
- exit 1
- ;;
- esac
-done
-
-start_api() {
- echo 'Starting API server...'
- cd "$WORKSPACE" \
- && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
- && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
- && export ARVADOS_TEST_API_INSTALLED="$$" \
- && (env | egrep ^ARVADOS)
-}
-
-start_nginx_proxy_services() {
- echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py start_keep_proxy \
- && python sdk/python/tests/run_test_server.py start_keep-web \
- && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
- && python sdk/python/tests/run_test_server.py start_nginx \
- && export ARVADOS_TEST_PROXY_SERVICES=1
-}
-
-stop_services() {
- if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
- unset ARVADOS_TEST_PROXY_SERVICES
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
- && python sdk/python/tests/run_test_server.py stop_keep-web \
- && python sdk/python/tests/run_test_server.py stop_keep_proxy
- fi
- if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
- unset ARVADOS_TEST_API_HOST
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop
- fi
-}
-
-interrupt() {
- failures+=("($(basename $0) interrupted)")
- exit_cleanly
-}
-trap interrupt INT
-
-sanity_checks
-
-echo "WORKSPACE=$WORKSPACE"
-
-if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
- # Jenkins expects us to use this by default.
- CONFIGSRC="$HOME/arvados-api-server"
-fi
-
-# Clean up .pyc files that may exist in the workspace
-cd "$WORKSPACE"
-find -name '*.pyc' -delete
-
-if [[ -z "$temp" ]]; then
- temp="$(mktemp -d)"
-fi
-
-# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
-do
- if [[ -z "${!tmpdir}" ]]; then
- eval "$tmpdir"="$temp/$tmpdir"
- fi
- if ! [[ -d "${!tmpdir}" ]]; then
- mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)"
- fi
-done
-
-setup_ruby_environment() {
- if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
- source "$HOME/.rvm/scripts/rvm"
- using_rvm=true
- elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
- source "/usr/local/rvm/scripts/rvm"
- using_rvm=true
- else
- using_rvm=false
- fi
-
- if [[ "$using_rvm" == true ]]; then
- # If rvm is in use, we can't just put separate "dependencies"
- # and "gems-under-test" paths to GEM_PATH: passenger resets
- # the environment to the "current gemset", which would lose
- # our GEM_PATH and prevent our test suites from running ruby
- # programs (for example, the Workbench test suite could not
- # boot an API server or run arv). Instead, we have to make an
- # rvm gemset and use it for everything.
-
- [[ `type rvm | head -n1` == "rvm is a function" ]] \
- || fatal 'rvm check'
-
- # Put rvm's favorite path back in first place (overriding
- # virtualenv, which just put itself there). Ignore rvm's
- # complaint about not being in first place already.
- rvm use @default 2>/dev/null
-
- # Create (if needed) and switch to an @arvados-tests
- # gemset. (Leave the choice of ruby to the caller.)
- rvm use @arvados-tests --create \
- || fatal 'rvm gemset setup'
-
- rvm env
- else
- # When our "bundle install"s need to install new gems to
- # satisfy dependencies, we want them to go where "gem install
- # --user-install" would put them. (However, if the caller has
- # already set GEM_HOME, we assume that's where dependencies
- # should be installed, and we should leave it alone.)
-
- if [ -z "$GEM_HOME" ]; then
- user_gempath="$(gem env gempath)"
- export GEM_HOME="${user_gempath%%:*}"
- fi
- PATH="$(gem env gemdir)/bin:$PATH"
-
- # When we build and install our own gems, we install them in our
- # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
- # PATH so integration tests prefer them over other versions that
- # happen to be installed in $user_gempath, system dirs, etc.
-
- tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
- PATH="$tmpdir_gem_home/bin:$PATH"
- export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
-
- echo "Will install dependencies to $(gem env gemdir)"
- echo "Will install arvados gems to $tmpdir_gem_home"
- echo "Gem search path is GEM_PATH=$GEM_PATH"
- fi
-}
-
-with_test_gemset() {
- if [[ "$using_rvm" == true ]]; then
- "$@"
- else
- GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
- fi
-}
-
-gem_uninstall_if_exists() {
- if gem list "$1\$" | egrep '^\w'; then
- gem uninstall --force --all --executables "$1"
- fi
-}
-
-setup_virtualenv() {
- local venvdest="$1"; shift
- if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
- virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
- fi
- "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7'
- # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
-}
-
-export PERLINSTALLBASE
-export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
-
-export GOPATH
-mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
- || fatal "symlink failed"
-
-setup_virtualenv "$VENVDIR" --python python2.7
-. "$VENVDIR/bin/activate"
-
-# Needed for run_test_server.py which is used by certain (non-Python) tests.
-pip freeze 2>/dev/null | egrep ^PyYAML= \
- || pip install PyYAML >/dev/null \
- || fatal "pip install PyYAML failed"
-
-# Preinstall forked version of libcloud, because nodemanager "pip install"
-# won't pick it up by default.
-pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
- || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
- || fatal "pip install apache-libcloud failed"
-
-# This will help people who reuse --temp dirs when we upgrade to llfuse 0.42
-if egrep -q 'llfuse.*>= *0\.42' "$WORKSPACE/services/fuse/setup.py"; then
- # Uninstall old llfuse, because services/fuse "pip install" won't
- # upgrade it by default.
- if pip freeze | egrep '^llfuse==0\.41\.'; then
- yes | pip uninstall 'llfuse<0.42'
- fi
-fi
-
-# Deactivate Python 2 virtualenv
-deactivate
-
-# If Python 3 is available, set up its virtualenv in $VENV3DIR.
-# Otherwise, skip dependent tests.
-PYTHON3=$(which python3)
-if [ "0" = "$?" ]; then
- setup_virtualenv "$VENV3DIR" --python python3
-else
- PYTHON3=
- skip[services/dockercleaner]=1
- cat >&2 <<EOF
-
-Warning: python3 could not be found
-services/dockercleaner install and tests will be skipped
-
-EOF
-fi
-
-# Reactivate Python 2 virtualenv
-. "$VENVDIR/bin/activate"
-
-# Note: this must be the last time we change PATH, otherwise rvm will
-# whine a lot.
-setup_ruby_environment
-
-echo "PATH is $PATH"
-
-if ! which bundler >/dev/null
-then
- gem install --user-install bundler || fatal 'Could not install bundler'
-fi
-
-checkexit() {
- if [[ "$1" != "0" ]]; then
- title "!!!!!! $2 FAILED !!!!!!"
- failures+=("$2 (`timer`)")
- else
- successes+=("$2 (`timer`)")
- fi
-}
-
-timer_reset() {
- t0=$SECONDS
-}
-
-timer() {
- echo -n "$(($SECONDS - $t0))s"
-}
-
-do_test() {
- while ! do_test_once ${@} && [[ "$retry" == 1 ]]
- do
- read -p 'Try again? [Y/n] ' x
- if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
- then
- break
- fi
- done
-}
-
-do_test_once() {
- unset result
- if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
- then
- title "Running $1 tests"
- timer_reset
- if [[ "$2" == "go" ]]
- then
- covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
- coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
- # We do "go get -t" here to catch compilation errors
- # before trying "go test". Otherwise, coverage-reporting
- # mode makes Go show the wrong line numbers when reporting
- # compilation errors.
- if [[ -n "${testargs[$1]}" ]]
- then
- # "go test -check.vv giturl" doesn't work, but this
- # does:
- cd "$WORKSPACE/$1" && \
- go get -t "git.curoverse.com/arvados.git/$1" && \
- go test ${coverflags[@]} ${testargs[$1]}
- else
- # The above form gets verbose even when testargs is
- # empty, so use this form in such cases:
- go get -t "git.curoverse.com/arvados.git/$1" && \
- go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
- fi
- result="$?"
- go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
- rm "$WORKSPACE/tmp/.$covername.tmp"
- elif [[ "$2" == "pip" ]]
- then
- # $3 can name a path directory for us to use, including trailing
- # slash; e.g., the bin/ subdirectory of a virtualenv.
- cd "$WORKSPACE/$1" \
- && "${3}python" setup.py test ${testargs[$1]}
- elif [[ "$2" != "" ]]
- then
- "test_$2"
- else
- "test_$1"
- fi
- result=${result:-$?}
- checkexit $result "$1 tests"
- title "End of $1 tests (`timer`)"
- return $result
- else
- title "Skipping $1 tests"
- fi
-}
-
-do_install() {
- if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
- then
- title "Running $1 install"
- timer_reset
- if [[ "$2" == "go" ]]
- then
- go get -t "git.curoverse.com/arvados.git/$1"
- elif [[ "$2" == "pip" ]]
- then
- # $3 can name a path directory for us to use, including trailing
- # slash; e.g., the bin/ subdirectory of a virtualenv.
-
- # Need to change to a different directory after creating
- # the source dist package to avoid a pip bug.
- # see https://arvados.org/issues/5766 for details.
-
- # Also need to install twice, because if it believes the package is
- # already installed, pip it won't install it. So the first "pip
- # install" ensures that the dependencies are met, the second "pip
- # install" ensures that we've actually installed the local package
- # we just built.
- cd "$WORKSPACE/$1" \
- && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
- && cd "$WORKSPACE" \
- && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
- elif [[ "$2" != "" ]]
- then
- "install_$2"
- else
- "install_$1"
- fi
- checkexit $? "$1 install"
- title "End of $1 install (`timer`)"
- else
- title "Skipping $1 install"
- fi
-}
-
-title () {
- txt="********** $1 **********"
- printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
-}
-
-bundle_install_trylocal() {
- (
- set -e
- echo "(Running bundle install --local. 'could not find package' messages are OK.)"
- if ! bundle install --local --no-deployment; then
- echo "(Running bundle install again, without --local.)"
- bundle install --no-deployment
- fi
- bundle package --all
- )
-}
-
-install_doc() {
- cd "$WORKSPACE/doc" \
- && bundle_install_trylocal \
- && rm -rf .site
-}
-do_install doc
-
-install_gem() {
- gemname=$1
- srcpath=$2
- with_test_gemset gem_uninstall_if_exists "$gemname" \
- && cd "$WORKSPACE/$srcpath" \
- && bundle_install_trylocal \
- && gem build "$gemname.gemspec" \
- && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
-}
-
-install_ruby_sdk() {
- install_gem arvados sdk/ruby
-}
-do_install sdk/ruby ruby_sdk
-
-install_perl_sdk() {
- cd "$WORKSPACE/sdk/perl" \
- && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
- && make install INSTALLDIRS=perl
-}
-do_install sdk/perl perl_sdk
-
-install_cli() {
- install_gem arvados-cli sdk/cli
-}
-do_install sdk/cli cli
-
-install_login-sync() {
- install_gem arvados-login-sync services/login-sync
-}
-do_install services/login-sync login-sync
-
-# Install the Python SDK early. Various other test suites (like
-# keepproxy) bring up run_test_server.py, which imports the arvados
-# module. We can't actually *test* the Python SDK yet though, because
-# its own test suite brings up some of those other programs (like
-# keepproxy).
-declare -a pythonstuff
-pythonstuff=(
- sdk/pam
- sdk/python
- sdk/cwl
- services/fuse
- services/nodemanager
- tools/crunchstat-summary
- )
-for p in "${pythonstuff[@]}"
-do
- do_install "$p" pip
-done
-if [ -n "$PYTHON3" ]; then
- do_install services/dockercleaner pip "$VENV3DIR/bin/"
-fi
-
-install_apiserver() {
- cd "$WORKSPACE/services/api" \
- && RAILS_ENV=test bundle_install_trylocal
-
- rm -f config/environments/test.rb
- cp config/environments/test.rb.example config/environments/test.rb
-
- if [ -n "$CONFIGSRC" ]
- then
- for f in database.yml application.yml
- do
- cp "$CONFIGSRC/$f" config/ || fatal "$f"
- done
- fi
-
- # Fill in a random secret_token and blob_signing_key for testing
- SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
- BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
-
- sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
- sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
-
- # Set up empty git repo (for git tests)
- GITDIR=$(mktemp -d)
- sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
-
- rm -rf $GITDIR
- mkdir -p $GITDIR/test
- cd $GITDIR/test \
- && git init \
- && git config user.email "jenkins at ci.curoverse.com" \
- && git config user.name "Jenkins, CI" \
- && touch tmp \
- && git add tmp \
- && git commit -m 'initial commit'
-
- # Clear out any lingering postgresql connections to the test
- # database, so that we can drop it. This assumes the current user
- # is a postgresql superuser.
- cd "$WORKSPACE/services/api" \
- && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
- && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
-
- cd "$WORKSPACE/services/api" \
- && RAILS_ENV=test bundle exec rake db:drop \
- && RAILS_ENV=test bundle exec rake db:setup \
- && RAILS_ENV=test bundle exec rake db:fixtures:load
-}
-do_install services/api apiserver
-
-declare -a gostuff
-gostuff=(
- sdk/go/arvadosclient
- sdk/go/blockdigest
- sdk/go/manifest
- sdk/go/streamer
- sdk/go/crunchrunner
- services/arv-git-httpd
- services/crunchstat
- services/keep-web
- services/keepstore
- sdk/go/keepclient
- services/keepproxy
- services/datamanager/summary
- services/datamanager/collection
- services/datamanager/keep
- services/datamanager
- services/crunch-dispatch-local
- services/crunch-run
- tools/keep-rsync
- )
-for g in "${gostuff[@]}"
-do
- do_install "$g" go
-done
-
-install_workbench() {
- cd "$WORKSPACE/apps/workbench" \
- && mkdir -p tmp/cache \
- && RAILS_ENV=test bundle_install_trylocal
-}
-do_install apps/workbench workbench
-
-test_doclinkchecker() {
- (
- set -e
- cd "$WORKSPACE/doc"
- ARVADOS_API_HOST=qr1hi.arvadosapi.com
- # Make sure python-epydoc is installed or the next line won't
- # do much good!
- PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
- )
-}
-do_test doc doclinkchecker
-
-stop_services
-
-test_apiserver() {
- cd "$WORKSPACE/services/api" \
- && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
-}
-do_test services/api apiserver
-
-# Shortcut for when we're only running apiserver tests. This saves a bit of time,
-# because we don't need to start up the api server for subsequent tests.
-if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
- rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
- exit_cleanly
-fi
-
-start_api
-
-test_ruby_sdk() {
- cd "$WORKSPACE/sdk/ruby" \
- && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
-}
-do_test sdk/ruby ruby_sdk
-
-test_cli() {
- cd "$WORKSPACE/sdk/cli" \
- && mkdir -p /tmp/keep \
- && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
-}
-do_test sdk/cli cli
-
-test_login-sync() {
- cd "$WORKSPACE/services/login-sync" \
- && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
-}
-do_test services/login-sync login-sync
-
-for p in "${pythonstuff[@]}"
-do
- do_test "$p" pip
-done
-do_test services/dockercleaner pip "$VENV3DIR/bin/"
-
-for g in "${gostuff[@]}"
-do
- do_test "$g" go
-done
-
-test_workbench() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
- && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
-}
-do_test apps/workbench workbench
-
-test_workbench_benchmark() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
- && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
-}
-do_test apps/workbench_benchmark workbench_benchmark
-
-test_workbench_profile() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
- && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
-}
-do_test apps/workbench_profile workbench_profile
-
-exit_cleanly
commit 2fecb120e5b9be7a85f209e90114731f3e95f709
Author: Ward Vandewege <ward at curoverse.com>
Date: Mon May 16 20:54:39 2016 -0400
We need a per-target last_upload timestamp file.
refs #9224
diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index b44cbdc..98c1ab2 100755
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -262,10 +262,10 @@ def build_suite_and_upload(target, since_timestamp, args):
def main(arguments, stdout=sys.stdout, stderr=sys.stderr):
args = parse_arguments(arguments)
setup_logger(stderr, args)
- ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
- '.last_upload'))
- last_upload_ts = ts_file.last_upload()
for target in args.targets:
+ ts_file = TimestampFile(os.path.join(args.workspace, 'packages',
+ '.last_upload_%s' % target))
+ last_upload_ts = ts_file.last_upload()
build_suite_and_upload(target, last_upload_ts, args)
ts_file.update()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list