[ARVADOS-DEV] created: 301f99d179c06fbb4d146f38e05530f5fb265510
git at public.curoverse.com
git at public.curoverse.com
Tue May 12 17:02:34 EDT 2015
at 301f99d179c06fbb4d146f38e05530f5fb265510 (commit)
commit 301f99d179c06fbb4d146f38e05530f5fb265510
Author: Brett Smith <brett at curoverse.com>
Date: Tue May 12 17:01:39 2015 -0400
3793: Build Docker cleaner service in run-build-packages.sh.
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 5ba1eb9..74d2730 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -130,27 +130,28 @@ build_and_scp_deb () {
PACKAGE=$1
shift
# The name of the package to build. Defaults to $PACKAGE.
- PACKAGE_NAME=$1
+ PACKAGE_NAME=${1:-$PACKAGE}
shift
# Optional: the vendor of the package. Should be "Curoverse, Inc." for
# packages of our own software. Passed to fpm --vendor.
VENDOR=$1
shift
# The type of source package. Passed to fpm -s. Default "python".
- PACKAGE_TYPE=$1
+ PACKAGE_TYPE=${1:-python}
shift
+
+ if [ "python3" = "$PACKAGE_TYPE" ]; then
+ # fpm does not actually support this package type. Instead we recognize
+ # it as a convenience shortcut to add several necessary arguments to
+ # fpm's command line later.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
+ --python-package-name-prefix python3
+ fi
# Optional: the package version number. Passed to fpm -v.
VERSION=$1
shift
- if [[ "$PACKAGE_NAME" == "" ]]; then
- PACKAGE_NAME=$PACKAGE
- fi
-
- if [[ "$PACKAGE_TYPE" == "" ]]; then
- PACKAGE_TYPE='python'
- fi
-
declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "deb" "-x" "usr/local/lib/python2.7/dist-packages/tests")
if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
@@ -461,6 +462,10 @@ cd $WORKSPACE/debs
# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
build_and_scp_deb $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+# The Docker image cleaner
+cd $WORKSPACE/debs
+build_and_scp_deb $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+
# A few dependencies
for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py virtualenv \
@@ -468,6 +473,7 @@ for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pycrypto backports.ssl_match_hostname; do
build_and_scp_deb "$deppkg"
done
+build_and_scp_deb docker-py python3-docker-py python3
# cwltool from common-workflow-language. We use this in arv-run-pipeline-instance.
# We use $WORKSPACE/common-workflow-language as the clean directory from which to build the cwltool package
commit acdfc466ec52fbbf394f0e0b00544e809e74cec8
Author: Brett Smith <brett at curoverse.com>
Date: Tue May 12 14:57:20 2015 -0400
3793: Add services/dockercleaner to run-tests.sh.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 91c7cac..8e58d67 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -56,6 +56,7 @@ apps/workbench_profile
doc
services/api
services/crunchstat
+services/dockercleaner
services/fuse
services/keepproxy
services/keepstore
@@ -79,6 +80,7 @@ unset $(env | cut -d= -f1 | grep \^ARVADOS_)
GITDIR=
GOPATH=
VENVDIR=
+VENV3DIR=
PYTHONPATH=
GEMHOME=
@@ -90,7 +92,7 @@ skip_install=
declare -A leave_temp
clear_temp() {
leaving=""
- for var in VENVDIR GOPATH GITDIR GEMHOME
+ for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
do
if [[ -z "${leave_temp[$var]}" ]]
then
@@ -221,6 +223,7 @@ do
;;
--leave-temp)
leave_temp[VENVDIR]=1
+ leave_temp[VENV3DIR]=1
leave_temp[GOPATH]=1
leave_temp[GEMHOME]=1
;;
@@ -295,7 +298,7 @@ cd "$WORKSPACE"
find -name '*.pyc' -delete
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
do
if [[ -n "${!tmpdir}" ]]; then
leave_temp[$tmpdir]=1
@@ -401,6 +404,23 @@ fi
echo "pip install -q PyYAML"
pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
+ || fatal "python3 virtualenv $VENV3DIR failed"
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
+
checkexit() {
if [[ "$1" != "0" ]]; then
title "!!!!!! $2 FAILED !!!!!!"
@@ -448,8 +468,10 @@ do_test_once() {
fi
elif [[ "$2" == "pip" ]]
then
- cd "$WORKSPACE/$1" \
- && python setup.py test ${testargs[$1]}
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
elif [[ "$2" != "" ]]
then
"test_$2"
@@ -475,6 +497,9 @@ do_install() {
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
# Need to change to a different directory after creating
# the source dist package to avoid a pip bug.
# see https://arvados.org/issues/5766 for details.
@@ -485,10 +510,10 @@ do_install() {
# install" ensures that we've actually install the local package
# we just built.
cd "$WORKSPACE/$1" \
- && python setup.py sdist rotate --keep=1 --match .tar.gz \
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
- && pip install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && pip install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
@@ -559,6 +584,9 @@ for p in "${pythonstuff[@]}"
do
do_install "$p" pip
done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
install_apiserver() {
cd "$WORKSPACE/services/api" \
@@ -677,6 +705,7 @@ for p in "${pythonstuff[@]}"
do
do_test "$p" pip
done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
for g in "${gostuff[@]}"
do
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list