[ARVADOS-DEV] updated: cc7fb93641cd735c8bd0ec744b1fa1b84a242626
git at public.curoverse.com
git at public.curoverse.com
Thu May 21 11:33:00 EDT 2015
Summary of changes:
discards 49d7df5e26de1e1f4a77d2c0dfa25d776515e2ee (commit)
discards a4b2f2a216800878e6dad3aa81fba92e11fb0431 (commit)
discards cfb3c4554ed688285e3cb6a827f13406f4ad80e3 (commit)
discards 2dae0755cb100b9fc81ba1768fc9233a17cc30ca (commit)
discards 0fbece8ea4acf263da8bc312ba3451c1adb7ea43 (commit)
discards 9fa534460eaa10ac16d1de355e45898517306ec5 (commit)
discards 08d02e15e140740902dfab5895295c08d4ca0d16 (commit)
discards 42e438d97a95d8e46f045acfaf82298d8bb3dcab (commit)
discards 303a6e2ad510ea2a4c36e7bc2b7ce789ce0b0a7a (commit)
discards 9d3fa25b005586925024bd7995f846df562cec65 (commit)
discards cb1c4d5827f2dba8531e55e384267ecb5ae19620 (commit)
discards 63e4023179b0b86ed2d7746fb1154ca20dff8aab (commit)
discards d16c7ef6c4dfe2501930eaa89297b2a31f1b5a3d (commit)
discards 089953257d8e91a3267b5f13374fd09e28c08887 (commit)
discards c4630bce4a7bf9260369b67faf4fbb4e7e549642 (commit)
discards 1341bb72ae74b1f151935bd1ee4564648de2246f (commit)
discards 8561679d3420531c237c67add8415fa0212af1ec (commit)
discards 0f3a97b13fb0003628df80e180cb3a2b945cd574 (commit)
discards 8b3d7b73503b41c2b55a62674a25b1d5c4a141bf (commit)
discards 858cd93950d0fd7b621e9e4656261856e60856ad (commit)
discards 034677aa057f89be2d5bf923d6956164396d11cb (commit)
discards acdfc466ec52fbbf394f0e0b00544e809e74cec8 (commit)
discards 274c7abf3aee7dee8bfe648f485087a5a0ab59ed (commit)
discards 3002e7a6bc67564e900a5da490ef4e6b3b43224f (commit)
discards 8b51595b8ee2a1a554d365e857dda5cfde440e53 (commit)
discards d64856940e5568d3b8bd83d60f36d13e7bc11fa2 (commit)
discards f9bc13bd47c5404da529d5a71153500d2ee755c7 (commit)
discards decff518125e54dc044170fb4e31a5b86d566bb5 (commit)
discards ccb48a84de0d9adff1809574aac4df77b9b4575c (commit)
discards 672107d387a86334aa70711d071bc3c2bf9c5502 (commit)
discards 40ee991675ff42fe531bb8f2d1bd44311e9fe370 (commit)
discards a00ef8633a3fbed5baecc5c47e9a330943dc8f8e (commit)
discards 3c0d0df8f6d287e83ea959881afedddc5d4b784a (commit)
via cc7fb93641cd735c8bd0ec744b1fa1b84a242626 (commit)
via 32f2a67ecfd7a5b4169d4e3fbc162306771f0d34 (commit)
via eae444a782a53e348fd5df5929cfb2713f0d6fa8 (commit)
via c4d584c524efe8c9c689a263bec54b99b3d89138 (commit)
via 31e86126eba36e27c824b769ddea304e52ceeb59 (commit)
via 23b42a86c646eed10830b4b34ec46173728140d0 (commit)
via 0f0793754990012c75393cd517c85c261579cddd (commit)
via a43a3fb3b393b125b7c6d4139375b4bd95890cfa (commit)
via 9a4afad134f563d41f2a0c84b4389698df07a8db (commit)
via 123d3d778051c90b6be39b0f0b25da24d025d382 (commit)
via 1553f4842dba02aaca9fd7e5a9ac426d7532c462 (commit)
via ffa8b415658e6d3ba3836c6e208fe35979de0bf4 (commit)
via a3465f97627494bde0f148e27559046308cbedda (commit)
via 5b7e2250d5802b20b73242ae4d0641b32f72cce7 (commit)
via 4bfcd9f168a3371b19bbd12429a8da37883256c6 (commit)
via 422349074a47c6cf6394234a3dba975c84b94b09 (commit)
via 72c3457917a25dede1695c8abda316aa4e5afd5e (commit)
via bf95f654a1587dceb6d2d27c03d6da84a5a2a92a (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (49d7df5e26de1e1f4a77d2c0dfa25d776515e2ee)
\
N -- N -- N (cc7fb93641cd735c8bd0ec744b1fa1b84a242626)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit cc7fb93641cd735c8bd0ec744b1fa1b84a242626
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 21 10:21:45 2015 -0400
3793: Fix how python3 fpm arguments are added.
The previous code would add arguments such that they could be
interpreted as the package version. Avoid this by adding arguments
after we're done consuming positional arguments.
Refs #3793.
no more trailing spaces
adding an empty csv in case of error
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
index ed5ee53..ce3bfed 100755
--- a/jenkins/create-plot-data-from-log.sh
+++ b/jenkins/create-plot-data-from-log.sh
@@ -12,7 +12,7 @@ usage() {
if [ $# -ne 3 ]
then
- usage
+ usage
exit 1
fi
@@ -33,7 +33,7 @@ fi
## MAXLINE is the amount of lines that will read after the pattern
## is match (the logfile could be hundred thousands lines long).
## 1000 should be safe enough to capture all the output of the individual test
-MAXLINES=1000
+MAXLINES=1000
## TODO: check $build and $file make sense
@@ -46,12 +46,14 @@ do
(zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail -n +1 | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$cleaned_test-$build.txt
result=$?
if [ $result -eq 0 ]
- then
+ then
echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
#echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
- else
+ else
echo "$test was't found on $file"
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ > $outputdir/$cleaned_test.csv
fi
done
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 95a507a..bb0877e 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -149,18 +149,18 @@ build_and_scp_deb () {
# The type of source package. Passed to fpm -s. Default "python".
PACKAGE_TYPE=${1:-python}
shift
+ # Optional: the package version number. Passed to fpm -v.
+ VERSION=$1
+ shift
+ # fpm does not actually support a python3 package type. Instead we recognize
+ # it as a convenience shortcut to add several necessary arguments to
+ # fpm's command line later, after we're done handling positional arguments.
if [ "python3" = "$PACKAGE_TYPE" ]; then
- # fpm does not actually support this package type. Instead we recognize
- # it as a convenience shortcut to add several necessary arguments to
- # fpm's command line later.
PACKAGE_TYPE=python
set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
--python-package-name-prefix python3 --depends python3
fi
- # Optional: the package version number. Passed to fpm -v.
- VERSION=$1
- shift
declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "deb" "-x" "usr/local/lib/python2.7/dist-packages/tests")
commit 32f2a67ecfd7a5b4169d4e3fbc162306771f0d34
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 21 10:03:36 2015 -0400
Improve packaging of third-party Python 3 packages.
* Fix the bug where we're passing in "python3" as the vendor argument
to build_and_scp_deb.
* Build more dependencies that docker-py needs.
* Automatically add a dependency on the python3 package.
Refs #3793.
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 3b9fc15..95a507a 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -156,7 +156,7 @@ build_and_scp_deb () {
# fpm's command line later.
PACKAGE_TYPE=python
set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
- --python-package-name-prefix python3
+ --python-package-name-prefix python3 --depends python3
fi
# Optional: the package version number. Passed to fpm -v.
VERSION=$1
@@ -483,7 +483,11 @@ for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pycrypto backports.ssl_match_hostname; do
build_and_scp_deb "$deppkg"
done
-build_and_scp_deb docker-py python3-docker-py python3
+# Python 3 dependencies
+for deppkg in docker-py six requests; do
+ # The empty string is the vendor argument: these aren't Curoverse software.
+ build_and_scp_deb "$deppkg" "python3-$deppkg" "" python3
+done
# cwltool from common-workflow-language. We use this in arv-run-pipeline-instance.
# We use $WORKSPACE/common-workflow-language as the clean directory from which to build the cwltool package
commit eae444a782a53e348fd5df5929cfb2713f0d6fa8
Author: Tom Clegg <tom at curoverse.com>
Date: Thu May 21 08:57:50 2015 -0400
5766: Remove "remove dependencies" step. Detect existing PyYAML faster.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index be35ae9..1a699a8 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -370,10 +370,6 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
. "$VENVDIR/bin/activate"
-# When re-using $VENVDIR, upgrade any packages (except arvados) that are
-# already installed
-pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=`
-
# Note: this must be the last time we change PATH, otherwise rvm will
# whine a lot.
setup_ruby_environment
@@ -386,8 +382,9 @@ then
fi
# Needed for run_test_server.py which is used by certain (non-Python) tests.
-echo "pip install -q PyYAML"
-pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+ || pip install PyYAML \
+ || fatal "pip install PyYAML failed"
# If Python 3 is available, set up its virtualenv in $VENV3DIR.
# Otherwise, skip dependent tests.
commit c4d584c524efe8c9c689a263bec54b99b3d89138
Author: Tom Clegg <tom at curoverse.com>
Date: Thu May 21 08:51:21 2015 -0400
Tidy up sanity checks. Add checks for fuse.h, pyconfig.h, nginx.
Suggest how to fix missing dependencies.
No issue #
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 70fd1a3..be35ae9 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -111,7 +111,7 @@ clear_temp() {
fatal() {
clear_temp
- echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+ echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
exit 1
}
@@ -145,44 +145,27 @@ exit_cleanly() {
}
sanity_checks() {
- # Make sure WORKSPACE is set
- if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
- fi
-
- # Make sure virtualenv is installed
- `virtualenv --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: virtualenv could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure go is installed
- `go env >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: go could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure gcc is installed
- `gcc --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: gcc could not be found"
- echo >&2
- exit 1
- fi
+ ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ echo Checking dependencies:
+ echo -n 'virtualenv: '
+ virtualenv --version \
+ || fatal "No virtualenv. Try: apt-get install virtualenv"
+ echo -n 'go: '
+ go version \
+ || fatal "No go binary. See http://golang.org/doc/install"
+ echo -n 'gcc: '
+ gcc --version | egrep ^gcc \
+ || fatal "No gcc. Try: apt-get install build-essential"
+ echo -n 'fuse.h: '
+ find /usr/include -wholename '*fuse/fuse.h' \
+ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'pyconfig.h: '
+ find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+ || fatal "No pyconfig.h. Try: apt-get install python-dev"
+ echo -n 'nginx: '
+ PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+ || fatal "No nginx. Try: apt-get install nginx"
}
rotate_logfile() {
commit 31e86126eba36e27c824b769ddea304e52ceeb59
Author: Nico Cesar <nico at nicocesar.com>
Date: Thu May 21 10:27:14 2015 -0400
6061: comment cleanup and final version tested. it works
closes #6061
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
index 26fd655..ed5ee53 100755
--- a/jenkins/create-plot-data-from-log.sh
+++ b/jenkins/create-plot-data-from-log.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
build=$1
file=$2
@@ -19,7 +19,7 @@ fi
if [ ! -e $file ]
then
usage
- echo "$file doesn't exists! exiting"
+ echo "$file doesn't exist! exiting"
exit 2
fi
if [ ! -w $outputdir ]
@@ -30,7 +30,9 @@ then
fi
#------------------------------
-## max lines that a test will output
+## MAXLINE is the amount of lines that will read after the pattern
+## is match (the logfile could be hundred thousands lines long).
+## 1000 should be safe enough to capture all the output of the individual test
MAXLINES=1000
## TODO: check $build and $file make sense
@@ -40,14 +42,15 @@ for test in \
test_Create,_show,_and_update_description_for_large_collection_with_manifest_text_of_100000 \
test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
do
-(zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$test-$build.txt
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ (zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail -n +1 | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$cleaned_test-$build.txt
result=$?
if [ $result -eq 0 ]
then
- echo processing $outputdir/$test-$build.txt creating $outputdir/$test.csv
- echo $(grep ^Completed $outputdir/$test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$test.csv
- echo $(grep ^Completed $outputdir/$test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$test.csv
- #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$test-$build.txt/*view*/ >> $outputdir/$test.properties
+ echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
+ #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
else
echo "$test was't found on $file"
fi
commit 23b42a86c646eed10830b4b34ec46173728140d0
Author: Nico Cesar <nico at nicocesar.com>
Date: Wed May 20 11:43:31 2015 -0400
6061: plot data
added create plot data script to exit_cleanly()
3 initial test to use
Matching more generic test names
refs #6061
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
index 60516a5..26fd655 100755
--- a/jenkins/create-plot-data-from-log.sh
+++ b/jenkins/create-plot-data-from-log.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
build=$1
file=$2
@@ -36,19 +36,19 @@ MAXLINES=1000
## TODO: check $build and $file make sense
for test in \
- test_Collection_page_renders_name \
- test_combine_selected_collections_into_new_collection \
- test_combine_selected_collection_files_into_new_collection_active_foo_collection_in_aproject_true \
- test_combine_selected_collection_files_into_new_collection_active_foo_file_false \
- test_combine_selected_collection_files_into_new_collection_project_viewer_foo_collection_in_aproject_false \
- test_combine_selected_collection_files_into_new_collection_project_viewer_foo_file_false \
test_Create_and_show_large_collection_with_manifest_text_of_20000000 \
- test_Create__show__and_update_description_for_large_collection_with_manifest_text_of_100000 \
+ test_Create,_show,_and_update_description_for_large_collection_with_manifest_text_of_100000 \
test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
do
- zgrep -A$MAXLINES "^CollectionsTest: $test" $file | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$test-$build.txt
- echo processing $outputdir/$test-$build.txt creating $outputdir/$test.csv
- echo $(grep ^Completed $test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$test.csv
- echo $(grep ^Completed $test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$test.csv
- #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$test-$build.txt/*view*/ >> $outputdir/$test.properties
+(zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$test-$build.txt
+ result=$?
+ if [ $result -eq 0 ]
+ then
+ echo processing $outputdir/$test-$build.txt creating $outputdir/$test.csv
+ echo $(grep ^Completed $outputdir/$test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$test.csv
+ echo $(grep ^Completed $outputdir/$test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$test.csv
+ #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$test-$build.txt/*view*/ >> $outputdir/$test.properties
+ else
+ echo "$test was't found on $file"
+ fi
done
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 375c4e0..70fd1a3 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -135,6 +135,7 @@ report_outcomes() {
exit_cleanly() {
trap - INT
+ create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
stop_services
rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
@@ -190,7 +191,6 @@ rotate_logfile() {
if [[ -f "$1/$2" ]]; then
THEDATE=`date +%Y%m%d%H%M%S`
mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
- ./create-plot-data-from-log.sh $BUILD_NUMBER "$1/$THEDATE-$BUILD_NUMBER-$2" "$1"
gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
fi
}
commit 0f0793754990012c75393cd517c85c261579cddd
Author: Nico Cesar <nico at nicocesar.com>
Date: Wed May 20 11:12:18 2015 -0400
adding 3 more test for the performance suite.
I'm guessing the names, lets see if they are good or not ...
refs #6061
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
index daf3ef8..60516a5 100755
--- a/jenkins/create-plot-data-from-log.sh
+++ b/jenkins/create-plot-data-from-log.sh
@@ -41,7 +41,10 @@ for test in \
test_combine_selected_collection_files_into_new_collection_active_foo_collection_in_aproject_true \
test_combine_selected_collection_files_into_new_collection_active_foo_file_false \
test_combine_selected_collection_files_into_new_collection_project_viewer_foo_collection_in_aproject_false \
- test_combine_selected_collection_files_into_new_collection_project_viewer_foo_file_false
+ test_combine_selected_collection_files_into_new_collection_project_viewer_foo_file_false \
+ test_Create_and_show_large_collection_with_manifest_text_of_20000000 \
+ test_Create__show__and_update_description_for_large_collection_with_manifest_text_of_100000 \
+ test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
do
zgrep -A$MAXLINES "^CollectionsTest: $test" $file | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$test-$build.txt
echo processing $outputdir/$test-$build.txt creating $outputdir/$test.csv
commit a43a3fb3b393b125b7c6d4139375b4bd95890cfa
Author: Nico Cesar <nico at nicocesar.com>
Date: Tue May 19 14:44:18 2015 -0400
adding the creation of plot data that jenkins plot pluging will be able to read
refs #6061
diff --git a/jenkins/create-plot-data-from-log.sh b/jenkins/create-plot-data-from-log.sh
new file mode 100755
index 0000000..daf3ef8
--- /dev/null
+++ b/jenkins/create-plot-data-from-log.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+build=$1
+file=$2
+outputdir=$3
+
+usage() {
+ echo "./$0 build_number file_to_parse output_dir"
+ echo "this script will use the build output to generate *csv and *txt"
+ echo "for jenkins plugin plot https://github.com/jenkinsci/plot-plugin/"
+}
+
+if [ $# -ne 3 ]
+then
+ usage
+ exit 1
+fi
+
+if [ ! -e $file ]
+then
+ usage
+ echo "$file doesn't exists! exiting"
+ exit 2
+fi
+if [ ! -w $outputdir ]
+then
+ usage
+ echo "$outputdir isn't writeable! exiting"
+ exit 3
+fi
+
+#------------------------------
+## max lines that a test will output
+MAXLINES=1000
+
+## TODO: check $build and $file make sense
+
+for test in \
+ test_Collection_page_renders_name \
+ test_combine_selected_collections_into_new_collection \
+ test_combine_selected_collection_files_into_new_collection_active_foo_collection_in_aproject_true \
+ test_combine_selected_collection_files_into_new_collection_active_foo_file_false \
+ test_combine_selected_collection_files_into_new_collection_project_viewer_foo_collection_in_aproject_false \
+ test_combine_selected_collection_files_into_new_collection_project_viewer_foo_file_false
+do
+ zgrep -A$MAXLINES "^CollectionsTest: $test" $file | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$test-$build.txt
+ echo processing $outputdir/$test-$build.txt creating $outputdir/$test.csv
+ echo $(grep ^Completed $test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$test.csv
+ echo $(grep ^Completed $test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$test.csv
+ #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$test-$build.txt/*view*/ >> $outputdir/$test.properties
+done
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 8e58d67..375c4e0 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -185,10 +185,12 @@ sanity_checks() {
}
rotate_logfile() {
+ # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
# $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
if [[ -f "$1/$2" ]]; then
THEDATE=`date +%Y%m%d%H%M%S`
mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
+ ./create-plot-data-from-log.sh $BUILD_NUMBER "$1/$THEDATE-$BUILD_NUMBER-$2" "$1"
gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
fi
}
commit 9a4afad134f563d41f2a0c84b4389698df07a8db
Author: Brett Smith <brett at curoverse.com>
Date: Tue May 12 17:01:39 2015 -0400
3793: Build Docker cleaner service in run-build-packages.sh.
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 5ba1eb9..3b9fc15 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -81,6 +81,16 @@ if [[ "$?" != 0 ]]; then
exit 1
fi
+if ! easy_install3 --version >/dev/null; then
+ cat >&2 <<EOF
+$helpmessage
+
+Error: easy_install3 (from python3-setuptools) not found
+
+EOF
+ exit 1
+fi
+
if [[ "$DEBUG" != 0 ]]; then
echo "Workspace is $WORKSPACE"
fi
@@ -130,27 +140,28 @@ build_and_scp_deb () {
PACKAGE=$1
shift
# The name of the package to build. Defaults to $PACKAGE.
- PACKAGE_NAME=$1
+ PACKAGE_NAME=${1:-$PACKAGE}
shift
# Optional: the vendor of the package. Should be "Curoverse, Inc." for
# packages of our own software. Passed to fpm --vendor.
VENDOR=$1
shift
# The type of source package. Passed to fpm -s. Default "python".
- PACKAGE_TYPE=$1
+ PACKAGE_TYPE=${1:-python}
shift
+
+ if [ "python3" = "$PACKAGE_TYPE" ]; then
+ # fpm does not actually support this package type. Instead we recognize
+ # it as a convenience shortcut to add several necessary arguments to
+ # fpm's command line later.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
+ --python-package-name-prefix python3
+ fi
# Optional: the package version number. Passed to fpm -v.
VERSION=$1
shift
- if [[ "$PACKAGE_NAME" == "" ]]; then
- PACKAGE_NAME=$PACKAGE
- fi
-
- if [[ "$PACKAGE_TYPE" == "" ]]; then
- PACKAGE_TYPE='python'
- fi
-
declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "deb" "-x" "usr/local/lib/python2.7/dist-packages/tests")
if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
@@ -461,6 +472,10 @@ cd $WORKSPACE/debs
# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
build_and_scp_deb $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+# The Docker image cleaner
+cd $WORKSPACE/debs
+build_and_scp_deb $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+
# A few dependencies
for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py virtualenv \
@@ -468,6 +483,7 @@ for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pycrypto backports.ssl_match_hostname; do
build_and_scp_deb "$deppkg"
done
+build_and_scp_deb docker-py python3-docker-py python3
# cwltool from common-workflow-language. We use this in arv-run-pipeline-instance.
# We use $WORKSPACE/common-workflow-language as the clean directory from which to build the cwltool package
commit 123d3d778051c90b6be39b0f0b25da24d025d382
Author: Brett Smith <brett at curoverse.com>
Date: Tue May 12 14:57:20 2015 -0400
3793: Add services/dockercleaner to run-tests.sh.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 91c7cac..8e58d67 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -56,6 +56,7 @@ apps/workbench_profile
doc
services/api
services/crunchstat
+services/dockercleaner
services/fuse
services/keepproxy
services/keepstore
@@ -79,6 +80,7 @@ unset $(env | cut -d= -f1 | grep \^ARVADOS_)
GITDIR=
GOPATH=
VENVDIR=
+VENV3DIR=
PYTHONPATH=
GEMHOME=
@@ -90,7 +92,7 @@ skip_install=
declare -A leave_temp
clear_temp() {
leaving=""
- for var in VENVDIR GOPATH GITDIR GEMHOME
+ for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
do
if [[ -z "${leave_temp[$var]}" ]]
then
@@ -221,6 +223,7 @@ do
;;
--leave-temp)
leave_temp[VENVDIR]=1
+ leave_temp[VENV3DIR]=1
leave_temp[GOPATH]=1
leave_temp[GEMHOME]=1
;;
@@ -295,7 +298,7 @@ cd "$WORKSPACE"
find -name '*.pyc' -delete
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
do
if [[ -n "${!tmpdir}" ]]; then
leave_temp[$tmpdir]=1
@@ -401,6 +404,23 @@ fi
echo "pip install -q PyYAML"
pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
+ || fatal "python3 virtualenv $VENV3DIR failed"
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
+
checkexit() {
if [[ "$1" != "0" ]]; then
title "!!!!!! $2 FAILED !!!!!!"
@@ -448,8 +468,10 @@ do_test_once() {
fi
elif [[ "$2" == "pip" ]]
then
- cd "$WORKSPACE/$1" \
- && python setup.py test ${testargs[$1]}
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
elif [[ "$2" != "" ]]
then
"test_$2"
@@ -475,6 +497,9 @@ do_install() {
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
# Need to change to a different directory after creating
# the source dist package to avoid a pip bug.
# see https://arvados.org/issues/5766 for details.
@@ -485,10 +510,10 @@ do_install() {
# install" ensures that we've actually install the local package
# we just built.
cd "$WORKSPACE/$1" \
- && python setup.py sdist rotate --keep=1 --match .tar.gz \
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
- && pip install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && pip install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
@@ -559,6 +584,9 @@ for p in "${pythonstuff[@]}"
do
do_install "$p" pip
done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
install_apiserver() {
cd "$WORKSPACE/services/api" \
@@ -677,6 +705,7 @@ for p in "${pythonstuff[@]}"
do
do_test "$p" pip
done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
for g in "${gostuff[@]}"
do
commit 1553f4842dba02aaca9fd7e5a9ac426d7532c462
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Apr 22 11:07:16 2015 -0400
5766: Make sure it doesn't try to fetch arvados packages from PyPi, make sure
we only install the local ones.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index ef2e40d..91c7cac 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -18,6 +18,7 @@ Options:
--skip-install Do not run any install steps. Just run tests.
You should provide GOPATH, GEMHOME, and VENVDIR options
from a previous invocation if you use this option.
+--only-install Run specific install step
WORKSPACE=path Arvados source tree to test.
CONFIGSRC=path Dir with api server config files to copy into source tree.
(If none given, leave config files alone in source tree.)
@@ -214,6 +215,10 @@ do
--skip-install)
skip_install=1
;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
--leave-temp)
leave_temp[VENVDIR]=1
leave_temp[GOPATH]=1
@@ -377,6 +382,10 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
. "$VENVDIR/bin/activate"
+# When re-using $VENVDIR, upgrade any packages (except arvados) that are
+# already installed
+pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=`
+
# Note: this must be the last time we change PATH, otherwise rvm will
# whine a lot.
setup_ruby_environment
@@ -390,7 +399,7 @@ fi
# Needed for run_test_server.py which is used by certain (non-Python) tests.
echo "pip install -q PyYAML"
-pip install -q PyYAML || fatal "pip install PyYAML failed"
+pip install --quiet PyYAML || fatal "pip install PyYAML failed"
checkexit() {
if [[ "$1" != "0" ]]; then
@@ -457,7 +466,7 @@ do_test_once() {
}
do_install() {
- if [[ -z "$skip_install" ]]
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
then
title "Running $1 install"
timer_reset
@@ -466,9 +475,20 @@ do_install() {
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it belives the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually install the local package
+ # we just built.
cd "$WORKSPACE/$1" \
&& python setup.py sdist rotate --keep=1 --match .tar.gz \
- && pip install -q --upgrade dist/*.tar.gz
+ && cd "$WORKSPACE" \
+ && pip install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && pip install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
commit ffa8b415658e6d3ba3836c6e208fe35979de0bf4
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 8 22:09:22 2015 -0400
5416: Support testargs for go bits, like "services/arv-git-httpd_test=-check.vv".
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 6f5dd38..ef2e40d 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -27,6 +27,9 @@ sdk/python_test="--test-suite test.test_keep_locator"
Restrict Python SDK tests to the given class
apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+ Show all log messages, even when tests pass (also works
+ with services/keepstore_test etc.)
ARVADOS_DEBUG=1
Print more debug messages
envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
@@ -424,7 +427,16 @@ do_test_once() {
timer_reset
if [[ "$2" == "go" ]]
then
- go test ${testargs[$1]} "git.curoverse.com/arvados.git/$1"
+ if [[ -n "${testargs[$1]}" ]]
+ then
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && go test ${testargs[$1]}
+ else
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go test "git.curoverse.com/arvados.git/$1"
+ fi
elif [[ "$2" == "pip" ]]
then
cd "$WORKSPACE/$1" \
commit a3465f97627494bde0f148e27559046308cbedda
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 8 10:17:16 2015 -0400
5416: Run keepproxy, arv-git-httpd, and SSL proxy services during Workbench test suites.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 6700712..6f5dd38 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -130,7 +130,7 @@ report_outcomes() {
exit_cleanly() {
trap - INT
rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
- stop_api
+ stop_services
rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
report_outcomes
clear_temp
@@ -243,7 +243,23 @@ start_api() {
&& (env | egrep ^ARVADOS)
}
-stop_api() {
+start_nginx_proxy_services() {
+ echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py start_nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1
+}
+
+stop_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ unset ARVADOS_TEST_PROXY_SERVICES
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy
+ fi
if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
unset ARVADOS_TEST_API_HOST
cd "$WORKSPACE" \
@@ -595,7 +611,7 @@ test_doclinkchecker() {
}
do_test doc doclinkchecker
-stop_api
+stop_services
test_apiserver() {
cd "$WORKSPACE/services/api" \
@@ -636,19 +652,22 @@ do
done
test_workbench() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench workbench
test_workbench_benchmark() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
}
do_test apps/workbench_benchmark workbench_benchmark
test_workbench_profile() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
do_test apps/workbench_profile workbench_profile
commit 5b7e2250d5802b20b73242ae4d0641b32f72cce7
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 2 20:15:58 2015 -0400
5416: Terminate connections on the configured test database (not necessarily arvados_test).
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index f835f95..6700712 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -548,9 +548,11 @@ install_apiserver() {
&& git add tmp \
&& git commit -m 'initial commit'
- # Clear out any lingering postgresql connections to arvados_test, so that we can drop it
- # This assumes the current user is a postgresql superuser
- psql arvados_test -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'arvados_test';" 2>/dev/null
+ # Clear out any lingering postgresql connections to the test
+ # database, so that we can drop it. This assumes the current user
+ # is a postgresql superuser.
+ test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
+ psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
commit 4bfcd9f168a3371b19bbd12429a8da37883256c6
Author: Ward Vandewege <ward at curoverse.com>
Date: Mon Apr 6 09:45:40 2015 -0400
run-tests.sh can now run the arv-git-httpd tests too.
No issue #
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index c81f3bd..f835f95 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -56,6 +56,7 @@ services/fuse
services/keepproxy
services/keepstore
services/nodemanager
+services/arv-git-httpd
sdk/cli
sdk/python
sdk/ruby
commit 422349074a47c6cf6394234a3dba975c84b94b09
Author: Ward Vandewege <ward at curoverse.com>
Date: Fri Apr 3 11:11:26 2015 -0400
Be consistent in our package and binary naming:
arv-git-httpd becomes arvados-git-httpd
No issue #
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 62bd6f9..5ba1eb9 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -426,7 +426,7 @@ fi
go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
cd $WORKSPACE/debs
-build_and_scp_deb $GOPATH/bin/arv-git-httpd=/usr/bin/arv-git-httpd arv-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
+build_and_scp_deb $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
# crunchstat
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
commit 72c3457917a25dede1695c8abda316aa4e5afd5e
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Mar 17 11:01:55 2015 -0400
5416: Add --retry flag: when a test suite fails, prompt to try that suite again.
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 38d8318..c81f3bd 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -215,6 +215,9 @@ do
leave_temp[GOPATH]=1
leave_temp[GEMHOME]=1
;;
+ --retry)
+ retry=1
+ ;;
*_test=*)
suite="${arg%%_test=*}"
args="${arg#*=}"
@@ -370,11 +373,11 @@ echo "pip install -q PyYAML"
pip install -q PyYAML || fatal "pip install PyYAML failed"
checkexit() {
- if [[ "$?" != "0" ]]; then
- title "!!!!!! $1 FAILED !!!!!!"
- failures+=("$1 (`timer`)")
+ if [[ "$1" != "0" ]]; then
+ title "!!!!!! $2 FAILED !!!!!!"
+ failures+=("$2 (`timer`)")
else
- successes+=("$1 (`timer`)")
+ successes+=("$2 (`timer`)")
fi
}
@@ -387,6 +390,17 @@ timer() {
}
do_test() {
+ while ! do_test_once ${@} && [[ "$retry" == 1 ]]
+ do
+ read -p 'Try again? [Y/n] ' x
+ if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
+ then
+ break
+ fi
+ done
+}
+
+do_test_once() {
if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
then
title "Running $1 tests"
@@ -404,8 +418,10 @@ do_test() {
else
"test_$1"
fi
- checkexit "$1 tests"
+ result="$?"
+ checkexit $result "$1 tests"
title "End of $1 tests (`timer`)"
+ return $result
else
title "Skipping $1 tests"
fi
@@ -430,7 +446,7 @@ do_install() {
else
"install_$1"
fi
- checkexit "$1 install"
+ checkexit $? "$1 install"
title "End of $1 install (`timer`)"
else
title "Skipping $1 install"
commit bf95f654a1587dceb6d2d27c03d6da84a5a2a92a
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Mar 17 11:00:45 2015 -0400
5416: Add arv-git-httpd.
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 006e3a4..62bd6f9 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -413,6 +413,21 @@ go get "git.curoverse.com/arvados.git/services/datamanager"
cd $WORKSPACE/debs
build_and_scp_deb $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
+# arv-git-httpd
+cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd"
+ARVGITHTTPD_VERSION=$(version_from_git)
+ARVGITHTTPD_TIMESTAMP=$(timestamp_from_git)
+
+if [[ "$GO_SDK_TIMESTAMP" -gt "$ARVGITHTTPD_TIMESTAMP" ]]; then
+ PKG_VERSION=$GO_SDK_VERSION
+else
+ PKG_VERSION=$ARVGITHTTPD_VERSION
+fi
+
+go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
+cd $WORKSPACE/debs
+build_and_scp_deb $GOPATH/bin/arv-git-httpd=/usr/bin/arv-git-httpd arv-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
+
# crunchstat
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
PKG_VERSION=$(version_from_git)
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 713ffda..38d8318 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -544,6 +544,7 @@ do_install services/api apiserver
declare -a gostuff
gostuff=(
+ services/arv-git-httpd
services/crunchstat
services/keepstore
services/keepproxy
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list