[ARVADOS-DEV] updated: 575ffaebb8e92cede788640f6616b0b3f97bb103
Git user
git at public.curoverse.com
Fri Jun 15 15:23:19 EDT 2018
Summary of changes:
jenkins/run-cwl-test.sh | 86 ++++++++++++++++++++++++++++++++++---------------
1 file changed, 60 insertions(+), 26 deletions(-)
via 575ffaebb8e92cede788640f6616b0b3f97bb103 (commit)
from 99fb28d9d051baee0f0cb0588913676535082914 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 575ffaebb8e92cede788640f6616b0b3f97bb103
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Fri Jun 15 15:22:11 2018 -0400
* Add a 'local' mode to jenkins/run-cwl-test.sh, where arvados-cwl-runner runs
locally
* Make fewer assumptions about the remote Arvados cluster.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/jenkins/run-cwl-test.sh b/jenkins/run-cwl-test.sh
index 4ad9ebd..3bd7f8b 100755
--- a/jenkins/run-cwl-test.sh
+++ b/jenkins/run-cwl-test.sh
@@ -9,7 +9,8 @@ set -o pipefail
DEBUG=0
SSH_PORT=22
JOBS=1
-ACCT=ci
+ACCT=
+LOCAL=0
function usage {
echo >&2
@@ -24,6 +25,7 @@ function usage {
echo >&2 " -h, --help Display this help and exit"
echo >&2 " -s, --scopes Print required scopes to run tests"
echo >&2 " -j, --jobs <jobs> Allow N jobs at once; 1 job with no arg."
+ echo >&2 " -l, --local Run arvados-cwl-runner locally, not on shell.<identifier>"
echo >&2
}
@@ -62,8 +64,8 @@ function print_scopes {
}
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hdp:s \
- --long help,scopes,debug,port:,acct:,jobs: \
+TEMP=`getopt -o hdlp:sj: \
+ --long help,scopes,debug,local,port:,acct:,jobs: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -90,6 +92,10 @@ do
-j | --jobs)
JOBS="$2"; shift 2
;;
+ -l | --local)
+ LOCAL=1
+ shift
+ ;;
--)
shift
break
@@ -137,32 +143,60 @@ function run_command() {
return_var=$2
command=$3
- title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' on $node"
- TMP_FILE=`mktemp`
- if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
- ECODE=$?
+ if [[ "$LOCAL" == "0" ]]; then
+ title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
+ TMP_FILE=`mktemp`
+ if [[ "$DEBUG" != "0" ]]; then
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
+ ECODE=$?
+ else
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
+ ECODE=$?
+ fi
+
+ if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
+ # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
+ # a shell node that is down.
+ title "ERROR running command on $node: exit code $ECODE"
+ if [[ "$DEBUG" == "0" ]]; then
+ title "Command output follows:"
+ cat $TMP_FILE
+ fi
+ fi
+ if [[ "$ECODE" == "255" ]]; then
+ title "Connection denied or timed out"
+ fi
else
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
- ECODE=$?
- fi
+ title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' locally"
+ TMP_FILE=`mktemp`
+ if [[ "$DEBUG" != "0" ]]; then
+ bash -c "$command" | tee $TMP_FILE
+ ECODE=$?
+ else
+ bash -c "$command" > $TMP_FILE 2>&1
+ ECODE=$?
+ fi
- if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
- # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
- # a shell node that is down.
- title "ERROR running command on $node: exit code $ECODE"
- if [[ "$DEBUG" == "0" ]]; then
- title "Command output follows:"
- cat $TMP_FILE
+ if [[ "$ECODE" != "0" ]]; then
+ title "ERROR running command locally: exit code $ECODE"
+ if [[ "$DEBUG" == "0" ]]; then
+ title "Command output follows:"
+ cat $TMP_FILE
+ fi
fi
fi
- if [[ "$ECODE" == "255" ]]; then
- title "Connection denied or timed out"
- fi
rm -f $TMP_FILE
eval "$return_var=$ECODE"
}
+if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
+ ACCT=$USER
+fi
+
+if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
+ ACCT=ci
+fi
+
title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
. $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
@@ -181,27 +215,27 @@ if [[ "$ECODE" != "0" ]]; then
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum --disable-reuse \"\$@\"' > arvados-cwl-runner-with-checksum.sh; chmod 755 arvados-cwl-runner-with-checksum.sh"
+run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum --disable-reuse \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
if [[ "$ECODE" != "0" ]]; then
echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
+run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
if [[ "$ECODE" != "0" ]]; then
echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN git clone --depth 1 https://git.$IDENTIFIER.arvadosapi.com/arvados.git; fi"
+run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/curoverse/arvados.git; fi"
if [[ "$ECODE" != "0" ]]; then
- echo "Failed to git clone --depth 1 git at git.$IDENTIFIER.arvadosapi.com:arvados.git"
+ echo "Failed to git clone --depth 1 https://github.com/curoverse/arvados.git"
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN && git pull && ./arvados-tests.sh"
+run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
exit $ECODE
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list