[ARVADOS-DEV] updated: b5c641cae15249d888c7bb41c937338af8f7102d

[git at public.curoverse.com]

Summary of changes:
 jenkins/run-deploy.sh | 72 +++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 53 insertions(+), 19 deletions(-)

       via  b5c641cae15249d888c7bb41c937338af8f7102d (commit)
      from  27d5f6ed785b6e127cf9a2f1145fac21627a9a89 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b5c641cae15249d888c7bb41c937338af8f7102d
Author: Ward Vandewege <ward at curoverse.com>
Date:   Wed Nov 19 08:57:18 2014 -0500

    Refactor deploy script a bit; make it less sensitive to transient errors
    we don't care about.
    
    No issue #

diff --git a/jenkins/run-deploy.sh b/jenkins/run-deploy.sh
index 599dff3..d4e31f1 100755
--- a/jenkins/run-deploy.sh
+++ b/jenkins/run-deploy.sh
@@ -1,15 +1,37 @@
 #!/bin/bash
 
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Deploy Arvados to a cluster
+
+Syntax:
+        WORKSPACE=/path/to/arvados $(basename $0) <identifier> <deploy_repo_name>
+
+Options:
+
+identifier             Arvados cluster name
+deploy_repo_name       Name for the repository with the (capistrano) deploy scripts
+
+WORKSPACE=path         Path to the Arvados source tree to deploy from
+
+EOF
+
+
 IDENTIFIER=$1
 DEPLOY_REPO=$2
 
-if [[ "$IDENTIFIER" == '' ]]; then
-  echo "Syntax: $0 <identifier> <deploy_repo_name>"
+if [[ "$IDENTIFIER" == '' || "$DEPLOY_REPO" == '' ]]; then
+  echo >&2 "$helpmessage"
+  echo >&2
   exit 1
 fi
 
-if [[ "$DEPLOY_REPO" == '' ]]; then
-  echo "Syntax: $0 <identifier> <deploy_repo_name>"
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+  echo >&2 "$helpmessage"
+  echo >&2
+  echo >&2 "Error: WORKSPACE environment variable not set"
+  echo >&2
   exit 1
 fi
 
@@ -33,6 +55,29 @@ rvm use default
 # Just say what version of ruby we're running
 ruby --version
 
+function run_puppet() {
+  node=$1
+  return_var=$2
+
+  TMP_FILE=`mktemp`
+  ssh -t -p2222 -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t" | tee $TMP_FILE
+
+  ECODE=$?
+  RESULT=$(cat $TMP_FILE)
+
+  if [[ "$ECODE" != "255" && ! ("$RESULT" =~ 'already in progress') && "$ECODE" != "2" && "$ECODE" != "0"  ]]; then
+    # Puppet exists 255 if the connection timed out. Just ignore that, it's possible that this node is
+    #   a compute node that was being shut down.
+    # Puppet exits 2 if there are changes. For real!
+    # Puppet prints 'Notice: Run of Puppet configuration client already in progress' if another puppet process
+    #   was already running
+    echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
+  fi
+  rm -f $TMP_FILE
+  echo
+  eval "$return_var=$ECODE"
+}
+
 function ensure_symlink() {
   if [[ ! -L $WORKSPACE/$1 ]]; then
     ln -s $WORKSPACE/$DEPLOY_REPO/$1 $WORKSPACE/$1
@@ -148,15 +193,8 @@ if [[ "$COMPRESSED_NODE_LIST" != '' ]]; then
   SUM_ECODE=0
   for node in $COMPUTE_NODES; do
     echo "Updating $node.$IDENTIFIER"
-    RESULT=`ssh -p2222  -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t"`
-    ECODE=$?
-    if [[ "$ECODE" != "255" && ! ("$RESULT" =~ 'already in progress') && "$ECODE" != "2" && "$ECODE" != "0"  ]]; then
-      # 255 -> connection timed out. Just ignore that, it's possible the compute node was being shut down.
-      # Puppet exits '2' if there are changes. For real!
-      # Puppet prints 'Notice: Run of Puppet configuration client already in progress' if another puppet process was already running
-      SUM_ECODE=$(($SUM_ECODE + $ECODE))
-      echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
-    fi
+    run_puppet $node ECODE
+    SUM_ECODE=$(($SUM_ECODE + $ECODE))
   done
 
   if [[ "$SUM_ECODE" != "0" ]]; then
@@ -169,9 +207,7 @@ title "Update compute node(s) complete"
 
 title "Update shell"
 
-ssh -p2222 root at shell.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
+run_puppet shell ECODE
 
 if [[ "$ECODE" == "2" ]]; then
   # Puppet exits '2' if there are changes. For real!
@@ -187,9 +223,7 @@ title "Update shell complete"
 
 title "Update keep0"
 
-ssh -p2222 root at keep0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
+run_puppet keep0 ECODE
 
 if [[ "$ECODE" == "2" ]]; then
   # Puppet exits '2' if there are changes. For real!

-----------------------------------------------------------------------


hooks/post-receive
--