[ARVADOS-DEV] updated: 1c44466e5983383785eaf25f3b5da01936cd2cad

[git at public.curoverse.com]

Summary of changes:
 jenkins/run-deploy.sh | 43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

       via  1c44466e5983383785eaf25f3b5da01936cd2cad (commit)
      from  a013dc7e4f0d91499d7349eee553a771551cd13e (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 1c44466e5983383785eaf25f3b5da01936cd2cad
Author: Ward Vandewege <ward at curoverse.com>
Date:   Wed Oct 22 09:35:49 2014 -0400

    Make run-deploy.sh a bit smarter about which compute nodes are actually
    up.
    
    Note: there are still race conditions to be addressed here. Or maybe
    better, we probably just shouldn't care about compute node puppet runs
    not completing succesfully. Either the node will get shut down by node
    manager, or a regular hourly puppet run will DTRT.
    
    No issue #

diff --git a/jenkins/run-deploy.sh b/jenkins/run-deploy.sh
index a7ffac7..a55678b 100755
--- a/jenkins/run-deploy.sh
+++ b/jenkins/run-deploy.sh
@@ -136,24 +136,35 @@ fi
 
 title "Deploying workbench complete"
 
-# Update compute0
-title "Update compute0"
-
-ssh -p2222 root at compute0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
-
-if [[ "$ECODE" == "2" ]]; then
-  # Puppet exits '2' if there are changes. For real!
-  ECODE=0
-fi
-
-if [[ "$ECODE" != "0" ]]; then
-  title "!!!!!! Update compute0 FAILED !!!!!!"
-  EXITCODE=$(($EXITCODE + $ECODE))
+# Update compute node(s)
+title "Update compute node(s)"
+
+# Get list of nodes that are up
+COMPRESSED_NODE_LIST=`ssh -p2222 root@$IDENTIFIER -C "sinfo --long -p crypto -r -o "%N" -h"`
+
+if [[ "$COMPRESSED_NODE_LIST" != '' ]]; then
+  COMPUTE_NODES=`ssh -p2222 root@$IDENTIFIER -C "scontrol show hostname $COMPRESSED_NODE_LIST"`
+
+  SUM_ECODE=0
+  for node in $COMPUTE_NODES; do
+    echo "Updating $node.$IDENTIFIER"
+    ssh -p2222  -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t"
+    ECODE=$?
+    if [[ "$ECODE" != "255" && "$ECODE" != "2" && "$ECODE" != "0"  ]]; then
+      # 255 -> connection timed out. Just ignore that, it's possible the compute node was being shut down.
+      # Puppet exits '2' if there are changes. For real!
+      SUM_ECODE=$(($SUM_ECODE + $ECODE))
+      echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
+    fi
+  done
+
+  if [[ "$SUM_ECODE" != "0" ]]; then
+    title "!!!!!! Update compute node(s) FAILED !!!!!!"
+    EXITCODE=$(($EXITCODE + $SUM_ECODE))
+  fi
 fi
 
-title "Update compute0 complete"
+title "Update compute node(s) complete"
 
 title "Update shell"
 

-----------------------------------------------------------------------


hooks/post-receive
--