[ARVADOS-DEV] created: e4376aca8fd1e81a03b8534cab6cbd07220c45b9

Git user git at public.arvados.org
Tue Oct 12 18:05:18 UTC 2021


        at  e4376aca8fd1e81a03b8534cab6cbd07220c45b9 (commit)


commit e4376aca8fd1e81a03b8534cab6cbd07220c45b9
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Oct 12 14:03:09 2021 -0400

    18264: clean up the way we run the CWL tests. Only run the upstream
    suite from a new run-cwl-test-suite.sh script (the arvados CWL tests are
    run separately now, directly from CI). Remove old run-cwl-test.sh script.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/jenkins/run-cwl-test-suite.sh b/jenkins/run-cwl-test-suite.sh
new file mode 100755
index 0000000..cad0c34
--- /dev/null
+++ b/jenkins/run-cwl-test-suite.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -o pipefail
+set -e
+
+DEBUG=0
+JOBS=1
+
+function usage {
+    echo >&2
+    echo >&2 "usage: $0 [options] <identifier>"
+    echo >&2
+    echo >&2 "   <identifier>                 Arvados cluster name"
+    echo >&2
+    echo >&2 "$0 options:"
+    echo >&2 "  -d, --debug                   Enable debug output"
+    echo >&2 "  -h, --help                    Display this help and exit"
+    echo >&2 "  -s, --scopes                  Print required scopes to run tests"
+    echo >&2 "  -j, --jobs <jobs>             Allow N jobs at once; 1 job with no arg."
+    echo >&2
+}
+
+function print_scopes {
+    echo >&2 " Required scope for the token used to run the tests:"
+    echo >&2
+    echo >&2 " arv api_client_authorization create_system_auth     --scopes "
+    echo >&2 "[\"GET /arvados/v1/virtual_machines\","
+    echo >&2 "\"GET /arvados/v1/keep_services\","
+    echo >&2 "\"GET /arvados/v1/keep_services/\","
+    echo >&2 "\"GET /arvados/v1/groups\","
+    echo >&2 "\"GET /arvados/v1/groups/\","
+    echo >&2 "\"GET /arvados/v1/links\","
+    echo >&2 "\"GET /arvados/v1/collections\","
+    echo >&2 "\"POST /arvados/v1/collections\","
+    echo >&2 "\"POST /arvados/v1/links\","
+    echo >&2 "\"GET /arvados/v1/users/current\","
+    echo >&2 "\"POST /arvados/v1/users/current\","
+    echo >&2 "\"GET /arvados/v1/jobs\","
+    echo >&2 "\"POST /arvados/v1/jobs\","
+    echo >&2 "\"GET /arvados/v1/pipeline_instances\","
+    echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
+    echo >&2 "\"POST /arvados/v1/pipeline_instances\","
+    echo >&2 "\"GET /arvados/v1/collections/\","
+    echo >&2 "\"POST /arvados/v1/collections/\","
+    echo >&2 "\"GET /arvados/v1/container_requests\","
+    echo >&2 "\"GET /arvados/v1/container_requests/\","
+    echo >&2 "\"POST /arvados/v1/container_requests\","
+    echo >&2 "\"POST /arvados/v1/container_requests/\","
+    echo >&2 "\"GET /arvados/v1/containers\","
+    echo >&2 "\"GET /arvados/v1/containers/\","
+    echo >&2 "\"GET /arvados/v1/repositories\","
+    echo >&2 "\"GET /arvados/v1/repositories/\","
+    echo >&2 "\"GET /arvados/v1/logs\" ]"
+    echo >&2
+}
+
+# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+TEMP=`getopt -o hdlp:sj: \
+    --long help,scopes,debug,jobs: \
+    -n "$0" -- "$@"`
+
+if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while [ $# -ge 1 ]
+do
+    case $1 in
+        -d | --debug)
+            DEBUG=1
+            shift
+            ;;
+        -s | --scopes)
+            print_scopes
+            exit 0
+            ;;
+        -j | --jobs)
+            JOBS="$2"; shift 2
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+
+title () {
+  date=`date +'%Y-%m-%d %H:%M:%S'`
+  printf "%s\n" "$date $1"
+}
+
+title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
+if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
+  title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
+  exit 1
+fi
+
+if [[ ! -e common-workflow-language ]]; then
+  git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git
+fi
+
+printf "%s\n%s\n" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 "$@"' > arvados-cwl-runner-with-checksum.sh
+chmod 755 arvados-cwl-runner-with-checksum.sh
+
+cd common-workflow-language
+git pull
+./run_test.sh -j$JOBS --timeout=900 RUNNER=../arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint
diff --git a/jenkins/run-cwl-test.sh b/jenkins/run-cwl-test.sh
deleted file mode 100755
index 742ddcb..0000000
--- a/jenkins/run-cwl-test.sh
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -o pipefail
-# extglob needs to be enabled for the suppression of tokens in the output
-shopt -s extglob
-
-DEBUG=0
-SSH_PORT=22
-JOBS=1
-ACCT=
-LOCAL=0
-
-function usage {
-    echo >&2
-    echo >&2 "usage: $0 [options] <identifier>"
-    echo >&2
-    echo >&2 "   <identifier>                 Arvados cluster name"
-    echo >&2
-    echo >&2 "$0 options:"
-    echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
-    echo >&2 "      --acct <username>         Account to log in with"
-    echo >&2 "  -d, --debug                   Enable debug output"
-    echo >&2 "  -h, --help                    Display this help and exit"
-    echo >&2 "  -s, --scopes                  Print required scopes to run tests"
-    echo >&2 "  -j, --jobs <jobs>             Allow N jobs at once; 1 job with no arg."
-    echo >&2 "  -l, --local                   Run arvados-cwl-runner locally, not on shell.<identifier>"
-    echo >&2
-}
-
-function print_scopes {
-    echo >&2 " Required scope for the token used to run the tests:"
-    echo >&2
-    echo >&2 " arv api_client_authorization create_system_auth     --scopes "
-    echo >&2 "[\"GET /arvados/v1/virtual_machines\","
-    echo >&2 "\"GET /arvados/v1/keep_services\","
-    echo >&2 "\"GET /arvados/v1/keep_services/\","
-    echo >&2 "\"GET /arvados/v1/groups\","
-    echo >&2 "\"GET /arvados/v1/groups/\","
-    echo >&2 "\"GET /arvados/v1/links\","
-    echo >&2 "\"GET /arvados/v1/collections\","
-    echo >&2 "\"POST /arvados/v1/collections\","
-    echo >&2 "\"POST /arvados/v1/links\","
-    echo >&2 "\"GET /arvados/v1/users/current\","
-    echo >&2 "\"POST /arvados/v1/users/current\","
-    echo >&2 "\"GET /arvados/v1/jobs\","
-    echo >&2 "\"POST /arvados/v1/jobs\","
-    echo >&2 "\"GET /arvados/v1/pipeline_instances\","
-    echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
-    echo >&2 "\"POST /arvados/v1/pipeline_instances\","
-    echo >&2 "\"GET /arvados/v1/collections/\","
-    echo >&2 "\"POST /arvados/v1/collections/\","
-    echo >&2 "\"GET /arvados/v1/container_requests\","
-    echo >&2 "\"GET /arvados/v1/container_requests/\","
-    echo >&2 "\"POST /arvados/v1/container_requests\","
-    echo >&2 "\"POST /arvados/v1/container_requests/\","
-    echo >&2 "\"GET /arvados/v1/containers\","
-    echo >&2 "\"GET /arvados/v1/containers/\","
-    echo >&2 "\"GET /arvados/v1/repositories\","
-    echo >&2 "\"GET /arvados/v1/repositories/\","
-    echo >&2 "\"GET /arvados/v1/logs\" ]"
-    echo >&2
-}
-
-# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hdlp:sj: \
-    --long help,scopes,debug,local,port:,acct:,jobs: \
-    -n "$0" -- "$@"`
-
-if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
-# Note the quotes around `$TEMP': they are essential!
-eval set -- "$TEMP"
-
-while [ $# -ge 1 ]
-do
-    case $1 in
-        -p | --port)
-            SSH_PORT="$2"; shift 2
-            ;;
-        --acct)
-            ACCT="$2"; shift 2
-            ;;
-        -d | --debug)
-            DEBUG=1
-            shift
-            ;;
-        -s | --scopes)
-            print_scopes
-            exit 0
-            ;;
-        -j | --jobs)
-            JOBS="$2"; shift 2
-            ;;
-        -l | --local)
-            LOCAL=1
-            shift
-            ;;
-        --)
-            shift
-            break
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-done
-
-IDENTIFIER=$1
-
-if [[ "$IDENTIFIER" == '' ]]; then
-  usage
-  exit 1
-fi
-
-EXITCODE=0
-
-COLUMNS=80
-
-PUPPET_AGENT='
-now() { date +%s; }
-let endtime="$(now) + 600"
-while [ "$endtime" -gt "$(now)" ]; do
-    puppet agent --test --detailed-exitcodes
-    agent_exitcode=$?
-    if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
-        break
-    else
-        sleep 10s
-    fi
-done
-exit ${agent_exitcode:-99}
-'
-
-title () {
-  date=`date +'%Y-%m-%d %H:%M:%S'`
-  printf "%s\n" "$date $1"
-}
-
-function run_command() {
-  node=$1
-  return_var=$2
-  command=$3
-
-  if [[ "$LOCAL" == "0" ]]; then
-    title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
-    TMP_FILE=`mktemp`
-    if [[ "$DEBUG" != "0" ]]; then
-      echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
-      ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
-      ECODE=$?
-    else
-      ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
-      ECODE=$?
-    fi
-
-    if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
-      # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
-      #   a shell node that is down.
-      title "ERROR running command on $node: exit code $ECODE"
-      if [[ "$DEBUG" == "0" ]]; then
-        title "Command output follows:"
-        cat $TMP_FILE
-      fi
-    fi
-    if [[ "$ECODE" == "255" ]]; then
-      title "Connection denied or timed out"
-    fi
-  else
-    title "Running '${command/ARVADOS_API_TOKEN=*([-a-z0-9\/])/ARVADOS_API_TOKEN=suppressed}' locally"
-    TMP_FILE=`mktemp`
-    if [[ "$DEBUG" != "0" ]]; then
-      bash -c "$command" | tee $TMP_FILE
-      ECODE=$?
-    else
-      bash -c "$command" > $TMP_FILE 2>&1
-      ECODE=$?
-    fi
-
-    if [[ "$ECODE" != "0"  ]]; then
-      title "ERROR running command locally: exit code $ECODE"
-      if [[ "$DEBUG" == "0" ]]; then
-        title "Command output follows:"
-        cat $TMP_FILE
-      fi
-    fi
-  fi
-  rm -f $TMP_FILE
-  eval "$return_var=$ECODE"
-}
-
-if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
-  ACCT=$USER
-fi
-
-if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
-  ACCT=ci
-fi
-
-title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
-if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
-  . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
-else
-  title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
-fi
-if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
-  title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
-  exit 1
-fi
-
-run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
-
-if [[ "$ECODE" != "0" ]]; then
-  echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
-  exit $ECODE
-fi
-
-run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
-
-if [[ "$ECODE" != "0" ]]; then
-  echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
-  exit $ECODE
-fi
-
-run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint"
-
-if [[ "$ECODE" != "0" ]]; then
-  echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
-  exit $ECODE
-fi
-
-run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi"
-
-if [[ "$ECODE" != "0" ]]; then
-  echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git"
-  exit $ECODE
-fi
-
-run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
-
-exit $ECODE

commit 6151fc5df0845ba51e94e5ea22bceb31c947cdfc
Author: Ward Vandewege <ward at curii.com>
Date:   Wed Oct 6 19:17:02 2021 -0400

    Fix output when a token is suppressed.
    
    No issue #
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/jenkins/run-cwl-test.sh b/jenkins/run-cwl-test.sh
index cf279f5..742ddcb 100755
--- a/jenkins/run-cwl-test.sh
+++ b/jenkins/run-cwl-test.sh
@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 set -o pipefail
+# extglob needs to be enabled for the suppression of tokens in the output
+shopt -s extglob
 
 DEBUG=0
 SSH_PORT=22
@@ -168,7 +170,7 @@ function run_command() {
       title "Connection denied or timed out"
     fi
   else
-    title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' locally"
+    title "Running '${command/ARVADOS_API_TOKEN=*([-a-z0-9\/])/ARVADOS_API_TOKEN=suppressed}' locally"
     TMP_FILE=`mktemp`
     if [[ "$DEBUG" != "0" ]]; then
       bash -c "$command" | tee $TMP_FILE

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list