[ARVADOS-DEV] updated: f55df5188d670fa5b27b7f711349661266c1f125
Git user
git at public.curoverse.com
Tue Mar 5 16:11:19 EST 2019
Summary of changes:
jenkins/run_upload_packages_testing.py | 0
jenkins/testing_to_stable_publish_packages.sh | 101 ++++++++++++++++++++++++++
jenkins/upload_packages_testing_to_stable.sh | 60 +++++++++++++++
3 files changed, 161 insertions(+)
mode change 100644 => 100755 jenkins/run_upload_packages_testing.py
create mode 100755 jenkins/testing_to_stable_publish_packages.sh
create mode 100755 jenkins/upload_packages_testing_to_stable.sh
via f55df5188d670fa5b27b7f711349661266c1f125 (commit)
from c7681a6c220cb6c4a41bf3bc33408c8a73242651 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f55df5188d670fa5b27b7f711349661266c1f125
Author: Fernando Monserrat <fernando.monserrat at gmail.com>
Date: Tue Mar 5 18:11:06 2019 -0300
issue #14891 add scripts to publish packages from testing to stable
Arvados-DCO-1.1-Signed-off-by: Fernando Monserrat <fmonserrat at veritasgenetics.com>
diff --git a/jenkins/run_upload_packages_testing.py b/jenkins/run_upload_packages_testing.py
old mode 100644
new mode 100755
diff --git a/jenkins/testing_to_stable_publish_packages.sh b/jenkins/testing_to_stable_publish_packages.sh
new file mode 100755
index 0000000..678256f
--- /dev/null
+++ b/jenkins/testing_to_stable_publish_packages.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# This script publishes packages from our testing repo to the prod repo (#11572 and #11878)
+# Parameters (both are mandatory):
+# --packages list of packages, comma separated, to move from dev_repo_dir to prod_repo_dir
+# --distros list of distros, comma separated, to which to publish packages
+
+APT_REPO_BASE_DIR="/var/lib/freight"
+RPM_REPO_BASE_DIR="/var/www/rpm.arvados.org"
+
+### MAIN ####################################################################
+[ $# -lt 2 ] && {
+ echo "usage: "
+ echo " $0 --distros <distroA,distroB,...,distroN> --packages <packageA,packageB,...,packageN>"
+ echo " (both parameters are mandatory)"
+ echo ""
+ exit 1
+}
+
+# Parse options
+while [ ${#} -gt 0 ]; do
+ case ${1} in
+ --packages) packages="$2"; shift 2 ;;
+ --distros) distros="$2"; shift 2 ;;
+ *) echo "$0: Unrecognized option: $1" >&2; exit 1;
+ esac
+done
+
+# Make sure the variables are set or provide an example of how to use them
+if [ -z "${packages}" ]; then
+ echo "You must provide a comma-separated list of packages to publish, ie."
+ echo "* Debian: --packages=arvados-ws_0.1.20170906144951.22418ed6e-1_amd64.deb,keep-exercise_*,*170906144951*"
+ echo "* Centos: --packages=arvados-ws_0.1.20170906144951.22418ed6e-1.x86_64.rpm,keep-exercise_0.1.20170906144951.22418ed6e-1.x86_64.rpm,crunch-dispatch-local_0.1.20170906144951.22418ed6e-1.x86_64.rpm"
+ exit 254
+fi
+if [ -z "${distros}" ]; then
+ echo "You must provide a space-separated list of LSB distribution codenames to which you want to publish to, ie."
+ echo "* Debian: --distros=jessie,xenial,stretch,etc."
+ echo "* Centos: --distros=centos7,etc."
+ exit 255
+fi
+
+DIST_LIST=$(echo ${distros} | sed s/,/' '/g |tr '[:upper:]' '[:lower:]')
+PACKAGES=$(echo ${packages} | sed s/,/' '/g)
+
+if ( echo ${DIST_LIST} |grep -q centos ); then
+ for DISTNAME in ${DIST_LIST}; do
+ case ${DISTNAME} in
+ 'centos7')
+ DIST_DIR_TEST='7/testing/x86_64'
+ DIST_DIR_PROD='7/os/x86_64'
+ ;;
+ *)
+ echo "Only centos7 is accepted right now"
+ exit 253
+ ;;
+ esac
+ cd ${RPM_REPO_BASE_DIR}
+ mkdir -p ${RPM_REPO_BASE_DIR}/CentOS/${DIST_DIR_PROD}
+ echo "Copying packages ..."
+ for P in ${PACKAGES}; do
+ cp ${RPM_REPO_BASE_DIR}/CentOS/${DIST_DIR_TEST}/${P} ${RPM_REPO_BASE_DIR}/CentOS/${DIST_DIR_PROD}/
+ if [ $? -ne 0 ]; then
+ FAILED_PACKAGES="${FAILED_PACKAGES} ${P}"
+ fi
+ done
+ echo "Recreating repo CentOS/${DIST_DIR_PROD} ..."
+ createrepo ${RPM_REPO_BASE_DIR}/CentOS/${DIST_DIR_PROD}
+ done
+else
+ for DISTNAME in ${DIST_LIST}; do
+ ADDED=()
+ cd ${APT_REPO_BASE_DIR}
+ mkdir -p ${APT_REPO_BASE_DIR}/apt/${DISTNAME}
+ echo "Copying packages ..."
+ for P in ${PACKAGES}; do
+ cp ${APT_REPO_BASE_DIR}/apt/${DISTNAME}-testing/${P} ${APT_REPO_BASE_DIR}/apt/${DISTNAME}/
+ if [ $? -ne 0 ]; then
+ FAILED_PACKAGES="${FAILED_PACKAGES} ${P}"
+ else
+ TMP=`ls -C1 ${APT_REPO_BASE_DIR}/apt/${DISTNAME}-testing/${P}`
+ ADDED+=( "${TMP[@]}" )
+ fi
+ done
+ for P in "${ADDED[@]}"; do
+ freight add apt/${DISTNAME}/$(basename ${P}) apt/${DISTNAME}
+ done
+ echo "Recreating repo apt/${DISTNAME} ..."
+ freight cache apt/${DISTNAME}
+ done
+fi
+
+if [ "${FAILED_PACKAGES}" != "" ]; then
+ echo "PACKAGES THAT FAILED TO PUBLISH"
+ echo "${FAILED_PACKAGES}"
+else
+ echo "All packages published correctly"
+fi
diff --git a/jenkins/upload_packages_testing_to_stable.sh b/jenkins/upload_packages_testing_to_stable.sh
new file mode 100755
index 0000000..0e38606
--- /dev/null
+++ b/jenkins/upload_packages_testing_to_stable.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# This script publishes packages from our dev repo to the prod repo (#11572)
+# Parameters: list of packages, space separated, to move from *-testing to *
+# under /var/lib/freight/apt/ in host public.curoverse.com
+
+APT_REPO_SERVER="apt.arvados.org"
+RPM_REPO_SERVER="rpm.arvados.org"
+
+DEBUG=1
+SSH_PORT=2222
+ECODE=0
+
+# Convert package list into a regex for publising
+# Make sure the variables are set or provide an example of how to use them
+if [ -z "${PACKAGES_TO_PUBLISH}" ]; then
+ echo "You must provide a list of packages to publish, as obtained with https://dev.arvados.org/projects/ops/wiki/Updating_clusters#Gathering-package-versions."
+ exit 254
+fi
+if [ -z "${LSB_DISTRIB_CODENAMES}" ]; then
+ echo "You must provide a space-separated list of LSB distribution codenames to which you want to publish to, ie."
+ echo "* Debian: jessie, xenial, stretch, etc."
+ echo "* Centos: centos7 (the only one currently supported.)"
+ exit 255
+fi
+
+# Sanitize the vars in a way suitable to be used by the remote 'publish_packages.sh' script
+# Just to make copying a single line, and not having to loop over it
+PACKAGES_LIST=$(echo ${PACKAGES_TO_PUBLISH} | sed 's/[a-z-]*-gem: [0-9\.]*//g; s/versions://g; s/\([0-9]\)[$, ]/\1* /g; s/[[:blank:]]\+/,/g; s/^,//g; s/:,/*/g')
+
+DISTROS=$(echo "${LSB_DISTRIB_CODENAMES}"|sed s/[[:space:]]/,/g |tr '[:upper:]' '[:lower:]')
+
+if ( echo ${LSB_DISTRIB_CODENAMES} |grep -q centos ); then
+ REPO_SERVER=${RPM_REPO_SERVER}
+else
+ REPO_SERVER=${APT_REPO_SERVER}
+fi
+
+REMOTE_CMD="/usr/local/bin/testing_to_stable_publish_packages.sh --distros ${DISTROS} --packages ${PACKAGES_LIST}"
+
+# Now we execute it remotely
+TMP_FILE=`mktemp`
+
+ssh -t \
+ -l jenkinsapt \
+ -p $SSH_PORT \
+ -o "StrictHostKeyChecking no" \
+ -o "ConnectTimeout 5" \
+ ${REPO_SERVER} \
+ "${REMOTE_CMD}" | tee ${TMP_FILE}
+
+grep -q "FAILED TO PUBLISH" ${TMP_FILE}
+if [ $? -eq 0 ]; then
+ ECODE=1
+fi
+rm -f ${TMP_FILE}
+exit ${ECODE}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list