[ARVADOS-DEV] created: 6c26aa2c194f60b2823da49166741203524a0b48
git at public.curoverse.com
git at public.curoverse.com
Wed Jun 3 16:28:28 EDT 2015
at 6c26aa2c194f60b2823da49166741203524a0b48 (commit)
commit 6c26aa2c194f60b2823da49166741203524a0b48
Author: Ward Vandewege <ward at curoverse.com>
Date: Wed Jun 3 16:27:48 2015 -0400
Build API server and Workbench debian packages.
Refs #6096
diff --git a/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh b/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh
new file mode 100755
index 0000000..cf32fdb
--- /dev/null
+++ b/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+RELEASE_PATH=/var/www/arvados-api/current
+SHARED_PATH=/var/www/arvados-api/shared
+CONFIG_PATH=/etc/arvados/api/
+
+echo "Assumption: nginx is configured to serve `hostname` from /var/www/`hostname`/current"
+echo "Assumption: /var/www/`hostname` is symlinked to /var/www/arvados-api"
+echo "Assumption: configuration files are in /etc/arvados/api/"
+echo "Assumption: nginx and passenger run as the www-data user"
+echo
+
+echo "Copying files from $CONFIG_PATH"
+cp -f $CONFIG_PATH/database.yml $RELEASE_PATH/config/database.yml
+cp -f $RELEASE_PATH/config/environments/production.rb.example $RELEASE_PATH/config/environments/production.rb
+cp -f $CONFIG_PATH/application.yml $RELEASE_PATH/config/application.yml
+cp -f $CONFIG_PATH/omniauth.rb $RELEASE_PATH/config/initializers/omniauth.rb
+echo "Done."
+
+# Before we do anything else, make sure some directories and files are in place
+if [[ ! -e $SHARED_PATH/log ]]; then mkdir -p $SHARED_PATH/log; fi
+if [[ ! -e $RELEASE_PATH/tmp ]]; then mkdir -p $RELEASE_PATH/tmp; fi
+if [[ ! -e $RELEASE_PATH/log ]]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi
+if [[ ! -e $SHARED_PATH/log/production.log ]]; then touch $SHARED_PATH/log/production.log; fi
+
+echo "Running bundle install"
+(cd $RELEASE_PATH && RAILS_ENV=production bundle install --path $SHARED_PATH/vendor_bundle)
+echo "Done."
+
+echo "Precompiling assets"
+# precompile assets; thankfully this does not take long
+(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake assets:precompile)
+echo "Done."
+
+echo "Ensuring directory and file permissions"
+# Ensure correct ownership of a few files
+chown www-data:www-data $RELEASE_PATH/config/environment.rb
+chown www-data:www-data $RELEASE_PATH/config.ru
+chown www-data:www-data $RELEASE_PATH/config/database.yml
+chown www-data:www-data $RELEASE_PATH/Gemfile.lock
+chown -R www-data:www-data $RELEASE_PATH/tmp
+chown -R www-data:www-data $SHARED_PATH/log
+chown www-data:www-data $RELEASE_PATH/db/structure.sql
+chmod 644 $SHARED_PATH/log/*
+echo "Done."
+
+echo "Running sanity check"
+(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake config:check)
+echo "Done."
+
+SANITY_CHECK_EXIT_CODE=$?
+
+if [[ "$SANITY_CHECK_EXIT_CODE" != "0" ]]; then
+ echo "Sanity check failed, aborting. Please roll back to the previous version of the package."
+ echo "The database has not been migrated yet, so reinstalling the previous version is safe."
+ exit $SANITY_CHECK_EXIT_CODE
+fi
+
+echo "Starting db:migrate"
+(cd $RELEASE_PATH && bundle exec rake RAILS_ENV=production db:migrate)
+echo "Done."
+
+echo "Restarting nginx"
+service nginx restart
+echo "Done."
diff --git a/jenkins/arvados-api-server-extras/postinst.sh b/jenkins/arvados-api-server-extras/postinst.sh
new file mode 100644
index 0000000..1aeffe9
--- /dev/null
+++ b/jenkins/arvados-api-server-extras/postinst.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+cd /var/www/arvados-api
+
+chown -R www-data:www-data tmp >/dev/null 2>&1
+chown -R www-data:www-data log >/dev/null 2>&1
+chown www-data:www-data db/structure.sql >/dev/null 2>&1
+chmod 644 log/* >/dev/null 2>&1
+
+# Errors above are not serious
+exit 0
+
diff --git a/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh b/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh
new file mode 100755
index 0000000..33e05d3
--- /dev/null
+++ b/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+RELEASE_PATH=/var/www/arvados-workbench/current
+SHARED_PATH=/var/www/arvados-workbench/shared
+CONFIG_PATH=/etc/arvados/workbench/
+
+echo "Assumption: nginx is configured to serve workbench.`hostname` from /var/www/workbench.`hostname`/current"
+echo "Assumption: /var/www/`hostname` is symlinked to /var/www/arvados-workbench"
+echo "Assumption: configuration files are in /etc/arvados/workbench/"
+echo "Assumption: nginx and passenger run as the www-data user"
+echo
+
+echo "Copying files from $CONFIG_PATH"
+cp -f $CONFIG_PATH/application.yml $RELEASE_PATH/config/application.yml
+cp -f $RELEASE_PATH/config/environments/production.rb.example $RELEASE_PATH/config/environments/production.rb
+echo "Done."
+
+# Before we do anything else, make sure some directories and files are in place
+if [[ ! -e $SHARED_PATH/log ]]; then mkdir -p $SHARED_PATH/log; fi
+if [[ ! -e $RELEASE_PATH/tmp ]]; then mkdir -p $RELEASE_PATH/tmp; fi
+if [[ ! -e $RELEASE_PATH/log ]]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi
+if [[ ! -e $SHARED_PATH/log/production.log ]]; then touch $SHARED_PATH/log/production.log; fi
+
+echo "Running bundle install"
+(cd $RELEASE_PATH && RAILS_ENV=production bundle install --path $SHARED_PATH/vendor_bundle)
+echo "Done."
+
+# We do not need to precompile assets, they are already part of the package.
+
+echo "Ensuring directory and file permissions"
+chown www-data:www-data $RELEASE_PATH/config/environment.rb
+chown www-data:www-data $RELEASE_PATH/config.ru
+chown www-data:www-data $RELEASE_PATH/config/database.yml
+chown www-data:www-data $RELEASE_PATH/Gemfile.lock
+chown -R www-data:www-data $RELEASE_PATH/tmp
+chown -R www-data:www-data $SHARED_PATH/log
+chown www-data:www-data $RELEASE_PATH/db/schema.rb
+chmod 644 $SHARED_PATH/log/*
+echo "Done."
+
+echo "Running sanity check"
+(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake config:check)
+echo "Done."
+
+SANITY_CHECK_EXIT_CODE=$?
+
+if [[ "$SANITY_CHECK_EXIT_CODE" != "0" ]]; then
+ echo "Sanity check failed, aborting. Please roll back to the previous version of the package."
+ exit $SANITY_CHECK_EXIT_CODE
+fi
+
+# We do not need to run db:migrate because Workbench is stateless
+
+echo "Restarting nginx"
+service nginx restart
+echo "Done."
+
diff --git a/jenkins/arvados-workbench-extras/postinst.sh b/jenkins/arvados-workbench-extras/postinst.sh
new file mode 100644
index 0000000..d0aa9b0
--- /dev/null
+++ b/jenkins/arvados-workbench-extras/postinst.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+cd /var/www/arvados-workbench
+
+chown -R www-data:www-data tmp >/dev/null 2>&1
+chown -R www-data:www-data log >/dev/null 2>&1
+chown www-data:www-data db/schema.rb >/dev/null 2>&1
+chmod 644 log/* >/dev/null 2>&1
+
+# Errors above are not serious
+exit 0
+
diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 691e93b..c5ae462 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -9,10 +9,16 @@ Syntax:
Options:
---upload Upload packages (default: false)
---scp-user USERNAME Scp user for apt server (only required when --upload is specified)
---apt-server HOSTNAME Apt server hostname (only required when --upload is specified)
---debug Output debug information (default: false)
+--upload
+ Upload packages (default: false)
+--scp-user USERNAME
+ Scp user for apt server (only required when --upload is specified)
+--apt-server HOSTNAME
+ Apt server hostname (only required when --upload is specified)
+--build-bundle-packages (default: false)
+ Build api server and workbench packages with vendor/bundle included
+--debug
+ Output debug information (default: false)
WORKSPACE=path Path to the Arvados source tree to build packages from
@@ -45,6 +51,9 @@ do
--upload)
UPLOAD=1
;;
+ --build-bundle-packages)
+ BUILD_BUNDLE_PACKAGES=1
+ ;;
*)
echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
exit 1
@@ -91,7 +100,16 @@ EOF
exit 1
fi
+RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
+RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
+if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
if [[ "$DEBUG" != 0 ]]; then
+ echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
echo "Workspace is $WORKSPACE"
fi
@@ -192,6 +210,15 @@ build_and_scp_deb () {
FPM_RESULTS=$("${COMMAND_ARR[@]}")
FPM_EXIT_CODE=$?
+ verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS
+}
+
+# verify build results and scp debs, if needed
+verify_and_scp_deb () {
+ FPM_EXIT_CODE=$1
+ shift
+ FPM_RESULTS=$@
+
FPM_PACKAGE_NAME=''
if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.deb) ]]; then
FPM_PACKAGE_NAME=${BASH_REMATCH[1]}
@@ -270,7 +297,7 @@ if [[ "$DEBUG" != 0 ]]; then
echo
fi
-if type rvm-exec 2>/dev/null; then
+if type rvm-exec >/dev/null 2>&1; then
FPM_GEM_PREFIX=$(rvm-exec system gem environment gemdir)
else
FPM_GEM_PREFIX=$(gem environment gemdir)
@@ -286,7 +313,7 @@ if [[ "$DEBUG" != 0 ]]; then
gem build arvados.gemspec
else
# -q appears to be broken in gem version 2.2.2
- gem build arvados.gemspec -q >/dev/null
+ gem build arvados.gemspec -q >/dev/null 2>&1
fi
if [[ "$UPLOAD" != 0 ]]; then
@@ -483,12 +510,127 @@ for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
pycrypto backports.ssl_match_hostname; do
build_and_scp_deb "$deppkg"
done
+
# Python 3 dependencies
for deppkg in docker-py six requests; do
# The empty string is the vendor argument: these aren't Curoverse software.
build_and_scp_deb "$deppkg" "python3-$deppkg" "" python3
done
+# Build the API server package
+
+cd "$WORKSPACE/services/api"
+
+API_VERSION=$(version_from_git)
+PACKAGE_NAME=arvados-api-server
+
+if [[ ! -d "$WORKSPACE/services/api/tmp" ]]; then
+ mkdir $WORKSPACE/services/api/tmp
+fi
+
+BUNDLE_OUTPUT=`bundle install --path vendor/bundle`
+
+if [[ "$DEBUG" != 0 ]]; then
+ echo $BUNDLE_OUTPUT
+fi
+
+/usr/bin/git rev-parse HEAD > git-commit.version
+
+cd $WORKSPACE/debs
+
+# Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now,
+# we do that in the upgrade script.
+# TODO: add bogus database.yml file so we can precompile the assets and put them in the
+# package. Then remove that database.yml file again. It has to be a valid file though.
+#RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile
+
+# This is the complete package with vendor/bundle included.
+# It's big, so we do not build it by default.
+if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "$PACKAGE_NAME" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
+
+ if [[ "$DEBUG" != 0 ]]; then
+ echo
+ echo "${COMMAND_ARR[@]}"
+ echo
+ fi
+
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+ verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS
+fi
+
+# Build the 'bare' package without vendor/bundle.
+declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "${PACKAGE_NAME}-bare" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
+
+if [[ "$DEBUG" != 0 ]]; then
+ echo
+ echo "${COMMAND_ARR[@]}"
+ echo
+fi
+
+FPM_RESULTS=$("${COMMAND_ARR[@]}")
+FPM_EXIT_CODE=$?
+verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS
+
+# API server package build done
+
+# Build the workbench server package
+
+cd "$WORKSPACE/apps/workbench"
+
+WORKBENCH_VERSION=$(version_from_git)
+PACKAGE_NAME=arvados-workbench
+
+if [[ ! -d "$WORKSPACE/apps/workbench/tmp" ]]; then
+ mkdir $WORKSPACE/apps/workbench/tmp
+fi
+
+BUNDLE_OUTPUT=`bundle install --path vendor/bundle`
+
+if [[ "$DEBUG" != 0 ]]; then
+ echo $BUNDLE_OUTPUT
+fi
+
+/usr/bin/git rev-parse HEAD > git-commit.version
+
+RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+cd $WORKSPACE/debs
+
+# This is the complete package with vendor/bundle included.
+# It's big, so we do not build it by default.
+if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
+
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "$PACKAGE_NAME" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/tmp" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
+
+ if [[ "$DEBUG" != 0 ]]; then
+ echo
+ echo "${COMMAND_ARR[@]}"
+ echo
+ fi
+
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+ verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS
+fi
+
+# Build the 'bare' package without vendor/bundle.
+
+declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "${PACKAGE_NAME}-bare" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/tmp" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
+
+if [[ "$DEBUG" != 0 ]]; then
+ echo
+ echo "${COMMAND_ARR[@]}"
+ echo
+fi
+
+FPM_RESULTS=$("${COMMAND_ARR[@]}")
+FPM_EXIT_CODE=$?
+verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS
+
+# Workbench package build done
+
# Finally, publish the packages, if necessary
if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then
ssh -p2222 $APTUSER@$APTSERVER -t "cd tmp && ls -laF *deb && freight add *deb apt/wheezy && freight cache && rm -f *deb"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list