[ARVADOS] updated: 1.3.0-1461-g366a2efdd
Git user
git at public.curoverse.com
Mon Aug 12 20:39:20 UTC 2019
Summary of changes:
build/run-build-packages.sh | 4 ++--
build/run-library.sh | 52 ++++++++++++++++++++++-----------------------
2 files changed, 28 insertions(+), 28 deletions(-)
discards ed318cfa38fb3e46b5e8b802d2642acd47d0e20e (commit)
via 366a2efdd0ac4630f4381f3b47d70ef155ed2df4 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (ed318cfa38fb3e46b5e8b802d2642acd47d0e20e)
\
N -- N -- N (366a2efdd0ac4630f4381f3b47d70ef155ed2df4)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 366a2efdd0ac4630f4381f3b47d70ef155ed2df4
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Fri Aug 9 15:25:23 2019 -0400
15516: Fix the arvados-server buildtime dependency for the arvados-workbench
package, and the runtime dependency for arvados-workbench and
arvados-api-server.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/apps/workbench/fpm-info.sh b/apps/workbench/fpm-info.sh
index 48913a14d..22ec1ba14 100644
--- a/apps/workbench/fpm-info.sh
+++ b/apps/workbench/fpm-info.sh
@@ -4,9 +4,9 @@
case "$TARGET" in
centos*)
- fpm_depends+=(git arvados-server)
+ fpm_depends+=(git)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++ arvados-server)
+ fpm_depends+=(git g++)
;;
esac
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 610dac90f..3d3ea6f0f 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -37,10 +37,6 @@ RUN rpm -ivh epel-release-latest-7.noarch.rpm
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
# The version of setuptools that comes with CentOS is way too old
RUN pip install --upgrade setuptools
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index f8104472b..c403d79bc 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -34,9 +34,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index 63c119c84..90f340e66 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index 5e4d06767..1adff7400 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 5c7dc342e..378c9bbfa 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -261,7 +261,6 @@ else
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
-set -x
# Build packages.
if docker run \
--rm \
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ae677a12c..8df8f981d 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -358,13 +358,16 @@ mkdir cwltest/bin && touch cwltest/bin/cwltest
fpm_build_virtualenv "cwltest" "cwltest"
rm -rf "$WORKSPACE/cwltest"
+calculate_go_package_version arvados_server_version cmd/arvados-server
+arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
# Build the API server package
test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
if [[ "$?" == "0" ]]; then
handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
# Build the workbench server package
@@ -372,6 +375,22 @@ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
if [[ "$?" == "0" ]] ; then
(
set -e
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
cd "$WORKSPACE/apps/workbench"
# We need to bundle to be ready even when we build a package without vendor directory
@@ -404,7 +423,7 @@ if [[ "$?" == "0" ]] ; then
handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
fi
diff --git a/build/run-library.sh b/build/run-library.sh
index b4c2d1ddc..af95cdc60 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -103,6 +103,37 @@ handle_ruby_gem() {
fi
}
+calculate_go_package_version() {
+ local -n __returnvar="$1"; shift
+ local src_path="$1"; shift
+
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+ (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
+
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
+ local version="$(version_from_git)"
+ local timestamp="$(timestamp_from_git)"
+
+ # Update the version number and build a new package if the vendor
+ # bundle has changed, or the command imports anything from the
+ # Arvados SDK and the SDK has changed.
+ declare -a checkdirs=(vendor)
+ if grep -qr git.curoverse.com/arvados .; then
+ checkdirs+=(sdk/go lib)
+ fi
+ for dir in ${checkdirs[@]}; do
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
+ ts="$(timestamp_from_git)"
+ if [[ "$ts" -gt "$timestamp" ]]; then
+ version=$(version_from_git)
+ timestamp="$ts"
+ fi
+ done
+
+ __returnvar="$version"
+}
+
# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
package_go_binary() {
local src_path="$1"; shift
@@ -110,46 +141,27 @@ package_go_binary() {
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
debug_echo "package_go_binary $src_path as $prog"
local basename="${src_path##*/}"
-
- mkdir -p "$GOPATH/src/git.curoverse.com"
- ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
- (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
-
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
- local version="$(version_from_git)"
- local timestamp="$(timestamp_from_git)"
-
- # Update the version number and build a new package if the vendor
- # bundle has changed, or the command imports anything from the
- # Arvados SDK and the SDK has changed.
- declare -a checkdirs=(vendor)
- if grep -qr git.curoverse.com/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
- for dir in ${checkdirs[@]}; do
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
- ts="$(timestamp_from_git)"
- if [[ "$ts" -gt "$timestamp" ]]; then
- version=$(version_from_git)
- timestamp="$ts"
- fi
- done
+ calculate_go_package_version go_package_version $src_path
cd $WORKSPACE/packages/$TARGET
- test_package_presence $prog $version go
+ test_package_presence $prog $go_package_version go
if [[ "$?" != "0" ]]; then
return 1
fi
- go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
+ go get -ldflags "-X main.version=${go_package_version}" "git.curoverse.com/arvados.git/$src_path"
local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
@@ -161,7 +173,7 @@ package_go_binary() {
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
default_iteration() {
@@ -213,47 +225,62 @@ test_rails_package_presence() {
test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
-test_package_presence() {
- local pkgname="$1"; shift
- local version="$1"; shift
- local pkgtype="$1"; shift
- local iteration="$1"; shift
- local arch="$1"; shift
+get_complete_package_name() {
+ local -n __returnvar="$1"; shift
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ "$iteration" == "" ]]; then
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ fi
- if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
- return 1
- fi
+ if [[ "$arch" == "" ]]; then
+ rpm_architecture="x86_64"
+ deb_architecture="amd64"
- if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ if [[ "$pkgtype" =~ ^(src)$ ]]; then
+ rpm_architecture="noarch"
+ deb_architecture="all"
fi
- if [[ "$arch" == "" ]]; then
+ # These python packages have binary components
+ if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
rpm_architecture="x86_64"
deb_architecture="amd64"
+ fi
+ else
+ rpm_architecture=$arch
+ deb_architecture=$arch
+ fi
- if [[ "$pkgtype" =~ ^(src)$ ]]; then
- rpm_architecture="noarch"
- deb_architecture="all"
- fi
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
+ fi
+ __returnvar=${complete_pkgname}
+}
- # These python packages have binary components
- if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+# Test if the package already exists, if not return 0, if it does return 1
+test_package_presence() {
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$pkgname" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ return 1
fi
- else
- rpm_architecture=$arch
- deb_architecture=$arch
fi
- if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
- else
- # rpm packages get iteration 1 if we don't supply one
- iteration=${iteration:-1}
- local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
- fi
+ local full_pkgname
+ get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch
# See if we can skip building the package, only if it already exists in the
# processed/ directory. If so, move it back to the packages directory to make
@@ -274,32 +301,32 @@ test_package_presence() {
fi
repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
- echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ echo ${repo_pkg_list} |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+ echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${full_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
+ echo "Package $full_pkgname exists, not rebuilding!"
return 1
else
- echo "Package $complete_pkgname not found, building"
+ echo "Package $full_pkgname not found, building"
return 0
fi
else
centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
repo_pkg_list=$(curl -s -o - ${centos_repo})
- echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ echo ${repo_pkg_list} |grep -q ${full_pkgname}
if [ $? -eq 0 ]; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+ echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" ${centos_repo}${full_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
+ echo "Package $full_pkgname exists, not rebuilding!"
return 1
else
- echo "Package $complete_pkgname not found, building"
+ echo "Package $full_pkgname not found, building"
return 0
fi
fi
@@ -681,7 +708,11 @@ fpm_build () {
shift
if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$PACKAGE_NAME" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
diff --git a/services/api/fpm-info.sh b/services/api/fpm-info.sh
index 47e1e4bb1..1d5891ed6 100644
--- a/services/api/fpm-info.sh
+++ b/services/api/fpm-info.sh
@@ -6,9 +6,9 @@ fpm_depends+=('git >= 1.7.10')
case "$TARGET" in
centos*)
- fpm_depends+=(libcurl-devel postgresql-devel arvados-server)
+ fpm_depends+=(libcurl-devel postgresql-devel)
;;
debian* | ubuntu*)
- fpm_depends+=(libcurl-ssl-dev libpq-dev g++ arvados-server)
+ fpm_depends+=(libcurl-ssl-dev libpq-dev g++)
;;
esac
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list