[ARVADOS] created: 1.3.0-1462-g2522f92cc
Git user
git at public.curoverse.com
Fri Aug 9 19:26:21 UTC 2019
at 2522f92cc25e55e57a31af7a59ebdbb066743445 (commit)
commit 2522f92cc25e55e57a31af7a59ebdbb066743445
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Fri Aug 9 15:25:23 2019 -0400
15516: Fix the arvados-server buildtime dependency for the arvados-workbench
package, and the runtime dependency for arvados-workbench and
arvados-api-server.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/apps/workbench/fpm-info.sh b/apps/workbench/fpm-info.sh
index 48913a14d..22ec1ba14 100644
--- a/apps/workbench/fpm-info.sh
+++ b/apps/workbench/fpm-info.sh
@@ -4,9 +4,9 @@
case "$TARGET" in
centos*)
- fpm_depends+=(git arvados-server)
+ fpm_depends+=(git)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++ arvados-server)
+ fpm_depends+=(git g++)
;;
esac
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 610dac90f..3d3ea6f0f 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -37,10 +37,6 @@ RUN rpm -ivh epel-release-latest-7.noarch.rpm
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
# The version of setuptools that comes with CentOS is way too old
RUN pip install --upgrade setuptools
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index f8104472b..c403d79bc 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -34,9 +34,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index 63c119c84..90f340e66 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index 5e4d06767..1adff7400 100644
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 5c7dc342e..378c9bbfa 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -261,7 +261,6 @@ else
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
-set -x
# Build packages.
if docker run \
--rm \
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ae677a12c..45c1c843a 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -358,13 +358,16 @@ mkdir cwltest/bin && touch cwltest/bin/cwltest
fpm_build_virtualenv "cwltest" "cwltest"
rm -rf "$WORKSPACE/cwltest"
+arvados_server_version=$(calculate_go_package_version cmd/arvados-server)
+arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
# Build the API server package
test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
if [[ "$?" == "0" ]]; then
handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
# Build the workbench server package
@@ -372,6 +375,22 @@ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
if [[ "$?" == "0" ]] ; then
(
set -e
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ arvados_server_pkgname=$(get_complete_package_name arvados-server ${arvados_server_version} go)
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
cd "$WORKSPACE/apps/workbench"
# We need to bundle to be ready even when we build a package without vendor directory
@@ -404,7 +423,7 @@ if [[ "$?" == "0" ]] ; then
handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
fi
diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh
index 962940b79..acc2de29f 100755
--- a/build/run-build-test-packages-one-target.sh
+++ b/build/run-build-test-packages-one-target.sh
@@ -108,7 +108,7 @@ title "Start test packages"
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
- $WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}" --test-packages
+ $WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}" --test-packages --force-test
else
echo "Skipping package upload, there were errors building the packages"
fi
diff --git a/build/run-library.sh b/build/run-library.sh
index b4c2d1ddc..92b8c3d12 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -103,6 +103,36 @@ handle_ruby_gem() {
fi
}
+calculate_go_package_version() {
+ local src_path="$1"; shift
+
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+ (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
+
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
+ local version="$(version_from_git)"
+ local timestamp="$(timestamp_from_git)"
+
+ # Update the version number and build a new package if the vendor
+ # bundle has changed, or the command imports anything from the
+ # Arvados SDK and the SDK has changed.
+ declare -a checkdirs=(vendor)
+ if grep -qr git.curoverse.com/arvados .; then
+ checkdirs+=(sdk/go lib)
+ fi
+ for dir in ${checkdirs[@]}; do
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
+ ts="$(timestamp_from_git)"
+ if [[ "$ts" -gt "$timestamp" ]]; then
+ version=$(version_from_git)
+ timestamp="$ts"
+ fi
+ done
+
+ echo ${version}
+}
+
# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
package_go_binary() {
local src_path="$1"; shift
@@ -110,37 +140,19 @@ package_go_binary() {
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
debug_echo "package_go_binary $src_path as $prog"
local basename="${src_path##*/}"
- mkdir -p "$GOPATH/src/git.curoverse.com"
- ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
- (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
-
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
- local version="$(version_from_git)"
- local timestamp="$(timestamp_from_git)"
-
- # Update the version number and build a new package if the vendor
- # bundle has changed, or the command imports anything from the
- # Arvados SDK and the SDK has changed.
- declare -a checkdirs=(vendor)
- if grep -qr git.curoverse.com/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
- for dir in ${checkdirs[@]}; do
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
- ts="$(timestamp_from_git)"
- if [[ "$ts" -gt "$timestamp" ]]; then
- version=$(version_from_git)
- timestamp="$ts"
- fi
- done
+ local version=$(calculate_go_package_version $src_path)
cd $WORKSPACE/packages/$TARGET
test_package_presence $prog $version go
@@ -213,47 +225,62 @@ test_rails_package_presence() {
test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
-test_package_presence() {
- local pkgname="$1"; shift
- local version="$1"; shift
- local pkgtype="$1"; shift
- local iteration="$1"; shift
- local arch="$1"; shift
+get_complete_package_name() {
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ "$iteration" == "" ]]; then
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ fi
- if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
- return 1
- fi
+ if [[ "$arch" == "" ]]; then
+ rpm_architecture="x86_64"
+ deb_architecture="amd64"
- if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ if [[ "$pkgtype" =~ ^(src)$ ]]; then
+ rpm_architecture="noarch"
+ deb_architecture="all"
fi
- if [[ "$arch" == "" ]]; then
+ # These python packages have binary components
+ if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
rpm_architecture="x86_64"
deb_architecture="amd64"
+ fi
+ else
+ rpm_architecture=$arch
+ deb_architecture=$arch
+ fi
- if [[ "$pkgtype" =~ ^(src)$ ]]; then
- rpm_architecture="noarch"
- deb_architecture="all"
- fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
+ else
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
+ fi
+
+ echo $complete_pkgname
+}
- # These python packages have binary components
- if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+# Test if the package already exists, if not return 0, if it does return 1
+test_package_presence() {
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$pkgname" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ return 1
fi
- else
- rpm_architecture=$arch
- deb_architecture=$arch
fi
- if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
- else
- # rpm packages get iteration 1 if we don't supply one
- iteration=${iteration:-1}
- local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
- fi
+ local complete_pkgname=$(get_complete_package_name $pkgname $version $pkgtype $iteration $arch)
# See if we can skip building the package, only if it already exists in the
# processed/ directory. If so, move it back to the packages directory to make
@@ -276,8 +303,8 @@ test_package_presence() {
repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
echo ${repo_pkg_list} |grep -q ${complete_pkgname}
if [ $? -eq 0 ] ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+ echo "Package $complete_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${complete_pkgname}" http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
echo "Package $complete_pkgname exists, not rebuilding!"
@@ -292,8 +319,8 @@ test_package_presence() {
repo_pkg_list=$(curl -s -o - ${centos_repo})
echo ${repo_pkg_list} |grep -q ${complete_pkgname}
if [ $? -eq 0 ]; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+ echo "Package $complete_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${complete_pkgname}" ${centos_repo}${complete_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
echo "Package $complete_pkgname exists, not rebuilding!"
@@ -681,7 +708,11 @@ fpm_build () {
shift
if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$PACKAGE_NAME" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
diff --git a/services/api/fpm-info.sh b/services/api/fpm-info.sh
index 47e1e4bb1..1d5891ed6 100644
--- a/services/api/fpm-info.sh
+++ b/services/api/fpm-info.sh
@@ -6,9 +6,9 @@ fpm_depends+=('git >= 1.7.10')
case "$TARGET" in
centos*)
- fpm_depends+=(libcurl-devel postgresql-devel arvados-server)
+ fpm_depends+=(libcurl-devel postgresql-devel)
;;
debian* | ubuntu*)
- fpm_depends+=(libcurl-ssl-dev libpq-dev g++ arvados-server)
+ fpm_depends+=(libcurl-ssl-dev libpq-dev g++)
;;
esac
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list