[ARVADOS] updated: 1.1.1-171-g53d70ab
Git user
git at public.curoverse.com
Thu Dec 7 09:15:10 EST 2017
Summary of changes:
apps/workbench/test/unit/user_test.rb | 14 ++
build/package-build-dockerfiles/centos7/Dockerfile | 2 +
build/package-build-dockerfiles/debian8/Dockerfile | 2 +
build/package-build-dockerfiles/debian9/Dockerfile | 2 +
.../ubuntu1404/Dockerfile | 2 +
.../ubuntu1604/Dockerfile | 2 +
build/run-build-packages-one-target.sh | 22 +-
build/run-build-packages.sh | 38 ++--
build/run-library.sh | 3 +-
build/run-tests.sh | 4 +-
doc/_config.yml | 2 +-
.../methods/container_requests.html.textile.liquid | 16 +-
doc/api/methods/containers.html.textile.liquid | 2 +-
doc/install/install-keepstore.html.textile.liquid | 2 +-
...uid => arvados-sync-groups.html.textile.liquid} | 12 +-
.../controllers/arvados/v1/schema_controller.rb | 1 +
services/api/app/models/container.rb | 3 +-
services/api/app/models/container_request.rb | 7 +-
services/api/config/application.default.yml | 7 +
services/api/test/fixtures/containers.yml | 2 +-
services/api/test/unit/container_request_test.rb | 43 +++-
services/api/test/unit/container_test.rb | 36 +++
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 37 ++++
.../crunch-dispatch-slurm_test.go | 246 +++++++++++++--------
services/crunch-dispatch-slurm/squeue.go | 42 +++-
tools/arv-sync-groups/.gitignore | 1 -
tools/arvbox/lib/arvbox/docker/api-setup.sh | 1 +
tools/sync-groups/.gitignore | 1 +
.../sync-groups.go} | 6 +-
.../sync-groups_test.go} | 0
30 files changed, 407 insertions(+), 151 deletions(-)
rename doc/user/topics/{arv-sync-groups.html.textile.liquid => arvados-sync-groups.html.textile.liquid} (77%)
delete mode 100644 tools/arv-sync-groups/.gitignore
create mode 100644 tools/sync-groups/.gitignore
rename tools/{arv-sync-groups/arv-sync-groups.go => sync-groups/sync-groups.go} (98%)
rename tools/{arv-sync-groups/arv-sync-groups_test.go => sync-groups/sync-groups_test.go} (100%)
via 53d70abf2749de0ad0eb29eb23da6ea084202545 (commit)
via e3265b24845a640810d2b46a8c70c63930c3891d (commit)
via a0623db24a21d5971f47f9e3abfbad6eff795cc7 (commit)
via b5e3317470650046cbc92095d9f1f2f788563bb9 (commit)
via e44c8ae2f57ce98c5e64f531539b8963f1a1ff00 (commit)
via 190b1365c4074d86ae1bb0b9b07c3626a8671fac (commit)
via ee8558ddd313b037e94f3342cee304f8bd3e5e66 (commit)
via ceb1d9dc3bb2f80faf1a005a21ea687108841f40 (commit)
via 50b36937d0bd921a9c3dc27080e0a3d7591b465a (commit)
via 606168825167357b59022e8f893ff1dbb02c7a6c (commit)
via 0cbcf8cb892c6011ae162e840e1892602d8af940 (commit)
via b13f48c0c80f3df65dffe0a3ba610ef12d65c2f7 (commit)
via 7341c69d9cf26fe4e1ffba898041d73e5f1c237f (commit)
via 61f82e9c0d9d543b86bd4fedfed2d5330eaf34c5 (commit)
via 0632129bf7861a6ca3ef4ff4e21006a5f11e8be0 (commit)
via c50dc8052ca72ac1ead3236546e5f3d98beaeead (commit)
via c53ad9bc61054880ae6d68db8b868556c7278a78 (commit)
via 0403d3bcd2003b0f88e0140ecfa8a62dae9bfd18 (commit)
via bcf069c6a726e219bc40224653268d87776e54aa (commit)
via b8f92203bfa5766f9aa8bfb728bb4a1580e7d6d1 (commit)
via d7b909ceb2646ef0637daaea70c768b799670c60 (commit)
via 5fae6a137301f1a127ef4cccfc6bce86a6a9506c (commit)
via c4e6d3c9a8a55460c4ee663e66ea1093c6088d4f (commit)
via eaf301421b5a71e8344688723c3852e7bd5154b8 (commit)
via 0910e857cbde6a23802ba026c3e5811e9d7b1d97 (commit)
via 1cd144fe8da5fdc1d2e2e0ba2a3dcf47fc717251 (commit)
via 0b1508df7ee4526340e8834422dd49ced63ee8d9 (commit)
via 10993634c308dc62dd5e60f7d8d8ebf0391dd050 (commit)
via 28560112303491b46c46377bc051e605e36a0927 (commit)
via 0bc641e077a050b3f86957514fae0e4d8ba1770b (commit)
via ab3d39a8fdbc76425f48988c27187f3b54967ff9 (commit)
via c108a18d477c34c7f89d9a27af34751133329786 (commit)
from 905283aa36b36d51c2c4376489cd97b34b488104 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 53d70abf2749de0ad0eb29eb23da6ea084202545
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 7 02:39:49 2017 -0500
12721: Install bundles in builder images.
Avoids downloading/building the entire bundle every time a Rails
package is built.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index cf120c9..c2fdfee 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -32,5 +32,7 @@ RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index b9998c6..739244d 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 28ba9a3..a6e5e88 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -31,5 +31,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index 9e77ad3..55b9899 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index e4673c8..92aee31 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ed06469..915a331 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -620,7 +620,7 @@ if [[ "$?" == "0" ]] ; then
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+ bundle install --system >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
commit e3265b24845a640810d2b46a8c70c63930c3891d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 7 02:37:23 2017 -0500
12721: Omit cache dirs for platform A in package for platform B.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index 8a9c887..6d46eb1 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -318,6 +318,7 @@ handle_rails_package() {
switches+=(-x "$exclude_root/$exclude")
done
fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
rm -rf "$scripts_dir"
}
commit a0623db24a21d5971f47f9e3abfbad6eff795cc7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 6 18:16:26 2017 -0500
12721: Fix dev-version build.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 0e7e399..ef7862c 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -86,12 +86,15 @@ while [ $# -gt 0 ]; do
test_packages=1
;;
--build-version)
- if ! [[ "$2" =~ (.*)-(.*) ]]; then
+ if [[ -z "$2" ]]; then
+ :
+ elif ! [[ "$2" =~ (.*)-(.*) ]]; then
echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
exit 1
+ else
+ ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+ ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
fi
- ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
- ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
shift
;;
--)
@@ -106,7 +109,9 @@ done
set -e
-echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
commit b5e3317470650046cbc92095d9f1f2f788563bb9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 6 18:11:34 2017 -0500
12721: Don't rebuild libcloud if package already exists.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index a64a7dc..ed06469 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -484,8 +484,9 @@ if [[ "$?" == "0" ]]; then
fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
- # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
LIBCLOUD_DIR=$(mktemp -d)
(
cd $LIBCLOUD_DIR
@@ -497,7 +498,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONL
handle_python_package
DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
)
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
rm -rf $LIBCLOUD_DIR
fi
commit e44c8ae2f57ce98c5e64f531539b8963f1a1ff00
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 6 17:57:21 2017 -0500
12721: Fix ignored iteration numbers in build.list.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index bee559b..a64a7dc 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -421,16 +421,16 @@ fi
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 2
fi
# The PAM module
@@ -468,25 +468,20 @@ fi
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-declare -a iterargs=()
-if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
- dockercleaner_iteration=3
- iterargs+=(--iteration "$dockercleaner_iteration")
-else
- dockercleaner_iteration=
-fi
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" "${iterargs[@]}"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
if [[ "$?" == "0" ]]; then
rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
diff --git a/build/run-library.sh b/build/run-library.sh
index 9898067..8a9c887 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -221,7 +221,7 @@ test_package_presence() {
return 1
fi
- if [[ "$iteration" == "" || -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ if [[ "$iteration" == "" ]]; then
iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
fi
commit 190b1365c4074d86ae1bb0b9b07c3626a8671fac
Merge: 905283a ee8558d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 6 17:33:10 2017 -0500
12721: Merge branch 'master' into 12721-build-package-version
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc build/run-build-packages-one-target.sh
index f2e5147,0db3051..0e7e399
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@@ -106,10 -95,17 +106,19 @@@ don
set -e
+echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
+ set +e
+ /usr/bin/which createrepo >/dev/null
+ if [[ "$?" != "0" ]]; then
+ echo >&2
+ echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo"
+ echo >&2
+ exit 1
+ fi
+ set -e
createrepo $WORKSPACE/packages/$TARGET
fi
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list