[ARVADOS] updated: 1.1.1-171-g53d70ab

Git user git at public.curoverse.com
Thu Dec 7 09:15:10 EST 2017


Summary of changes:
 apps/workbench/test/unit/user_test.rb              |  14 ++
 build/package-build-dockerfiles/centos7/Dockerfile |   2 +
 build/package-build-dockerfiles/debian8/Dockerfile |   2 +
 build/package-build-dockerfiles/debian9/Dockerfile |   2 +
 .../ubuntu1404/Dockerfile                          |   2 +
 .../ubuntu1604/Dockerfile                          |   2 +
 build/run-build-packages-one-target.sh             |  22 +-
 build/run-build-packages.sh                        |  38 ++--
 build/run-library.sh                               |   3 +-
 build/run-tests.sh                                 |   4 +-
 doc/_config.yml                                    |   2 +-
 .../methods/container_requests.html.textile.liquid |  16 +-
 doc/api/methods/containers.html.textile.liquid     |   2 +-
 doc/install/install-keepstore.html.textile.liquid  |   2 +-
 ...uid => arvados-sync-groups.html.textile.liquid} |  12 +-
 .../controllers/arvados/v1/schema_controller.rb    |   1 +
 services/api/app/models/container.rb               |   3 +-
 services/api/app/models/container_request.rb       |   7 +-
 services/api/config/application.default.yml        |   7 +
 services/api/test/fixtures/containers.yml          |   2 +-
 services/api/test/unit/container_request_test.rb   |  43 +++-
 services/api/test/unit/container_test.rb           |  36 +++
 .../crunch-dispatch-slurm/crunch-dispatch-slurm.go |  37 ++++
 .../crunch-dispatch-slurm_test.go                  | 246 +++++++++++++--------
 services/crunch-dispatch-slurm/squeue.go           |  42 +++-
 tools/arv-sync-groups/.gitignore                   |   1 -
 tools/arvbox/lib/arvbox/docker/api-setup.sh        |   1 +
 tools/sync-groups/.gitignore                       |   1 +
 .../sync-groups.go}                                |   6 +-
 .../sync-groups_test.go}                           |   0
 30 files changed, 407 insertions(+), 151 deletions(-)
 rename doc/user/topics/{arv-sync-groups.html.textile.liquid => arvados-sync-groups.html.textile.liquid} (77%)
 delete mode 100644 tools/arv-sync-groups/.gitignore
 create mode 100644 tools/sync-groups/.gitignore
 rename tools/{arv-sync-groups/arv-sync-groups.go => sync-groups/sync-groups.go} (98%)
 rename tools/{arv-sync-groups/arv-sync-groups_test.go => sync-groups/sync-groups_test.go} (100%)

       via  53d70abf2749de0ad0eb29eb23da6ea084202545 (commit)
       via  e3265b24845a640810d2b46a8c70c63930c3891d (commit)
       via  a0623db24a21d5971f47f9e3abfbad6eff795cc7 (commit)
       via  b5e3317470650046cbc92095d9f1f2f788563bb9 (commit)
       via  e44c8ae2f57ce98c5e64f531539b8963f1a1ff00 (commit)
       via  190b1365c4074d86ae1bb0b9b07c3626a8671fac (commit)
       via  ee8558ddd313b037e94f3342cee304f8bd3e5e66 (commit)
       via  ceb1d9dc3bb2f80faf1a005a21ea687108841f40 (commit)
       via  50b36937d0bd921a9c3dc27080e0a3d7591b465a (commit)
       via  606168825167357b59022e8f893ff1dbb02c7a6c (commit)
       via  0cbcf8cb892c6011ae162e840e1892602d8af940 (commit)
       via  b13f48c0c80f3df65dffe0a3ba610ef12d65c2f7 (commit)
       via  7341c69d9cf26fe4e1ffba898041d73e5f1c237f (commit)
       via  61f82e9c0d9d543b86bd4fedfed2d5330eaf34c5 (commit)
       via  0632129bf7861a6ca3ef4ff4e21006a5f11e8be0 (commit)
       via  c50dc8052ca72ac1ead3236546e5f3d98beaeead (commit)
       via  c53ad9bc61054880ae6d68db8b868556c7278a78 (commit)
       via  0403d3bcd2003b0f88e0140ecfa8a62dae9bfd18 (commit)
       via  bcf069c6a726e219bc40224653268d87776e54aa (commit)
       via  b8f92203bfa5766f9aa8bfb728bb4a1580e7d6d1 (commit)
       via  d7b909ceb2646ef0637daaea70c768b799670c60 (commit)
       via  5fae6a137301f1a127ef4cccfc6bce86a6a9506c (commit)
       via  c4e6d3c9a8a55460c4ee663e66ea1093c6088d4f (commit)
       via  eaf301421b5a71e8344688723c3852e7bd5154b8 (commit)
       via  0910e857cbde6a23802ba026c3e5811e9d7b1d97 (commit)
       via  1cd144fe8da5fdc1d2e2e0ba2a3dcf47fc717251 (commit)
       via  0b1508df7ee4526340e8834422dd49ced63ee8d9 (commit)
       via  10993634c308dc62dd5e60f7d8d8ebf0391dd050 (commit)
       via  28560112303491b46c46377bc051e605e36a0927 (commit)
       via  0bc641e077a050b3f86957514fae0e4d8ba1770b (commit)
       via  ab3d39a8fdbc76425f48988c27187f3b54967ff9 (commit)
       via  c108a18d477c34c7f89d9a27af34751133329786 (commit)
      from  905283aa36b36d51c2c4376489cd97b34b488104 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 53d70abf2749de0ad0eb29eb23da6ea084202545
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Dec 7 02:39:49 2017 -0500

    12721: Install bundles in builder images.
    
    Avoids downloading/building the entire bundle every time a Rails
    package is built.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index cf120c9..c2fdfee 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -32,5 +32,7 @@ RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
 
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
 ENV WORKSPACE /arvados
 CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index b9998c6..739244d 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
 
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 28ba9a3..a6e5e88 100644
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -31,5 +31,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
 
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index 9e77ad3..55b9899 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
 
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index e4673c8..92aee31 100644
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -29,5 +29,7 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
 
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
 ENV WORKSPACE /arvados
 CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ed06469..915a331 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -620,7 +620,7 @@ if [[ "$?" == "0" ]] ; then
 
       # We need to bundle to be ready even when we build a package without vendor directory
       # because asset compilation requires it.
-      bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+      bundle install --system >"$STDOUT_IF_DEBUG"
 
       # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
       # and we want that in the package, so it's easier to not exclude the tmp directory

commit e3265b24845a640810d2b46a8c70c63930c3891d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Dec 7 02:37:23 2017 -0500

    12721: Omit cache dirs for platform A in package for platform B.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-library.sh b/build/run-library.sh
index 8a9c887..6d46eb1 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -318,6 +318,7 @@ handle_rails_package() {
         switches+=(-x "$exclude_root/$exclude")
     done
     fpm_build "${pos_args[@]}" "${switches[@]}" \
+              -x "$exclude_root/vendor/cache-*" \
               -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
     rm -rf "$scripts_dir"
 }

commit a0623db24a21d5971f47f9e3abfbad6eff795cc7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Dec 6 18:16:26 2017 -0500

    12721: Fix dev-version build.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 0e7e399..ef7862c 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -86,12 +86,15 @@ while [ $# -gt 0 ]; do
             test_packages=1
             ;;
         --build-version)
-            if ! [[ "$2" =~ (.*)-(.*) ]]; then
+            if [[ -z "$2" ]]; then
+                :
+            elif ! [[ "$2" =~ (.*)-(.*) ]]; then
                 echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
                 exit 1
+            else
+                ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+                ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
             fi
-            ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
-            ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
             shift
             ;;
         --)
@@ -106,7 +109,9 @@ done
 
 set -e
 
-echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+    echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
 
 if [[ -n "$test_packages" ]]; then
     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then

commit b5e3317470650046cbc92095d9f1f2f788563bb9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Dec 6 18:11:34 2017 -0500

    12721: Don't rebuild libcloud if package already exists.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index a64a7dc..ed06469 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -484,8 +484,9 @@ if [[ "$?" == "0" ]]; then
   fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
 fi
 
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
-  # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
   LIBCLOUD_DIR=$(mktemp -d)
   (
       cd $LIBCLOUD_DIR
@@ -497,7 +498,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONL
       handle_python_package
       DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
   )
-  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
   rm -rf $LIBCLOUD_DIR
 fi
 

commit e44c8ae2f57ce98c5e64f531539b8963f1a1ff00
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Dec 6 17:57:21 2017 -0500

    12721: Fix ignored iteration numbers in build.list.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index bee559b..a64a7dc 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -421,16 +421,16 @@ fi
 #
 # Ward, 2016-03-17
 saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
 if [[ "$?" == "0" ]]; then
-  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
 fi
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
 cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
 if [[ "$?" == "0" ]]; then
-  fpm_build cwltool "" "" python $cwltoolversion
+  fpm_build cwltool "" "" python $cwltoolversion --iteration 2
 fi
 
 # The PAM module
@@ -468,25 +468,20 @@ fi
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
 dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-declare -a iterargs=()
-if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
-    dockercleaner_iteration=3
-    iterargs+=(--iteration "$dockercleaner_iteration")
-else
-    dockercleaner_iteration=
-fi
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
 if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" "${iterargs[@]}"
+  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
 fi
 
 # The Arvados crunchstat-summary tool
 cd $WORKSPACE/packages/$TARGET
 crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
 if [[ "$?" == "0" ]]; then
   rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
 fi
 
 if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
diff --git a/build/run-library.sh b/build/run-library.sh
index 9898067..8a9c887 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -221,7 +221,7 @@ test_package_presence() {
         return 1
     fi
 
-    if [[ "$iteration" == "" || -n "$ARVADOS_BUILDING_VERSION" ]]; then
+    if [[ "$iteration" == "" ]]; then
         iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
     fi
 

commit 190b1365c4074d86ae1bb0b9b07c3626a8671fac
Merge: 905283a ee8558d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Dec 6 17:33:10 2017 -0500

    12721: Merge branch 'master' into 12721-build-package-version
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --cc build/run-build-packages-one-target.sh
index f2e5147,0db3051..0e7e399
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@@ -106,10 -95,17 +106,19 @@@ don
  
  set -e
  
 +echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
 +
  if [[ -n "$test_packages" ]]; then
      if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
+ 	set +e
+ 	/usr/bin/which createrepo >/dev/null
+ 	if [[ "$?" != "0" ]]; then
+ 		echo >&2
+ 		echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo"
+ 		echo >&2
+ 		exit 1
+ 	fi
+ 	set -e
          createrepo $WORKSPACE/packages/$TARGET
      fi
  

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list