[ARVADOS] updated: 935c8f81f0a041a58bbb45602ef44cdd6673163a

Git user git at public.curoverse.com
Fri Jan 13 22:39:05 EST 2017


Summary of changes:
 apps/workbench/test/integration/projects_test.rb   |   8 -
 build/build.list                                   |  44 +++
 build/run-build-packages.sh                        | 369 ++++++++++++---------
 build/run-library.sh                               |  88 ++++-
 sdk/cwl/arvados_cwl/crunch_script.py               |   7 +-
 sdk/cwl/setup.py                                   |   6 +-
 sdk/go/arvados/collection.go                       |  11 +-
 sdk/python/arvados/commands/run.py                 |  10 +-
 sdk/python/tests/test_events.py                    |  56 +++-
 .../api/app/controllers/application_controller.rb  |  28 +-
 .../arvados/v1/collections_controller.rb           |   6 +-
 services/api/app/models/collection.rb              |   2 +
 services/api/app/models/user.rb                    |  90 +++--
 services/api/config/application.rb                 |  11 +-
 ...t_finished_at_on_finished_pipeline_instances.rb |   5 +
 services/api/db/structure.sql                      |   4 +-
 services/api/lib/create_permission_view.sql        |  41 +++
 .../arvados/v1/collections_controller_test.rb      |  37 ++-
 .../arvados/v1/groups_controller_test.rb           |   5 +-
 services/api/test/unit/user_test.rb                |  10 +-
 services/keep-balance/collection.go                |   2 +-
 services/keepstore/azure_blob_volume.go            |  14 +-
 services/keepstore/volume_unix.go                  | 157 ++++++---
 services/keepstore/volume_unix_test.go             |  61 ++++
 24 files changed, 752 insertions(+), 320 deletions(-)
 create mode 100644 build/build.list
 create mode 100644 services/api/db/migrate/20170105160302_set_finished_at_on_finished_pipeline_instances.rb
 create mode 100644 services/api/lib/create_permission_view.sql

       via  935c8f81f0a041a58bbb45602ef44cdd6673163a (commit)
       via  1ccb1dbb6a3d342a647e2f5ab8d0f3d99d048a6b (commit)
       via  7f105aaf983d67ef2c6f08bc4f18eb5d32ffc0c7 (commit)
       via  52c8ac88a924ffe9771876ed8e8399b801d79a56 (commit)
       via  eddb8f6851899df30ff40d053870e4cbf3c123fe (commit)
       via  99b571c9050f28525427c100f9072159a3145bc7 (commit)
       via  b97d14dbd377cc73854eefe77d6450859a41f6b7 (commit)
       via  a6f289904b3960a9ffaf4be453ce82d3757fba23 (commit)
       via  f892f08b305dbac15d80b350c7a9192a5a7b2eaf (commit)
       via  6b30f7c9a223cc1d22974085f6df7aa62393cc55 (commit)
       via  1d19121f1216a20fa5083ada76d5b77f7901a9d5 (commit)
       via  bab78d47d93fa6e0bf2e0d41bcdcd65799f683e4 (commit)
       via  863280201573355f380c59226015373885b50617 (commit)
       via  293f8a1d393d7fbfd12ed6c62260aa1b24c2e0fe (commit)
       via  49db3b740b861688eff2a872c8f69f65ee893ed2 (commit)
       via  1d34696de2fb46133e1d969d8d2aed1de3da2ff2 (commit)
       via  7ad611f636e849e255b0f66ca6d7a027193b65dc (commit)
       via  daafdb4c939f265b4604711d0fc946a830d9d54e (commit)
       via  6951a378515a726fa178e0d1554c72044a7f5f88 (commit)
       via  2cddd92e172cc20d0b7c1c8013b64fb32056f3df (commit)
       via  ba300c9e82f9fd2f853a7bfb4e0df58685071d0c (commit)
       via  8e6cd14b7884a691a110110b0f366577437c6d9e (commit)
       via  532be5dc8d48ec4839aca38032fa0b64ddf7cd75 (commit)
       via  a6afada6f023621d2725fccf31a214a6d9e0a3e3 (commit)
       via  fe1b0b43931dcefbf9308dc7b0a3639a4410ca53 (commit)
       via  87275ec769831e04982a714049ee9d67b6342d68 (commit)
       via  07a94c1d7f2fc3b0c18504c46de03abeec5da8fc (commit)
       via  3aee47f04fddb82b1887044de86f8824e9e2f5f2 (commit)
       via  b7ff345255551a5187ae7b2846b704a3c800256d (commit)
       via  4a5f17bc047eea3cc65ccee62315f2215ad5e936 (commit)
       via  837949653b69e357cfa90fb0b8855a37e9c406d7 (commit)
       via  f301a70fdcfda9872965835b26d1400a53d584a1 (commit)
       via  72007cdcdcc228f1ba04b6963fdd9864b30bf281 (commit)
       via  ccb95499d6eb46957cda2f060d0b931df2dd82d4 (commit)
       via  a714babda599431fd509dff6dee87ab30a1793eb (commit)
       via  4f4f6f5fe9367d5cd6c57070fd8a223efc87cb21 (commit)
       via  a57f8602e4a89e526464587c78e91c2086aae8d8 (commit)
       via  53d04f5af565604b9cf5fd2e84d75549f8bb1c4f (commit)
       via  333f65cfbdc3cc4e19fc5029de409522d707b319 (commit)
       via  99992d17f51e59ff80d52d10cca2102389ee2d9b (commit)
       via  e2bfcb8bc5f030828754215aff1955f5ab4bd45a (commit)
      from  6da5c2c733a3a3194887502d5cdb5a8446fbf7e0 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 935c8f81f0a041a58bbb45602ef44cdd6673163a
Merge: 1ccb1db 7f105aa
Author: Ward Vandewege <ward at curoverse.com>
Date:   Fri Jan 13 22:38:38 2017 -0500

    Merge branch 'master' into 10858-build-packages-improvements
    
    refs #10858


commit 1ccb1dbb6a3d342a647e2f5ab8d0f3d99d048a6b
Author: Ward Vandewege <ward at curoverse.com>
Date:   Fri Jan 13 22:36:12 2017 -0500

    Refactor the build-packages scripts a bit to make them much more
    efficient when there is nothing to do.
    
    refs #10858

diff --git a/build/build.list b/build/build.list
new file mode 100644
index 0000000..4b60e42
--- /dev/null
+++ b/build/build.list
@@ -0,0 +1,44 @@
+#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
+debian8,ubuntu1204,centos7|python-gflags|2.0|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|google-api-python-client|1.4.2|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|rsa|3.4.2|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|uritemplate|3.0.0|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|httplib2|0.9.2|3|python|all
+debian8,ubuntu1204,centos7|ws4py|0.3.5|2|python|all
+debian8,ubuntu1204,centos7|pykka|1.2.1|2|python|all
+debian8,ubuntu1204,ubuntu1404|six|1.10.0|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|ciso8601|1.0.3|3|python|amd64
+debian8,ubuntu1204,centos7|pycrypto|2.6.1|3|python|amd64
+debian8,ubuntu1204,ubuntu1404|backports.ssl_match_hostname|3.5.0.1|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|llfuse|0.41.1|3|python|amd64
+debian8,ubuntu1204,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
+debian8,ubuntu1204,ubuntu1404,centos7|pyyaml|3.12|2|python|amd64
+debian8,ubuntu1204,ubuntu1404,centos7|rdflib|4.2.1|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|mistune|0.7.3|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|typing|3.5.3.0|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|avro|1.8.1|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
+debian8,ubuntu1204,ubuntu1404,centos7|cachecontrol|0.11.7|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|pathlib2|2.1.0|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|docker-py|1.7.2|2|python3|all
+debian8,ubuntu1204,centos7|six|1.10.0|2|python3|all
+debian8,ubuntu1204,ubuntu1404,centos7|requests|2.12.4|2|python3|all
+debian8,ubuntu1204,ubuntu1404,centos7|websocket-client|0.37.0|2|python3|all
+ubuntu1204|requests|2.12.4|2|python|all
+ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
+ubuntu1204,centos7|isodate|0.5.4|2|python|all
+centos7|daemon|2.1.1|2|python|all
+centos7|pbr|0.11.1|2|python|all
+centos7|pyparsing|2.1.10|2|python|all
+centos7|sparqlwrapper|1.8.0|2|python|all
+centos7|html5lib|0.9999999|2|python|all
+centos7|keepalive|0.5|2|python|all
+all|lockfile|0.12.2|2|python|all|--epoch 1
+all|ruamel.yaml|0.13.7|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+all|cwltest|1.0.20160907111242|2|python|all
+all|rdflib-jsonld|0.4.0|2|python|all
+all|futures|3.0.5|2|python|all
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 40e3b43..490e430 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -32,6 +32,8 @@ DEBUG=${ARVADOS_DEBUG:-0}
 TARGET=debian8
 COMMAND=
 
+RAILS_PACKAGE_ITERATION=7
+
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
     help,build-bundle-packages,debug,target:,only-build: \
     -- "" "$@")
@@ -103,34 +105,12 @@ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 case "$TARGET" in
     debian8)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six \
-            ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol 'pathlib2==2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1204)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six \
-            ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro isodate ruamel.ordereddict
-            cachecontrol requests 'pathlib2==2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1404)
         FORMAT=deb
-        PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
-            google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
-            rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol 'pathlib2==2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
         ;;
     centos7)
         FORMAT=rpm
@@ -141,15 +121,6 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
         PYTHON3_PREFIX=/opt/rh/python33/root/usr
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka  \
-            ciso8601 pycrypto 'pycurl<7.21.5' \
-            python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml contextlib2 \
-            'rdflib>=4.2.0' shellescape mistune typing avro \
-            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
-            ruamel.ordereddict cachecontrol 'pathlib2==2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -222,21 +193,29 @@ fi
 debug_echo -e "\nPerl packages\n"
 
 if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
-cd "$WORKSPACE/sdk/perl"
+  cd "$WORKSPACE/sdk/perl"
+  libarvados_perl_version="$(version_from_git)"
 
-if [[ -e Makefile ]]; then
-  make realclean >"$STDOUT_IF_DEBUG"
-fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-    -delete
-rm -rf install
-
-perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
-    make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
-    fpm_build install/lib/=/usr/share libarvados-perl \
-    "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
-    "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
-    mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  cd $WORKSPACE/packages/$TARGET
+  test_package_presence libarvados-perl "$libarvados_perl_version"
+
+  if [[ "$?" == "0" ]]; then
+    cd "$WORKSPACE/sdk/perl"
+
+    if [[ -e Makefile ]]; then
+      make realclean >"$STDOUT_IF_DEBUG"
+    fi
+    find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+        -delete
+    rm -rf install
+
+    perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+        make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+        fpm_build install/lib/=/usr/share libarvados-perl \
+        "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+        "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+        mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  fi
 fi
 
 # Ruby gems
@@ -273,27 +252,33 @@ handle_python_package
 
 # arvados-src
 (
-    set -e
-
     cd "$WORKSPACE"
     COMMIT_HASH=$(format_last_commit_here "%H")
+    arvados_src_version="$(version_from_git)"
 
-    SRC_BUILD_DIR=$(mktemp -d)
-    # mktemp creates the directory with 0700 permissions by default
-    chmod 755 $SRC_BUILD_DIR
-    git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
-    cd "$SRC_BUILD_DIR"
+    cd $WORKSPACE/packages/$TARGET
+    test_package_presence arvados-src $arvados_src_version src
 
-    # go into detached-head state
-    git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
-    echo "$COMMIT_HASH" >git-commit.version
+    if [[ "$?" == "0" ]]; then
+      cd "$WORKSPACE"
+      SRC_BUILD_DIR=$(mktemp -d)
+      # mktemp creates the directory with 0700 permissions by default
+      chmod 755 $SRC_BUILD_DIR
+      git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+      cd "$SRC_BUILD_DIR"
 
-    cd "$SRC_BUILD_DIR"
-    PKG_VERSION=$(version_from_git)
-    cd $WORKSPACE/packages/$TARGET
-    fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+      # go into detached-head state
+      git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+      echo "$COMMIT_HASH" >git-commit.version
+
+      cd "$SRC_BUILD_DIR"
+      PKG_VERSION=$(version_from_git)
+      cd $WORKSPACE/packages/$TARGET
+      fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+      rm -rf "$SRC_BUILD_DIR"
 
-    rm -rf "$SRC_BUILD_DIR"
+    fi
 )
 
 # On older platforms we need to publish a backport of libfuse >=2.9.2,
@@ -386,14 +371,21 @@ package_go_binary tools/keep-exercise keep-exercise \
 # 2014-05-15
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+fi
 
 # cwl-runner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
-
-fpm_build lockfile "" "" python 0.12.2 --epoch 1
+arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
+arvados_cwl_runner_iteration=3
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+fi
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -411,28 +403,27 @@ fpm_build lockfile "" "" python 0.12.2 --epoch 1
 #
 # Ward, 2016-03-17
 saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
-
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
-# Ward, 2016-05-26
-fpm_build ruamel.yaml "" "" python 0.13.7 --python-setup-py-arguments "--single-version-externally-managed"
-
-# Dependency of cwltool.  Fpm doesn't produce a package with the correct version
-# number unless we build it explicitly
-fpm_build cwltest "" "" python 1.0.20160907111242
+test_package_presence python-schema-salad "$saladversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+fi
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
 cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build cwltool "" "" python $cwltoolversion
-
-# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm_build rdflib-jsonld "" "" python 0.4.0
+test_package_presence python-cwltool "$cwltoolversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build cwltool "" "" python $cwltoolversion
+fi
 
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
     cd $WORKSPACE/packages/$TARGET
     rm -rf "$WORKSPACE/sdk/pam/build"
-    fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    libpam_arvados_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)
+    test_package_presence libpam-arvados "$libpam_arvados_version" python
+    if [[ "$?" == "0" ]]; then
+      fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$libpam_arvados_version" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    fi
 fi
 
 # The FUSE driver
@@ -440,37 +431,56 @@ fi
 # not omit the python- prefix first.
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)
+test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The node manager
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)
+test_package_presence arvados-node-manager "$nodemanager_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The Docker image cleaner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
+dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
+dockercleaner_iteration=3
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+fi
 
 # The Arvados crunchstat-summary tool
 cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+if [[ "$?" == "0" ]]; then
+  rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
+  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+fi
 
-# Forked libcloud
-LIBCLOUD_DIR=$(mktemp -d)
-(
-    cd $LIBCLOUD_DIR
-    git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
-    git checkout apache-libcloud-$LIBCLOUD_PIN
-    # libcloud is absurdly noisy without -q, so force -q here
-    OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
-    DASHQ_UNLESS_DEBUG=-q
-    handle_python_package
-    DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
-)
-fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
-rm -rf $LIBCLOUD_DIR
+if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
+  # Forked libcloud
+  LIBCLOUD_DIR=$(mktemp -d)
+  (
+      cd $LIBCLOUD_DIR
+      git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+      git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
+      # libcloud is absurdly noisy without -q, so force -q here
+      OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+      DASHQ_UNLESS_DEBUG=-q
+      handle_python_package
+      DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+  )
+  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+  rm -rf $LIBCLOUD_DIR
+fi
 
 # Python 2 dependencies
 declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
@@ -482,99 +492,134 @@ case "$?" in
     *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
 esac
 
-for deppkg in "${PYTHON_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+while read -r line || [[ -n "$line" ]]; do
+#  echo "Text read from file: $line"
+  if [[ "$line" =~ ^# ]]; then
+    continue
+  fi
+  IFS='|'; arr=($line); unset IFS
 
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
-        continue
-    fi
+  dist=${arr[0]}
 
-    case "$deppkg" in
-        httplib2|google-api-python-client)
+  IFS=',';dists=($dist); unset IFS
+
+  MATCH=0
+  for d in "${dists[@]}"; do
+    if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
+      MATCH=1
+    fi
+  done
+
+  if [[ "$MATCH" != "1" ]]; then
+    continue
+  fi
+  name=${arr[1]}
+  version=${arr[2]}
+  iteration=${arr[3]}
+  pkgtype=${arr[4]}
+  arch=${arr[5]}
+  extra=${arr[6]}
+
+  if [[ "$pkgtype" == "python" ]]; then
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+  else
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+  fi
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
+      continue
+  fi
+
+  case "$name" in
+      httplib2|google-api-python-client)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
             # Work around 0640 permissions on some package files.
             # See #7591 and #7991.
             pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
                 set -e
                 cd "$pyfpm_workdir"
-                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
                 # Sometimes pip gives us a tarball, sometimes a zip file...
-                DOWNLOADED=`ls $deppkg-*`
+                DOWNLOADED=`ls $name-*`
                 [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
                 [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
-                cd "$deppkg"-*/
+                cd "$name"-*/
                 "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
                 chmod -R go+rX .
                 set +e
-                fpm_build . "$outname" "" python "" --iteration 3
+                fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" $extra
                 # The upload step uses the package timestamp to determine
-                # whether it's new.  --no-clobber plays nice with that.
+                # if it is new.  --no-clobber plays nice with that.
                 mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
             )
             if [ 0 != "$?" ]; then
-                echo "ERROR: $deppkg build process failed"
+                echo "ERROR: $name build process failed"
                 EXITCODE=1
             fi
             if [ -n "$pyfpm_workdir" ]; then
                 rm -rf "$pyfpm_workdir"
             fi
-            ;;
-        *)
-            fpm_build "$deppkg" "$outname"
-            ;;
-    esac
-done
-
-# Python 3 dependencies
-for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
-    # The empty string is the vendor argument: these aren't Curoverse software.
-    fpm_build "$deppkg" "$outname" "" python3
-done
+          fi
+          ;;
+      *)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
+            fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" $extra
+          fi
+          ;;
+  esac
+
+done <`dirname "$(readlink -f "$0")"`"/build.list"
 
 # Build the API server package
-handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
-    "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-    --description="Arvados API server - Arvados is a free and open source platform for big data science." \
-    --license="GNU Affero General Public License, version 3.0"
-
-# Build the workbench server package
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then
-(
-    set -e
-    cd "$WORKSPACE/apps/workbench"
-
-    # We need to bundle to be ready even when we build a package without vendor directory
-    # because asset compilation requires it.
-    bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
-
-    # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-    # and we want that in the package, so it's easier to not exclude the tmp directory
-    # from the package - empty it instead.
-    rm -rf tmp
-    mkdir tmp
-
-    # Set up application.yml and production.rb so that asset precompilation works
-    \cp config/application.yml.example config/application.yml -f
-    \cp config/environments/production.rb.example config/environments/production.rb -f
-    sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
-
-    RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
-
-    # Remove generated configuration files so they don't go in the package.
-    rm config/application.yml config/environments/production.rb
-)
-fi
-
-if [[ "$?" != "0" ]]; then
-  echo "ERROR: Asset precompilation failed"
-  EXITCODE=1
-else
-  handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+if [[ "$?" == "0" ]]; then
+  handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
       "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-      --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+      --description="Arvados API server - Arvados is a free and open source platform for big data science." \
       --license="GNU Affero General Public License, version 3.0"
 fi
 
+# Build the workbench server package
+test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+if [[ "$?" == "0" ]] ; then
+  (
+      set -e
+      cd "$WORKSPACE/apps/workbench"
+
+      # We need to bundle to be ready even when we build a package without vendor directory
+      # because asset compilation requires it.
+      bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+      # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+      # and we want that in the package, so it's easier to not exclude the tmp directory
+      # from the package - empty it instead.
+      rm -rf tmp
+      mkdir tmp
+
+      # Set up application.yml and production.rb so that asset precompilation works
+      \cp config/application.yml.example config/application.yml -f
+      \cp config/environments/production.rb.example config/environments/production.rb -f
+      sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+      # Remove generated configuration files so they don't go in the package.
+      rm config/application.yml config/environments/production.rb
+  )
+
+  if [[ "$?" != "0" ]]; then
+    echo "ERROR: Asset precompilation failed"
+    EXITCODE=1
+  else
+    handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+        --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+        --license="GNU Affero General Public License, version 3.0"
+  fi
+fi
+
 # clean up temporary GOPATH
 rm -rf "$GOPATH"
 
diff --git a/build/run-library.sh b/build/run-library.sh
index 23e09d4..267b641 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -113,6 +113,12 @@ package_go_binary() {
     fi
 
     cd $WORKSPACE/packages/$TARGET
+    test_package_presence $prog $version go
+
+    if [[ "$?" != "0" ]]; then
+      return 1
+    fi
+
     go get "git.curoverse.com/arvados.git/$src_path"
 
     declare -a switches=()
@@ -131,11 +137,16 @@ package_go_binary() {
 default_iteration() {
     local package_name="$1"; shift
     local package_version="$1"; shift
+    local package_type="$1"; shift
     local iteration=1
     if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
            [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
         iteration=2
     fi
+    if [[ $package_type =~ ^python ]]; then
+      # Fix --iteration for #9242.
+      iteration=2
+    fi
     echo $iteration
 }
 
@@ -149,13 +160,80 @@ _build_rails_package_scripts() {
     done
 }
 
+test_rails_package_presence() {
+  local pkgname="$1"; shift
+  local srcdir="$1"; shift
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+    return 1
+  fi
+
+  tmppwd=`pwd`
+
+  cd $srcdir
+
+  local version="$(version_from_git)"
+
+  cd $tmppwd
+
+  test_package_presence $pkgname $version rails $RAILS_PACKAGE_ITERATION
+}
+
+test_package_presence() {
+    local pkgname="$1"; shift
+    local version="$1"; shift
+    local pkgtype="$1"; shift
+    local iteration="$1"; shift
+    local arch="$1"; shift
+
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+        return 1
+    fi
+
+    if [[ "$iteration" == "" ]]; then
+      iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+    fi
+
+    if [[ "$arch" == "" ]]; then
+      rpm_architecture="x86_64"
+      deb_architecture="amd64"
+
+      if [[ "$pkgtype" =~ ^(python|python3)$ ]]; then
+        rpm_architecture="all"
+        deb_architecture="all"
+      fi
+
+      # These packages have binary components
+      if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
+        rpm_architecture="x86_64"
+        deb_architecture="amd64"
+      fi
+    else
+      rpm_architecture=$arch
+      deb_architecture=$arch
+    fi
+
+    if [[ "$FORMAT" == "deb" ]]; then
+      local complete_pkgname=$pkgname"_"$version"-"$iteration"_"$deb_architecture".deb"
+    else
+      local complete_pkgname="$pkgname-$version-$iteration.$rpm_architecture.rpm"
+    fi
+
+    if [[ -e "$complete_pkgname" ]]; then
+      echo "Package $complete_pkgname exists, not rebuilding!"
+      return 1
+    else
+      echo "Package $complete_pkgname not found, building"
+      return 0
+    fi
+}
+
 handle_rails_package() {
     local pkgname="$1"; shift
 
     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
         return 0
     fi
-
     local srcdir="$1"; shift
     local license_path="$1"; shift
     local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
@@ -178,7 +256,7 @@ handle_rails_package() {
     local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
                        "$(cat "$version_file")")
     local license_arg="$license_path=$railsdir/$(basename "$license_path")"
-    local -a switches=(--iteration=7
+    local -a switches=(--iteration=$RAILS_PACKAGE_ITERATION
                        --after-install "$scripts_dir/postinst"
                        --before-remove "$scripts_dir/prerm"
                        --after-remove "$scripts_dir/postrm")
@@ -225,7 +303,7 @@ fpm_build () {
       return 0
   fi
 
-  local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")"
+  local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
 
   case "$PACKAGE_TYPE" in
       python)
@@ -239,8 +317,6 @@ fpm_build () {
               --python-install-data . \
               --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
               --depends "$PYTHON2_PACKAGE"
-          # Fix --iteration for #9242.
-          default_iteration_value=$(($default_iteration_value + 1))
           ;;
       python3)
           # fpm does not actually support a python3 package type.  Instead
@@ -256,8 +332,6 @@ fpm_build () {
               --python-install-data . \
               --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
               --depends "$PYTHON3_PACKAGE"
-          # Fix --iteration for #9242.
-          default_iteration_value=$(($default_iteration_value + 1))
           ;;
   esac
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list