[ARVADOS] updated: 47adcb535cff8f6121f8f55b2121973cbc7b0480
Git user
git at public.curoverse.com
Fri May 20 15:51:42 EDT 2016
Summary of changes:
backports/python-ciso8601/fpm-info.sh | 2 +-
backports/python-llfuse/fpm-info.sh | 2 +-
backports/python-pycrypto/fpm-info.sh | 2 +-
backports/python-pycurl/fpm-info.sh | 2 +-
build/run-build-packages.sh | 12 ++-
build/run-library.sh | 24 +++--
crunch_scripts/cwl-runner | 16 +++-
sdk/cwl/arvados_cwl/__init__.py | 124 ++++++++++++++++----------
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_job.py | 11 ++-
sdk/cwl/tests/test_submit.py | 12 +--
sdk/cwl/tests/tool/submit_tool.cwl | 1 +
sdk/cwl/tests/wf/inputs_test.cwl | 1 +
sdk/cwl/tests/wf/submit_wf.cwl | 1 +
services/datamanager/collection/collection.go | 14 +--
15 files changed, 141 insertions(+), 85 deletions(-)
discards b4dd34d1fedfc2fc85869962d4ea4da446076adb (commit)
discards a9c85879f8dba8aa6bd417cb5bdc60fd888a822e (commit)
via 47adcb535cff8f6121f8f55b2121973cbc7b0480 (commit)
via 6e03a31b8d4bc8990fd61d25e64a143b5fd18dcc (commit)
via cffa542e4cb069b40764c889613c249e88af62d0 (commit)
via 7a430017d745cab4458aa03f620e7925a50b7d06 (commit)
via 08cc91f0493dd8dfe27046faf02c2c907e50443e (commit)
via 3c8c81609c8e9b3b422477002e8a348aabafb3b9 (commit)
via ece22330313613cf4ce8f2b0bd6db4f00bdd08fc (commit)
via e0fac5213f40fc00946f8ec5e4df42bebdf756d2 (commit)
via 5011827dbbe6b17606bf83338b99dd53789ae9cc (commit)
via 0bce01f575822851da09f252b38d7009ed7a5ab0 (commit)
via df8d0badeb97036564a80e89569c8e62ede22163 (commit)
via 1cde74c26fd6a7793a7bb1a9bd748ce06e37c455 (commit)
via 53deed9cb00b94a69a3be1f959cbfda342646718 (commit)
via c8d9a898cde654b53200bda0b0ef8b406dd71739 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (b4dd34d1fedfc2fc85869962d4ea4da446076adb)
\
N -- N -- N (47adcb535cff8f6121f8f55b2121973cbc7b0480)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 47adcb535cff8f6121f8f55b2121973cbc7b0480
Author: Brett Smith <brett at curoverse.com>
Date: Fri May 20 11:10:41 2016 -0400
9242: Python packages install libraries to the distro path.
This avoids breaking dependent packages that expect to find files in
the same place.
diff --git a/backports/python-ciso8601/fpm-info.sh b/backports/python-ciso8601/fpm-info.sh
index 42f31e3..c9665be 100644
--- a/backports/python-ciso8601/fpm-info.sh
+++ b/backports/python-ciso8601/fpm-info.sh
@@ -8,4 +8,4 @@ case "$TARGET" in
esac
# FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)
diff --git a/backports/python-llfuse/fpm-info.sh b/backports/python-llfuse/fpm-info.sh
index a7d9398..9fc0098 100644
--- a/backports/python-llfuse/fpm-info.sh
+++ b/backports/python-llfuse/fpm-info.sh
@@ -10,4 +10,4 @@ case "$TARGET" in
esac
# FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)
diff --git a/backports/python-pycrypto/fpm-info.sh b/backports/python-pycrypto/fpm-info.sh
index 52df465..65a83a0 100644
--- a/backports/python-pycrypto/fpm-info.sh
+++ b/backports/python-pycrypto/fpm-info.sh
@@ -11,4 +11,4 @@ case "$TARGET" in
esac
# FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)
diff --git a/backports/python-pycurl/fpm-info.sh b/backports/python-pycurl/fpm-info.sh
index e4fbf59..5bfb010 100644
--- a/backports/python-pycurl/fpm-info.sh
+++ b/backports/python-pycurl/fpm-info.sh
@@ -1,5 +1,5 @@
# FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)
case "$TARGET" in
centos6)
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index c1dcbeb..5b4efb0 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -86,9 +86,13 @@ PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=s
# You can customize them as needed in distro sections below.
PYTHON2_PACKAGE=python$PYTHON2_VERSION
PYTHON2_PKG_PREFIX=python
+PYTHON2_PREFIX=/usr
+PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/dist-packages
PYTHON3_PACKAGE=python$PYTHON3_VERSION
PYTHON3_PKG_PREFIX=python3
+PYTHON3_PREFIX=/usr
+PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
## End Debian Python defaults.
case "$TARGET" in
@@ -134,8 +138,12 @@ case "$TARGET" in
FORMAT=rpm
PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON2_PREFIX=/opt/rh/python27/root/usr
+ PYTHON2_INSTALL_LIB=lib64/python$PYTHON2_VERSION
PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON3_PREFIX=/opt/rh/python33/root/usr
+ PYTHON3_INSTALL_LIB=lib64/python$PYTHON3_VERSION
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
@@ -404,7 +412,7 @@ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'C
# cwl-runner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 2
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3
# schema_salad. This is a python dependency of arvados-cwl-runner,
# but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -421,13 +429,13 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover
# So we build this thing separately.
#
# Ward, 2016-03-17
-fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-schema-salad" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.7.20160316203940 schema_salad
+fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-schema-salad" --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.7.20160316203940 schema_salad
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-cwltool" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.0.20160427142240 cwltool
+fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-cwltool" --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.0.20160427142240 cwltool
# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --verbose --log info -n "${PYTHON2_PKG_PREFIX}-rdflib-jsonld" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 0.3.0 rdflib-jsonld
+fpm --maintainer='Ward Vandewege <ward at curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --verbose --log info -n "${PYTHON2_PKG_PREFIX}-rdflib-jsonld" --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 0.3.0 rdflib-jsonld
# The PAM module
if [[ $TARGET =~ debian|ubuntu ]]; then
@@ -501,9 +509,7 @@ for deppkg in "${PYTHON_BACKPORTS[@]}"; do
"python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
chmod -R go+rX .
set +e
- # --iteration 2 provides an upgrade for previously built
- # buggy packages.
- fpm_build . "$outname" "" python "" --iteration 2
+ fpm_build . "$outname" "" python "" --iteration 3
# The upload step uses the package timestamp to determine
# whether it's new. --no-clobber plays nice with that.
mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
diff --git a/build/run-library.sh b/build/run-library.sh
index 8d97ada..c7bc9538 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -192,6 +192,8 @@ fpm_build () {
VERSION=$1
shift
+ local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")"
+
case "$PACKAGE_TYPE" in
python)
# All Arvados Python2 packages depend on Python 2.7.
@@ -199,7 +201,12 @@ fpm_build () {
set -- "$@" --python-bin python2.7 \
--python-easyinstall "$EASY_INSTALL2" \
--python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --prefix "$PYTHON2_PREFIX" \
+ --python-install-lib "$PYTHON2_INSTALL_LIB" \
+ --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
--depends "$PYTHON2_PACKAGE"
+ # Fix --iteration for #9242.
+ default_iteration_value=$(($default_iteration_value + 1))
;;
python3)
# fpm does not actually support a python3 package type. Instead
@@ -210,19 +217,21 @@ fpm_build () {
set -- "$@" --python-bin python3 \
--python-easyinstall "$EASY_INSTALL3" \
--python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --prefix "$PYTHON3_PREFIX" \
+ --python-install-lib "$PYTHON3_INSTALL_LIB" \
+ --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
--depends "$PYTHON3_PACKAGE"
+ # Fix --iteration for #9242.
+ default_iteration_value=$(($default_iteration_value + 1))
;;
esac
declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward at curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
- if [ python = "$PACKAGE_TYPE" ]; then
- COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
- if [ deb = "$FORMAT" ]; then
- # Dependencies are built from setup.py. Since setup.py will never
- # refer to Debian package iterations, it doesn't make sense to
- # enforce those in the .deb dependencies.
- COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
- fi
+ if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
+ # Dependencies are built from setup.py. Since setup.py will never
+ # refer to Debian package iterations, it doesn't make sense to
+ # enforce those in the .deb dependencies.
+ COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
fi
if [[ "${DEBUG:-0}" != "0" ]]; then
@@ -242,7 +251,7 @@ fpm_build () {
fi
# We can always add an --iteration here. If another one is specified in $@,
# that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")")
+ COMMAND_ARR+=(--iteration "$default_iteration_value")
# Append --depends X and other arguments specified by fpm-info.sh in
# the package source dir. These are added last so they can override
commit 6e03a31b8d4bc8990fd61d25e64a143b5fd18dcc
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 19 15:41:16 2016 -0400
9242: Refactor Python constant definitions in r-b-p.
There are about to be more of them, which will make this a real space
savings.
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index c16141a..c1dcbeb 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -82,13 +82,18 @@ declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
PYTHON2_VERSION=2.7
PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+## These defaults are suitable for any Debian-based distribution.
+# You can customize them as needed in distro sections below.
+PYTHON2_PACKAGE=python$PYTHON2_VERSION
+PYTHON2_PKG_PREFIX=python
+
+PYTHON3_PACKAGE=python$PYTHON3_VERSION
+PYTHON3_PKG_PREFIX=python3
+## End Debian Python defaults.
+
case "$TARGET" in
debian7)
FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
@@ -99,10 +104,6 @@ case "$TARGET" in
;;
debian8)
FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
@@ -113,10 +114,6 @@ case "$TARGET" in
;;
ubuntu1204)
FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
@@ -127,10 +124,6 @@ case "$TARGET" in
;;
ubuntu1404)
FORMAT=deb
- PYTHON2_PACKAGE=python$PYTHON2_VERSION
- PYTHON2_PKG_PREFIX=python
- PYTHON3_PACKAGE=python$PYTHON3_VERSION
- PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list