[ARVADOS] updated: 35abd530f3fbcc1893d7b08fecafbc5211e24f88
Git user
git at public.curoverse.com
Tue May 31 16:36:51 EDT 2016
Summary of changes:
build/package-build-dockerfiles/Makefile | 8 +-
build/package-build-dockerfiles/centos6/Dockerfile | 2 +-
build/package-build-dockerfiles/debian7/Dockerfile | 2 +-
build/package-build-dockerfiles/debian8/Dockerfile | 2 +-
.../ubuntu1204/Dockerfile | 2 +-
.../ubuntu1404/Dockerfile | 2 +-
build/run-build-packages-one-target.sh | 3 +-
build/run-build-packages.sh | 9 +-
build/run-tests.sh | 8 +-
sdk/go/arvados/client.go | 169 ++++++
sdk/go/arvados/client_test.go | 83 +++
sdk/go/arvados/collection.go | 62 ++
sdk/go/arvados/doc.go | 12 +
sdk/go/arvados/duration.go | 31 +
sdk/go/arvados/keep_block.go | 15 +
sdk/go/arvados/keep_service.go | 123 ++++
sdk/go/arvados/resource_list.go | 25 +
sdk/go/arvados/resource_list_test.go | 21 +
sdk/go/arvados/user.go | 17 +
services/keep-balance/balance.go | 638 +++++++++++++++++++++
services/keep-balance/balance_run_test.go | 374 ++++++++++++
services/keep-balance/balance_test.go | 255 ++++++++
services/keep-balance/block_state.go | 95 +++
services/keep-balance/change_set.go | 75 +++
services/keep-balance/change_set_test.go | 35 ++
services/keep-balance/collection.go | 95 +++
services/keep-balance/integration_test.go | 92 +++
services/keep-balance/keep_service.go | 76 +++
services/keep-balance/main.go | 156 +++++
services/keep-balance/main_test.go | 43 ++
services/keep-balance/time_me.go | 14 +
services/keep-balance/usage.go | 83 +++
services/keepstore/keepstore.go | 7 +-
33 files changed, 2621 insertions(+), 13 deletions(-)
create mode 100644 sdk/go/arvados/client.go
create mode 100644 sdk/go/arvados/client_test.go
create mode 100644 sdk/go/arvados/collection.go
create mode 100644 sdk/go/arvados/doc.go
create mode 100644 sdk/go/arvados/duration.go
create mode 100644 sdk/go/arvados/keep_block.go
create mode 100644 sdk/go/arvados/keep_service.go
create mode 100644 sdk/go/arvados/resource_list.go
create mode 100644 sdk/go/arvados/resource_list_test.go
create mode 100644 sdk/go/arvados/user.go
create mode 100644 services/keep-balance/balance.go
create mode 100644 services/keep-balance/balance_run_test.go
create mode 100644 services/keep-balance/balance_test.go
create mode 100644 services/keep-balance/block_state.go
create mode 100644 services/keep-balance/change_set.go
create mode 100644 services/keep-balance/change_set_test.go
create mode 100644 services/keep-balance/collection.go
create mode 100644 services/keep-balance/integration_test.go
create mode 100644 services/keep-balance/keep_service.go
create mode 100644 services/keep-balance/main.go
create mode 100644 services/keep-balance/main_test.go
create mode 100644 services/keep-balance/time_me.go
create mode 100644 services/keep-balance/usage.go
discards 046a1aa4efc2a60fcc1304aeaf843cde5a6b6516 (commit)
discards 59082eebcc258a864662a8714a065e5879af9f3e (commit)
via 35abd530f3fbcc1893d7b08fecafbc5211e24f88 (commit)
via 8bfc966f5c2715a169e88448e470285f35c1e477 (commit)
via 55919d0afc1e87a79700dc4bd6808b870c40bcdc (commit)
via c900f416c36cd74675c5bf4c33ad1dbe5d1e78fa (commit)
via 27f948d88bbdd5be848011871d2b592d7057ece1 (commit)
via b002129afda08bbb4fdbed6e629858a5c298c068 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (046a1aa4efc2a60fcc1304aeaf843cde5a6b6516)
\
N -- N -- N (35abd530f3fbcc1893d7b08fecafbc5211e24f88)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 35abd530f3fbcc1893d7b08fecafbc5211e24f88
Author: Brett Smith <brett at curoverse.com>
Date: Tue May 31 16:35:53 2016 -0400
9242: Update Python module paths for CentOS 6.
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 34b1b1f..389936a 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -139,11 +139,11 @@ case "$TARGET" in
PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
PYTHON2_PREFIX=/opt/rh/python27/root/usr
- PYTHON2_INSTALL_LIB=lib64/python$PYTHON2_VERSION
+ PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
PYTHON3_PREFIX=/opt/rh/python33/root/usr
- PYTHON3_INSTALL_LIB=lib64/python$PYTHON3_VERSION
+ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
commit 8bfc966f5c2715a169e88448e470285f35c1e477
Author: Brett Smith <brett at curoverse.com>
Date: Fri May 27 18:34:56 2016 -0400
9309: Add packages and tests for CentOS 7.
diff --git a/backports/python-apache-libcloud/fpm-info.sh b/backports/python-apache-libcloud/fpm-info.sh
new file mode 100644
index 0000000..0333bdf
--- /dev/null
+++ b/backports/python-apache-libcloud/fpm-info.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+case $TARGET in
+ centos7)
+ # fpm incorrectly transforms the dependency name in this case.
+ fpm_depends+=(python-backports-ssl_match_hostname)
+ fpm_args+=(--python-disable-dependency backports.ssl-match-hostname)
+ ;;
+esac
diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 3482886..2180b87 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -1,9 +1,13 @@
-all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
+all: centos6/generated centos7/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
centos6/generated: common-generated-all
test -d centos6/generated || mkdir centos6/generated
cp -rlt centos6/generated common-generated/*
+centos7/generated: common-generated-all
+ test -d centos7/generated || mkdir centos7/generated
+ cp -rlt centos7/generated common-generated/*
+
debian7/generated: common-generated-all
test -d debian7/generated || mkdir debian7/generated
cp -rlt debian7/generated common-generated/*
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
new file mode 100644
index 0000000..a37c97f
--- /dev/null
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -0,0 +1,25 @@
+FROM centos:7
+MAINTAINER Brett Smith <brett at curoverse.com>
+
+# Install build dependencies provided in base distribution
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python33
+RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
+
+ENV WORKSPACE /arvados
+CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile
new file mode 100644
index 0000000..6bc40bf
--- /dev/null
+++ b/build/package-test-dockerfiles/centos7/Dockerfile
@@ -0,0 +1,15 @@
+FROM centos:7
+MAINTAINER Brett Smith <brett at curoverse.com>
+
+RUN yum -q -y install scl-utils centos-release-scl which tar
+
+# Install RVM
+RUN touch /var/lib/rpm/* && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundle && \
+ /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
diff --git a/build/package-test-dockerfiles/centos7/localrepo.repo b/build/package-test-dockerfiles/centos7/localrepo.repo
new file mode 100644
index 0000000..ebb8765
--- /dev/null
+++ b/build/package-test-dockerfiles/centos7/localrepo.repo
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos7
+gpgcheck=0
+enabled=1
diff --git a/build/package-testing/rpm-common-test-packages.sh b/build/package-testing/rpm-common-test-packages.sh
new file mode 100755
index 0000000..4d0c32b
--- /dev/null
+++ b/build/package-testing/rpm-common-test-packages.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eu
+
+target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b')
+
+yum -q clean all
+touch /var/lib/rpm/*
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
+
+yum install --assumeyes $1
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
+
+diff "$ARV_PACKAGES_DIR/$1".{before,after} >"$ARV_PACKAGES_DIR/$1.diff" || true
+
+# Enable any Software Collections that the package depended on.
+if [[ -d /opt/rh ]]; then
+ # We have to stage the list to a file, because `ls | while read` would
+ # make a subshell, causing the `source` lines to have no effect.
+ scl_list=$(mktemp)
+ ls /opt/rh >"$scl_list"
+
+ # SCL scripts aren't designed to run with -eu.
+ set +eu
+ while read scl; do
+ source scl_source enable "$scl"
+ done <"$scl_list"
+ set -eu
+ rm "$scl_list"
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
+
+find -name '*.so' | while read so; do
+ echo -e "\n== Packages dependencies for $so =="
+ ldd "$so" \
+ | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+done
+
+exec /jenkins/package-testing/common-test-packages.sh "$1"
diff --git a/build/package-testing/test-package-arvados-api-server.sh b/build/package-testing/test-package-arvados-api-server.sh
index e975448..66d5969 100755
--- a/build/package-testing/test-package-arvados-api-server.sh
+++ b/build/package-testing/test-package-arvados-api-server.sh
@@ -7,7 +7,7 @@ case "$TARGET" in
apt-get install -y nginx
dpkg-reconfigure arvados-api-server
;;
- centos6)
+ centos*)
yum install --assumeyes httpd
yum reinstall --assumeyes arvados-api-server
;;
diff --git a/build/package-testing/test-package-arvados-sso-server.sh b/build/package-testing/test-package-arvados-sso-server.sh
index c1a377e..ab0b225 100755
--- a/build/package-testing/test-package-arvados-sso-server.sh
+++ b/build/package-testing/test-package-arvados-sso-server.sh
@@ -18,7 +18,7 @@ case "$TARGET" in
debian*|ubuntu*)
FORMAT=deb
;;
- centos6)
+ centos*)
FORMAT=rpm
;;
*)
@@ -75,12 +75,9 @@ fi
if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then
# We need to set up our database configuration now.
if [[ "$FORMAT" == "rpm" ]]; then
- # postgres packaging on CentOS6 is kind of primitive, needs an initdb
service postgresql initdb
- if [ "$TARGET" = "centos6" ]; then
- sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
- sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
- fi
+ sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
+ sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
fi
service postgresql start
diff --git a/build/package-testing/test-package-arvados-workbench.sh b/build/package-testing/test-package-arvados-workbench.sh
index 1be4dea..5deb1a0 100755
--- a/build/package-testing/test-package-arvados-workbench.sh
+++ b/build/package-testing/test-package-arvados-workbench.sh
@@ -7,7 +7,7 @@ case "$TARGET" in
apt-get install -y nginx
dpkg-reconfigure arvados-workbench
;;
- centos6)
+ centos*)
yum install --assumeyes httpd
yum reinstall --assumeyes arvados-workbench
;;
diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh
deleted file mode 100755
index 4e05364..0000000
--- a/build/package-testing/test-packages-centos6.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-yum -q clean all
-touch /var/lib/rpm/*
-
-export ARV_PACKAGES_DIR=/arvados/packages/centos6
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
-
-yum install --assumeyes $1
-
-rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
-
-set +e
-diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff"
-set -e
-
-SCL=""
-if scl enable python27 true 2>/dev/null ; then
- SCL="scl enable python27"
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
-
-shared=$(find -name '*.so')
-if test -n "$shared" ; then
- for so in $shared ; do
- echo
- echo "== Packages dependencies for $so =="
- $SCL ldd "$so" \
- | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
- done
-fi
-
-if test -n "$SCL" ; then
- exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'"
-else
- exec /jenkins/package-testing/common-test-packages.sh "$1"
-fi
diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh
new file mode 120000
index 0000000..64ef604
--- /dev/null
+++ b/build/package-testing/test-packages-centos6.sh
@@ -0,0 +1 @@
+rpm-common-test-packages.sh
\ No newline at end of file
diff --git a/build/package-testing/test-packages-centos7.sh b/build/package-testing/test-packages-centos7.sh
new file mode 120000
index 0000000..64ef604
--- /dev/null
+++ b/build/package-testing/test-packages-centos7.sh
@@ -0,0 +1 @@
+rpm-common-test-packages.sh
\ No newline at end of file
diff --git a/build/run-build-packages-sso.sh b/build/run-build-packages-sso.sh
index cc673a6..b5dcdfc 100755
--- a/build/run-build-packages-sso.sh
+++ b/build/run-build-packages-sso.sh
@@ -80,7 +80,7 @@ case "$TARGET" in
ubuntu1404)
FORMAT=deb
;;
- centos6)
+ centos6|centos7)
FORMAT=rpm
;;
*)
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 5490562..34b1b1f 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -155,6 +155,25 @@ case "$TARGET" in
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
export PYCURL_SSL_LIBRARY=nss
;;
+ centos7)
+ FORMAT=rpm
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON3_PREFIX=/opt/rh/python33/root/usr
+ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
+ PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka pyexecjs jsonschema \
+ ciso8601 pycrypto 'pycurl<7.21.5' \
+ python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
+ 'rdflib>=4.2.0' shellescape mistune typing avro \
+ isodate pyparsing sparqlwrapper html5lib keepalive)
+ PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+ export PYCURL_SSL_LIBRARY=nss
+ ;;
*)
echo -e "$0: Unknown target '$TARGET'.\n" >&2
exit 1
commit 55919d0afc1e87a79700dc4bd6808b870c40bcdc
Author: Brett Smith <brett at curoverse.com>
Date: Fri May 27 17:50:40 2016 -0400
9309: Look for fpm-info in backports/$PACKAGE_NAME.
This lets us define additional fpm flags when we build a non-dir
package from a source directory.
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7631718..5490562 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -432,17 +432,17 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover
# So we build this thing separately.
#
# Ward, 2016-03-17
-fpm_build schema_salad schema_salad "" python 1.7.20160316203940
+fpm_build schema_salad "" "" python 1.7.20160316203940
# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
# Ward, 2016-05-26
-fpm_build ruamel.yaml ruamel.yaml "" python "" --python-setup-py-arguments "--single-version-externally-managed"
+fpm_build ruamel.yaml "" "" python "" --python-setup-py-arguments "--single-version-externally-managed"
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-fpm_build cwltool cwltool "" python 1.0.20160427142240
+fpm_build cwltool "" "" python 1.0.20160427142240
# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm_build rdflib-jsonld rdflib-jsonld "" python 0.3.0
+fpm_build rdflib-jsonld "" "" python 0.3.0
# The PAM module
if [[ $TARGET =~ debian|ubuntu ]]; then
diff --git a/build/run-library.sh b/build/run-library.sh
index c7bc9538..58a177e 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -178,8 +178,8 @@ fpm_build () {
# pip).
PACKAGE=$1
shift
- # The name of the package to build. Defaults to $PACKAGE.
- PACKAGE_NAME=${1:-$PACKAGE}
+ # The name of the package to build.
+ PACKAGE_NAME=$1
shift
# Optional: the vendor of the package. Should be "Curoverse, Inc." for
# packages of our own software. Passed to fpm --vendor.
@@ -238,7 +238,7 @@ fpm_build () {
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
+ if [[ -n "$PACKAGE_NAME" ]]; then
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fi
@@ -265,6 +265,9 @@ fpm_build () {
"${PACKAGE%%=/*}"
# backports ("llfuse==0.41.1" => "backports/python-llfuse")
"${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
+ if [[ -n "$PACKAGE_NAME" ]]; then
+ fpm_dirs+=("${WORKSPACE}/backports/${PACKAGE_NAME}")
+ fi
for pkgdir in "${fpm_dirs[@]}"; do
fpminfo="$pkgdir/fpm-info.sh"
if [[ -e "$fpminfo" ]]; then
@@ -391,4 +394,3 @@ report_outcomes() {
done
fi
}
-
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list