[ARVADOS] created: c90a8dac53ef90927d5992a5d082901854ca806b
Git user
git at public.curoverse.com
Tue Jan 31 16:23:47 EST 2017
at c90a8dac53ef90927d5992a5d082901854ca806b (commit)
commit c90a8dac53ef90927d5992a5d082901854ca806b
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Dec 21 09:56:52 2016 -0500
5439: Run tests in a publishable docker image
diff --git a/apps/workbench/test/controllers/search_controller_test.rb b/apps/workbench/test/controllers/search_controller_test.rb
index a09d966..f5e8d64 100644
--- a/apps/workbench/test/controllers/search_controller_test.rb
+++ b/apps/workbench/test/controllers/search_controller_test.rb
@@ -13,7 +13,7 @@ class SearchControllerTest < ActionController::TestCase
format: :js,
title: 'Search',
action_name: 'Show',
- action_href: url_for(host: 'localhost', controller: :actions, action: :show),
+ action_href: url_for(host: '0.0.0.0', controller: :actions, action: :show),
action_data: {}.to_json,
}, session_for(:active)
assert_response :success
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
index a6a85b5..cf6850f 100644
--- a/apps/workbench/test/integration/repositories_browse_test.rb
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -42,7 +42,7 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
gitsslport = File.read(portfile)
Repository.any_instance.
stubs(:http_fetch_url).
- returns "https://localhost:#{gitsslport}/#{repo['name']}.git"
+ returns "https://0.0.0.0:#{gitsslport}/#{repo['name']}.git"
commit_sha1 = '1de84a854e2b440dc53bf42f8548afa4c17da332'
visit page_with_token('active', "/repositories/#{repo['uuid']}/commit/#{commit_sha1}")
assert_text "Date: Tue Mar 18 15:55:28 2014 -0400"
diff --git a/apps/workbench/test/integration_helper.rb b/apps/workbench/test/integration_helper.rb
index 3d92585..fd79542 100644
--- a/apps/workbench/test/integration_helper.rb
+++ b/apps/workbench/test/integration_helper.rb
@@ -157,8 +157,8 @@ module KeepWebConfig
def use_keep_web_config
@kwport = getport 'keep-web-ssl'
@kwdport = getport 'keep-web-dl-ssl'
- Rails.configuration.keep_web_url = "https://localhost:#{@kwport}/c=%{uuid_or_pdh}"
- Rails.configuration.keep_web_download_url = "https://localhost:#{@kwdport}/c=%{uuid_or_pdh}"
+ Rails.configuration.keep_web_url = "https://0.0.0.0:#{@kwport}/c=%{uuid_or_pdh}"
+ Rails.configuration.keep_web_download_url = "https://0.0.0.0:#{@kwdport}/c=%{uuid_or_pdh}"
CollectionsController.any_instance.expects(:file_enumerator).never
end
end
diff --git a/build/builder/.gitignore b/build/builder/.gitignore
new file mode 100644
index 0000000..7d52e05
--- /dev/null
+++ b/build/builder/.gitignore
@@ -0,0 +1 @@
+Dockerfile.generated
diff --git a/build/builder/Dockerfile.append b/build/builder/Dockerfile.append
new file mode 100644
index 0000000..b1697ed
--- /dev/null
+++ b/build/builder/Dockerfile.append
@@ -0,0 +1,40 @@
+USER root
+ENV LANG en_US.utf8
+RUN echo ${LANG} UTF-8 | tee -a /etc/locale.gen && locale-gen -a
+RUN pg_dropcluster --stop 9.4 main; pg_createcluster --start 9.4 main
+ADD pg_boot /usr/local/sbin/
+ADD pg_shutdown /usr/local/sbin/
+RUN pg_boot && sudo -u postgres psql -c "create user arvados with createdb encrypted password 'xxxxxxxx'" && service postgresql stop
+ADD sudo_sv_updown /etc/sudoers.d/
+
+RUN echo user_allow_other > /etc/fuse.conf
+
+RUN addgroup --system fuse && adduser --gecos '' --disabled-password builder && adduser builder fuse
+RUN mkdir /workspace /src && chown builder:builder /workspace /src
+
+# checkout-and-build uses rsync
+RUN ${APT_GET_INSTALL} rsync && apt-get clean
+
+# Otherwise "bundle package" dies trying to read cached gems:
+RUN chmod -R a+rX /usr/local/lib/ruby/gems/2.3.0/cache
+
+# Otherwise sinfo waits 60 seconds for it to appear:
+RUN touch /etc/slurm-llnl/slurm.conf
+
+USER builder
+ENV PATH $PATH:/home/builder/.gem/ruby/2.3.0/bin
+RUN gem install --user bundler
+RUN cd /src && git clone --no-checkout git://git.curoverse.com/arvados.git && cd arvados && git gc
+WORKDIR /src/arvados
+
+USER root
+ADD checkout-and-test /usr/local/bin/
+
+USER builder
+# Seed ~/.gem with gem dependencies
+RUN checkout-and-test master --only install && sudo pg_shutdown
+
+# services/api tests assume tmp/pids/ already exists. Fix services/api and remove this.
+RUN mkdir -p /workspace/services/api/tmp/pids
+
+CMD TMPDIR=/tmp/arvados-build/tmp checkout-and-test
diff --git a/build/builder/Makefile b/build/builder/Makefile
new file mode 100644
index 0000000..d1f863c
--- /dev/null
+++ b/build/builder/Makefile
@@ -0,0 +1,4 @@
+arvados/builder:
+ cat ../../tools/arvbox/lib/arvbox/docker/Dockerfile.base | perl -ne 'print unless /VOLUME|var.lib.gems/; exit if /ARG arvados_version/;' > Dockerfile.generated
+ cat Dockerfile.append >>Dockerfile.generated
+ docker build --tag=arvados/builder --file=Dockerfile.generated .
diff --git a/build/builder/README.md b/build/builder/README.md
new file mode 100644
index 0000000..e6e3aef
--- /dev/null
+++ b/build/builder/README.md
@@ -0,0 +1,63 @@
+# purpose
+
+The arvados/builder image runs tests for a specified (recent or near future) version of arvados.
+
+The image comes with most of the dependencies already installed, in order to reduce per-build time and network traffic.
+
+(Most Ruby and Python dependencies can be updated at build time as needed. In that sense it's better, but usually not necessary, to keep builder up to date.)
+
+# building
+
+Make a docker image called "arvados/builder" suitable for using locally or pushing to dockerhub.
+
+```
+make builder
+```
+
+Build dependencies change over time, so sometimes it is necessary to build a new (or old) `arvados/builder` image that is compatible with the version of arvados you want to build.
+
+Generate a new image:
+
+```
+git checkout master
+make builder
+```
+
+Confirm the new image passes tests using its bundled version:
+
+```
+docker run -it arvados/builder
+```
+
+# using
+
+Run the test suites using the source tree baked into the builder image itself (which might be out of date):
+
+```
+docker run -it arvados/builder
+```
+
+Run the test suites using a specific git commit (first fetching the latest commits from git.curoverse.com _unless_ the given commit is a full sha1 and is already present in the baked-in git history):
+
+```
+docker run -it arvados/builder master
+```
+
+# developer hacks
+
+The focus of arvados/builder is running test suites in a well-defined environment in order to establish that a sha1-addressable version passes its tests.
+
+There are some opportunities to use the builder image in other ways as part of your development cycle, though.
+
+Run the test suites using a source tree on the host (which might have uncommitted local changes):
+
+```
+docker run -it -v /PATH/TO/LOCAL/ARVADOS:/src/arvados:ro arvados/builder
+```
+
+Add `-v /PATH/TO/LOCAL/CACHE:/tmp` to preserve installed bits between runs. For example, to test whether your `services/api` changes affect `services/arv-git-httpd` integration tests:
+
+```
+docker run -it -v /PATH/TO/LOCAL/CACHE:/tmp arvados/builder -v /HOST/SRC/ARVADOS:/src/arvados:ro "" --only install --only-install services/api
+docker run -it -v /PATH/TO/LOCAL/CACHE:/tmp arvados/builder -v /HOST/SRC/ARVADOS:/src/arvados:ro "" --skip-install --only services/arv-git-httpd
+```
diff --git a/build/builder/checkout-and-test b/build/builder/checkout-and-test
new file mode 100755
index 0000000..82b9713
--- /dev/null
+++ b/build/builder/checkout-and-test
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+sudo pg_boot
+if [[ -n "$WITH_DOCKER" ]]; then
+ sudo service docker start
+fi
+
+SRC=/src/arvados
+export WORKSPACE="/workspace"
+export TMPDIR=/tmp
+
+mkdir -p "$WORKSPACE"
+if [[ -e "$SRC/.gitignore" ]]; then
+ exclude_gitignore=(--exclude-from "$SRC/.gitignore")
+else
+ exclude_gitignore=()
+fi
+
+rsync -a --exclude packages --exclude cache --exclude tmp --exclude services/api/log --exclude apps/workbench/log --exclude 'sdk/*/Gemfile.lock' "${exclude_gitignore[@]}" "$SRC/" "$WORKSPACE/"
+
+cd "$WORKSPACE"
+ref="$1"; shift || true
+if [[ -n "$ref" ]]; then
+ if [[ "${#ref}" != 40 ]] || ! git checkout "$ref"; then
+ git checkout --detach HEAD
+ git fetch origin '+*:*'
+ git checkout "$ref"
+ fi
+fi
+
+cp services/api/config/database.yml{.example,}
+build/run-tests.sh --temp "$TMPDIR" "$@"
diff --git a/build/builder/pg_boot b/build/builder/pg_boot
new file mode 100755
index 0000000..613da53
--- /dev/null
+++ b/build/builder/pg_boot
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+service postgresql start
+echo -n waiting for postgresql to start accepting connections: >&2
+while ! pg_isready >/dev/null; do
+ echo -n . >&2
+ sleep 0.2
+done
+echo " ready." >&2
diff --git a/build/builder/pg_shutdown b/build/builder/pg_shutdown
new file mode 100755
index 0000000..5dad4c8
--- /dev/null
+++ b/build/builder/pg_shutdown
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+service postgresql stop
diff --git a/build/builder/sudo_sv_updown b/build/builder/sudo_sv_updown
new file mode 100644
index 0000000..435d15a
--- /dev/null
+++ b/build/builder/sudo_sv_updown
@@ -0,0 +1,5 @@
+Defaults lecture=never
+builder ALL=(ALL:ALL) NOPASSWD: /usr/local/sbin/pg_boot
+builder ALL=(ALL:ALL) NOPASSWD: /usr/local/sbin/pg_shutdown
+builder ALL=(ALL:ALL) NOPASSWD: /usr/sbin/service docker start
+builder ALL=(ALL:ALL) NOPASSWD: /usr/sbin/service docker stop
diff --git a/build/run-tests.sh b/build/run-tests.sh
index e0e1ce2..79df561 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -814,14 +814,14 @@ test_apiserver() {
}
do_test services/api apiserver
-# Shortcut for when we're only running apiserver tests. This saves a bit of time,
-# because we don't need to start up the api server for subsequent tests.
-if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
- rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
- exit_cleanly
-fi
-
-start_api
+# Avoid starting up the api server if there are no integration tests to do.
+case "$only" in
+ install|services/api)
+ ;;
+ *)
+ start_api
+ ;;
+esac
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
diff --git a/sdk/cli/arvados-cli.gemspec b/sdk/cli/arvados-cli.gemspec
index 0e11f96..0eeee57 100644
--- a/sdk/cli/arvados-cli.gemspec
+++ b/sdk/cli/arvados-cli.gemspec
@@ -1,4 +1,4 @@
-if not File.exists?('/usr/bin/git') then
+if not File.exist?('/usr/bin/git') then
STDERR.puts "\nGit binary not found, aborting. Please install git and run gem build from a checked out copy of the git repository.\n\n"
exit
end
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
s.authors = ["Arvados Authors"]
s.email = 'gem-dev at curoverse.com'
#s.bindir = '.'
- s.licenses = ['Apache License, Version 2.0']
+ s.licenses = ['Apache-2.0']
s.files = ["bin/arv", "bin/arv-run-pipeline-instance",
"bin/arv-crunch-job", "bin/arv-tag", "bin/crunch-job",
"LICENSE-2.0.txt"]
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 0066040..ff75d90 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -3,6 +3,12 @@ error_log stderr info; # Yes, must be specified here _and_ cmdline
events {
}
http {
+ client_body_temp_path {{TMPDIR}}/nginx_body;
+ proxy_temp_path {{TMPDIR}}/nginx_proxy_temp;
+ proxy_cache_path {{TMPDIR}}/nginx_proxy_cache keys_zone=cache:1m;
+ fastcgi_temp_path {{TMPDIR}}/nginx_fastcgi_temp;
+ uwsgi_temp_path {{TMPDIR}}/nginx_uwsgi_temp;
+ scgi_temp_path {{TMPDIR}}/nginx_scgi_temp;
access_log {{ACCESSLOG}} combined;
upstream arv-git-http {
server localhost:{{GITPORT}};
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index b969b12..6e3340d 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -602,6 +602,7 @@ def run_nginx():
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
+ nginxconf['TMPDIR'] = TEST_TMPDIR
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
diff --git a/sdk/ruby/arvados.gemspec b/sdk/ruby/arvados.gemspec
index 2c4e60e..84e24f4 100644
--- a/sdk/ruby/arvados.gemspec
+++ b/sdk/ruby/arvados.gemspec
@@ -1,4 +1,4 @@
-if not File.exists?('/usr/bin/git') then
+if not File.exist?('/usr/bin/git') then
STDERR.puts "\nGit binary not found, aborting. Please install git and run gem build from a checked out copy of the git repository.\n\n"
exit
end
@@ -14,7 +14,7 @@ Gem::Specification.new do |s|
s.description = "Arvados client library, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
s.email = 'gem-dev at curoverse.com'
- s.licenses = ['Apache License, Version 2.0']
+ s.licenses = ['Apache-2.0']
s.files = ["lib/arvados.rb", "lib/arvados/google_api_client.rb",
"lib/arvados/collection.rb", "lib/arvados/keep.rb",
"README", "LICENSE-2.0.txt"]
diff --git a/services/api/test/integration/websocket_test.rb b/services/api/test/integration/websocket_test.rb
index a9993b2..e8b8809 100644
--- a/services/api/test/integration/websocket_test.rb
+++ b/services/api/test/integration/websocket_test.rb
@@ -75,9 +75,9 @@ class WebsocketTest < ActionDispatch::IntegrationTest
EM.run do
if token
- ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket?api_token=#{api_client_authorizations(token).api_token}")
+ ws = Faye::WebSocket::Client.new("ws://0.0.0.0:#{@@port}/websocket?api_token=#{api_client_authorizations(token).api_token}")
else
- ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket")
+ ws = Faye::WebSocket::Client.new("ws://0.0.0.0:#{@@port}/websocket")
end
ws.on :open do |event|
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index a894350..8d80383 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -1,10 +1,10 @@
FROM debian:8
ENV DEBIAN_FRONTEND noninteractive
+ENV APT_GET_INSTALL "apt-get -yq install --no-install-recommends -o Acquire::Retries=6"
-RUN apt-get clean && \
- apt-get update && \
- apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
+RUN apt-get update && \
+ ${APT_GET_INSTALL} \
postgresql-9.4 git build-essential runit curl libpq-dev \
libcurl4-openssl-dev libssl-dev zlib1g-dev libpcre3-dev \
openssh-server python-setuptools netcat-traditional \
@@ -20,25 +20,23 @@ RUN apt-get clean && \
ENV GOVERSION 1.7.3
# Install golang binary
-RUN cd /usr/local && \
- curl -O http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz && \
- tar -xzf go${GOVERSION}.linux-amd64.tar.gz && \
- rm go${GOVERSION}.linux-amd64.tar.gz
+RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
+ tar -C /usr/local -xzf -
ENV PATH ${PATH}:/usr/local/go/bin
-ENV RUBYVERSION 2.1.8
+ENV RUBYVERSION_MINOR 2.3
+ENV RUBYVERSION 2.3.1
# Install Ruby from source
-RUN cd /usr/local/lib && \
- curl -O http://cache.ruby-lang.org/pub/ruby/2.1/ruby-${RUBYVERSION}.tar.gz && \
- tar xzf ruby-${RUBYVERSION}.tar.gz && \
+RUN cd /tmp && \
+ curl -f http://cache.ruby-lang.org/pub/ruby/${RUBYVERSION_MINOR}/ruby-${RUBYVERSION}.tar.gz | tar -xzf - && \
cd ruby-${RUBYVERSION} && \
./configure --disable-install-doc && \
make && \
make install && \
- cd /usr/local/lib && \
- rm -rf ruby-${RUBYVERSION}.tar.gz ruby-${RUBYVERSION}
+ cd /tmp && \
+ rm -rf ruby-${RUBYVERSION}
ENV GEM_HOME /var/lib/gems
ENV GEM_PATH /var/lib/gems
@@ -54,7 +52,8 @@ RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E8
RUN mkdir -p /etc/apt/sources.list.d && \
echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \
apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get -yq install docker-engine=1.9.1-0~jessie
+ ${APT_GET_INSTALL} docker-engine=1.9.1-0~jessie && \
+ apt-get clean
RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql
@@ -62,11 +61,11 @@ RUN cd /root && \
GOPATH=$PWD go get github.com/curoverse/runsvinit && \
install bin/runsvinit /usr/local/bin
-RUN set -e && \
- PJS=phantomjs-1.9.7-linux-x86_64 && \
- curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
- tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
- ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+ENV PJS phantomjs-1.9.8-linux-x86_64
+RUN (curl -fL http://cache.arvados.org/${PJS}.tar.bz2 || \
+ curl -fL https://bitbucket.org/ariya/phantomjs/downloads/${PJS}.tar.bz2) | \
+ tar -C /usr/local -xjf - && \
+ ln -s ../${PJS}/bin/phantomjs /usr/local/bin/
RUN pip install -U setuptools
diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh
index 9ef3792..3cdd3c8 100755
--- a/tools/arvbox/lib/arvbox/docker/createusers.sh
+++ b/tools/arvbox/lib/arvbox/docker/createusers.sh
@@ -26,8 +26,8 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
/var/lib/nginx /var/log/nginx /etc/ssl/private \
/var/lib/gopath /var/lib/pip
- mkdir -p /var/lib/gems/ruby/2.1.0
- chown arvbox:arvbox -R /var/lib/gems/ruby/2.1.0
+ mkdir -p /var/lib/gems/ruby/2.3.0
+ chown arvbox:arvbox -R /var/lib/gems/ruby/2.3.0
mkdir -p /tmp/crunch0 /tmp/crunch1
chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index c248773..b74ce13 100755
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -63,7 +63,7 @@ fi
if ! [[ -z "$waiting" ]] ; then
if ps x | grep -v grep | grep "bundle install" > /dev/null; then
- gemcount=$(ls /var/lib/gems/ruby/2.1.0/gems 2>/dev/null | wc -l)
+ gemcount=$(ls /var/lib/gems/ruby/2.3.0/gems 2>/dev/null | wc -l)
gemlockcount=0
for l in /usr/src/arvados/services/api/Gemfile.lock \
diff --git a/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh b/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh
index 84d9904..a81ad8e 100755
--- a/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh
+++ b/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh
@@ -1,4 +1,4 @@
#!/bin/sh
-while ! psql postgres -c\\du >/dev/null 2>/dev/null ; do
- sleep 1
+while ! pg_isready >/dev/null; do
+ sleep 0.2
done
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list