[ARVADOS] updated: ab371a54c3522c06fbc82c9838a24c6f53122a6e
git at public.curoverse.com
git at public.curoverse.com
Wed Jul 16 10:06:19 EDT 2014
Summary of changes:
.../_show_advanced_cli_example.html.erb | 2 +-
.../app/views/application/_show_api.html.erb | 2 +-
.../app/views/authorized_keys/_show_help.html.erb | 2 +-
.../notifications/_ssh_key_notification.html.erb | 2 +-
.../app/views/repositories/_show_help.html.erb | 2 +-
.../app/views/virtual_machines/_show_help.html.erb | 2 +-
docker/api/Dockerfile | 8 +-
docker/api/arvados-clients.yml.in | 5 +
docker/api/crunch-dispatch-run.sh | 3 +
docker/api/setup-gitolite.sh | 55 +
docker/api/setup.sh.in | 2 -
docker/api/supervisor.conf | 12 +
docker/api/update-gitolite.rb | 166 ++
docker/base/Dockerfile | 4 +
docker/{compute => base}/apt.arvados.org.list | 0
docker/build_tools/Makefile | 2 +
docker/compute/Dockerfile | 8 +-
docker/compute/setup.sh.in | 2 +
docker/compute/supervisor.conf | 3 +
docker/workbench/Dockerfile | 3 +
sdk/cli/bin/crunch-job | 44 +-
.../api/app/controllers/application_controller.rb | 8 +
.../app/controllers/arvados/v1/jobs_controller.rb | 80 +-
.../app/controllers/arvados/v1/users_controller.rb | 11 +
services/api/app/models/arvados_model.rb | 15 +-
services/api/app/models/commit.rb | 8 +-
services/api/app/models/job_task.rb | 12 +-
services/api/config/application.rb | 1 +
.../20140709172343_job_task_serial_qsequence.rb | 11 +
services/api/db/schema.rb | 443 -----
services/api/db/structure.sql | 1974 ++++++++++++++++++++
services/api/lib/record_filters.rb | 27 +-
.../test/fixtures/api_client_authorizations.yml | 6 +
services/api/test/fixtures/jobs.yml | 2 +-
services/api/test/fixtures/links.yml | 28 +
services/api/test/fixtures/pipeline_templates.yml | 2 +-
services/api/test/fixtures/users.yml | 10 +
.../arvados/v1/groups_controller_test.rb | 77 +-
.../arvados/v1/job_reuse_controller_test.rb | 91 +
.../functional/arvados/v1/users_controller_test.rb | 100 +
services/api/test/integration/jobs_api_test.rb | 19 +
services/api/test/integration/permissions_test.rb | 16 +-
services/api/test/integration/select_test.rb | 2 +-
services/api/test/unit/job_task_test.rb | 15 +-
44 files changed, 2727 insertions(+), 560 deletions(-)
create mode 100644 docker/api/arvados-clients.yml.in
create mode 100755 docker/api/setup-gitolite.sh
create mode 100755 docker/api/update-gitolite.rb
rename docker/{compute => base}/apt.arvados.org.list (100%)
create mode 100644 services/api/db/migrate/20140709172343_job_task_serial_qsequence.rb
delete mode 100644 services/api/db/schema.rb
create mode 100644 services/api/db/structure.sql
via ab371a54c3522c06fbc82c9838a24c6f53122a6e (commit)
via 150f495e96c43f5d7e0df58352c5b20cc9e83ccd (commit)
via 76c61ffb15b51c00a174a6f73a96ab84df69f0f6 (commit)
via f85c6cf14fada8b2ad4b6b7e6e7e9cf9852ce83d (commit)
via 29542968833ad233b21fc7fe2f86b1ba88334da2 (commit)
via 5c5cb0a226a165c0028bb2c55cba8f4dcc359be1 (commit)
via ac97e7087df8b48ee289c696b47973618a40ce73 (commit)
via 16bf6c2ad0f88183198701cc6242240f9b194a7b (commit)
via 07f7373ef75ae9c574060e2a61c3749dc7c6ffe3 (commit)
via be6e47b1274d4b4e80afe51593222f99d90bac66 (commit)
via cb63ae17322aa698f6636d39f43c6ad8f35c6fc5 (commit)
via 56303d3c734fa0ed9d5f077aac60b97596506efc (commit)
via 1b4adbdc0711468971f619527cb250e8b743084b (commit)
via 02de894919a054c31384aeda99a2c2de3a09ee60 (commit)
via 5a3fcd947082b668f2d6941b88f0cf6399d4ad1e (commit)
via 76116fc706e2580e530d40b0e5f85783289150bf (commit)
via 4a7ca048ea3f26fa8c417001bd5b9d2c1ab62e71 (commit)
via 8b185ea4dda14333a099964589d8d57aef3f1403 (commit)
via 85706e064ed0e0e1742ff5fbbc61733552afc827 (commit)
from ee0896bf5bf1f7f4ebdc5168e928c28d9f06eaeb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ab371a54c3522c06fbc82c9838a24c6f53122a6e
Merge: 150f495 76c61ff
Author: Ward Vandewege <ward at curoverse.com>
Date: Tue Jul 15 15:28:16 2014 -0400
Merge branch 'master' into 3219-further-docker-improvements
commit 150f495e96c43f5d7e0df58352c5b20cc9e83ccd
Author: Ward Vandewege <ward at curoverse.com>
Date: Tue Jul 15 15:27:04 2014 -0400
Add local git repository support.
refs #3219
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index ce5755d..ad85315 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -10,7 +10,7 @@ MAINTAINER Tim Pierce <twp at curoverse.com>
RUN apt-get update && \
apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
- libjson-perl && \
+ libjson-perl cron && \
git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
ADD munge.key /etc/munge/
@@ -66,10 +66,16 @@ RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid
ADD keep_server_0.json /root/
ADD keep_server_1.json /root/
+# Set up update-gitolite.rb
+RUN mkdir /usr/local/arvados/config -p
+ADD generated/arvados-clients.yml /usr/local/arvados/config/
+ADD update-gitolite.rb /usr/local/arvados/
+
# Supervisor.
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
ADD ssh.sh /usr/local/bin/ssh.sh
ADD generated/setup.sh /usr/local/bin/setup.sh
+ADD setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
ADD crunch-dispatch-run.sh /usr/local/bin/crunch-dispatch-run.sh
ADD apache2_foreground.sh /etc/apache2/foreground.sh
diff --git a/docker/api/arvados-clients.yml.in b/docker/api/arvados-clients.yml.in
new file mode 100644
index 0000000..babfc4e
--- /dev/null
+++ b/docker/api/arvados-clients.yml.in
@@ -0,0 +1,5 @@
+production:
+ gitolite_url: 'git at api:gitolite-admin.git'
+ gitolite_tmp: 'gitolite-tmp'
+ arvados_api_host: 'api'
+ arvados_api_token: '@@API_SUPERUSER_SECRET@@'
diff --git a/docker/api/setup-gitolite.sh b/docker/api/setup-gitolite.sh
new file mode 100755
index 0000000..c55fdee
--- /dev/null
+++ b/docker/api/setup-gitolite.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+ssh-keygen -q -N '' -t rsa -f /root/.ssh/id_rsa
+
+useradd git
+mkdir /home/git
+
+# Set up gitolite repository
+cp ~root/.ssh/id_rsa.pub ~git/root-authorized_keys.pub
+chown git:git /home/git -R
+su - git -c "mkdir -p ~/bin"
+
+su - git -c "git clone git://github.com/sitaramc/gitolite"
+su - git -c "gitolite/install -ln ~/bin"
+su - git -c "PATH=/home/git/bin:$PATH gitolite setup -pk ~git/root-authorized_keys.pub"
+
+# Now set up the gitolite repo(s) we use
+mkdir -p /usr/local/arvados/gitolite-tmp/
+# Make ssh store the host key
+ssh -o "StrictHostKeyChecking no" git at api info
+# Now check out the tree
+git clone git at api:gitolite-admin.git /usr/local/arvados/gitolite-tmp/gitolite-admin/
+cd /usr/local/arvados/gitolite-tmp/gitolite-admin
+mkdir keydir/arvados
+mkdir conf/admin
+mkdir conf/auto
+echo "
+
+ at arvados_git_user = arvados_git_user
+
+repo @all
+ RW+ = @arvados_git_user
+
+" > conf/admin/arvados.conf
+echo '
+include "auto/*.conf"
+include "admin/*.conf"
+' >> conf/gitolite.conf
+
+#su - git -c "ssh-keygen -t rsa"
+cp /root/.ssh/id_rsa.pub keydir/arvados/arvados_git_user.pub
+# Replace the 'root' key with the user key, just in case
+cp /root/.ssh/authorized_keys keydir/root-authorized_keys.pub
+# But also make sure we have the root key installed so it can access all keys
+git add keydir/root-authorized_keys.pub
+git add keydir/arvados/arvados_git_user.pub
+git add conf/admin/arvados.conf
+git add keydir/arvados/
+git add conf/gitolite.conf
+git commit -a -m 'git server setup'
+git push
+
+echo "ARVADOS_API_HOST_INSECURE=yes" > /etc/cron.d/gitolite-update
+echo "*/5 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index fd4cf07..f107109 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -9,7 +9,6 @@ export ARVADOS_API_HOST_INSECURE=yes
export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
# All users group
-
prefix=`arv --format=uuid user current | cut -d- -f1`
read -rd $'\000' newgroup <<EOF; arv group create --group "$newgroup"
{
@@ -44,4 +43,3 @@ grep -q keep_server_1 /tmp/keep_service.list
if [[ "$?" != "0" ]]; then
arv keep_service create --keep-service "$(cat /root/keep_server_1.json)"
fi
-
diff --git a/docker/api/supervisor.conf b/docker/api/supervisor.conf
index a1dacac..e85bb72 100644
--- a/docker/api/supervisor.conf
+++ b/docker/api/supervisor.conf
@@ -14,16 +14,28 @@ stopsignal=6
[program:munge]
user=root
command=/etc/init.d/munge start
+startsecs=0
[program:slurm]
user=root
command=/etc/init.d/slurm-llnl start
+startsecs=0
+
+[program:cron]
+user=root
+command=/etc/init.d/cron start
+startsecs=0
[program:setup]
user=root
command=/usr/local/bin/setup.sh
startsecs=0
+[program:setup-gitolite]
+user=root
+command=/usr/local/bin/setup-gitolite.sh
+startsecs=0
+
[program:crunch-dispatch]
user=root
command=/usr/local/bin/crunch-dispatch-run.sh
diff --git a/docker/api/update-gitolite.rb b/docker/api/update-gitolite.rb
new file mode 100755
index 0000000..1623a8b
--- /dev/null
+++ b/docker/api/update-gitolite.rb
@@ -0,0 +1,166 @@
+#!/usr/bin/env ruby
+
+###################################################################
+# THIS FILE IS MANAGED BY PUPPET -- CHANGES WILL BE OVERWRITTEN #
+###################################################################
+
+require 'rubygems'
+require 'pp'
+require 'arvados'
+require 'active_support/all'
+
+# This script does the actual gitolite config management on disk.
+#
+# Ward Vandewege <ward at curoverse.com>
+
+# Default is development
+production = ARGV[0] == "production"
+
+ENV["RAILS_ENV"] = "development"
+ENV["RAILS_ENV"] = "production" if production
+
+DEBUG = 1
+
+# load and merge in the environment-specific application config info
+# if present, overriding base config parameters as specified
+path = File.dirname(__FILE__) + '/config/arvados-clients.yml'
+if File.exists?(path) then
+ cp_config = YAML.load_file(path)[ENV['RAILS_ENV']]
+else
+ puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file"
+ exit 1
+end
+
+gitolite_url = cp_config['gitolite_url']
+gitolite_tmp = cp_config['gitolite_tmp']
+
+gitolite_admin = File.join(File.expand_path(File.dirname(__FILE__)) + '/' + gitolite_tmp + '/gitolite-admin')
+
+ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
+ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
+
+keys = ''
+
+seen = Hash.new
+
+def ensure_repo(name,permissions,user_keys,gitolite_admin)
+ tmp = ''
+ # Just in case...
+ name.gsub!(/[^a-z0-9]/i,'')
+
+ keys = Hash.new()
+
+ user_keys.each do |uuid,p|
+ p.each do |k|
+ next if k[:public_key].nil?
+ keys[uuid] = Array.new() if not keys.key?(uuid)
+
+ key = k[:public_key]
+ # Handle putty-style ssh public keys
+ key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ key.gsub!(/\n/,'')
+ key.strip
+
+ keys[uuid].push(key)
+ end
+ end
+
+ cf = gitolite_admin + '/conf/auto/' + name + '.conf'
+
+ conf = "\nrepo #{name}\n"
+
+ commit = false
+
+ seen = {}
+ permissions.sort.each do |uuid,v|
+ conf += "\t#{v[:gitolite_permissions]}\t= #{uuid.to_s}\n"
+
+ count = 0
+ keys.include?(uuid) and keys[uuid].each do |v|
+ kf = gitolite_admin + '/keydir/arvados/' + uuid.to_s + "@#{count}.pub"
+ seen[kf] = true
+ if !File.exists?(kf) or IO::read(kf) != v then
+ commit = true
+ f = File.new(kf + ".tmp",'w')
+ f.write(v)
+ f.close()
+ # File.rename will overwrite the destination file if it exists
+ File.rename(kf + ".tmp",kf);
+ end
+ count += 1
+ end
+ end
+
+ if !File.exists?(cf) or IO::read(cf) != conf then
+ commit = true
+ f = File.new(cf + ".tmp",'w')
+ f.write(conf)
+ f.close()
+ # this is about as atomic as we can make the replacement of the file...
+ File.unlink(cf) if File.exists?(cf)
+ File.rename(cf + ".tmp",cf);
+ end
+
+ return commit,seen
+end
+
+begin
+
+ pwd = Dir.pwd
+ # Get our local gitolite-admin repo up to snuff
+ if not File.exists?(File.dirname(__FILE__) + '/' + gitolite_tmp) then
+ Dir.mkdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp), 0700)
+ end
+ if not File.exists?(gitolite_admin) then
+ Dir.chdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp))
+ `git clone #{gitolite_url}`
+ else
+ Dir.chdir(gitolite_admin)
+ `git pull`
+ end
+ Dir.chdir(pwd)
+
+ arv = Arvados.new( { :suppress_ssl_warnings => false } )
+
+ permissions = arv.repository.get_all_permissions
+
+ repos = permissions[:repositories]
+ user_keys = permissions[:user_keys]
+
+ @commit = false
+
+ @seen = {}
+
+ repos.each do |r|
+ next if r[:name].nil?
+ (@c, at s) = ensure_repo(r[:name],r[:user_permissions],user_keys,gitolite_admin)
+ @seen.merge!(@s)
+ @commit = true if @c
+ end
+
+ # Clean up public key files that should not be present
+ Dir.glob(gitolite_admin + '/keydir/arvados/*.pub') do |key_file|
+ next if key_file =~ /arvados_git_user.pub$/
+ next if @seen.has_key?(key_file)
+ puts "Extra file #{key_file}"
+ @commit = true
+ Dir.chdir(gitolite_admin)
+ key_file.gsub!(/^#{gitolite_admin}\//,'')
+ `git rm #{key_file}`
+ end
+
+ if @commit then
+ message = "#{Time.now().to_s}: update from API"
+ Dir.chdir(gitolite_admin)
+ `git add --all`
+ `git commit -m '#{message}'`
+ `git push`
+ end
+
+rescue Exception => bang
+ puts "Error: " + bang.to_s
+ puts bang.backtrace.join("\n")
+ exit 1
+end
+
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index fd49b3c..172da98 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -54,6 +54,7 @@ COMPUTE_GENERATED = compute/generated/setup.sh
COMPUTE_GENERATED_IN = compute/setup.sh.in
API_GENERATED = \
+ api/generated/arvados-clients.yml \
api/generated/apache2_vhost \
api/generated/config_databases.sh \
api/generated/database.yml \
@@ -64,6 +65,7 @@ API_GENERATED = \
api/generated/superuser_token
API_GENERATED_IN = \
+ api/arvados-clients.yml.in \
api/apache2_vhost.in \
api/config_databases.sh.in \
api/database.yml.in \
diff --git a/docker/compute/supervisor.conf b/docker/compute/supervisor.conf
index af081df..f2cce3f 100644
--- a/docker/compute/supervisor.conf
+++ b/docker/compute/supervisor.conf
@@ -6,14 +6,17 @@ startsecs=0
[program:munge]
user=root
command=/etc/init.d/munge start
+startsecs=0
[program:slurm]
user=root
command=/etc/init.d/slurm-llnl start
+startsecs=0
[program:cron]
user=root
command=/etc/init.d/cron start
+startsecs=0
[program:setup]
user=root
diff --git a/docker/workbench/Dockerfile b/docker/workbench/Dockerfile
index 75e60c9..97ed013 100644
--- a/docker/workbench/Dockerfile
+++ b/docker/workbench/Dockerfile
@@ -4,7 +4,7 @@ FROM arvados/passenger
MAINTAINER Ward Vandewege <ward at curoverse.com>
# We need graphviz for the provenance graphs
-RUN apt-get update && apt-get install graphviz
+RUN apt-get update && apt-get -qqy install graphviz
# Update Arvados source
RUN /bin/mkdir -p /usr/src/arvados/apps
commit cb63ae17322aa698f6636d39f43c6ad8f35c6fc5
Author: Ward Vandewege <ward at curoverse.com>
Date: Mon Jul 14 14:30:53 2014 -0400
Compute nodes can now run jobs. Also, small fixes all over the place.
refs #3219
diff --git a/docker/api/crunch-dispatch-run.sh b/docker/api/crunch-dispatch-run.sh
index f422e78..5103b1d 100755
--- a/docker/api/crunch-dispatch-run.sh
+++ b/docker/api/crunch-dispatch-run.sh
@@ -14,6 +14,9 @@ export CRUNCH_JOB_BIN=/usr/src/arvados/services/crunch/crunch-job
export HOME=`pwd`
fuser -TERM -k $CRUNCH_DISPATCH_LOCKFILE || true
+# Give the compute nodes some time to start up
+sleep 5
+
cd /usr/src/arvados/services/api
export RAILS_ENV=production
/usr/local/rvm/bin/rvm-exec default bundle install
diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile
index f1defce..79cb424 100644
--- a/docker/base/Dockerfile
+++ b/docker/base/Dockerfile
@@ -22,6 +22,10 @@ RUN apt-get update && \
/usr/local/rvm/bin/rvm alias create default ruby && \
/bin/mkdir -p /usr/src/arvados
+ADD apt.arvados.org.list /etc/apt/sources.list.d/
+RUN apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7
+RUN apt-get update && apt-get -qqy install python-arvados-python-client
+
ADD generated/arvados.tar.gz /usr/src/arvados/
# Update gem. This (hopefully) fixes
diff --git a/docker/compute/apt.arvados.org.list b/docker/base/apt.arvados.org.list
similarity index 100%
rename from docker/compute/apt.arvados.org.list
rename to docker/base/apt.arvados.org.list
diff --git a/docker/compute/Dockerfile b/docker/compute/Dockerfile
index 4cc02aa..929c136 100644
--- a/docker/compute/Dockerfile
+++ b/docker/compute/Dockerfile
@@ -3,10 +3,7 @@
FROM arvados/slurm
MAINTAINER Ward Vandewege <ward at curoverse.com>
-ADD apt.arvados.org.list /etc/apt/sources.list.d/
-RUN apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7
-
-RUN apt-get update && apt-get -qqy install supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-python-client python-arvados-fuse cron
+RUN apt-get update && apt-get -qqy install supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-fuse cron
ADD fuse.conf /etc/fuse.conf
@@ -20,9 +17,6 @@ RUN apt-get update -qq && apt-get install -qqy lxc-docker
RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid 4005 crunch && usermod crunch -G fuse,docker && chown crunch:crunch /home/crunch
-# Fix /dev/fuse permissions/ownership
-RUN chmod 1660 /dev/fuse && chgrp fuse /dev/fuse
-
# Supervisor.
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
ADD ssh.sh /usr/local/bin/ssh.sh
diff --git a/docker/compute/setup.sh.in b/docker/compute/setup.sh.in
index ade905d..e107d80 100755
--- a/docker/compute/setup.sh.in
+++ b/docker/compute/setup.sh.in
@@ -16,3 +16,5 @@ echo "*/5 * * * * root /usr/bin/curl -k -d ping_secret=$PING_SECRET https://api/
# Send a ping now
/usr/bin/curl -k -d ping_secret=$PING_SECRET https://api/arvados/v1/nodes/$UUID/ping?ping_secret=$PING_SECRET
+# Just make sure /dev/fuse permissions are correct (the device appears after fuse is loaded)
+chmod 1660 /dev/fuse && chgrp fuse /dev/fuse
diff --git a/docker/workbench/Dockerfile b/docker/workbench/Dockerfile
index d9dfeff..75e60c9 100644
--- a/docker/workbench/Dockerfile
+++ b/docker/workbench/Dockerfile
@@ -3,6 +3,9 @@
FROM arvados/passenger
MAINTAINER Ward Vandewege <ward at curoverse.com>
+# We need graphviz for the provenance graphs
+RUN apt-get update && apt-get install graphviz
+
# Update Arvados source
RUN /bin/mkdir -p /usr/src/arvados/apps
ADD generated/workbench.tar.gz /usr/src/arvados/apps/
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list