[ARVADOS] updated: 6c6a37bf3d37e3116ae39536b1ea56b077de9dd1
git at public.curoverse.com
git at public.curoverse.com
Mon Mar 30 11:06:58 EDT 2015
Summary of changes:
docker/api/.gitolite.rc | 191 +++++++++++++++++++++++++
docker/api/Dockerfile | 1 +
docker/api/setup-gitolite.sh.in | 5 +-
docker/api/update-gitolite.rb | 252 ++++++++++++++++++++++-----------
services/api/script/crunch-dispatch.rb | 2 +-
5 files changed, 367 insertions(+), 84 deletions(-)
create mode 100644 docker/api/.gitolite.rc
via 6c6a37bf3d37e3116ae39536b1ea56b077de9dd1 (commit)
via 2c65feefb8c3fc8539e18939b444050435f58a80 (commit)
from 2013ac2400a36ea6154e84e1991944afc536c9a0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6c6a37bf3d37e3116ae39536b1ea56b077de9dd1
Author: Brett Smith <brett at curoverse.com>
Date: Mon Mar 30 11:06:51 2015 -0400
4253: Sync up Docker with our production Gitolite setup.
This updates our API server Docker image to store repositories by
UUID, with name aliases available. .gitolite.rc enables the aliasing,
and update-gitolite.rb generates the necessary configuration. This
makes it possible to test the recent repository changes in Docker.
diff --git a/docker/api/.gitolite.rc b/docker/api/.gitolite.rc
new file mode 100644
index 0000000..855e103
--- /dev/null
+++ b/docker/api/.gitolite.rc
@@ -0,0 +1,191 @@
+# configuration variables for gitolite
+
+# This file is in perl syntax. But you do NOT need to know perl to edit it --
+# just mind the commas, use single quotes unless you know what you're doing,
+# and make sure the brackets and braces stay matched up!
+
+# (Tip: perl allows a comma after the last item in a list also!)
+
+# HELP for commands can be had by running the command with "-h".
+
+# HELP for all the other FEATURES can be found in the documentation (look for
+# "list of non-core programs shipped with gitolite" in the master index) or
+# directly in the corresponding source file.
+
+my $repo_aliases;
+my $aliases_src = "$ENV{HOME}/.gitolite/arvadosaliases.pl";
+if ($ENV{HOME} && (-e $aliases_src)) {
+ $repo_aliases = do $aliases_src;
+}
+$repo_aliases ||= {};
+
+%RC = (
+
+ # ------------------------------------------------------------------
+
+ # default umask gives you perms of '0700'; see the rc file docs for
+ # how/why you might change this
+ UMASK => 0022,
+
+ # look for "git-config" in the documentation
+ GIT_CONFIG_KEYS => '',
+
+ # comment out if you don't need all the extra detail in the logfile
+ LOG_EXTRA => 1,
+
+ # roles. add more roles (like MANAGER, TESTER, ...) here.
+ # WARNING: if you make changes to this hash, you MUST run 'gitolite
+ # compile' afterward, and possibly also 'gitolite trigger POST_COMPILE'
+ ROLES => {
+ READERS => 1,
+ WRITERS => 1,
+ },
+
+ REPO_ALIASES => $repo_aliases,
+
+ # ------------------------------------------------------------------
+
+ # rc variables used by various features
+
+ # the 'info' command prints this as additional info, if it is set
+ # SITE_INFO => 'Please see http://blahblah/gitolite for more help',
+
+ # the 'desc' command uses this
+ # WRITER_CAN_UPDATE_DESC => 1,
+ # the 'readme' command uses this
+ # WRITER_CAN_UPDATE_README => 1,
+
+ # the CpuTime feature uses these
+ # display user, system, and elapsed times to user after each git operation
+ # DISPLAY_CPU_TIME => 1,
+ # display a warning if total CPU times (u, s, cu, cs) crosses this limit
+ # CPU_TIME_WARN_LIMIT => 0.1,
+
+ # the Mirroring feature needs this
+ # HOSTNAME => "foo",
+
+ # if you enabled 'Shell', you need this
+ # SHELL_USERS_LIST => "$ENV{HOME}/.gitolite.shell-users",
+
+ # ------------------------------------------------------------------
+
+ # suggested locations for site-local gitolite code (see cust.html)
+
+ # this one is managed directly on the server
+ # LOCAL_CODE => "$ENV{HOME}/local",
+
+ # or you can use this, which lets you put everything in a subdirectory
+ # called "local" in your gitolite-admin repo. For a SECURITY WARNING
+ # on this, see http://gitolite.com/gitolite/cust.html#pushcode
+ # LOCAL_CODE => "$rc{GL_ADMIN_BASE}/local",
+
+ # ------------------------------------------------------------------
+
+ # List of commands and features to enable
+
+ ENABLE => [
+
+ # COMMANDS
+
+ # These are the commands enabled by default
+ 'help',
+ 'desc',
+ 'info',
+ 'perms',
+ 'writable',
+
+ # Uncomment or add new commands here.
+ # 'create',
+ # 'fork',
+ # 'mirror',
+ # 'readme',
+ # 'sskm',
+ # 'D',
+
+ # These FEATURES are enabled by default.
+
+ # essential (unless you're using smart-http mode)
+ 'ssh-authkeys',
+
+ # creates git-config enties from gitolite.conf file entries like 'config foo.bar = baz'
+ 'git-config',
+
+ # creates git-daemon-export-ok files; if you don't use git-daemon, comment this out
+ 'daemon',
+
+ # creates projects.list file; if you don't use gitweb, comment this out
+ 'gitweb',
+
+ # These FEATURES are disabled by default; uncomment to enable. If you
+ # need to add new ones, ask on the mailing list :-)
+
+ # user-visible behaviour
+
+ # prevent wild repos auto-create on fetch/clone
+ # 'no-create-on-read',
+ # no auto-create at all (don't forget to enable the 'create' command!)
+ # 'no-auto-create',
+
+ # access a repo by another (possibly legacy) name
+ 'Alias',
+
+ # give some users direct shell access
+ # 'Shell',
+
+ # set default roles from lines like 'option default.roles-1 = ...', etc.
+ # 'set-default-roles',
+
+ # show more detailed messages on deny
+ # 'expand-deny-messages',
+
+ # system admin stuff
+
+ # enable mirroring (don't forget to set the HOSTNAME too!)
+ # 'Mirroring',
+
+ # allow people to submit pub files with more than one key in them
+ # 'ssh-authkeys-split',
+
+ # selective read control hack
+ # 'partial-copy',
+
+ # manage local, gitolite-controlled, copies of read-only upstream repos
+ # 'upstream',
+
+ # updates 'description' file instead of 'gitweb.description' config item
+ # 'cgit',
+
+ # allow repo-specific hooks to be added
+ # 'repo-specific-hooks',
+
+ # performance, logging, monitoring...
+
+ # be nice
+ # 'renice 10',
+
+ # log CPU times (user, system, cumulative user, cumulative system)
+ # 'CpuTime',
+
+ # syntactic_sugar for gitolite.conf and included files
+
+ # allow backslash-escaped continuation lines in gitolite.conf
+ # 'continuation-lines',
+
+ # create implicit user groups from directory names in keydir/
+ # 'keysubdirs-as-groups',
+
+ # allow simple line-oriented macros
+ # 'macros',
+
+ ],
+
+);
+
+# ------------------------------------------------------------------------------
+# per perl rules, this should be the last line in such a file:
+1;
+
+# Local variables:
+# mode: perl
+# End:
+# vim: set syn=perl:
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index 07acb63..e0b353c 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -68,6 +68,7 @@ ADD keep_proxy.json /root/
# Set up update-gitolite.rb
RUN mkdir /usr/local/arvados/config -p
ADD generated/arvados-clients.yml /usr/local/arvados/config/
+ADD .gitolite.rc /usr/local/arvados/config/
ADD update-gitolite.rb /usr/local/arvados/
# Supervisor.
diff --git a/docker/api/setup-gitolite.sh.in b/docker/api/setup-gitolite.sh.in
index 92014f9..ec55482 100755
--- a/docker/api/setup-gitolite.sh.in
+++ b/docker/api/setup-gitolite.sh.in
@@ -13,10 +13,7 @@ su - git -c "mkdir -p ~/bin"
su - git -c "git clone git://github.com/sitaramc/gitolite"
su - git -c "gitolite/install -ln ~/bin"
su - git -c "PATH=/home/git/bin:$PATH gitolite setup -pk ~git/root-authorized_keys.pub"
-
-# Make sure the repositories are created in such a way that they are readable
-# by the api server
-sed -i 's/0077/0022/g' /home/git/.gitolite.rc
+install -o git -g git -m 600 /usr/local/arvados/config/.gitolite.rc /home/git/
# And make sure that the existing repos are equally readable, or the API server commit model will freak out...
chmod 755 /home/git/repositories
diff --git a/docker/api/update-gitolite.rb b/docker/api/update-gitolite.rb
index 2c46a0d..8247931 100755
--- a/docker/api/update-gitolite.rb
+++ b/docker/api/update-gitolite.rb
@@ -3,7 +3,7 @@
require 'rubygems'
require 'pp'
require 'arvados'
-require 'active_support/all'
+require 'tempfile'
require 'yaml'
# This script does the actual gitolite config management on disk.
@@ -29,9 +29,12 @@ else
end
gitolite_url = cp_config['gitolite_url']
-gitolite_tmp = cp_config['gitolite_tmp']
+gitolite_arvados_git_user_key = cp_config['gitolite_arvados_git_user_key']
-gitolite_admin = File.join(File.expand_path(File.dirname(__FILE__)) + '/' + gitolite_tmp + '/gitolite-admin')
+gitolite_tmpdir = File.join(File.absolute_path(File.dirname(__FILE__)),
+ cp_config['gitolite_tmp'])
+gitolite_admin = File.join(gitolite_tmpdir, 'gitolite-admin')
+gitolite_keydir = File.join(gitolite_admin, 'keydir', 'arvados')
ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
@@ -41,118 +44,209 @@ else
ENV.delete('ARVADOS_API_HOST_INSECURE')
end
-keys = ''
+def ensure_directory(path, mode)
+ begin
+ Dir.mkdir(path, mode)
+ rescue Errno::EEXIST
+ end
+end
-seen = Hash.new
+def replace_file(path, contents)
+ unlink_now = true
+ dirname, basename = File.split(path)
+ new_file = Tempfile.new([basename, ".tmp"], dirname)
+ begin
+ new_file.write(contents)
+ new_file.flush
+ File.rename(new_file, path)
+ unlink_now = false
+ ensure
+ new_file.close(unlink_now)
+ end
+end
-def ensure_repo(name,permissions,user_keys,gitolite_admin)
- tmp = ''
- # Just in case...
- name.gsub!(/[^a-z0-9]/i,'')
+def file_has_contents?(path, contents)
+ begin
+ IO.read(path) == contents
+ rescue Errno::ENOENT
+ false
+ end
+end
- keys = Hash.new()
+module TrackCommitState
+ module ClassMethods
+ # Note that all classes that include TrackCommitState will have
+ # @@need_commit = true if any of them set it. Since this flag reports
+ # a boolean state of the underlying git repository, that's OK in the
+ # current implementation.
+ @@need_commit = false
- user_keys.each do |uuid,p|
- p.each do |k|
- next if k[:public_key].nil?
- keys[uuid] = Array.new() if not keys.key?(uuid)
+ def changed?
+ @@need_commit
+ end
- key = k[:public_key]
- # Handle putty-style ssh public keys
- key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
- key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
- key.gsub!(/\n/,'')
- key.strip
+ def ensure_in_git(path, contents)
+ unless file_has_contents?(path, contents)
+ replace_file(path, contents)
+ system("git", "add", path)
+ @@need_commit = true
+ end
+ end
+ end
+
+ def ensure_in_git(path, contents)
+ self.class.ensure_in_git(path, contents)
+ end
- keys[uuid].push(key)
+ def self.included(base)
+ base.extend(ClassMethods)
+ end
+end
+
+class UserSSHKeys
+ include TrackCommitState
+
+ def initialize(user_keys_map, key_dir)
+ @user_keys_map = user_keys_map
+ @key_dir = key_dir
+ @installed = {}
+ end
+
+ def install(filename, pubkey)
+ unless pubkey.nil?
+ key_path = File.join(@key_dir, filename)
+ ensure_in_git(key_path, pubkey)
end
+ @installed[filename] = true
end
- cf = gitolite_admin + '/conf/auto/' + name + '.conf'
+ def ensure_keys_for_user(user_uuid)
+ return unless key_list = @user_keys_map.delete(user_uuid)
+ key_list.map { |k| k[:public_key] }.compact.each_with_index do |pubkey, ii|
+ # Handle putty-style ssh public keys
+ pubkey.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ pubkey.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ pubkey.gsub!(/\n/,'')
+ pubkey.strip!
+ install("#{user_uuid}@#{ii}.pub", pubkey)
+ end
+ end
- conf = "\nrepo #{name}\n"
+ def installed?(filename)
+ @installed[filename]
+ end
+end
- commit = false
+class Repository
+ include TrackCommitState
- seen = {}
- permissions.sort.each do |uuid,v|
- conf += "\t#{v[:gitolite_permissions]}\t= #{uuid.to_s}\n"
+ @@aliases = {}
- count = 0
- keys.include?(uuid) and keys[uuid].each do |v|
- kf = gitolite_admin + '/keydir/arvados/' + uuid.to_s + "@#{count}.pub"
- seen[kf] = true
- if !File.exists?(kf) or IO::read(kf) != v then
- commit = true
- f = File.new(kf + ".tmp",'w')
- f.write(v)
- f.close()
- # File.rename will overwrite the destination file if it exists
- File.rename(kf + ".tmp",kf);
- end
- count += 1
+ def initialize(arv_repo, user_keys)
+ @arv_repo = arv_repo
+ @user_keys = user_keys
+ end
+
+ def self.ensure_system_config(conf_root)
+ ensure_in_git(File.join(conf_root, "conf", "gitolite.conf"),
+ %Q{include "auto/*.conf"\ninclude "admin/*.conf"\n})
+ ensure_in_git(File.join(conf_root, "arvadosaliases.pl"), alias_config)
+
+ conf_path = File.join(conf_root, "conf", "admin", "arvados.conf")
+ conf_file = %Q{
+ at arvados_git_user = arvados_git_user
+
+repo gitolite-admin
+ RW = @arvados_git_user
+
+}
+ ensure_directory(File.dirname(conf_path), 0755)
+ ensure_in_git(conf_path, conf_file)
+ end
+
+ def ensure_config(conf_root)
+ if name and (File.exist?(auto_conf_path(conf_root, name)))
+ # This gitolite installation knows the repository by name, rather than
+ # UUID. Leave it configured that way until a separate migration is run.
+ basename = name
+ else
+ basename = uuid
+ @@aliases[name] = uuid unless name.nil?
+ end
+ conf_file = "\nrepo #{basename}\n"
+ @arv_repo[:user_permissions].sort.each do |user_uuid, perm|
+ conf_file += "\t#{perm[:gitolite_permissions]}\t= #{user_uuid}\n"
+ @user_keys.ensure_keys_for_user(user_uuid)
end
+ ensure_in_git(auto_conf_path(conf_root, basename), conf_file)
+ end
+
+ private
+
+ def auto_conf_path(conf_root, basename)
+ File.join(conf_root, "conf", "auto", "#{basename}.conf")
+ end
+
+ def uuid
+ @arv_repo[:uuid]
end
- if !File.exists?(cf) or IO::read(cf) != conf then
- commit = true
- f = File.new(cf + ".tmp",'w')
- f.write(conf)
- f.close()
- # this is about as atomic as we can make the replacement of the file...
- File.unlink(cf) if File.exists?(cf)
- File.rename(cf + ".tmp",cf);
+ def name
+ if @arv_repo[:name].nil?
+ nil
+ else
+ @clean_name ||=
+ @arv_repo[:name].sub(/^[^A-Za-z]+/, "").gsub(/[^\w\.\/]/, "")
+ end
end
- return commit,seen
+ def self.alias_config
+ conf_s = "{\n"
+ @@aliases.sort.each do |(repo_name, repo_uuid)|
+ conf_s += "\t'#{repo_name}' \t=> '#{repo_uuid}',\n"
+ end
+ conf_s += "};\n"
+ conf_s
+ end
end
begin
-
- pwd = Dir.pwd
# Get our local gitolite-admin repo up to snuff
- if not File.exists?(File.dirname(__FILE__) + '/' + gitolite_tmp) then
- Dir.mkdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp), 0700)
- end
if not File.exists?(gitolite_admin) then
- Dir.chdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp))
+ ensure_directory(gitolite_tmpdir, 0700)
+ Dir.chdir(gitolite_tmpdir)
`git clone #{gitolite_url}`
+ Dir.chdir(gitolite_admin)
else
Dir.chdir(gitolite_admin)
`git pull`
end
- Dir.chdir(pwd)
-
- arv = Arvados.new( { :suppress_ssl_warnings => false } )
+ arv = Arvados.new
permissions = arv.repository.get_all_permissions
- repos = permissions[:repositories]
- user_keys = permissions[:user_keys]
-
- @commit = false
+ ensure_directory(gitolite_keydir, 0700)
+ user_ssh_keys = UserSSHKeys.new(permissions[:user_keys], gitolite_keydir)
+ # Make sure the arvados_git_user key is installed
+ user_ssh_keys.install('arvados_git_user.pub', gitolite_arvados_git_user_key)
- @seen = {}
-
- repos.each do |r|
- next if r[:name].nil?
- (@c, at s) = ensure_repo(r[:name],r[:user_permissions],user_keys,gitolite_admin)
- @seen.merge!(@s)
- @commit = true if @c
+ permissions[:repositories].each do |repo_record|
+ repo = Repository.new(repo_record, user_ssh_keys)
+ repo.ensure_config(gitolite_admin)
end
+ Repository.ensure_system_config(gitolite_admin)
# Clean up public key files that should not be present
- Dir.glob(gitolite_admin + '/keydir/arvados/*.pub') do |key_file|
- next if key_file =~ /arvados_git_user.pub$/
- next if @seen.has_key?(key_file)
- puts "Extra file #{key_file}"
- @commit = true
- Dir.chdir(gitolite_admin)
- key_file.gsub!(/^#{gitolite_admin}\//,'')
- `git rm #{key_file}`
+ Dir.chdir(gitolite_keydir)
+ stale_keys = Dir.glob('*.pub').reject do |key_file|
+ user_ssh_keys.installed?(key_file)
+ end
+ if stale_keys.any?
+ stale_keys.each { |key_file| puts "Extra file #{key_file}" }
+ system("git", "rm", "--quiet", *stale_keys)
end
- if @commit then
+ if UserSSHKeys.changed? or Repository.changed? or stale_keys.any?
message = "#{Time.now().to_s}: update from API"
Dir.chdir(gitolite_admin)
`git add --all`
@@ -160,7 +254,7 @@ begin
`git push`
end
-rescue Exception => bang
+rescue => bang
puts "Error: " + bang.to_s
puts bang.backtrace.join("\n")
exit 1
commit 2c65feefb8c3fc8539e18939b444050435f58a80
Author: Brett Smith <brett at curoverse.com>
Date: Mon Mar 30 11:04:28 2015 -0400
4253: Fixup crunch-dispatch again.
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index c07ddb0..249582e 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -66,7 +66,7 @@ class Dispatcher
end
@repo_root = Rails.configuration.git_repositories_dir
- @arvados_repo_path = Repository.where(name: "arvados").server_path
+ @arvados_repo_path = Repository.where(name: "arvados").first.server_path
@authorizations = {}
@did_recently = {}
@fetched_commits = {}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list