[ARVADOS] created: 68c70c54a32ebbf22c563a4b4dd6e7cefe49126e
git at public.curoverse.com
git at public.curoverse.com
Fri Aug 14 17:42:03 EDT 2015
at 68c70c54a32ebbf22c563a4b4dd6e7cefe49126e (commit)
commit 68c70c54a32ebbf22c563a4b4dd6e7cefe49126e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Aug 14 17:37:26 2015 -0400
6781: Add arvados-login-sync section to shell server install page.
diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid
index 506894e..57141a0 100644
--- a/doc/install/install-shell-server.html.textile.liquid
+++ b/doc/install/install-shell-server.html.textile.liquid
@@ -58,3 +58,83 @@ Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate
<code>~$ <span class="userinput">sudo git config --system 'credential.https://git.<b>uuid_prefix.your.domain</b>/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
</pre>
</notextile>
+
+h2. Install arvados-login-sync
+
+This program makes it possible for Arvados users to log in to the shell server -- subject to permissions assigned by the Arvados administrator -- using the SSH keys they upload to Workbench. It sets up login accounts, updates group membership, and adds users' public keys to the appropriate @authorized_keys@ files.
+
+Create an Arvados virtual_machine object representing this shell server. This will assign a UUID.
+
+<notextile>
+<pre>
+<code>apiserver:~$ <span class="userinput">arv --format=uuid virtual_machine create --virtual-machine '{"hostname":"<b>your.shell.server.hostname</b>"}'</span>
+zzzzz-2x53u-zzzzzzzzzzzzzzz</code>
+</pre>
+</notextile>
+
+Create a token that is allowed to read login information for this VM.
+
+<notextile>
+<pre>
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/"]}'
+{
+ ...
+ "api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
+ ...
+}</code>
+</pre>
+</notextile>
+
+Note the UUID and the API token output by the above commands: you will need them in a minute.
+
+Install the arvados-login-sync program.
+
+If you're using RVM:
+
+<notextile>
+<pre>
+<code>shellserver:~$ <span class="userinput">sudo -i `which rvm-exec` default gem install arvados-login-sync</span></code>
+</pre>
+</notextile>
+
+If you're not using RVM:
+
+<notextile>
+<pre>
+<code>shellserver:~$ <span class="userinput">sudo -i gem install arvados-login-sync</span></code>
+</pre>
+</notextile>
+
+Install cron.
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install cron</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install cron</span>
+</code></pre>
+</notextile>
+
+Configure cron to run the @arvados-login-sync@ program every 2 minutes:
+
+<notextile>
+<pre>
+<code>shellserver:~$ <span class="userinput">sudo tee <<'EOF' >/etc/cron.d/arvados-login-sync
+ARVADOS_API_HOST="<strong>uuid_prefix.your.domain</strong>"
+ARVADOS_API_TOKEN="<strong>the_token_you_created_above</strong>"
+ARVADOS_VIRTUAL_MACHINE_UUID="<strong>zzzzz-2x53u-zzzzzzzzzzzzzzz</strong>"
+*/2 * * * * root arvados-login-sync
+EOF</span>
+~$ <span class="userinput">sudo chmod 600 /etc/cron.d/arvados-login-sync</span></code>
+</pre>
+</notextile>
+
+You should now be able to give a user permission to log in to the shell server:
+# The user must upload an SSH public key: Workbench → Account menu → "SSH keys" item → "Add new SSH key" button.
+# As an admin user, you must give the user permission to log in: Workbench → Admin menu → "Users" item → "Show" button → "Admin" tab → "Setup shell account" button.
commit eda3946fc638257cb8281875acb4910f2a22e097
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Aug 14 00:44:52 2015 -0400
6781: Add test cases and packaging support.
diff --git a/.gitignore b/.gitignore
index 0871c89..6cbcccc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,9 @@
.rvmrc
*~
*.pyc
+*.gem
+*.rpm
+*.deb
docker/*/generated
docker/config.yml
doc/.site
diff --git a/services/login-sync/.gitignore b/services/login-sync/.gitignore
new file mode 100644
index 0000000..cec3cb5
--- /dev/null
+++ b/services/login-sync/.gitignore
@@ -0,0 +1,2 @@
+*.gem
+Gemfile.lock
diff --git a/services/login-sync/Gemfile b/services/login-sync/Gemfile
new file mode 100644
index 0000000..ffeab22
--- /dev/null
+++ b/services/login-sync/Gemfile
@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gemspec
+group :test, :performance do
+ gem 'minitest', '>= 5.0.0'
+ gem 'mocha', require: false
+ gem 'rake'
+end
diff --git a/services/login-sync/Rakefile b/services/login-sync/Rakefile
new file mode 100644
index 0000000..cf4652f
--- /dev/null
+++ b/services/login-sync/Rakefile
@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+ t.libs << 'test'
+end
+
+desc 'Run tests'
+task default: :test
diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync
index b9cecb6..2e407d8 100755
--- a/services/login-sync/bin/arvados-login-sync
+++ b/services/login-sync/bin/arvados-login-sync
@@ -81,7 +81,7 @@ begin
groups << "fuse"
groups.select! { |name| gids[name] }
# Create new user
- next unless system("/usr/sbin/useradd", "-m",
+ next unless system("useradd", "-m",
"-c", l[:username],
"-s", "/bin/bash",
"-G", groups.join(","),
diff --git a/services/login-sync/test/binstub_new_user/useradd b/services/login-sync/test/binstub_new_user/useradd
new file mode 100755
index 0000000..173bc1d
--- /dev/null
+++ b/services/login-sync/test/binstub_new_user/useradd
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+stub="${0##*/}"
+
+# Record what actually happened in the "spy" file
+echo "$stub $*" >> "$ARVADOS_LOGIN_SYNC_TMPDIR/spy"
+
+# Exit 0 if this command was listed in the "succeed" file
+exec fgrep -qx -- "$stub $*" "$ARVADOS_LOGIN_SYNC_TMPDIR/succeed"
diff --git a/services/login-sync/test/stubs.rb b/services/login-sync/test/stubs.rb
new file mode 100644
index 0000000..62d952f
--- /dev/null
+++ b/services/login-sync/test/stubs.rb
@@ -0,0 +1,52 @@
+require 'etc'
+require 'mocha/mini_test'
+require 'ostruct'
+
+module Stubs
+ # These Etc mocks help only when we run arvados-login-sync in-process.
+
+ def setup
+ super
+ ENV['ARVADOS_VIRTUAL_MACHINE_UUID'] = 'testvm2.shell'
+ Etc.stubs(:to_enum).with(:passwd).returns stubpasswd.map { |x| OpenStruct.new x }
+ Etc.stubs(:to_enum).with(:group).returns stubgroup.map { |x| OpenStruct.new x }
+ end
+
+ def stubpasswd
+ [{name: 'root', uid: 0}]
+ end
+
+ def stubgroup
+ [{name: 'root', gid: 0}]
+ end
+
+ # These child-ENV tricks help only when we run arvados-login-sync as a subprocess.
+
+ def setup
+ super
+ @env_was = Hash[ENV]
+ @tmpdir = Dir.mktmpdir
+ end
+
+ def teardown
+ FileUtils.remove_dir(@tmpdir)
+ ENV.select! { |k| @env_was.has_key? k }
+ @env_was.each do |k,v| ENV[k]=v end
+ super
+ end
+
+ def stubenv opts={}
+ # Use UUID of testvm2.shell fixture, unless otherwise specified by test case.
+ Hash[ENV].merge('ARVADOS_VIRTUAL_MACHINE_UUID' => 'zzzzz-2x53u-382brsig8rp3065',
+ 'ARVADOS_LOGIN_SYNC_TMPDIR' => @tmpdir)
+ end
+
+ def invoke_sync opts={}
+ env = stubenv.merge(opts[:env] || {})
+ (opts[:binstubs] || []).each do |binstub|
+ env['PATH'] = File.absolute_path('../binstub_'+binstub, __FILE__) + ':' + env['PATH']
+ end
+ login_sync_path = File.absolute_path '../../bin/arvados-login-sync', __FILE__
+ system env, login_sync_path
+ end
+end
diff --git a/services/login-sync/test/test_add_user.rb b/services/login-sync/test/test_add_user.rb
new file mode 100644
index 0000000..c7d92b7
--- /dev/null
+++ b/services/login-sync/test/test_add_user.rb
@@ -0,0 +1,32 @@
+require 'minitest/autorun'
+
+require 'stubs'
+
+class TestAddUser < Minitest::Test
+ include Stubs
+
+ def test_useradd_error
+ # binstub_new_user/useradd will exit non-zero because its args
+ # won't match any line in this empty file:
+ File.open(@tmpdir+'/succeed', 'w') do |f| end
+ invoke_sync binstubs: ['new_user']
+ spied = File.read(@tmpdir+'/spy')
+ assert_match %r{useradd -m -c active -s /bin/bash -G fuse active}, spied
+ assert_match %r{useradd -m -c adminroot -s /bin/bash -G docker,fuse adminroot}, spied
+ end
+
+ def test_useradd_success
+ # binstub_new_user/useradd will exit non-zero because its args
+ # won't match any line in this empty file:
+ File.open(@tmpdir+'/succeed', 'w') do |f|
+ f.puts 'useradd -m -c active -s /bin/bash -G fuse active'
+ f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,fuse adminroot'
+ end
+ $stderr.puts "*** Expect crash in dir_s_mkdir:"
+ invoke_sync binstubs: ['new_user']
+ assert !$?.success?
+ spied = File.read(@tmpdir+'/spy')
+ # Expect a crash after adding one user, because Dir.mkdir({home}) fails.
+ assert_match %r{^useradd -m -c [^\n]+\n$}s, spied
+ end
+end
commit dbc73778d068929e2d5f2fe3a422438990a698d8
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Aug 13 11:56:21 2015 -0400
6781: Use env vars instead of yaml config file.
diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync
index 641c510..b9cecb6 100755
--- a/services/login-sync/bin/arvados-login-sync
+++ b/services/login-sync/bin/arvados-login-sync
@@ -7,50 +7,31 @@ require 'etc'
require 'fileutils'
require 'yaml'
-# This script does the actual account/key management on disk for the shell machine(s).
-#
-# Ward Vandewege <ward at curoverse.com>
-
-# Default is development
-production = ARGV[0] == "production"
-
-ENV["RAILS_ENV"] = "development"
-ENV["RAILS_ENV"] = "production" if production
-
-DEBUG = 1
-
-# load and merge in the environment-specific application config info
-# if present, overriding base config parameters as specified
-path = File.dirname(__FILE__) + '/config/arvados-clients.yml'
-if File.exists?(path) then
- cp_config = YAML.load_file(path)[ENV['RAILS_ENV']]
-else
- puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file"
- exit 1
+req_envs = %w(ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_VIRTUAL_MACHINE_UUID)
+req_envs.each do |k|
+ unless ENV[k]
+ abort "Fatal: These environment vars must be set: #{req_envs}"
+ end
end
-shell_hostname = cp_config['arvados_shell_hostname']
-
-ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
-ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
-
keys = ''
seen = Hash.new
begin
-
uids = Hash[Etc.to_enum(:passwd).map { |ent| [ent.name, ent.uid] }]
gids = Hash[Etc.to_enum(:group).map { |ent| [ent.name, ent.gid] }]
- arv = Arvados.new( { :suppress_ssl_warnings => false } )
+ arv = Arvados.new({ :suppress_ssl_warnings => false })
+
+ vm_uuid = ENV['ARVADOS_VIRTUAL_MACHINE_UUID']
begin
- logins = arv.virtual_machine.get_all_logins(limit: 10000, uuid: cp_config['vm_uuid'])[:items]
+ logins = arv.virtual_machine.get_all_logins(limit: 10000, uuid: vm_uuid)[:items]
rescue
- logins = arv.virtual_machine.logins(:uuid => cp_config['vm_uuid'])[:items]
+ logins = arv.virtual_machine.logins(:uuid => vm_uuid)[:items]
end
logins = [] if logins.nil?
- logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:hostname] != shell_hostname }
+ logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:virtual_machine_uuid] != vm_uuid }
# No system users
uid_min = 1000
commit 39a62064a95b3909080c0ab9fbb82a09a7bdaaee
Author: Ward Vandewege <ward at curoverse.com>
Date: Thu Aug 13 10:46:57 2015 -0400
6781: Add arvados-login-sync (was update-shell-accounts.rb)
diff --git a/services/login-sync/arvados-login-sync.gemspec b/services/login-sync/arvados-login-sync.gemspec
new file mode 100644
index 0000000..2f95a55
--- /dev/null
+++ b/services/login-sync/arvados-login-sync.gemspec
@@ -0,0 +1,24 @@
+if not File.exists?('/usr/bin/git') then
+ STDERR.puts "\nGit binary not found, aborting. Please install git and run gem build from a checked out copy of the git repository.\n\n"
+ exit
+end
+
+git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H .`.chomp.split(":")
+git_timestamp = Time.at(git_timestamp.to_i).utc
+
+Gem::Specification.new do |s|
+ s.name = 'arvados-login-sync'
+ s.version = "0.1.#{git_timestamp.strftime('%Y%m%d%H%M%S')}"
+ s.date = git_timestamp.strftime("%Y-%m-%d")
+ s.summary = "Set up local login accounts for Arvados users"
+ s.description = "Creates and updates local login accounts for Arvados users. Built from git commit #{git_hash}"
+ s.authors = ["Arvados Authors"]
+ s.email = 'gem-dev at curoverse.com'
+ s.licenses = ['GNU Affero General Public License, version 3.0']
+ s.files = ["bin/arvados-login-sync"]
+ s.executables << "arvados-login-sync"
+ s.required_ruby_version = '>= 2.1.0'
+ s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.20150615153458'
+ s.homepage =
+ 'https://arvados.org'
+end
diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync
new file mode 100755
index 0000000..641c510
--- /dev/null
+++ b/services/login-sync/bin/arvados-login-sync
@@ -0,0 +1,134 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'pp'
+require 'arvados'
+require 'etc'
+require 'fileutils'
+require 'yaml'
+
+# This script does the actual account/key management on disk for the shell machine(s).
+#
+# Ward Vandewege <ward at curoverse.com>
+
+# Default is development
+production = ARGV[0] == "production"
+
+ENV["RAILS_ENV"] = "development"
+ENV["RAILS_ENV"] = "production" if production
+
+DEBUG = 1
+
+# load and merge in the environment-specific application config info
+# if present, overriding base config parameters as specified
+path = File.dirname(__FILE__) + '/config/arvados-clients.yml'
+if File.exists?(path) then
+ cp_config = YAML.load_file(path)[ENV['RAILS_ENV']]
+else
+ puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file"
+ exit 1
+end
+
+shell_hostname = cp_config['arvados_shell_hostname']
+
+ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
+ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
+
+keys = ''
+
+seen = Hash.new
+
+begin
+
+ uids = Hash[Etc.to_enum(:passwd).map { |ent| [ent.name, ent.uid] }]
+ gids = Hash[Etc.to_enum(:group).map { |ent| [ent.name, ent.gid] }]
+ arv = Arvados.new( { :suppress_ssl_warnings => false } )
+
+ begin
+ logins = arv.virtual_machine.get_all_logins(limit: 10000, uuid: cp_config['vm_uuid'])[:items]
+ rescue
+ logins = arv.virtual_machine.logins(:uuid => cp_config['vm_uuid'])[:items]
+ end
+ logins = [] if logins.nil?
+ logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:hostname] != shell_hostname }
+
+ # No system users
+ uid_min = 1000
+ open("/etc/login.defs", encoding: "utf-8") do |login_defs|
+ login_defs.each_line do |line|
+ next unless match = /^UID_MIN\s+(\S+)$/.match(line)
+ if match[1].start_with?("0x")
+ base = 16
+ elsif match[1].start_with?("0")
+ base = 8
+ else
+ base = 10
+ end
+ new_uid_min = match[1].to_i(base)
+ uid_min = new_uid_min if (new_uid_min > 0)
+ end
+ end
+ logins.reject! { |l| (uids[l[:username]] || 65535) < uid_min }
+
+ keys = Hash.new()
+
+ # Collect all keys
+ logins.each do |l|
+ keys[l[:username]] = Array.new() if not keys.has_key?(l[:username])
+ key = l[:public_key]
+ # Handle putty-style ssh public keys
+ key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ key.gsub!(/\n/,'')
+ key.strip
+
+ keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ end
+
+ seen = Hash.new()
+ devnull = open("/dev/null", "w")
+
+ logins.each do |l|
+ next if seen[l[:username]]
+ seen[l[:username]] = true if not seen.has_key?(l[:username])
+ @homedir = "/home/#{l[:username]}"
+
+ unless uids[l[:username]]
+ STDERR.puts "Creating account #{l[:username]}"
+ groups = l[:groups] || []
+ # Adding users to the FUSE group has long been hardcoded behavior.
+ groups << "fuse"
+ groups.select! { |name| gids[name] }
+ # Create new user
+ next unless system("/usr/sbin/useradd", "-m",
+ "-c", l[:username],
+ "-s", "/bin/bash",
+ "-G", groups.join(","),
+ l[:username],
+ out: devnull)
+ end
+ # Create .ssh directory if necessary
+ userdotssh = File.join(@homedir, ".ssh")
+ Dir.mkdir(userdotssh) if !File.exists?(userdotssh)
+ @key = "#######################################################################################
+# THIS FILE IS MANAGED BY #{$0} -- CHANGES WILL BE OVERWRITTEN #
+#######################################################################################\n\n"
+ @key += keys[l[:username]].join("\n") + "\n"
+ userauthkeys = File.join(userdotssh, "authorized_keys")
+ if !File.exists?(userauthkeys) or IO::read(userauthkeys) != @key then
+ f = File.new(userauthkeys, 'w')
+ f.write(@key)
+ f.close()
+ end
+ FileUtils.chown_R(l[:username], l[:username], userdotssh)
+ File.chmod(0700, userdotssh)
+ File.chmod(0750, @homedir)
+ end
+
+ devnull.close
+rescue Exception => bang
+ puts "Error: " + bang.to_s
+ puts bang.backtrace.join("\n")
+ exit 1
+end
+
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list