[ARVADOS] created: a28d3645016c76a8f7e0e4b3c6237a73e9f0f3e3
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 24 16:56:43 EST 2014
at a28d3645016c76a8f7e0e4b3c6237a73e9f0f3e3 (commit)
commit a28d3645016c76a8f7e0e4b3c6237a73e9f0f3e3
Author: Brett Smith <brett at curoverse.com>
Date: Mon Nov 24 16:55:38 2014 -0500
4027: Crunch installs jobs' requested arvados_sdk_version.
* crunch-dispatch fetches the requested SDK version into its internal
git repository, just like it does for the Crunch script. Refactored
crunch-dispatch to make that code reusable.
* crunch-job's main script archives the sdk subdirectory as of that
commit, sending it along to compute nodes in the same .tar as the
Crunch script, under .arvados.sdk.
* crunch-job's __DATA__ dispatch section looks for the SDK under
.arvados.sdk, and installs it as much as possible.
Since I was messing with it so much already, I changed the semantics
of crunch-job's __DATA__ section: it is now either in installation
mode or run mode, based on whether there's anything in @ARGV. I
confirmed that this is consistent with current calls to the section.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 0d35d53..8528dd7 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -335,13 +335,9 @@ if (!$have_slurm)
must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
}
-
-my $build_script;
-do {
- local $/ = undef;
- $build_script = <DATA>;
-};
+my $build_script = handle_readall(\*DATA);
my $nodelist = join(",", @node);
+my $git_tar_count = 0;
if (!defined $no_clear_tmp) {
# Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
@@ -363,8 +359,49 @@ if (!defined $no_clear_tmp) {
Log (undef, "Cleanup command exited ".exit_status_s($?));
}
+# If this job requires a Docker image, install that.
+my $docker_bin = "/usr/bin/docker.io";
+my ($docker_locator, $docker_stream, $docker_hash);
+if ($docker_locator = $Job->{docker_image_locator}) {
+ ($docker_stream, $docker_hash) = find_docker_image($docker_locator);
+ if (!$docker_hash)
+ {
+ croak("No Docker image hash found from locator $docker_locator");
+ }
+ $docker_stream =~ s/^\.//;
+ my $docker_install_script = qq{
+if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then
+ arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load
+fi
+};
+ my $docker_pid = fork();
+ if ($docker_pid == 0)
+ {
+ srun (["srun", "--nodelist=" . join(',', @node)],
+ ["/bin/sh", "-ec", $docker_install_script]);
+ exit ($?);
+ }
+ while (1)
+ {
+ last if $docker_pid == waitpid (-1, WNOHANG);
+ freeze_if_want_freeze ($docker_pid);
+ select (undef, undef, undef, 0.1);
+ }
+ if ($? != 0)
+ {
+ croak("Installing Docker image from $docker_locator exited "
+ .exit_status_s($?));
+ }
+
+ if ($Job->{arvados_sdk_version}) {
+ # The job also specifies an Arvados SDK version. Add the SDKs to the
+ # tar file for the build script to install.
+ add_git_archive("git", "--git-dir=$git_dir", "archive",
+ "--prefix=.arvados.sdk/",
+ $Job->{arvados_sdk_version}, "sdk");
+ }
+}
-my $git_archive;
if (!defined $git_dir && $Job->{'script_version'} =~ m{^/}) {
# If script_version looks like an absolute path, *and* the --git-dir
# argument was not given -- which implies we were not invoked by
@@ -518,12 +555,10 @@ else {
}
$ENV{"CRUNCH_SRC_COMMIT"} = $commit;
- $git_archive = `$gitcmd archive ''\Q$commit\E`;
- if ($?) {
- croak("Error: $gitcmd archive exited ".exit_status_s($?));
- }
+ add_git_archive("$gitcmd archive ''\Q$commit\E");
}
+my $git_archive = combined_git_archive();
if (!defined $git_archive) {
Log(undef, "Skip install phase (no git archive)");
if ($have_slurm) {
@@ -553,48 +588,10 @@ else {
}
my $install_exited = $?;
Log (undef, "Install script exited ".exit_status_s($install_exited));
- exit (1) if $install_exited != 0;
-}
-
-if (!$have_slurm)
-{
- # Grab our lock again (we might have deleted and re-created CRUNCH_TMP above)
- must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
-}
-
-# If this job requires a Docker image, install that.
-my $docker_bin = "/usr/bin/docker.io";
-my ($docker_locator, $docker_stream, $docker_hash);
-if ($docker_locator = $Job->{docker_image_locator}) {
- ($docker_stream, $docker_hash) = find_docker_image($docker_locator);
- if (!$docker_hash)
- {
- croak("No Docker image hash found from locator $docker_locator");
- }
- $docker_stream =~ s/^\.//;
- my $docker_install_script = qq{
-if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then
- arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load
-fi
-};
- my $docker_pid = fork();
- if ($docker_pid == 0)
- {
- srun (["srun", "--nodelist=" . join(',', @node)],
- ["/bin/sh", "-ec", $docker_install_script]);
- exit ($?);
- }
- while (1)
- {
- last if $docker_pid == waitpid (-1, WNOHANG);
- freeze_if_want_freeze ($docker_pid);
- select (undef, undef, undef, 0.1);
- }
- if ($? != 0)
- {
- croak("Installing Docker image from $docker_locator exited "
- .exit_status_s($?));
+ foreach my $tar_filename (map { tar_filename_n($_); } (1..$git_tar_count)) {
+ unlink($tar_filename);
}
+ exit (1) if $install_exited != 0;
}
foreach (qw (script script_version script_parameters runtime_constraints))
@@ -1728,17 +1725,85 @@ sub exit_status_s {
return $s;
}
+sub handle_readall {
+ # Pass in a glob reference to a file handle.
+ # Read all its contents and return them as a string.
+ my $fh_glob_ref = shift;
+ local $/ = undef;
+ return <$fh_glob_ref>;
+}
+
+sub tar_filename_n {
+ my $n = shift;
+ return sprintf("%s/git.%s.%d.tar", $ENV{CRUNCH_TMP}, $job_id, $n);
+}
+
+sub add_git_archive {
+ # Pass in a git archive command as a string or list, a la system().
+ # This method will save its output to be included in the archive sent to the
+ # build script.
+ my $git_input;
+ $git_tar_count++;
+ if (!open(GIT_ARCHIVE, ">", tar_filename_n($git_tar_count))) {
+ croak("Failed to save git archive: $!");
+ }
+ my $git_pid = open2(">&GIT_ARCHIVE", $git_input, @_);
+ close($git_input);
+ waitpid($git_pid, 0);
+ close(GIT_ARCHIVE);
+ if ($?) {
+ croak("Failed to save git archive: git exited " . exit_status_s($?));
+ }
+}
+
+sub combined_git_archive {
+ # Combine all saved tar archives into a single archive, then return its
+ # contents in a string. Return undef if no archives have been saved.
+ if ($git_tar_count < 1) {
+ return undef;
+ }
+ my $base_tar_name = tar_filename_n(1);
+ foreach my $tar_to_append (map { tar_filename_n($_); } (2..$git_tar_count)) {
+ my $tar_exit = system("tar", "-Af", $base_tar_name, $tar_to_append);
+ if ($tar_exit != 0) {
+ croak("Error preparing build archive: tar -A exited " .
+ exit_status_s($tar_exit));
+ }
+ }
+ if (!open(GIT_TAR, "<", $base_tar_name)) {
+ croak("Could not open build archive: $!");
+ }
+ my $tar_contents = handle_readall(\*GIT_TAR);
+ close(GIT_TAR);
+ return $tar_contents;
+}
+
__DATA__
#!/usr/bin/perl
-
-# checkout-and-build
+#
+# This is crunch-job's internal dispatch script. crunch-job running on the API
+# server invokes this script on individual compute nodes, or localhost if we're
+# running a job locally. It gets called in two modes:
+#
+# * No arguments: Installation mode. Read a tar archive from the DATA
+# file handle; it includes the Crunch script's source code, and
+# maybe SDKs as well. Those should be installed in the proper
+# locations.
+#
+# * With arguments: Crunch script run mode. This script should set up the
+# environment, then run the command specified in the arguments.
use Fcntl ':flock';
use File::Path qw( make_path remove_tree );
+use POSIX qw(getcwd);
+
+# Map SDK subdirectories to the path environments they belong to.
+my %SDK_ENVVARS = ("perl/lib" => "PERLLIB", "ruby/lib" => "RUBYLIB");
my $destdir = $ENV{"CRUNCH_SRC"};
my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
my $repo = $ENV{"CRUNCH_SRC_URL"};
+my $install_dir = $ENV{"CRUNCH_INSTALL"} || (getcwd() . "/opt");
my $job_work = $ENV{"JOB_WORK"};
my $task_work = $ENV{"TASK_WORK"};
@@ -1753,17 +1818,31 @@ if ($task_work) {
remove_tree($task_work, {keep_root => 1});
}
-my @git_archive_data = <DATA>;
-if (!@git_archive_data) {
- # Nothing to extract -> nothing to install.
- run_argv_and_exit();
+if (@ARGV) {
+ if (-e "$install_dir/bin/activate") {
+ my $orig_argv = join(" ", map { quotemeta($_); } @ARGV);
+ @ARGV = ("/bin/sh", "-ec",
+ ". \Q$install_dir/bin/activate\E; exec $orig_argv");
+ }
+ while (my ($sdk_dir, $sdk_envkey) = each(%SDK_ENVVARS)) {
+ my $sdk_path = "$install_dir/$sdk_dir";
+ if (-d $sdk_path) {
+ if ($ENV{$sdk_envkey}) {
+ $ENV{$sdk_envkey} = "$sdk_path:" . $ENV{$sdk_envkey};
+ } else {
+ $ENV{$sdk_envkey} = $sdk_path;
+ }
+ }
+ }
+ exec(@ARGV);
+ die "Cannot exec `@ARGV`: $!";
}
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
flock L, LOCK_EX;
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
# This version already installed -> nothing to do.
- run_argv_and_exit();
+ exit(0);
}
unlink "$destdir.commit";
@@ -1772,21 +1851,31 @@ open STDOUT, ">", "$destdir.log";
open STDERR, ">&STDOUT";
mkdir $destdir;
-open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
-print TARX @git_archive_data;
+open TARX, "|-", "tar", "-xC", $destdir;
+{
+ local $/ = undef;
+ print TARX <DATA>;
+}
if(!close(TARX)) {
- die "'tar -C $destdir -xf -' exited $?: $!";
+ die "'tar -xC $destdir' exited $?: $!";
}
-my $pwd;
-chomp ($pwd = `pwd`);
-my $install_dir = $ENV{"CRUNCH_INSTALL"} || "$pwd/opt";
mkdir $install_dir;
-for my $src_path ("$destdir/arvados/sdk/python") {
- if (-d $src_path) {
- shell_or_die ("virtualenv", $install_dir);
- shell_or_die ("cd $src_path && ./build.sh && $install_dir/bin/python setup.py install");
+my $sdk_root = "$destdir/.arvados.sdk/sdk";
+if (-d $sdk_root) {
+ if (can_run("virtualenv")) {
+ shell_or_die("virtualenv", "--python=python2.7", "--system-site-packages",
+ $install_dir);
+ shell_or_die("$install_dir/bin/pip", "install", "$sdk_root/python");
+ }
+
+ foreach my $sdk_lang (map { (split /\//, $_, 2)[0]; } keys(%SDK_ENVVARS)) {
+ if (-d "$sdk_root/$sdk_lang") {
+ if (!rename("$sdk_root/$sdk_lang", "$install_dir/$sdk_lang")) {
+ die "Failed to install $sdk_lang SDK: $!";
+ }
+ }
}
}
@@ -1807,16 +1896,12 @@ if ($commit) {
close L;
-run_argv_and_exit();
-
-sub run_argv_and_exit
-{
- if (@ARGV) {
- exec(@ARGV);
- die "Cannot exec `@ARGV`: $!";
- } else {
- exit 0;
- }
+sub can_run {
+ my $command_name = shift;
+ open(my $which, "-|", "which", $command_name);
+ while (<$which>) { }
+ close($which);
+ return ($? == 0);
}
sub shell_or_die
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index ebd5165..697c85d 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -47,16 +47,38 @@ require 'open3'
class Dispatcher
include ApplicationHelper
+ def initialize
+ @crunch_job_bin = (ENV['CRUNCH_JOB_BIN'] || `which arv-crunch-job`.strip)
+ if @crunch_job_bin.empty?
+ raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
+ end
+
+ @arvados_internal = Rails.configuration.git_internal_dir
+ if not File.exists? @arvados_internal
+ $stderr.puts `mkdir -p #{@arvados_internal.shellescape} && git init --bare #{@arvados_internal.shellescape}`
+ raise "No internal git repository available" unless ($? == 0)
+ end
+
+ @repo_root = Rails.configuration.git_repositories_dir
+ @authorizations = {}
+ @did_recently = {}
+ @fetched_commits = {}
+ @git_tags = {}
+ @node_state = {}
+ @pipe_auth_tokens = {}
+ @running = {}
+ @todo = []
+ @todo_pipelines = []
+ end
+
def sysuser
return act_as_system_user
end
def refresh_todo
- @todo = []
if $options[:jobs]
@todo = Job.queue.select(&:repository)
end
- @todo_pipelines = []
if $options[:pipelines]
@todo_pipelines = PipelineInstance.queue
end
@@ -108,7 +130,6 @@ class Dispatcher
def update_node_status
return unless Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/
- @node_state ||= {}
slurm_status.each_pair do |hostname, slurmdata|
next if @node_state[hostname] == slurmdata
begin
@@ -217,6 +238,96 @@ class Dispatcher
end
end
+ def git_cmd(cmd_s)
+ "git --git-dir=#{@arvados_internal.shellescape} #{cmd_s}"
+ end
+
+ def get_authorization(job)
+ if @authorizations[job.uuid] and
+ @authorizations[job.uuid].user.uuid != job.modified_by_user_uuid
+ # We already made a token for this job, but we need a new one
+ # because modified_by_user_uuid has changed (the job will run
+ # as a different user).
+ @authorizations[job.uuid].update_attributes expires_at: Time.now
+ @authorizations[job.uuid] = nil
+ end
+ if not @authorizations[job.uuid]
+ auth = ApiClientAuthorization.
+ new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
+ api_client_id: 0)
+ if not auth.save
+ $stderr.puts "dispatch: auth.save failed for #{job.uuid}"
+ else
+ @authorizations[job.uuid] = auth
+ end
+ end
+ @authorizations[job.uuid]
+ end
+
+ def get_commit(repo_name, commit_hash)
+ # @fetched_commits[V]==true if we know commit V exists in the
+ # arvados_internal git repository.
+ if !@fetched_commits[commit_hash]
+ src_repo = File.join(@repo_root, repo_name + '.git')
+ if not File.exists? src_repo
+ src_repo = File.join(@repo_root, repo_name, '.git')
+ if not File.exists? src_repo
+ fail_job job, "No #{repo_name}.git or #{repo_name}/.git at #{@repo_root}"
+ return nil
+ end
+ end
+
+ # check if the commit needs to be fetched or not
+ commit_rev = `#{git_cmd("rev-list -n1 #{commit_hash.shellescape} 2>/dev/null")}`.chomp
+ unless $? == 0 and commit_rev == commit_hash
+ # commit does not exist in internal repository, so import the source repository using git fetch-pack
+ cmd = git_cmd("fetch-pack --no-progress --all #{src_repo.shellescape}")
+ $stderr.puts "dispatch: #{cmd}"
+ $stderr.puts `#{cmd}`
+ unless $? == 0
+ fail_job job, "git fetch-pack failed"
+ return nil
+ end
+ end
+ @fetched_commits[commit_hash] = true
+ end
+ @fetched_commits[commit_hash]
+ end
+
+ def tag_commit(commit_hash, tag_name)
+ # @git_tags[T]==V if we know commit V has been tagged T in the
+ # arvados_internal repository.
+ if not @git_tags[tag_name]
+ cmd = git_cmd("tag #{tag_name.shellescape} #{commit_hash.shellescape} 2>/dev/null")
+ $stderr.puts "dispatch: #{cmd}"
+ $stderr.puts `#{cmd}`
+ unless $? == 0
+ # git tag failed. This may be because the tag already exists, so check for that.
+ tag_rev = `#{git_cmd("rev-list -n1 #{tag_name.shellescape}")}`.chomp
+ if $? == 0
+ # We got a revision back
+ if tag_rev != commit_hash
+ # Uh oh, the tag doesn't point to the revision we were expecting.
+ # Someone has been monkeying with the job record and/or git.
+ fail_job job, "Existing tag #{tag_name} points to commit #{tag_rev} but expected commit #{commit_hash}"
+ return nil
+ end
+ # we're okay (fall through to setting @git_tags below)
+ else
+ # git rev-list failed for some reason.
+ fail_job job, "'git tag' for #{tag_name} failed but did not find any existing tag using 'git rev-list'"
+ return nil
+ end
+ end
+ # 'git tag' was successful, or there is an existing tag that points to the same revision.
+ @git_tags[tag_name] = commit_hash
+ elsif @git_tags[tag_name] != commit_hash
+ fail_job job, "Existing tag #{tag_name} points to commit #{@git_tags[tag_name]} but this job uses commit #{commit_hash}"
+ return nil
+ end
+ @git_tags[tag_name]
+ end
+
def start_jobs
@todo.each do |job|
next if @running[job.uuid]
@@ -259,108 +370,19 @@ class Dispatcher
"GEM_PATH=#{ENV['GEM_PATH']}")
end
- @authorizations ||= {}
- if @authorizations[job.uuid] and
- @authorizations[job.uuid].user.uuid != job.modified_by_user_uuid
- # We already made a token for this job, but we need a new one
- # because modified_by_user_uuid has changed (the job will run
- # as a different user).
- @authorizations[job.uuid].update_attributes expires_at: Time.now
- @authorizations[job.uuid] = nil
- end
- if not @authorizations[job.uuid]
- auth = ApiClientAuthorization.
- new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
- api_client_id: 0)
- if not auth.save
- $stderr.puts "dispatch: auth.save failed"
- next
- end
- @authorizations[job.uuid] = auth
- end
-
- crunch_job_bin = (ENV['CRUNCH_JOB_BIN'] || `which arv-crunch-job`.strip)
- if crunch_job_bin == ''
- raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
- end
-
- arvados_internal = Rails.configuration.git_internal_dir
- if not File.exists? arvados_internal
- $stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
- end
-
- git = "git --git-dir=#{arvados_internal.shellescape}"
-
- # @fetched_commits[V]==true if we know commit V exists in the
- # arvados_internal git repository.
- @fetched_commits ||= {}
- if !@fetched_commits[job.script_version]
-
- repo_root = Rails.configuration.git_repositories_dir
- src_repo = File.join(repo_root, job.repository + '.git')
- if not File.exists? src_repo
- src_repo = File.join(repo_root, job.repository, '.git')
- if not File.exists? src_repo
- fail_job job, "No #{job.repository}.git or #{job.repository}/.git at #{repo_root}"
- next
- end
- end
-
- # check if the commit needs to be fetched or not
- commit_rev = `#{git} rev-list -n1 #{job.script_version.shellescape} 2>/dev/null`.chomp
- unless $? == 0 and commit_rev == job.script_version
- # commit does not exist in internal repository, so import the source repository using git fetch-pack
- cmd = "#{git} fetch-pack --no-progress --all #{src_repo.shellescape}"
- $stderr.puts "dispatch: #{cmd}"
- $stderr.puts `#{cmd}`
- unless $? == 0
- fail_job job, "git fetch-pack failed"
- next
- end
- end
- @fetched_commits[job.script_version] = true
- end
-
- # @job_tags[J]==V if we know commit V has been tagged J in the
- # arvados_internal repository. (J is a job UUID, V is a commit
- # sha1.)
- @job_tags ||= {}
- if not @job_tags[job.uuid]
- cmd = "#{git} tag #{job.uuid.shellescape} #{job.script_version.shellescape} 2>/dev/null"
- $stderr.puts "dispatch: #{cmd}"
- $stderr.puts `#{cmd}`
- unless $? == 0
- # git tag failed. This may be because the tag already exists, so check for that.
- tag_rev = `#{git} rev-list -n1 #{job.uuid.shellescape}`.chomp
- if $? == 0
- # We got a revision back
- if tag_rev != job.script_version
- # Uh oh, the tag doesn't point to the revision we were expecting.
- # Someone has been monkeying with the job record and/or git.
- fail_job job, "Existing tag #{job.uuid} points to commit #{tag_rev} but expected commit #{job.script_version}"
- next
- end
- # we're okay (fall through to setting @job_tags below)
- else
- # git rev-list failed for some reason.
- fail_job job, "'git tag' for #{job.uuid} failed but did not find any existing tag using 'git rev-list'"
- next
- end
- end
- # 'git tag' was successful, or there is an existing tag that points to the same revision.
- @job_tags[job.uuid] = job.script_version
- elsif @job_tags[job.uuid] != job.script_version
- fail_job job, "Existing tag #{job.uuid} points to commit #{@job_tags[job.uuid]} but this job uses commit #{job.script_version}"
- next
+ ready = (get_authorization(job) and
+ get_commit(job.repository, job.script_version) and
+ tag_commit(job.script_version, job.uuid))
+ if ready and job.arvados_sdk_version
+ ready = (get_commit("arvados", job.arvados_sdk_version) and
+ tag_commit(job.arvados_sdk_version, "#{job.uuid}-arvados-sdk"))
end
+ next unless ready
- cmd_args << crunch_job_bin
- cmd_args << '--job-api-token'
- cmd_args << @authorizations[job.uuid].api_token
- cmd_args << '--job'
- cmd_args << job.uuid
- cmd_args << '--git-dir'
- cmd_args << arvados_internal
+ cmd_args += [@crunch_job_bin,
+ '--job-api-token', @authorizations[job.uuid].api_token,
+ '--job', job.uuid,
+ '--git-dir', @arvados_internal]
$stderr.puts "dispatch: #{cmd_args.join ' '}"
@@ -652,8 +674,6 @@ class Dispatcher
def run
act_as_system_user
- @running ||= {}
- @pipe_auth_tokens ||= { }
$stderr.puts "dispatch: ready"
while !$signal[:term] or @running.size > 0
read_pipes
@@ -688,7 +708,6 @@ class Dispatcher
protected
def did_recently(thing, min_interval)
- @did_recently ||= {}
if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
@did_recently[thing] = Time.now
false
commit 300a50084617b6278ed3a91845b3e9f4ccf4c2d6
Author: Brett Smith <brett at curoverse.com>
Date: Mon Nov 24 15:53:44 2014 -0500
4027: Bugfix update-gitolite.rb in Docker.
* Load a YAML library.
* Support ARVADOS_API_HOST_INSECURE, and set it in normal Docker use.
diff --git a/docker/api/arvados-clients.yml.in b/docker/api/arvados-clients.yml.in
index babfc4e..59ff352 100644
--- a/docker/api/arvados-clients.yml.in
+++ b/docker/api/arvados-clients.yml.in
@@ -3,3 +3,4 @@ production:
gitolite_tmp: 'gitolite-tmp'
arvados_api_host: 'api'
arvados_api_token: '@@API_SUPERUSER_SECRET@@'
+ arvados_api_host_insecure: true
diff --git a/docker/api/update-gitolite.rb b/docker/api/update-gitolite.rb
index 779099a..2c46a0d 100755
--- a/docker/api/update-gitolite.rb
+++ b/docker/api/update-gitolite.rb
@@ -4,6 +4,7 @@ require 'rubygems'
require 'pp'
require 'arvados'
require 'active_support/all'
+require 'yaml'
# This script does the actual gitolite config management on disk.
#
@@ -34,6 +35,11 @@ gitolite_admin = File.join(File.expand_path(File.dirname(__FILE__)) + '/' + gito
ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
+if cp_config['arvados_api_host_insecure']
+ ENV['ARVADOS_API_HOST_INSECURE'] = 'true'
+else
+ ENV.delete('ARVADOS_API_HOST_INSECURE')
+end
keys = ''
commit 7f521ceda9485d18a0948ea79af4f009dba9ce93
Author: Brett Smith <brett at curoverse.com>
Date: Mon Nov 24 15:53:00 2014 -0500
4027: Revamp SSH use in our Docker images.
* Don't install or run SSH in most of our Docker images. `docker
exec` is now preferred to inspect running images.
* Do run SSH on the API server, always, for Gitolite.
There is a feature regression here: the user's SSH key is not
automatically installed on the shell account. This needs to be fixed
another way. In the meantime, it's not difficult to run
`docker exec -ti --user=self shell /bin/bash`, and you can clone the
repository from the host system.
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index ee9198e..abd2114 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -8,7 +8,7 @@ RUN apt-get update -qq
RUN apt-get install -qqy \
procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
- libjson-perl cron
+ libjson-perl cron openssh-server
ADD munge.key /etc/munge/
RUN chown munge:munge /etc/munge/munge.key && chmod 600 /etc/munge/munge.key
@@ -71,7 +71,6 @@ ADD update-gitolite.rb /usr/local/arvados/
# Supervisor.
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
-ADD ssh.sh /usr/local/bin/ssh.sh
ADD generated/setup.sh /usr/local/bin/setup.sh
ADD generated/setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
ADD crunch-dispatch-run.sh /usr/local/bin/crunch-dispatch-run.sh
diff --git a/docker/api/ssh.sh b/docker/api/ssh.sh
deleted file mode 100755
index 664414b..0000000
--- a/docker/api/ssh.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-echo $ENABLE_SSH
-
-# Start ssh daemon if requested via the ENABLE_SSH env variable
-if [[ ! "$ENABLE_SSH" =~ (0|false|no|f|^$) ]]; then
-echo "STARTING"
- /etc/init.d/ssh start
-fi
-
diff --git a/docker/api/supervisor.conf b/docker/api/supervisor.conf
index e684ff8..b01dc1c 100644
--- a/docker/api/supervisor.conf
+++ b/docker/api/supervisor.conf
@@ -1,6 +1,6 @@
[program:ssh]
user=root
-command=/usr/local/bin/ssh.sh
+command=/etc/init.d/ssh start
startsecs=0
[program:postgres]
diff --git a/docker/arvdock b/docker/arvdock
index 31e405b..142ba27 100755
--- a/docker/arvdock
+++ b/docker/arvdock
@@ -1,6 +1,5 @@
#!/bin/bash
-ENABLE_SSH=false
DOCKER=`which docker.io`
if [[ "$DOCKER" == "" ]]; then
@@ -22,7 +21,6 @@ function usage {
echo >&2 " -v, --vm Shell server"
echo >&2 " -n, --nameserver Nameserver"
echo >&2 " -k, --keep Keep servers"
- echo >&2 " --ssh Enable SSH access to server containers"
echo >&2 " -h, --help Display this help and exit"
echo >&2
echo >&2 " If no options are given, the action is applied to all servers."
@@ -65,11 +63,6 @@ function start_container {
fi
local image=$5
- if $ENABLE_SSH
- then
- args="$args -e ENABLE_SSH=$ENABLE_SSH"
- fi
-
`$DOCKER ps |grep -P "$name[^/]" -q`
if [[ "$?" == "0" ]]; then
echo "You have a running container with name $name -- skipping."
@@ -86,15 +79,6 @@ function start_container {
if [ "$?" != "0" -o "$container" = "" ]; then
echo "Unable to start container"
exit 1
- fi
- if [ "$name" == "compute" -o "$ENABLE_SSH" != "false" ];
- then
- ip=$(ip_address $container )
- echo
- echo "You can ssh into the container with:"
- echo
- echo " ssh root@$ip"
- echo
else
echo "Started container: $container"
fi
@@ -155,7 +139,7 @@ function do_start {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
local TEMP=`getopt -o d::s::a::cw::nkvh \
- --long doc::,sso::,api::,compute,workbench::,nameserver,keep,vm,help,ssh \
+ --long doc::,sso::,api::,compute,workbench::,nameserver,keep,vm,help \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -206,11 +190,6 @@ function do_start {
start_keep=true
shift
;;
- --ssh)
- # ENABLE_SSH is a global variable
- ENABLE_SSH=true
- shift
- ;;
--)
shift
break
diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile
index 2959d50..325b779 100644
--- a/docker/base/Dockerfile
+++ b/docker/base/Dockerfile
@@ -14,12 +14,11 @@ ADD apt.arvados.org.list /etc/apt/sources.list.d/
RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
RUN apt-get update -qq
-RUN apt-get install -qqy openssh-server apt-utils git curl \
+RUN apt-get install -qqy apt-utils git curl \
libcurl3 libcurl3-gnutls libcurl4-openssl-dev locales \
postgresql-server-dev-9.1 python-arvados-python-client
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
- /bin/mkdir -p /root/.ssh && \
/bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
/usr/sbin/locale-gen && \
curl -L https://get.rvm.io | bash -s stable && \
@@ -36,6 +35,3 @@ RUN /usr/local/rvm/bin/rvm-exec default gem update --system && \
/usr/local/rvm/bin/rvm-exec default bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
/usr/local/rvm/bin/rvm-exec default bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
/usr/local/rvm/bin/rvm-exec default bundle install --gemfile=/usr/src/arvados/doc/Gemfile
-
-ADD generated/id_rsa.pub /root/.ssh/authorized_keys
-RUN chown root:root /root/.ssh/authorized_keys
diff --git a/docker/build_tools/build.rb b/docker/build_tools/build.rb
index a79157f..e8f5809 100755
--- a/docker/build_tools/build.rb
+++ b/docker/build_tools/build.rb
@@ -85,7 +85,6 @@ def main options
config['ARVADOS_USER_NAME'] = user_name
config['API_HOSTNAME'] = generate_api_hostname
config['API_WORKBENCH_ADDRESS'] = 'false'
- config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
config.each_key do |var|
config_out.write "#{var}: #{config[var]}\n"
end
@@ -166,22 +165,6 @@ def docker_ok?(docker_path)
return system "#{docker_path} images > /dev/null 2>&1"
end
-# find_or_create_ssh_key arvados_name
-# Returns the SSH public key appropriate for this Arvados instance,
-# generating one if necessary.
-#
-def find_or_create_ssh_key arvados_name
- ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
- unless File.exists? ssh_key_file
- system 'ssh-keygen',
- '-f', ssh_key_file,
- '-C', "arvados@#{arvados_name}",
- '-P', ''
- end
-
- return "#{ssh_key_file}.pub"
-end
-
# install_docker
# Determines which Docker package is suitable for this Linux distro
# and installs it, resolving any dependencies.
diff --git a/docker/build_tools/config.rb b/docker/build_tools/config.rb
index d8bf256..296bc20 100755
--- a/docker/build_tools/config.rb
+++ b/docker/build_tools/config.rb
@@ -72,12 +72,3 @@ Dir.glob(globdir + '/*.in') do |template_file|
end
end
end
-
-# Copy the ssh public key file to base/generated (if a path is given)
-generated_dir = File.join('base/generated')
-Dir.mkdir(generated_dir) unless Dir.exists? generated_dir
-if (!config['PUBLIC_KEY_PATH'].nil? and
- File.readable? config['PUBLIC_KEY_PATH'])
- FileUtils.cp(config['PUBLIC_KEY_PATH'],
- File.join(generated_dir, 'id_rsa.pub'))
-end
diff --git a/docker/compute/Dockerfile b/docker/compute/Dockerfile
index 1dd3889..462115c 100644
--- a/docker/compute/Dockerfile
+++ b/docker/compute/Dockerfile
@@ -18,7 +18,6 @@ RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid
# Supervisor.
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
-ADD ssh.sh /usr/local/bin/ssh.sh
ADD generated/setup.sh /usr/local/bin/setup.sh
ADD wrapdocker /usr/local/bin/wrapdocker.sh
diff --git a/docker/compute/ssh.sh b/docker/compute/ssh.sh
deleted file mode 100755
index 664414b..0000000
--- a/docker/compute/ssh.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-echo $ENABLE_SSH
-
-# Start ssh daemon if requested via the ENABLE_SSH env variable
-if [[ ! "$ENABLE_SSH" =~ (0|false|no|f|^$) ]]; then
-echo "STARTING"
- /etc/init.d/ssh start
-fi
-
diff --git a/docker/compute/supervisor.conf b/docker/compute/supervisor.conf
index 7fc34fc..615e55a 100644
--- a/docker/compute/supervisor.conf
+++ b/docker/compute/supervisor.conf
@@ -1,8 +1,3 @@
-[program:ssh]
-user=root
-command=/usr/local/bin/ssh.sh
-startsecs=0
-
[program:munge]
user=root
command=/etc/init.d/munge start
diff --git a/docker/config.yml.example b/docker/config.yml.example
index 6ba5bcf..4210ec3 100644
--- a/docker/config.yml.example
+++ b/docker/config.yml.example
@@ -1,12 +1,6 @@
# Configuration for the Rails databases (database names,
# usernames and passwords).
-# Path to a public ssh key you would like to have installed in the
-# root account of the generated images. This is useful for debugging.
-# To enable the SSH server, set the ENABLE_SSH environment variable to
-# true when starting the container.
-PUBLIC_KEY_PATH:
-
# Username for your Arvados user. This will be used as your shell login name
# as well as the name for your git repository.
ARVADOS_USER_NAME:
diff --git a/docker/doc/apache2_foreground.sh b/docker/doc/apache2_foreground.sh
index c3facfe..fc6028e 100755
--- a/docker/doc/apache2_foreground.sh
+++ b/docker/doc/apache2_foreground.sh
@@ -3,10 +3,5 @@
read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
trap "kill -TERM -$pgrp; exit" EXIT TERM KILL SIGKILL SIGTERM SIGQUIT
-# Start ssh daemon if requested via the ENABLE_SSH env variable
-if [[ ! "$ENABLE_SSH" =~ (0|false|no|f|^$) ]]; then
- /etc/init.d/ssh start
-fi
-
source /etc/apache2/envvars
/usr/sbin/apache2 -D FOREGROUND
diff --git a/docker/shell/Dockerfile b/docker/shell/Dockerfile
index 539ff94..8235159 100644
--- a/docker/shell/Dockerfile
+++ b/docker/shell/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update -qq
RUN apt-get install -qqy \
python-pip python-pyvcf python-gflags python-google-api-python-client \
python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse \
- crunchstat python-arvados-fuse cron vim supervisor
+ crunchstat python-arvados-fuse cron vim supervisor openssh-server
ADD fuse.conf /etc/fuse.conf
RUN chmod 644 /etc/fuse.conf
diff --git a/docker/shell/setup.sh.in b/docker/shell/setup.sh.in
index 2815201..03beb4b 100755
--- a/docker/shell/setup.sh.in
+++ b/docker/shell/setup.sh.in
@@ -5,8 +5,6 @@ USER_NAME="@@ARVADOS_USER_NAME@@"
useradd $USER_NAME -s /bin/bash
mkdir /home/$USER_NAME/.ssh -p
-cp ~root/.ssh/authorized_keys /home/$USER_NAME/.ssh/authorized_keys
-
# Install our token
mkdir -p /home/$USER_NAME/.config/arvados;
echo "ARVADOS_API_HOST=api" >> /home/$USER_NAME/.config/arvados/settings.conf
diff --git a/docker/slurm/supervisor.conf b/docker/slurm/supervisor.conf
index 6563b54..64f86b1 100644
--- a/docker/slurm/supervisor.conf
+++ b/docker/slurm/supervisor.conf
@@ -1,8 +1,3 @@
-[program:ssh]
-user=root
-command=/usr/local/bin/ssh.sh
-startsecs=0
-
[program:munge]
user=root
command=/etc/init.d/munge start
@@ -10,5 +5,3 @@ command=/etc/init.d/munge start
[program:slurm]
user=root
command=/etc/init.d/slurm-llnl start
-
-
diff --git a/docker/sso/apache2_foreground.sh b/docker/sso/apache2_foreground.sh
index c3facfe..fc6028e 100755
--- a/docker/sso/apache2_foreground.sh
+++ b/docker/sso/apache2_foreground.sh
@@ -3,10 +3,5 @@
read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
trap "kill -TERM -$pgrp; exit" EXIT TERM KILL SIGKILL SIGTERM SIGQUIT
-# Start ssh daemon if requested via the ENABLE_SSH env variable
-if [[ ! "$ENABLE_SSH" =~ (0|false|no|f|^$) ]]; then
- /etc/init.d/ssh start
-fi
-
source /etc/apache2/envvars
/usr/sbin/apache2 -D FOREGROUND
diff --git a/docker/workbench/apache2_foreground.sh b/docker/workbench/apache2_foreground.sh
index 37650ab..5475ff0 100755
--- a/docker/workbench/apache2_foreground.sh
+++ b/docker/workbench/apache2_foreground.sh
@@ -3,13 +3,8 @@
read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
trap "kill -TERM -$pgrp; exit" EXIT TERM KILL SIGKILL SIGTERM SIGQUIT
-# Start ssh daemon if requested via the ENABLE_SSH env variable
-if [[ ! "$ENABLE_SSH" =~ (0|false|no|f|^$) ]]; then
- /etc/init.d/ssh start
-fi
-
# Override the default API server address if necessary.
-if [[ "$API_PORT_443_TCP_ADDR" != "" ]]; then
+if [[ "$API_PORT_443_TCP_ADDR" != "" ]]; then
sed -i "s/localhost:9900/$API_PORT_443_TCP_ADDR/" /usr/src/arvados/apps/workbench/config/application.yml
fi
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list