[ARVADOS] updated: b0bbd5ec9f29e1a7e297b4de4d19b238cf188f3b
git at public.curoverse.com
git at public.curoverse.com
Mon Dec 1 15:33:42 EST 2014
Summary of changes:
sdk/cli/bin/crunch-job | 33 ++++++++++++++++++++++-----------
services/api/script/crunch-dispatch.rb | 25 ++++++++++++++++---------
2 files changed, 38 insertions(+), 20 deletions(-)
via b0bbd5ec9f29e1a7e297b4de4d19b238cf188f3b (commit)
from 0e9c8241e683d331a8491bc3a7f5a734df4c0dd2 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b0bbd5ec9f29e1a7e297b4de4d19b238cf188f3b
Author: Brett Smith <brett at curoverse.com>
Date: Mon Dec 1 15:33:39 2014 -0500
4027: Crunch fixups suggested by code review.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 8528dd7..caf837b 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1788,10 +1788,12 @@ __DATA__
# * No arguments: Installation mode. Read a tar archive from the DATA
# file handle; it includes the Crunch script's source code, and
# maybe SDKs as well. Those should be installed in the proper
-# locations.
+# locations. This runs outside of any Docker container, so don't try to
+# introspect Crunch's runtime environment.
#
# * With arguments: Crunch script run mode. This script should set up the
-# environment, then run the command specified in the arguments.
+# environment, then run the command specified in the arguments. This runs
+# inside any Docker container.
use Fcntl ':flock';
use File::Path qw( make_path remove_tree );
@@ -1818,12 +1820,24 @@ if ($task_work) {
remove_tree($task_work, {keep_root => 1});
}
+### Crunch script run mode
if (@ARGV) {
- if (-e "$install_dir/bin/activate") {
+ my $python_src = "$install_dir/python";
+ my $venv_dir = "$job_work/.arvados.venv";
+ my $venv_built = -e "$venv_dir/bin/activate";
+ if ((!$venv_built) and (-d $python_src) and can_run("virtualenv")) {
+ shell_or_die("virtualenv", "--python=python2.7", "--system-site-packages",
+ $venv_dir);
+ shell_or_die("$venv_dir/bin/pip", "install", $python_src);
+ $venv_built = 1;
+ }
+
+ if ($venv_built) {
my $orig_argv = join(" ", map { quotemeta($_); } @ARGV);
@ARGV = ("/bin/sh", "-ec",
- ". \Q$install_dir/bin/activate\E; exec $orig_argv");
+ ". \Q$venv_dir/bin/activate\E; exec $orig_argv");
}
+
while (my ($sdk_dir, $sdk_envkey) = each(%SDK_ENVVARS)) {
my $sdk_path = "$install_dir/$sdk_dir";
if (-d $sdk_path) {
@@ -1834,10 +1848,12 @@ if (@ARGV) {
}
}
}
+
exec(@ARGV);
die "Cannot exec `@ARGV`: $!";
}
+### Installation mode
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
flock L, LOCK_EX;
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
@@ -1864,13 +1880,8 @@ mkdir $install_dir;
my $sdk_root = "$destdir/.arvados.sdk/sdk";
if (-d $sdk_root) {
- if (can_run("virtualenv")) {
- shell_or_die("virtualenv", "--python=python2.7", "--system-site-packages",
- $install_dir);
- shell_or_die("$install_dir/bin/pip", "install", "$sdk_root/python");
- }
-
- foreach my $sdk_lang (map { (split /\//, $_, 2)[0]; } keys(%SDK_ENVVARS)) {
+ foreach my $sdk_lang (("python",
+ map { (split /\//, $_, 2)[0]; } keys(%SDK_ENVVARS))) {
if (-d "$sdk_root/$sdk_lang") {
if (!rename("$sdk_root/$sdk_lang", "$install_dir/$sdk_lang")) {
die "Failed to install $sdk_lang SDK: $!";
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 697c85d..18c2cb8 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -238,8 +238,14 @@ class Dispatcher
end
end
- def git_cmd(cmd_s)
- "git --git-dir=#{@arvados_internal.shellescape} #{cmd_s}"
+ def stdout_s(cmd_a, opts={})
+ IO.popen(cmd_a, "r", opts) do |pipe|
+ return pipe.read.chomp
+ end
+ end
+
+ def git_cmd(*cmd_a)
+ ["git", "--git-dir=#{@arvados_internal.shellescape}"] + cmd_a
end
def get_authorization(job)
@@ -268,7 +274,7 @@ class Dispatcher
# @fetched_commits[V]==true if we know commit V exists in the
# arvados_internal git repository.
if !@fetched_commits[commit_hash]
- src_repo = File.join(@repo_root, repo_name + '.git')
+ src_repo = File.join(@repo_root, "#{repo_name}.git")
if not File.exists? src_repo
src_repo = File.join(@repo_root, repo_name, '.git')
if not File.exists? src_repo
@@ -278,12 +284,13 @@ class Dispatcher
end
# check if the commit needs to be fetched or not
- commit_rev = `#{git_cmd("rev-list -n1 #{commit_hash.shellescape} 2>/dev/null")}`.chomp
+ commit_rev = stdout_s(git_cmd("rev-list", "-n1", commit_hash),
+ err: "/dev/null")
unless $? == 0 and commit_rev == commit_hash
# commit does not exist in internal repository, so import the source repository using git fetch-pack
- cmd = git_cmd("fetch-pack --no-progress --all #{src_repo.shellescape}")
+ cmd = git_cmd("fetch-pack", "--no-progress", "--all", src_repo)
$stderr.puts "dispatch: #{cmd}"
- $stderr.puts `#{cmd}`
+ $stderr.puts(stdout_s(cmd))
unless $? == 0
fail_job job, "git fetch-pack failed"
return nil
@@ -298,12 +305,12 @@ class Dispatcher
# @git_tags[T]==V if we know commit V has been tagged T in the
# arvados_internal repository.
if not @git_tags[tag_name]
- cmd = git_cmd("tag #{tag_name.shellescape} #{commit_hash.shellescape} 2>/dev/null")
+ cmd = git_cmd("tag", tag_name, commit_hash)
$stderr.puts "dispatch: #{cmd}"
- $stderr.puts `#{cmd}`
+ $stderr.puts(stdout_s(cmd, err: "/dev/null"))
unless $? == 0
# git tag failed. This may be because the tag already exists, so check for that.
- tag_rev = `#{git_cmd("rev-list -n1 #{tag_name.shellescape}")}`.chomp
+ tag_rev = stdout_s(git_cmd("rev-list", "-n1", tag_name))
if $? == 0
# We got a revision back
if tag_rev != commit_hash
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list