[ARVADOS] created: eb8ae85898833a302b31f3f93ef4cfdf5c5a764d
git at public.curoverse.com
git at public.curoverse.com
Sun Oct 5 11:45:21 EDT 2014
at eb8ae85898833a302b31f3f93ef4cfdf5c5a764d (commit)
commit eb8ae85898833a302b31f3f93ef4cfdf5c5a764d
Author: Brett Smith <brett at curoverse.com>
Date: Sun Oct 5 11:45:16 2014 -0400
4012: crunch-job retries all API operations.
This will make jobs more robust against transient errors when talking
to the API server.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index f56099d..bed94d3 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -143,7 +143,7 @@ $SIG{'USR2'} = sub
my $arv = Arvados->new('apiVersion' => 'v1');
my $local_logfile;
-my $User = $arv->{'users'}->{'current'}->execute;
+my $User = retry_op(sub { $arv->{'users'}->{'current'}->execute; });
my $Job = {};
my $job_id;
@@ -151,7 +151,9 @@ my $dbh;
my $sth;
if ($job_has_uuid)
{
- $Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
+ $Job = retry_op(sub {
+ $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
+ });
if (!$force_unlock) {
# If some other crunch-job process has grabbed this job (or we see
# other evidence that the job is already underway) we exit
@@ -192,7 +194,7 @@ else
$Job->{'is_locked_by_uuid'} = $User->{'uuid'};
$Job->{'started_at'} = gmtime;
- $Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job);
+ $Job = retry_op(sub { $arv->{'jobs'}->{'create'}->execute('job' => $Job); });
$job_has_uuid = 1;
}
@@ -337,12 +339,14 @@ if (defined $Job->{thawedfromkey})
}
else
{
- my $first_task = $arv->{'job_tasks'}->{'create'}->execute('job_task' => {
- 'job_uuid' => $Job->{'uuid'},
- 'sequence' => 0,
- 'qsequence' => 0,
- 'parameters' => {},
- });
+ my $first_task = retry_op(sub {
+ $arv->{'job_tasks'}->{'create'}->execute('job_task' => {
+ 'job_uuid' => $Job->{'uuid'},
+ 'sequence' => 0,
+ 'qsequence' => 0,
+ 'parameters' => {},
+ });
+ });
push @jobstep, { 'level' => 0,
'failures' => 0,
'arvados_task' => $first_task,
@@ -861,8 +865,9 @@ else {
while (my $manifest_line = <$orig_manifest>) {
$orig_manifest_text .= $manifest_line;
}
- my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
- 'manifest_text' => $orig_manifest_text,
+ my $output = retry_op(sub {
+ $arv->{'collections'}->{'create'}->execute(
+ 'collection' => {'manifest_text' => $orig_manifest_text,});
});
Log(undef, "output uuid " . $output->{uuid});
Log(undef, "output hash " . $output->{portable_data_hash});
@@ -1004,13 +1009,15 @@ sub reapchildren
my $newtask_list = [];
my $newtask_results;
do {
- $newtask_results = $arv->{'job_tasks'}->{'list'}->execute(
- 'where' => {
- 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
- },
- 'order' => 'qsequence',
- 'offset' => scalar(@$newtask_list),
- );
+ $newtask_results = retry_op(sub {
+ $arv->{'job_tasks'}->{'list'}->execute(
+ 'where' => {
+ 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
+ },
+ 'order' => 'qsequence',
+ 'offset' => scalar(@$newtask_list),
+ );
+ });
push(@$newtask_list, @{$newtask_results->{items}});
} while (@{$newtask_results->{items}});
foreach my $arvados_task (@$newtask_list) {
@@ -1034,7 +1041,9 @@ sub check_refresh_wanted
if (@stat && $stat[9] > $latest_refresh) {
$latest_refresh = scalar time;
if ($job_has_uuid) {
- my $Job2 = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
+ my $Job2 = retry_op(sub {
+ $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
+ });
for my $attr ('cancelled_at',
'cancelled_by_user_uuid',
'cancelled_by_client_uuid',
@@ -1216,7 +1225,7 @@ sub collate_output
my ($child_out, $child_in);
my $pid = open2($child_out, $child_in, 'arv-put', '--raw',
- '--retries', put_retry_count());
+ '--retries', retry_count());
my $joboutput;
for (@jobstep)
{
@@ -1358,7 +1367,7 @@ sub save_meta
return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
$local_logfile->flush;
- my $retry_count = put_retry_count();
+ my $retry_count = retry_count();
my $cmd = "arv-put --portable-data-hash --retries $retry_count " .
"--filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename);
my $loglocator = `$cmd`;
@@ -1485,7 +1494,10 @@ sub find_docker_image {
# If not, return undef for both values.
my $locator = shift;
my ($streamname, $filename);
- if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
+ my $image = retry_op(sub {
+ $arv->{collections}->{get}->execute(uuid => $locator);
+ });
+ if ($image) {
foreach my $line (split(/\n/, $image->{manifest_text})) {
my @tokens = split(/\s+/, $line);
next if (!@tokens);
@@ -1506,12 +1518,20 @@ sub find_docker_image {
}
}
-sub put_retry_count {
- # Calculate a --retries argument for arv-put that will have it try
- # approximately as long as this Job has been running.
- my $stoptime = shift || time;
+sub retry_count {
+ # Calculate the number of times an operation should be retried,
+ # assuming exponential backoff, and that we're willing to retry as
+ # long as tasks have been running. Enforce a minimum of 3 retries.
+ my $endtime = $jobstep[-1]->{finishtime};
my $starttime = $jobstep[0]->{starttime};
- my $timediff = defined($starttime) ? ($stoptime - $starttime) : 1;
+ my $timediff;
+ if (!defined($starttime)) {
+ $timediff = 0;
+ } elsif (!defined($endtime)) {
+ $timediff = time - $starttime;
+ } else {
+ $timediff = ($endtime - $starttime) - (time - $endtime);
+ }
my $retries = 0;
while ($timediff >= 2) {
$retries++;
@@ -1520,6 +1540,25 @@ sub put_retry_count {
return ($retries > 3) ? $retries : 3;
}
+sub retry_op {
+ # Given a function reference, call it with the given arguments. If
+ # it dies, retry it with exponential backoff until it succeeds, or until
+ # the current retry_count is exhausted.
+ my $operation = shift;
+ foreach my $try_count (0..retry_count()) {
+ my $result = eval { $operation->(@_); };
+ if ($@) {
+ sleep(2 ** $try_count);
+ } else {
+ return $result;
+ }
+ }
+ # Ensure the error message ends in a newline, so Perl doesn't add
+ # retry_op's line number to it.
+ chomp $@;
+ die($@ . "\n");
+}
+
__DATA__
#!/usr/bin/perl
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list