[ARVADOS] updated: 538f4a849b02d8be2a91ad65edb4d127cce890c3

[git at public.curoverse.com]

Summary of changes:
 sdk/cli/bin/crunch-job | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

       via  538f4a849b02d8be2a91ad65edb4d127cce890c3 (commit)
       via  f90440a49970364bdcfb2cd8692caedbedddf42b (commit)
      from  d70f2d85958f988be3c227870d7adc578ee12c8a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 538f4a849b02d8be2a91ad65edb4d127cce890c3
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Feb 25 11:37:26 2015 -0500

    5283: crunch-job doesn't use freeze logic after a job fails.
    
    If the job has failed permanently, we want to go through all the
    end-of-job logic.  Previously, we were getting sidetracked into
    freeze_if_want_freeze, which skips some steps like setting the
    permanent job output record.

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 33c48b5..2c9fb43 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1037,7 +1037,6 @@ sub reapchildren
     if (!$temporary_fail || $Jobstep->{'failures'} >= 3) {
       # Give up on this task, and the whole job
       $main::success = 0;
-      $main::please_freeze = 1;
     }
     # Put this task back on the todo queue
     push @jobstep_todo, $jobstepid;

commit f90440a49970364bdcfb2cd8692caedbedddf42b
Author: Brett Smith <brett at curoverse.com>
Date:   Wed Feb 25 11:53:23 2015 -0500

    5283: Log more crunch-job output handling.
    
    Requested during code review.

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 0d49eae..33c48b5 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1056,7 +1056,9 @@ sub reapchildren
   $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
   $Jobstep->{'arvados_task'}->save;
   process_stderr ($jobstepid, $task_success);
-  Log ($jobstepid, "output " . $Jobstep->{'arvados_task'}->{output});
+  Log ($jobstepid, sprintf("output (%d bytes): %s",
+                           length($Jobstep->{'arvados_task'}->{output}),
+                           $Jobstep->{'arvados_task'}->{output}));
 
   close $reader{$jobstepid};
   delete $reader{$jobstepid};
@@ -1299,6 +1301,7 @@ print (arvados.api("v1").collections().
 }, retry_count());
 
   my $task_idx = -1;
+  my $manifest_size = 0;
   for (@jobstep)
   {
     ++$task_idx;
@@ -1315,6 +1318,8 @@ print (arvados.api("v1").collections().
         # There's been an error writing.  Stop the loop.
         # We'll log details about the exit code later.
         last;
+      } else {
+        $manifest_size += length($next_write);
       }
     } else {
       my $uuid = $_->{'arvados_task'}->{'uuid'};
@@ -1323,11 +1328,12 @@ print (arvados.api("v1").collections().
     }
   }
   close($child_in);
+  Log(undef, "collated output manifest is $manifest_size bytes");
 
   my $joboutput;
   my $s = IO::Select->new($child_out);
   if ($s->can_read(120)) {
-    sysread($child_out, $joboutput, 64 * 1024 * 1024);
+    sysread($child_out, $joboutput, 1024 * 1024);
     waitpid($pid, 0);
     if ($?) {
       Log(undef, "output collection creation exited " . exit_status_s($?));

-----------------------------------------------------------------------


hooks/post-receive
--