[ARVADOS] updated: ec51cd72eabf0f0bbebf7cd979d1a23a1319a416

[Git user]

Summary of changes:
 sdk/cli/bin/crunch-job | 59 +++++++++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 27 deletions(-)

       via  ec51cd72eabf0f0bbebf7cd979d1a23a1319a416 (commit)
       via  8b91f9737d6219321cacb3c94356f35b59bb9a43 (commit)
       via  30e5b921dc5e2a3c62e50e466d144c70055aa098 (commit)
       via  de4d6a927c41f417790fe4274eb56df4d487a8a4 (commit)
      from  cac5db66cba0d5dd97c8434853bcbf2ab19fbda5 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit ec51cd72eabf0f0bbebf7cd979d1a23a1319a416
Merge: cac5db6 8b91f97
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Mar 2 16:40:05 2016 -0500

    Merge branch '7263-batch-task-lookup' closes #7263


commit 8b91f9737d6219321cacb3c94356f35b59bb9a43
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Mar 2 16:35:47 2016 -0500

    7263: Let arvados library do the json encoding.

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index f727018..e473710 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1246,7 +1246,7 @@ sub reapchildren
     do {
       $newtask_results = api_call(
         "job_tasks/list",
-        'filters' => '[["created_by_job_task_uuid","in",['.join(',',map {"\"$_\""} @successful_task_uuids).']]]',
+        'filters' => [["created_by_job_task_uuid","in",\@successful_task_uuids]],
         'order' => 'qsequence',
         'offset' => scalar(@$newtask_list),
           );

commit 30e5b921dc5e2a3c62e50e466d144c70055aa098
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Mar 2 16:25:55 2016 -0500

    7263: Check new tasks after a task completes, even if others failed in the same reapchildren().

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 3425519..f727018 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1237,9 +1237,9 @@ sub reapchildren
     $progress_is_dirty = 1;
   }
 
-  if ($children_reaped > 0 && (scalar(@successful_task_uuids) == $children_reaped))
+  if (scalar(@successful_task_uuids) > 0)
   {
-    Log (undef, sprintf("Reaped %d children and all were successful, checking for new tasks from API server.", $children_reaped));
+    Log (undef, sprintf("%d tasks exited (%d succeeded), checking for new tasks from API server.", $children_reaped, scalar(@successful_task_uuids)));
     # Load new tasks
     my $newtask_list = [];
     my $newtask_results;

commit de4d6a927c41f417790fe4274eb56df4d487a8a4
Author: Joshua Randall <joshua.randall at sanger.ac.uk>
Date:   Mon Feb 29 23:10:47 2016 +0000

    7263: Updates reapchildren to perform API query for new tasks in batches
    
    Conflicts:
    	sdk/cli/bin/crunch-job

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 1a9dac3..3425519 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1134,7 +1134,9 @@ sub update_progress_stats
 sub reapchildren
 {
   my $children_reaped = 0;
-  while ((my $pid = waitpid (-1, WNOHANG)) > 0)
+  my @successful_task_uuids = ();
+
+  while((my $pid = waitpid (-1, WNOHANG)) > 0)
   {
     my $childstatus = $?;
 
@@ -1207,8 +1209,9 @@ sub reapchildren
       push @jobstep_todo, $jobstepidx;
       $Job->{'tasks_summary'}->{'failed'}++;
     }
-    else
+    else # task_success
     {
+      push @successful_task_uuids, $Jobstep->{'arvados_task'}->{uuid};
       ++$thisround_succeeded;
       $slot[$proc{$pid}->{slot}]->{node}->{losing_streak} = 0;
       $slot[$proc{$pid}->{slot}]->{node}->{hold_until} = 0;
@@ -1231,34 +1234,36 @@ sub reapchildren
     push @freeslot, $proc{$pid}->{slot};
     delete $proc{$pid};
 
-    if ($task_success) {
-      # Load new tasks
-      my $newtask_list = [];
-      my $newtask_results;
-      do {
-        $newtask_results = api_call(
-          "job_tasks/list",
-          'where' => {
-            'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
-          },
-          'order' => 'qsequence',
-          'offset' => scalar(@$newtask_list),
-            );
-        push(@$newtask_list, @{$newtask_results->{items}});
-      } while (@{$newtask_results->{items}});
-      foreach my $arvados_task (@$newtask_list) {
-        my $jobstep = {
-          'level' => $arvados_task->{'sequence'},
-          'failures' => 0,
-          'arvados_task' => $arvados_task
-        };
-        push @jobstep, $jobstep;
-        push @jobstep_todo, $#jobstep;
-      }
-    }
     $progress_is_dirty = 1;
   }
 
+  if ($children_reaped > 0 && (scalar(@successful_task_uuids) == $children_reaped))
+  {
+    Log (undef, sprintf("Reaped %d children and all were successful, checking for new tasks from API server.", $children_reaped));
+    # Load new tasks
+    my $newtask_list = [];
+    my $newtask_results;
+    do {
+      $newtask_results = api_call(
+        "job_tasks/list",
+        'filters' => '[["created_by_job_task_uuid","in",['.join(',',map {"\"$_\""} @successful_task_uuids).']]]',
+        'order' => 'qsequence',
+        'offset' => scalar(@$newtask_list),
+          );
+      push(@$newtask_list, @{$newtask_results->{items}});
+    } while (@{$newtask_results->{items}});
+    Log (undef, sprintf("Got %d new tasks from API server.", scalar(@$newtask_list)));
+    foreach my $arvados_task (@$newtask_list) {
+      my $jobstep = {
+        'level' => $arvados_task->{'sequence'},
+        'failures' => 0,
+        'arvados_task' => $arvados_task
+      };
+      push @jobstep, $jobstep;
+      push @jobstep_todo, $#jobstep;
+    }
+  }
+
   return $children_reaped;
 }
 

-----------------------------------------------------------------------


hooks/post-receive
--