[ARVADOS] updated: 6d3a88841b58eb02f6f6aedd38a40ed02cc42d1f

[git at public.curoverse.com]

Summary of changes:
 services/api/script/crunch-dispatch.rb | 52 ++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 22 deletions(-)

       via  6d3a88841b58eb02f6f6aedd38a40ed02cc42d1f (commit)
       via  3254f0a981f72decc49f2fbd43d4cffb47dfb4b4 (commit)
      from  2e76e6ae877fb528d666071075afe2bf4ab21035 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 6d3a88841b58eb02f6f6aedd38a40ed02cc42d1f
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Oct 21 13:29:05 2014 -0400

    4266: Fix OOM when stderr has a lot of data with no newlines.

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 601ec47..bbb2be9 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -450,8 +450,20 @@ class Dispatcher
         bufend = ''
         streambuf.lines("\n").each do |line|
           if not line.end_with? "\n"
-            bufend = line
-            break
+            if line.size > 2**20
+              # Without a limit here, we'll use 2x an arbitrary amount
+              # of memory, and waste a lot of time copying strings
+              # around, all without providing any feedback to anyone
+              # about what's going on _or_ hitting any of our throttle
+              # limits. Better to introduce a spurious line break once
+              # in a while:
+              line << "[...]\n"
+            else
+              # If line length is sane, we'll wait for the rest of the
+              # line to appear in the next read_pipes() call.
+              bufend = line
+              break
+            end
           end
           # rate_limit returns true or false as to whether to actually log
           # the line or not.  It also modifies "line" in place to replace
@@ -463,6 +475,9 @@ class Dispatcher
             j[:stderr_buf_to_flush] << pub_msg
           end
         end
+
+        # Leave the trailing incomplete line (if any) in streambuf for
+        # next time.
         streambuf.replace bufend
       end
       # Flush buffered logs to the logs table, if appropriate. We have

commit 3254f0a981f72decc49f2fbd43d4cffb47dfb4b4
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Oct 21 13:05:36 2014 -0400

    4266: Clean up streambuf.split(newline) loop.

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index c7ddaf2..601ec47 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -444,33 +444,26 @@ class Dispatcher
           next
         end
 
-        # Add to the stream buffer
+        # Append to incomplete line from previous read, if any
         streambuf << buf
 
-        # Check for at least one complete line
-        if streambuf.index "\n"
-          lines = streambuf.lines("\n").to_a
-
-          # check if the last line is partial or not
-          streambuf.replace(if streambuf[-1] == "\n"
-                              ''        # ends on a newline
-                            else
-                              lines.pop # Put the partial line back into the buffer
-                            end)
-
-          # Now spool the lines to the log output buffer
-          lines.each do |line|
-            # rate_limit returns true or false as to whether to actually log
-            # the line or not.  It also modifies "line" in place to replace
-            # it with an error if a logging limit is tripped.
-            if rate_limit j, line
-              $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
-              $stderr.puts line
-              pub_msg = "#{Time.now.ctime.to_s} #{line.strip}\n"
-              j[:stderr_buf_to_flush] << pub_msg
-            end
+        bufend = ''
+        streambuf.lines("\n").each do |line|
+          if not line.end_with? "\n"
+            bufend = line
+            break
+          end
+          # rate_limit returns true or false as to whether to actually log
+          # the line or not.  It also modifies "line" in place to replace
+          # it with an error if a logging limit is tripped.
+          if rate_limit j, line
+            $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
+            $stderr.puts line
+            pub_msg = "#{Time.now.ctime.to_s} #{line.strip}\n"
+            j[:stderr_buf_to_flush] << pub_msg
           end
         end
+        streambuf.replace bufend
       end
       # Flush buffered logs to the logs table, if appropriate. We have
       # to do this even if we didn't collect any new logs this time:

-----------------------------------------------------------------------


hooks/post-receive
--