[ARVADOS] created: f1c9f136ec77d1b48bc96e59df3c3c1368810ca7

[Git user]

        at  f1c9f136ec77d1b48bc96e59df3c3c1368810ca7 (commit)


commit f1c9f136ec77d1b48bc96e59df3c3c1368810ca7
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jul 6 10:50:43 2016 -0400

    9542: Avoid retrieving huge result sets from postgres.

diff --git a/services/api/lib/eventbus.rb b/services/api/lib/eventbus.rb
index aaeebdc..17ad1f3 100644
--- a/services/api/lib/eventbus.rb
+++ b/services/api/lib/eventbus.rb
@@ -132,15 +132,22 @@ class EventBus
           logs = logs.where(cond_id, *param_out)
         end
 
-        # Execute query and actually send the matching log rows
-        logs.each do |l|
-          if not ws.sent_ids.include?(l.id)
+        # Execute query and actually send the matching log rows. Load
+        # the full log records only when we're ready to send them,
+        # though: otherwise, (1) postgres has to build the whole
+        # result set and return it to us before we can send the first
+        # event, and (2) we store lots of records in memory while
+        # waiting to spool them out to the client. Both of these are
+        # troublesome when log records are large (e.g., a collection
+        # update contains both old and new manifest_text).
+        logs.select(:id).map(&:id).each do |id|
+          if not ws.sent_ids.include?(id)
             # only send if not a duplicate
-            ws.send(l.as_api_response.to_json)
+            ws.send(Log.find(id).as_api_response.to_json)
           end
           if not ws.last_log_id.nil?
             # record ids only when sending "catchup" messages, not notifies
-            ws.sent_ids << l.id
+            ws.sent_ids << id
           end
         end
         ws.last_log_id = nil

-----------------------------------------------------------------------


hooks/post-receive
--