[arvados] created: 2.5.0-267-gad18aa8b7

[git repository hosting]

        at  ad18aa8b7f032e5acf356bd97e1d62684c24e891 (commit)


commit ad18aa8b7f032e5acf356bd97e1d62684c24e891
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Fri Mar 10 13:19:09 2023 -0500

    20223: Use in_batches in used_by to tamp down memory usage
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index dc20d6ae7..1d6800636 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -183,11 +183,11 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where(output: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(output: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
 
-        Container.readable_by(*@read_users).where(log: loc.to_s).each do |c|
+        Container.readable_by(*@read_users).where(log: loc.to_s).in_batches(of: 15).each do |c|
           search_edges(visited, c.uuid, :search_up)
         end
       elsif direction == :search_down
@@ -207,7 +207,7 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
+        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).in_batches(of: 15).each do |c|
           if c.output != loc.to_s && c.log != loc.to_s
             search_edges(visited, c.uuid, :search_down)
           end
@@ -276,11 +276,11 @@ class Arvados::V1::CollectionsController < ApplicationController
               end
             end
 
-            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
 
-            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
+            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).in_batches(of: 15).each do |cr|
               search_edges(visited, cr.uuid, :search_up)
             end
           elsif direction == :search_down

commit 8b366554a0dced3b9423a6fa0109f168a521de74
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Fri Mar 10 13:13:48 2023 -0500

    20223: Make trash sweep use in_batches
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/services/api/app/controllers/sys_controller.rb b/services/api/app/controllers/sys_controller.rb
index a67b124bd..7d20cf77f 100644
--- a/services/api/app/controllers/sys_controller.rb
+++ b/services/api/app/controllers/sys_controller.rb
@@ -12,9 +12,11 @@ class SysController < ApplicationController
       # Sweep trashed collections
       Collection.
         where('delete_at is not null and delete_at < statement_timestamp()').
+        in_batches(of: 15).
         destroy_all
       Collection.
         where('is_trashed = false and trash_at < statement_timestamp()').
+        in_batches(of: 15).
         update_all('is_trashed = true')
 
       # Sweep trashed projects and their contents (as well as role
@@ -50,7 +52,7 @@ class SysController < ApplicationController
     skipped_classes = ['Group', 'User']
     ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
       if !skipped_classes.include?(klass.name) && klass.columns.collect(&:name).include?('owner_uuid')
-        klass.where({owner_uuid: p_uuid}).destroy_all
+        klass.where({owner_uuid: p_uuid}).in_batches(of: 15).destroy_all
       end
     end
     # Finally delete the project itself
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index b59604cb2..146cc16ec 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -831,19 +831,17 @@ class Container < ArvadosModel
           ContainerRequest.
             where(requesting_container_uuid: uuid,
                   state: ContainerRequest::Committed).
-            find_in_batches(batch_size: 10) do |batch|
-            batch.each do |cr|
-              leave_modified_by_user_alone do
-                cr.set_priority_zero
-                container_state = Container.where(uuid: cr.container_uuid).pluck(:state).first
-                if container_state == Container::Queued || container_state == Container::Locked
-                  # If the child container hasn't started yet, finalize the
-                  # child CR now instead of leaving it "on hold", i.e.,
-                  # Queued with priority 0.  (OTOH, if the child is already
-                  # running, leave it alone so it can get cancelled the
-                  # usual way, get a copy of the log collection, etc.)
-                  cr.update_attributes!(state: ContainerRequest::Final)
-                end
+            in_batches(of: 15).each do |cr|
+            leave_modified_by_user_alone do
+              cr.set_priority_zero
+              container_state = Container.where(uuid: cr.container_uuid).pluck(:state).first
+              if container_state == Container::Queued || container_state == Container::Locked
+                # If the child container hasn't started yet, finalize the
+                # child CR now instead of leaving it "on hold", i.e.,
+                # Queued with priority 0.  (OTOH, if the child is already
+                # running, leave it alone so it can get cancelled the
+                # usual way, get a copy of the log collection, etc.)
+                cr.update_attributes!(state: ContainerRequest::Final)
               end
             end
           end

-----------------------------------------------------------------------


hooks/post-receive
--