[ARVADOS] created: 1.1.4-281-g3b3794c

[Git user]

        at  3b3794c7e0e026a6338165b9e171925e384e6502 (commit)


commit 3b3794c7e0e026a6338165b9e171925e384e6502
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 18 14:29:34 2018 -0400

    13164: Lock tables in order when locking a container.
    
    Locking involves assigning auth_uuid, which involves looking up
    container requests, so both tables must be locked in order to avoid
    deadlock.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index e9d4f83..1dbdb57 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -316,11 +316,11 @@ class Container < ArvadosModel
     # (because state might have changed while acquiring the lock).
     check_lock_fail
     transaction do
-      begin
-        reload(lock: 'FOR UPDATE NOWAIT')
-      rescue
-        raise LockFailedError.new("cannot lock: other transaction in progress")
-      end
+      # Locking involves assigning auth_uuid, which involves looking
+      # up container requests, so we must lock both tables in the
+      # proper order to avoid deadlock.
+      ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+      reload
       check_lock_fail
       update_attributes!(state: Locked)
     end

commit 43b2200fd669c6eaa095c8adfd48462880aef551
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 18 14:02:29 2018 -0400

    13164: Lock tables in order for container updates.
    
    Otherwise, a container update can lock a container row, then perform
    container request queries, which can deadlock with another thread that
    locked container_requests first.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/api/app/controllers/arvados/v1/containers_controller.rb b/services/api/app/controllers/arvados/v1/containers_controller.rb
index 6ec92b0..fa29dbd 100644
--- a/services/api/app/controllers/arvados/v1/containers_controller.rb
+++ b/services/api/app/controllers/arvados/v1/containers_controller.rb
@@ -20,10 +20,13 @@ class Arvados::V1::ContainersController < ApplicationController
     show
   end
 
-  # Updates use row locking to resolve races between multiple
-  # dispatchers trying to lock the same container.
   def update
-    @object.with_lock do
+    # container updates can trigger container request lookups, which
+    # can deadlock if we don't lock the container_requests table
+    # first.
+    @object.transaction do
+      ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+      @object.reload
       super
     end
   end

-----------------------------------------------------------------------


hooks/post-receive
--