[ARVADOS] created: 1.1.4-263-g764334d

[Git user]

        at  764334dee966e04161d411a9feb0074b99faa147 (commit)


commit 764334dee966e04161d411a9feb0074b99faa147
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon May 14 10:28:36 2018 -0400

    13164: Lock tables when updating multiple containers + CRs.
    
    Avoid database inconsistencies after update races.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 8882b2c..e9d4f83 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -126,6 +126,7 @@ class Container < ArvadosModel
       # Update the priority of child container requests to match new
       # priority of the parent container (ignoring requests with no
       # container assigned, because their priority doesn't matter).
+      ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
       ContainerRequest.
         where(requesting_container_uuid: self.uuid,
               state: ContainerRequest::Committed).
@@ -541,6 +542,7 @@ class Container < ArvadosModel
     if self.state_changed? and self.final?
       act_as_system_user do
 
+        ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
         if self.state == Cancelled
           retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid)
         else
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index bc01b33..ac4415b 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -37,10 +37,10 @@ class ContainerRequest < ArvadosModel
   validate :check_update_whitelist
   validate :secret_mounts_key_conflict
   before_save :scrub_secret_mounts
-  after_save :update_priority
-  after_save :finalize_if_needed
   before_create :set_requesting_container_uuid
   before_destroy :set_priority_zero
+  after_save :update_priority
+  after_save :finalize_if_needed
 
   api_accessible :user, extend: :common do |t|
     t.add :command
@@ -86,7 +86,7 @@ class ContainerRequest < ArvadosModel
   AttrsPermittedAlways = [:owner_uuid, :state, :name, :description]
   AttrsPermittedBeforeCommit = [:command, :container_count_max,
   :container_image, :cwd, :environment, :filters, :mounts,
-  :output_path, :priority, :properties, :requesting_container_uuid,
+  :output_path, :priority, :properties,
   :runtime_constraints, :state, :container_uuid, :use_existing,
   :scheduling_parameters, :secret_mounts, :output_name, :output_ttl]
 
@@ -286,9 +286,9 @@ class ContainerRequest < ArvadosModel
   def update_priority
     return unless state_changed? || priority_changed? || container_uuid_changed?
     act_as_system_user do
+      ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
       Container.
         where('uuid in (?)', [self.container_uuid_was, self.container_uuid].compact).
-        lock(true).
         map(&:update_priority!)
     end
   end
@@ -298,14 +298,11 @@ class ContainerRequest < ArvadosModel
   end
 
   def set_requesting_container_uuid
-    return !new_record? if self.requesting_container_uuid   # already set
-
-    token_uuid = current_api_client_authorization.andand.uuid
-    container = Container.where('auth_uuid=?', token_uuid).order('created_at desc').first
-    if container
-      self.requesting_container_uuid = container.uuid
-      self.priority = container.priority > 0 ? 1 : 0
+    return if !current_api_client_authorization
+    ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+    if (c = Container.where('auth_uuid=?', current_api_client_authorization.uuid).select([:uuid, :priority]).first)
+      self.requesting_container_uuid = c.uuid
+      self.priority = c.priority>0 ? 1 : 0
     end
-    true
   end
 end
diff --git a/services/api/db/migrate/20180514135529_add_container_auth_uuid_index.rb b/services/api/db/migrate/20180514135529_add_container_auth_uuid_index.rb
new file mode 100644
index 0000000..56cafea
--- /dev/null
+++ b/services/api/db/migrate/20180514135529_add_container_auth_uuid_index.rb
@@ -0,0 +1,5 @@
+class AddContainerAuthUuidIndex < ActiveRecord::Migration
+  def change
+    add_index :containers, :auth_uuid
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index caf7683..0ab30c5 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -1879,6 +1879,13 @@ CREATE UNIQUE INDEX index_container_requests_on_uuid ON container_requests USING
 
 
 --
+-- Name: index_containers_on_auth_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_containers_on_auth_uuid ON containers USING btree (auth_uuid);
+
+
+--
 -- Name: index_containers_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
 --
 
@@ -3071,3 +3078,5 @@ INSERT INTO schema_migrations (version) VALUES ('20180313180114');
 
 INSERT INTO schema_migrations (version) VALUES ('20180501182859');
 
+INSERT INTO schema_migrations (version) VALUES ('20180514135529');
+
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index cc257cc..3483b87 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -261,10 +261,12 @@ class ContainerRequestTest < ActiveSupport::TestCase
 
     c = Container.find_by_uuid cr.container_uuid
     assert_operator 0, :<, c.priority
+    lock_and_run(c)
 
-    cr2 = create_minimal_req!
-    cr2.update_attributes!(priority: 10, state: "Committed", requesting_container_uuid: c.uuid, command: ["echo", "foo2"], container_count_max: 1)
-    cr2.reload
+    cr2 = with_container_auth(c) do
+      create_minimal_req!(priority: 10, state: "Committed", container_count_max: 1, command: ["echo", "foo2"])
+    end
+    assert_not_nil cr2.requesting_container_uuid
     assert_equal users(:active).uuid, cr2.modified_by_user_uuid
 
     c2 = Container.find_by_uuid cr2.container_uuid
@@ -613,7 +615,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
 
   test "requesting_container_uuid at create is not allowed" do
     set_user_from_auth :active
-    assert_raises(ActiveRecord::RecordNotSaved) do
+    assert_raises(ActiveRecord::RecordInvalid) do
       create_minimal_req!(state: "Uncommitted", priority: 1, requesting_container_uuid: 'youcantdothat')
     end
   end

-----------------------------------------------------------------------


hooks/post-receive
--