Merge pull request #17280 from d-m-u/adding_lock_to_retire_now

Add lock to retire_now start
(cherry picked from commit 2deffa5)

Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1570950

app/models/mixins/retirement_mixin.rb

@@ -1,9 +1,9 @@
 module RetirementMixin
   extend ActiveSupport::Concern
-
-  RETIRED  = 'retired'
-  RETIRING = 'retiring'
-  ERROR_RETIRING = 'error'
+  RETIREMENT_ERROR = 'error'.freeze
+  RETIREMENT_INITIALIZING = 'initializing'.freeze
+  RETIREMENT_RETIRED  = 'retired'.freeze
+  RETIREMENT_RETIRING = 'retiring'.freeze
 
   included do
     scope :scheduled_to_retire, -> { where(arel_table[:retires_on].not_eq(nil).or(arel_table[:retired].not_eq(true))) }
@@ -105,7 +105,7 @@ def raise_retire_audit_event(message)
   end
 
   def retirement_check
-    return if self.retired?
+    return if retired? || retiring? || retirement_initialized?
 
     if !retirement_warned? && retirement_warning_due?
       begin
@@ -124,14 +124,23 @@ def retire_now(requester = nil)
     if retired
       return if retired_validated?
       _log.info("#{retirement_object_title}: [#{name}], Retires On: [#{retires_on.strftime("%x %R %Z")}], was previously retired, but currently #{retired_invalid_reason}")
+    elsif retiring?
+      _log.info("#{retirement_object_title}: [#{name}] retirement in progress")
     else
-      update_attributes(:retirement_requester => requester)
-      event_name = "request_#{retirement_event_prefix}_retire"
-      _log.info("calling #{event_name}")
-      begin
-        raise_retirement_event(event_name, requester)
-      rescue => err
-        _log.log_backtrace(err)
+      lock do
+        reload
+        if error_retiring? || retirement_state.blank?
+          update_attributes(:retirement_state => "initializing", :retirement_requester => requester)
+          event_name = "request_#{retirement_event_prefix}_retire"
+          _log.info("calling #{event_name}")
+          begin
+            raise_retirement_event(event_name, requester)
+          rescue => err
+            _log.log_backtrace(err)
+          end
+        else
+          _log.info("#{retirement_object_title}: retirement for [#{name}] got updated while waiting to be unlocked and is now #{retirement_state}")
+        end
       end
     end
   end
@@ -148,7 +157,7 @@ def finish_retirement
   end
 
   def start_retirement
-    return if self.retired?
+    return if retired? || retiring?
     $log.info("Starting Retirement for [#{name}]")
     update_attributes(:retirement_state => "retiring")
   end
@@ -165,6 +174,10 @@ def retirement_base_model_name
     @retirement_base_model_name ||= self.class.base_model.name
   end
 
+  def retirement_initialized?
+    retirement_state == RETIREMENT_INITIALIZING
+  end
+
   def retirement_object_title
     @retirement_object_title ||= retirement_base_model_name
   end
@@ -201,11 +214,11 @@ def raise_audit_event(event_name, message, requester = nil)
   end
 
   def retiring?
-    retirement_state == RETIRING
+    retirement_state == RETIREMENT_RETIRING
   end
 
   def error_retiring?
-    retirement_state == ERROR_RETIRING
+    retirement_state == RETIREMENT_ERROR
   end
 
   private

spec/models/service/retirement_management_spec.rb

@@ -26,7 +26,26 @@
     expect(@service.retirement_state).to be_nil
     expect(MiqEvent).to receive(:raise_evm_event).once
     @service.retire_now
-    @service.reload
+    expect(@service.retirement_state).to eq('initializing')
+  end
+
+  it "#retire_now when called more than once" do
+    expect(@service.retirement_state).to be_nil
+    expect(MiqEvent).to receive(:raise_evm_event).once
+    3.times { @service.retire_now }
+    expect(@service.retirement_state).to eq('initializing')
+  end
+
+  it "#retire_now not called when already retiring" do
+    @service.update_attributes(:retirement_state => 'retiring')
+    expect(MiqEvent).to receive(:raise_evm_event).exactly(0).times
+    @service.retire_now
+  end
+
+  it "#retire_now not called when already retired" do
+    @service.update_attributes(:retirement_state => 'retired')
+    expect(MiqEvent).to receive(:raise_evm_event).exactly(0).times
+    @service.retire_now
   end
 
   it "#retire_now with userid" do
@@ -38,7 +57,6 @@
     expect(MiqEvent).to receive(:raise_evm_event).with(@service, event_name, event_hash, {}).once
 
     @service.retire_now('freddy')
-    @service.reload
   end
 
   it "#retire_now without userid" do
@@ -50,7 +68,6 @@
     expect(MiqEvent).to receive(:raise_evm_event).with(@service, event_name, event_hash, {}).once
 
     @service.retire_now
-    @service.reload
   end
 
   it "#retire warn" do

spec/models/vm/retirement_management_spec.rb

@@ -29,6 +29,25 @@
     expect(MiqEvent).to receive(:raise_evm_event).once
 
     @vm.retire_now
+    expect(@vm.retirement_state).to eq('initializing')
+  end
+
+  it "#retire_now when called more than once" do
+    expect(MiqEvent).to receive(:raise_evm_event).once
+    3.times { @vm.retire_now }
+    expect(@vm.retirement_state).to eq('initializing')
+  end
+
+  it "#retire_now not called when already retiring" do
+    @vm.update_attributes(:retirement_state => 'retiring')
+    expect(MiqEvent).to receive(:raise_evm_event).exactly(0).times
+    @vm.retire_now
+  end
+
+  it "#retire_now not called when already retired" do
+    @vm.update_attributes(:retirement_state => 'retired')
+    expect(MiqEvent).to receive(:raise_evm_event).exactly(0).times
+    @vm.retire_now
   end
 
   it "#retire_now with userid" do