Wait for ems workers to finish before destroying the ems

app/models/ext_management_system.rb

@@ -422,6 +422,46 @@ def enable!
     update!(:enabled => true)
   end
 
+  def self.destroy_queue(ids)
+    ids = Array.wrap(ids)
+    _log.info("Queuing destroy of #{name} with the following ids: #{ids.inspect}")
+    ids.each do |id|
+      schedule_destroy_queue(id)
+    end
+  end
+
+  # override destroy_queue from AsyncDeleteMixin
+  def destroy_queue
+    self.class.schedule_destroy_queue(id)
+  end
+
+  def self.schedule_destroy_queue(id, deliver_on = nil)
+    MiqQueue.put(
+      :class_name  => name,
+      :instance_id => id,
+      :method_name => "orchestrate_destroy",
+      :deliver_on  => deliver_on,
+    )
+  end
+
+  # Wait until all associated workers are dead to destroy this ems
+  def orchestrate_destroy
+    disable! if enabled?
+
+    if self.destroy == false
+      _log.info("Cant #{self.class.name} with id: #{id}, workers still in progress. Requeuing destroy...")
+      schedule_destroy_queue(id, :deliver_on => 15.seconds.from_now)
+    else
+      _log.info("#{self.class.name} with id: #{id} destroyed")
+    end
+  end
+
+  before_destroy :assert_no_queues_present
+
+  def assert_no_queues_present
+    throw(:abort) if MiqWorker.find_alive.where(:queue_name => queue_name).any?
+  end
+
   def disconnect_inv
     hosts.each { |h| h.disconnect_ems(self) }
     vms.each   { |v| v.disconnect_ems(self) }
@@ -431,6 +471,10 @@ def disconnect_inv
     resource_pools.destroy_all
   end
 
+  def queue_name
+    "ems_#{id}"
+  end
+
   def enforce_policy(target, event)
     inputs = {:ext_management_system => self}
     inputs[:vm]   = target if target.kind_of?(Vm)

app/models/mixins/per_ems_worker_mixin.rb

@@ -97,7 +97,7 @@ def queue_name_for_ems(ems)
       return "generic" if ems.kind_of?(Host) && ems.acts_as_ems?
 
       return ems unless ems.kind_of?(ExtManagementSystem)
-      "ems_#{ems.id}"
+      ems.queue_name
     end
 
     def ems_id_from_queue_name(queue_name)

spec/models/async_delete_mixin_spec.rb

@@ -33,9 +33,9 @@ def self.should_define_delete_queue_class_method
     end
   end
 
-  def self.should_queue_destroy_on_instance
+  def self.should_queue_destroy_on_instance(queue_method = "destroy")
     it "should queue up destroy on instance" do
-      cond = ["class_name = ? AND instance_id = ? AND method_name = ?", @obj.class.name, @obj.id, "destroy"]
+      cond = ["class_name = ? AND instance_id = ? AND method_name = ?", @obj.class.name, @obj.id, queue_method]
 
       expect { @obj.destroy_queue }.not_to raise_error
       expect(MiqQueue.where(cond).count).to eq(1)
@@ -48,10 +48,10 @@ def self.should_queue_destroy_on_instance
     end
   end
 
-  def self.should_queue_destroy_on_class_with_many_ids
+  def self.should_queue_destroy_on_class_with_many_ids(queue_method = "destroy")
     it "should queue up destroy on class method with many ids" do
       ids = @objects.collect(&:id)
-      cond = ["class_name = ? AND instance_id in (?) AND method_name = ?", @obj.class.name, ids, "destroy"]
+      cond = ["class_name = ? AND instance_id in (?) AND method_name = ?", @obj.class.name, ids, queue_method]
 
       expect { @obj.class.destroy_queue(ids) }.not_to raise_error
       expect(MiqQueue.where(cond).count).to eq(ids.length)
@@ -129,8 +129,8 @@ def self.should_queue_delete_on_class_with_many_ids
 
       should_define_destroy_queue_instance_method
       should_define_destroy_queue_class_method
-      should_queue_destroy_on_instance
-      should_queue_destroy_on_class_with_many_ids
+      should_queue_destroy_on_instance("orchestrate_destroy")
+      should_queue_destroy_on_class_with_many_ids("orchestrate_destroy")
 
       should_define_delete_queue_instance_method
       should_define_delete_queue_class_method

spec/models/ext_management_system_spec.rb

@@ -392,4 +392,19 @@
       expect(tenant.ext_management_systems).to include(ems)
     end
   end
+
+  context "destroy" do
+    it "destroys an ems with no active workers" do
+      ems = FactoryGirl.create(:ext_management_system)
+      ems.destroy
+      expect(ExtManagementSystem.count).to eq(0)
+    end
+
+    it "does not destroy an ems with active workers" do
+      ems = FactoryGirl.create(:ext_management_system)
+      FactoryGirl.create(:miq_ems_refresh_worker, :queue_name => ems.queue_name, :status => "started")
+      ems.destroy
+      expect(ExtManagementSystem.count).to eq(1)
+    end
+  end
 end