From b9309e7a8fa7e9b2977ee94cbf0ccc24fbd49749 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 14 Dec 2017 11:56:54 -0500 Subject: [PATCH 1/4] Create a task when destroying an ems When queueing a destroy on an EMS create a task and return the ID to be used by the caller to track when the destroy is complete. Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1525498 --- app/models/ext_management_system.rb | 29 ++++++--- spec/models/ext_management_system_spec.rb | 72 +++++++++++++++++++++-- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/app/models/ext_management_system.rb b/app/models/ext_management_system.rb index 13aab22ef99..0f90e8d2e64 100644 --- a/app/models/ext_management_system.rb +++ b/app/models/ext_management_system.rb @@ -444,33 +444,48 @@ def enable! # override destroy_queue from AsyncDeleteMixin def self.destroy_queue(ids) - find(Array.wrap(ids)).each(&:destroy_queue) + find(Array.wrap(ids)).map(&:destroy_queue) end def destroy_queue - _log.info("Queuing destroy of #{self.class.name} with id: #{id}") + msg = "Queuing destroy of #{self.class.name} with id: #{id}" + + _log.info(msg) + task = MiqTask.create( + :name => "Destroying #{self.class.name} with id: #{id}", + :state => MiqTask::STATE_QUEUED, + :status => MiqTask::STATUS_OK, + :message => msg, + ) + child_managers.each(&:destroy_queue) - self.class.schedule_destroy_queue(id) + self.class.schedule_destroy_queue(id, task.id) + + task.id end - def self.schedule_destroy_queue(id, deliver_on = nil) + def self.schedule_destroy_queue(id, task_id, deliver_on = nil) MiqQueue.put( :class_name => name, :instance_id => id, :method_name => "orchestrate_destroy", :deliver_on => deliver_on, + :args => [task_id], ) end # Wait until all associated workers are dead to destroy this ems - def orchestrate_destroy + def orchestrate_destroy(task_id) disable! if enabled? if self.destroy == false _log.info("Cannot destroy #{self.class.name} with id: #{id}, workers still in progress. Requeuing destroy...") - self.class.schedule_destroy_queue(id, 15.seconds.from_now) + self.class.schedule_destroy_queue(id, task_id, 15.seconds.from_now) else - _log.info("#{self.class.name} with id: #{id} destroyed") + msg = "#{self.class.name} with id: #{id} destroyed" + MiqTask.update_status(task_id, MiqTask::STATE_FINISHED, MiqTask::STATUS_OK, msg) + + _log.info(msg) end end diff --git a/spec/models/ext_management_system_spec.rb b/spec/models/ext_management_system_spec.rb index 73fc5f1781f..864b7efbb39 100644 --- a/spec/models/ext_management_system_spec.rb +++ b/spec/models/ext_management_system_spec.rb @@ -410,13 +410,73 @@ ems.destroy expect(ExtManagementSystem.count).to eq(1) end + end + + context "#queue_destroy" do + let(:server) { EvmSpecHelper.local_miq_server } + let(:zone) { server.zone } + + context "with no child managers" do + let(:ems) do + FactoryGirl.create(:ext_management_system, :zone => zone) + end + + it "returns a task" do + task_id = ems.destroy_queue + + queue_message = MiqQueue.first + status, message, result = queue_message.deliver + queue_message.delivered(status, message, result) + + task = MiqTask.find(task_id) + expect(task.state).to eq("Finished") + expect(task.status).to eq("Ok") + end + + it "re-schedules with active workers" do + FactoryGirl.create(:miq_ems_refresh_worker, :queue_name => ems.queue_name, :status => "started", :miq_server => server) + ems.destroy_queue + + expect(MiqQueue.count).to eq(1) + + queue_message = MiqQueue.first + status, message, result = queue_message.deliver + queue_message.delivered(status, message, result) + + expect(MiqQueue.count).to eq(1) + expect(MiqQueue.last.deliver_on).to_not be_nil + end + + it "destroys the ems when the active worker shuts down" do + refresh_worker = FactoryGirl.create(:miq_ems_refresh_worker, :queue_name => ems.queue_name, :status => "started", :miq_server => server) + ems.destroy_queue + + queue_message = MiqQueue.first + status, message, result = queue_message.deliver + queue_message.delivered(status, message, result) + + expect(ExtManagementSystem.count).to eq(1) + + refresh_worker.destroy + + queue_message = MiqQueue.first + status, message, result = queue_message.deliver + queue_message.delivered(status, message, result) - it "queues up destroy for child_managers" do - child_manager = FactoryGirl.create(:ext_management_system) - ems = FactoryGirl.create(:ext_management_system, :child_managers => [child_manager]) - expect(described_class).to receive(:schedule_destroy_queue).with(ems.id) - expect(described_class).to receive(:schedule_destroy_queue).with(child_manager.id) - described_class.destroy_queue(ems.id) + expect(ExtManagementSystem.count).to eq(0) + end + end + + context "with child managers" do + let(:child_manager) { FactoryGirl.create(:ext_management_system) } + let(:ems) { FactoryGirl.create(:ext_management_system, :zone => zone, :child_managers => [child_manager]) } + + it "queues up destroy for child_managers" do + described_class.destroy_queue(ems.id) + + expect(MiqQueue.count).to eq(2) + expect(MiqQueue.pluck(:instance_id)).to include(ems.id, child_manager.id) + end end end From c450468d4891e794734ba5d24c366996ccaa5909 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Mon, 18 Dec 2017 09:19:55 -0500 Subject: [PATCH 2/4] Extract queue message delivery into helper method --- spec/models/ext_management_system_spec.rb | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/spec/models/ext_management_system_spec.rb b/spec/models/ext_management_system_spec.rb index 864b7efbb39..93d725eee50 100644 --- a/spec/models/ext_management_system_spec.rb +++ b/spec/models/ext_management_system_spec.rb @@ -424,9 +424,7 @@ it "returns a task" do task_id = ems.destroy_queue - queue_message = MiqQueue.first - status, message, result = queue_message.deliver - queue_message.delivered(status, message, result) + deliver_queue_message task = MiqTask.find(task_id) expect(task.state).to eq("Finished") @@ -439,9 +437,7 @@ expect(MiqQueue.count).to eq(1) - queue_message = MiqQueue.first - status, message, result = queue_message.deliver - queue_message.delivered(status, message, result) + deliver_queue_message expect(MiqQueue.count).to eq(1) expect(MiqQueue.last.deliver_on).to_not be_nil @@ -451,17 +447,13 @@ refresh_worker = FactoryGirl.create(:miq_ems_refresh_worker, :queue_name => ems.queue_name, :status => "started", :miq_server => server) ems.destroy_queue - queue_message = MiqQueue.first - status, message, result = queue_message.deliver - queue_message.delivered(status, message, result) + deliver_queue_message expect(ExtManagementSystem.count).to eq(1) refresh_worker.destroy - queue_message = MiqQueue.first - status, message, result = queue_message.deliver - queue_message.delivered(status, message, result) + deliver_queue_message expect(ExtManagementSystem.count).to eq(0) end @@ -478,6 +470,11 @@ expect(MiqQueue.pluck(:instance_id)).to include(ems.id, child_manager.id) end end + + def deliver_queue_message(queue_message = MiqQueue.first) + status, message, result = queue_message.deliver + queue_message.delivered(status, message, result) + end end context "virtual column :supports_block_storage" do From d74ee55405d8562212d3c93d959105cb573fea8d Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Tue, 19 Dec 2017 08:22:00 -0500 Subject: [PATCH 3/4] Mark task as active while destroying an ems --- app/models/ext_management_system.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/ext_management_system.rb b/app/models/ext_management_system.rb index 0f90e8d2e64..e9b01b6e8c0 100644 --- a/app/models/ext_management_system.rb +++ b/app/models/ext_management_system.rb @@ -479,7 +479,11 @@ def orchestrate_destroy(task_id) disable! if enabled? if self.destroy == false - _log.info("Cannot destroy #{self.class.name} with id: #{id}, workers still in progress. Requeuing destroy...") + msg = "Cannot destroy #{self.class.name} with id: #{id}, workers still in progress. Requeuing destroy..." + MiqTask.update_status(task_id, MiqTask::STATE_ACTIVE, MiqTask::STATUS_OK, msg) + + _log.info(msg) + self.class.schedule_destroy_queue(id, task_id, 15.seconds.from_now) else msg = "#{self.class.name} with id: #{id} destroyed" From fbd94feb6236e98878939407db5edba703c329c5 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Wed, 20 Dec 2017 14:34:57 -0500 Subject: [PATCH 4/4] Order queue messages by ID for ems spec test --- spec/models/ext_management_system_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/ext_management_system_spec.rb b/spec/models/ext_management_system_spec.rb index 93d725eee50..9e8cddd5dda 100644 --- a/spec/models/ext_management_system_spec.rb +++ b/spec/models/ext_management_system_spec.rb @@ -471,7 +471,7 @@ end end - def deliver_queue_message(queue_message = MiqQueue.first) + def deliver_queue_message(queue_message = MiqQueue.order(:id).first) status, message, result = queue_message.deliver queue_message.delivered(status, message, result) end