Merge pull request #19216 from fdupont-redhat/v2v_state_machine_shutdown

[V2V] Add shutting_down_vm state to InfraConversionJob

app/models/infra_conversion_job.rb

@@ -30,19 +30,21 @@ def load_transitions
     self.state ||= 'initialize'
 
     {
-      :initializing                         => { 'initialize'         => 'waiting_to_start' },
-      :start                                => { 'waiting_to_start'   => 'started' },
-      :remove_snapshots                     => { 'started'            => 'removing_snapshots' },
-      :poll_remove_snapshots_complete       => { 'removing_snapshots' => 'removing_snapshots' },
+      :initializing                         => {'initialize'         => 'waiting_to_start'},
+      :start                                => {'waiting_to_start'   => 'started'},
+      :remove_snapshots                     => {'started'            => 'removing_snapshots'},
+      :poll_remove_snapshots_complete       => {'removing_snapshots' => 'removing_snapshots'},
       :wait_for_ip_address                  => {
         'removing_snapshots'     => 'waiting_for_ip_address',
         'waiting_for_ip_address' => 'waiting_for_ip_address'
       },
-      :run_migration_playbook               => { 'waiting_for_ip_address' => 'running_migration_playbook' },
-      :poll_run_migration_playbook_complete => { 'running_migration_playbook' => 'running_migration_playbook' },
+      :run_migration_playbook               => {'waiting_for_ip_address' => 'running_migration_playbook'},
+      :poll_run_migration_playbook_complete => {'running_migration_playbook' => 'running_migration_playbook'},
+      :shutdown_vm                          => {'running_migration_playbook' => 'shutting_down_vm' },
+      :poll_shutdown_vm_complete            => {'shutting_down_vm' => 'shutting_down_vm'},
       :poll_automate_state_machine          => {
-        'running_migration_playbook' => 'running_in_automate',
-        'running_in_automate'        => 'running_in_automate'
+        'shutting_down_vm'    => 'running_in_automate',
+        'running_in_automate' => 'running_in_automate'
       },
       :finish                               => {'*'                => 'finished'},
       :abort_job                            => {'*'                => 'aborting'},
@@ -74,6 +76,11 @@ def state_settings
         :weight      => 10,
         :max_retries => 6.hours / state_retry_interval
       },
+      :shutting_down_vm           => {
+        :description => "Shutting down virtual machine",
+        :weight      => 1,
+        :max_retries => 15.minutes / state_retry_interval
+      },
       :running_in_automate        => {
         :max_retries => 36.hours / state_retry_interval
       }
@@ -227,7 +234,7 @@ def remove_snapshots
 
   def poll_remove_snapshots_complete
     update_migration_task_progress(:on_entry)
-    raise 'Collapsing snapshots timed out' if polling_timeout
+    raise 'Removing snapshots timed out' if polling_timeout
 
     async_task = MiqTask.find(context[:async_task_id_removing_snapshots])
 
@@ -279,8 +286,7 @@ def run_migration_playbook
     end
 
     update_migration_task_progress(:on_exit)
-    handover_to_automate
-    queue_signal(:poll_automate_state_machine)
+    queue_signal(:shutdown_vm)
   rescue StandardError => error
     update_migration_task_progress(:on_error)
     abort_conversion(error.message, 'error')
@@ -301,13 +307,49 @@ def poll_run_migration_playbook_complete
       migration_task.update_options(:playbooks => playbooks_status)
       raise "Ansible playbook has failed (migration_phase=#{migration_phase})" if service_request.status == 'Error' && migration_phase == 'pre'
 
+      update_migration_task_progress(:on_exit)
+      return queue_signal(:shutdown_vm)
+    end
+
+    update_migration_task_progress(:on_retry)
+    queue_signal(:poll_run_migration_playbook_complete, :deliver_on => Time.now.utc + state_retry_interval)
+  rescue StandardError => error
+    update_migration_task_progress(:on_error)
+    abort_conversion(error.message, 'error')
+  end
+
+  def shutdown_vm
+    update_migration_task_progress(:on_entry)
+    unless target_vm.power_state == 'off'
+      if target_vm.supports_shutdown_guest?
+        target_vm.shutdown_guest
+      else
+        target_vm.stop
+      end
+      update_migration_task_progress(:on_exit)
+      return queue_signal(:poll_shutdown_vm_complete, :deliver_on => Time.now.utc + state_retry_interval)
+    end
+
+    update_migration_task_progress(:on_exit)
+    handover_to_automate
+    queue_signal(:poll_automate_state_machine)
+  rescue StandardError => error
+    update_migration_task_progress(:on_error)
+    abort_conversion(error.message, 'error')
+  end
+
+  def poll_shutdown_vm_complete
+    update_migration_task_progress(:on_entry)
+    return abort_conversion('Shutting down VM timed out', 'error') if polling_timeout
+
+    if target_vm.power_state == 'off'
       update_migration_task_progress(:on_exit)
       handover_to_automate
       return queue_signal(:poll_automate_state_machine)
     end
 
     update_migration_task_progress(:on_retry)
-    queue_signal(:poll_run_migration_playbook_complete, :deliver_on => Time.now.utc + state_retry_interval)
+    queue_signal(:poll_shutdown_vm_complete, :deliver_on => Time.now.utc + state_retry_interval)
   rescue StandardError => error
     update_migration_task_progress(:on_error)
     abort_conversion(error.message, 'error')

spec/models/infra_conversion_job_spec.rb

@@ -348,7 +348,7 @@
   end
 
   context 'state transitions' do
-    %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete poll_automate_state_machine finish abort_job cancel error].each do |signal|
+    %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete poll_automate_state_machine finish abort_job cancel error].each do |signal|
       shared_examples_for "allows #{signal} signal" do
         it signal.to_s do
           expect(job).to receive(signal.to_sym)
@@ -357,7 +357,7 @@
       end
     end
 
-    %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete poll_automate_state_machine].each do |signal|
+    %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete poll_automate_state_machine].each do |signal|
       shared_examples_for "doesn't allow #{signal} signal" do
         it signal.to_s do
           expect { job.signal(signal.to_sym) }.to raise_error(RuntimeError, /#{signal} is not permitted at state #{job.state}/)
@@ -381,6 +381,8 @@
       it_behaves_like 'doesn\'t allow wait_for_ip_address signal'
       it_behaves_like 'doesn\'t allow run_migration_playbook signal'
       it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
       it_behaves_like 'doesn\'t allow poll_automate_state_machine signal'
     end
 
@@ -400,6 +402,8 @@
       it_behaves_like 'doesn\'t allow wait_for_ip_address signal'
       it_behaves_like 'doesn\'t allow run_migration_playbook signal'
       it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
       it_behaves_like 'doesn\'t allow poll_automate_state_machine signal'
     end
 
@@ -419,6 +423,8 @@
       it_behaves_like 'doesn\'t allow remove_snapshots signal'
       it_behaves_like 'doesn\'t allow run_migration_playbook signal'
       it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
       it_behaves_like 'doesn\'t allow poll_automate_state_machine signal'
     end
 
@@ -438,6 +444,8 @@
       it_behaves_like 'doesn\'t allow remove_snapshots signal'
       it_behaves_like 'doesn\'t allow poll_remove_snapshots_complete signal'
       it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
       it_behaves_like 'doesn\'t allow poll_automate_state_machine signal'
     end
 
@@ -447,6 +455,27 @@
       end
 
       it_behaves_like 'allows poll_run_migration_playbook_complete signal'
+      it_behaves_like 'allows shutdown_vm signal'
+      it_behaves_like 'allows finish signal'
+      it_behaves_like 'allows abort_job signal'
+      it_behaves_like 'allows cancel signal'
+      it_behaves_like 'allows error signal'
+
+      it_behaves_like 'doesn\'t allow start signal'
+      it_behaves_like 'doesn\'t allow remove_snapshots signal'
+      it_behaves_like 'doesn\'t allow poll_remove_snapshots_complete signal'
+      it_behaves_like 'doesn\'t allow wait_for_ip_address signal'
+      it_behaves_like 'doesn\'t allow run_migration_playbook signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
+      it_behaves_like 'doesn\'t allow poll_automate_state_machine signal'
+    end
+
+    context 'shutting_down_vm' do
+      before do
+        job.state = 'shutting_down_vm'
+      end
+
+      it_behaves_like 'allows poll_shutdown_vm_complete signal'
       it_behaves_like 'allows poll_automate_state_machine signal'
       it_behaves_like 'allows finish signal'
       it_behaves_like 'allows abort_job signal'
@@ -458,6 +487,8 @@
       it_behaves_like 'doesn\'t allow poll_remove_snapshots_complete signal'
       it_behaves_like 'doesn\'t allow wait_for_ip_address signal'
       it_behaves_like 'doesn\'t allow run_migration_playbook signal'
+      it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
     end
 
     context 'running_in_automate' do
@@ -477,6 +508,8 @@
       it_behaves_like 'doesn\'t allow wait_for_ip_address signal'
       it_behaves_like 'doesn\'t allow run_migration_playbook signal'
       it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal'
+      it_behaves_like 'doesn\'t allow shutdown_vm signal'
+      it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal'
     end
   end
 
@@ -535,7 +568,7 @@
         job.context[:retries_removing_snapshots] = 960
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_error)
-        expect(job).to receive(:abort_conversion).with('Collapsing snapshots timed out', 'error')
+        expect(job).to receive(:abort_conversion).with('Removing snapshots timed out', 'error')
         job.signal(:poll_remove_snapshots_complete)
       end
 
@@ -615,9 +648,8 @@
           embedded_ansible_service_template.delete
           expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
           expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
-          expect(job).to receive(:queue_signal).with(:poll_automate_state_machine)
+          expect(job).to receive(:queue_signal).with(:shutdown_vm)
           job.signal(:run_migration_playbook)
-          expect(task.reload.options[:workflow_runner]).to eq('automate')
         end
       end
 
@@ -670,9 +702,8 @@
         embedded_ansible_service_request.update!(:request_state => 'finished', :status => 'Ok')
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
-        expect(job).to receive(:queue_signal).with(:poll_automate_state_machine)
+        expect(job).to receive(:queue_signal).with(:shutdown_vm)
         job.signal(:poll_run_migration_playbook_complete)
-        expect(task.reload.options[:workflow_runner]).to eq('automate')
       end
 
       it 'fails if service request is finished and migration_phase is "pre" and its status is Error' do
@@ -689,8 +720,77 @@
         embedded_ansible_service_request.update!(:state => 'finished', :status => 'Error', :message => 'Fake error message')
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
         expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
-        expect(job).to receive(:queue_signal).with(:poll_automate_state_machine)
+        expect(job).to receive(:queue_signal).with(:shutdown_vm)
         job.signal(:poll_run_migration_playbook_complete)
+      end
+    end
+
+    context '#shutdown_vm' do
+      before do
+        task.update_options(:migration_phase => 'pre')
+        job.state = 'running_migration_playbook'
+      end
+
+      it 'exits if VM is already off' do
+        vm_vmware.update!(:raw_power_state => 'poweredOff')
+        expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
+        expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
+        expect(job).to receive(:queue_signal).with(:poll_automate_state_machine)
+        job.signal(:shutdown_vm)
+        expect(task.reload.options[:workflow_runner]).to eq('automate')
+      end
+
+      it 'sends shutdown request to VM if VM supports shutdown_guest' do
+        vm_vmware.update!(:raw_power_state => 'poweredOn')
+        Timecop.freeze(2019, 2, 6) do
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
+          expect(job.migration_task.source).to receive(:shutdown_guest)
+          expect(job).to receive(:queue_signal).with(:poll_shutdown_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval)
+          job.signal(:shutdown_vm)
+        end
+      end
+
+      it 'sends stop request to VM if VM does not support shutdown_guest' do
+        vm_vmware.update!(:raw_power_state => 'unknown')
+        Timecop.freeze(2019, 2, 6) do
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
+          expect(job.migration_task.source).to receive(:stop)
+          expect(job).to receive(:queue_signal).with(:poll_shutdown_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval)
+          job.signal(:shutdown_vm)
+        end
+      end
+    end
+
+    context '#poll_shutdown_vm_complete' do
+      before do
+        task.update_options(:migration_phase => 'pre')
+        job.state = 'shutting_down_vm'
+      end
+
+      it 'abort_conversion when shutting_down_vm times out' do
+        job.context[:retries_shutting_down_vm] = 60
+        expect(job).to receive(:abort_conversion).with('Shutting down VM timed out', 'error')
+        job.signal(:poll_shutdown_vm_complete)
+      end
+
+      it 'retries if VM is not off' do
+        vm_vmware.update!(:raw_power_state => 'poweredOn')
+        Timecop.freeze(2019, 2, 6) do
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
+          expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry)
+          expect(job).to receive(:queue_signal).with(:poll_shutdown_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval)
+          job.signal(:poll_shutdown_vm_complete)
+        end
+      end
+
+      it 'exits if VM is off' do
+        vm_vmware.update!(:raw_power_state => 'poweredOff')
+        expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry)
+        expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit)
+        expect(job).to receive(:queue_signal).with(:poll_automate_state_machine)
+        job.signal(:poll_shutdown_vm_complete)
         expect(task.reload.options[:workflow_runner]).to eq('automate')
       end
     end