From ce3516ec387a8cffa2d88836368939ed22a98723 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 28 Aug 2019 22:52:03 +0200 Subject: [PATCH 1/3] Add transforming_vm state to InfraConversionJob --- app/models/infra_conversion_job.rb | 61 ++++++++-- spec/models/infra_conversion_job_spec.rb | 144 ++++++++++++++++++++++- 2 files changed, 193 insertions(+), 12 deletions(-) diff --git a/app/models/infra_conversion_job.rb b/app/models/infra_conversion_job.rb index 1b8ad2d704a..49303f70ec8 100644 --- a/app/models/infra_conversion_job.rb +++ b/app/models/infra_conversion_job.rb @@ -42,8 +42,10 @@ def load_transitions :poll_run_migration_playbook_complete => {'running_migration_playbook' => 'running_migration_playbook'}, :shutdown_vm => {'running_migration_playbook' => 'shutting_down_vm' }, :poll_shutdown_vm_complete => {'shutting_down_vm' => 'shutting_down_vm'}, + :transform_vm => {'shutting_down_vm' => 'transforming_vm'}, + :poll_transform_vm_complete => {'transforming_vm' => 'transforming_vm'}, :poll_automate_state_machine => { - 'shutting_down_vm' => 'running_in_automate', + 'transforming_vm' => 'running_in_automate', 'running_in_automate' => 'running_in_automate' }, :finish => {'*' => 'finished'}, @@ -81,6 +83,11 @@ def state_settings :weight => 1, :max_retries => 15.minutes / state_retry_interval }, + :transforming_vm => { + :description => "Converting disks", + :weight => 60, + :max_retries => 1.day / state_retry_interval + }, :running_in_automate => { :max_retries => 36.hours / state_retry_interval } @@ -127,8 +134,8 @@ def on_entry(state_hash, _) }.compact end - def on_retry(state_hash, state_progress = nil) - if state_progress.nil? + def on_retry(state_hash, state_progress = {}) + if state_progress.empty? state_hash[:percent] = context["retries_#{state}".to_sym].to_f / state_settings[state.to_sym][:max_retries].to_f * 100.0 else state_hash.merge!(state_progress) @@ -151,7 +158,7 @@ def on_error(state_hash, _) state_hash end - def update_migration_task_progress(state_phase, state_progress = nil) + def update_migration_task_progress(state_phase, state_progress = {}) progress = migration_task.options[:progress] || { :current_state => state, :percent => 0.0, :states => {} } state_hash = send(state_phase, progress[:states][state.to_sym], state_progress) progress[:states][state.to_sym] = state_hash @@ -332,7 +339,7 @@ def shutdown_vm update_migration_task_progress(:on_exit) handover_to_automate - queue_signal(:poll_automate_state_machine) + queue_signal(:transform_vm) rescue StandardError => error update_migration_task_progress(:on_error) abort_conversion(error.message, 'error') @@ -344,8 +351,7 @@ def poll_shutdown_vm_complete if target_vm.power_state == 'off' update_migration_task_progress(:on_exit) - handover_to_automate - return queue_signal(:poll_automate_state_machine) + return queue_signal(:transform_vm) end update_migration_task_progress(:on_retry) @@ -355,6 +361,47 @@ def poll_shutdown_vm_complete abort_conversion(error.message, 'error') end + def transform_vm + update_migration_task_progress(:on_entry) + migration_task.run_conversion + update_migration_task_progress(:on_exit) + queue_signal(:poll_transform_vm_complete, :deliver_on => Time.now.utc + state_retry_interval) + rescue StandardError => error + update_migration_task_progress(:on_error) + abort_conversion(error.message, 'error') + end + + def poll_transform_vm_complete + update_migration_task_progress(:on_entry) + return abort_conversion('Converting disks timed out', 'error') if polling_timeout + + migration_task.get_conversion_state + case migration_task.options[:virtv2v_status] + when 'active' + virtv2v_disks = migration_task.options[:virtv2v_disks] + converted_disks = virtv2v_disks.reject { |disk| disk[:percent].zero? } + if converted_disks.empty? + message = 'Disk transformation is initializing.' + percent = 1 + else + percent = 0 + converted_disks.each { |disk| percent += (disk[:percent].to_f * disk[:weight].to_f / 100.0) } + message = "Converting disks #{converted_disks.length} / #{virtv2v_disks.length} [#{percent.round(2)}%]." + end + update_migration_task_progress(:on_retry, :message => message, :percent => percent) + queue_signal(:poll_transform_vm_complete, :deliver_on => Time.now.utc + state_retry_interval) + when 'failed' + raise migration_task.options[:virtv2v_message] + when 'succeeded' + update_migration_task_progress(:on_exit) + handover_to_automate + queue_signal(:poll_automate_state_machine) + end + rescue StandardError => error + update_migration_task_progress(:on_error) + abort_conversion(error.message, 'error') + end + def poll_automate_state_machine return abort_conversion('Polling Automate state machine timed out', 'error') if polling_timeout diff --git a/spec/models/infra_conversion_job_spec.rb b/spec/models/infra_conversion_job_spec.rb index d3eaaaa14a9..c15527bc117 100644 --- a/spec/models/infra_conversion_job_spec.rb +++ b/spec/models/infra_conversion_job_spec.rb @@ -348,7 +348,7 @@ end context 'state transitions' do - %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete poll_automate_state_machine finish abort_job cancel error].each do |signal| + %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete transform_vm poll_transform_vm_complete poll_automate_state_machine finish abort_job cancel error].each do |signal| shared_examples_for "allows #{signal} signal" do it signal.to_s do expect(job).to receive(signal.to_sym) @@ -357,7 +357,7 @@ end end - %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete poll_automate_state_machine].each do |signal| + %w[start remove_snapshots poll_remove_snapshots_complete wait_for_ip_address run_migration_playbook poll_run_migration_playbook_complete shutdown_vm poll_shutdown_vm_complete transform_vm poll_transform_vm_complete poll_automate_state_machine].each do |signal| shared_examples_for "doesn't allow #{signal} signal" do it signal.to_s do expect { job.signal(signal.to_sym) }.to raise_error(RuntimeError, /#{signal} is not permitted at state #{job.state}/) @@ -383,6 +383,8 @@ it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end @@ -404,6 +406,8 @@ it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end @@ -425,6 +429,8 @@ it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end @@ -446,6 +452,8 @@ it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end @@ -467,6 +475,8 @@ it_behaves_like 'doesn\'t allow wait_for_ip_address signal' it_behaves_like 'doesn\'t allow run_migration_playbook signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end @@ -475,7 +485,30 @@ job.state = 'shutting_down_vm' end + it_behaves_like 'allows transform_vm signal' it_behaves_like 'allows poll_shutdown_vm_complete signal' + it_behaves_like 'allows finish signal' + it_behaves_like 'allows abort_job signal' + it_behaves_like 'allows cancel signal' + it_behaves_like 'allows error signal' + + it_behaves_like 'doesn\'t allow start signal' + it_behaves_like 'doesn\'t allow remove_snapshots signal' + it_behaves_like 'doesn\'t allow poll_remove_snapshots_complete signal' + it_behaves_like 'doesn\'t allow wait_for_ip_address signal' + it_behaves_like 'doesn\'t allow run_migration_playbook signal' + it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' + it_behaves_like 'doesn\'t allow shutdown_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' + it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' + end + + context 'transforming_vm' do + before do + job.state = 'transforming_vm' + end + + it_behaves_like 'allows poll_transform_vm_complete signal' it_behaves_like 'allows poll_automate_state_machine signal' it_behaves_like 'allows finish signal' it_behaves_like 'allows abort_job signal' @@ -489,6 +522,8 @@ it_behaves_like 'doesn\'t allow run_migration_playbook signal' it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' + it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' end context 'running_in_automate' do @@ -510,6 +545,8 @@ it_behaves_like 'doesn\'t allow poll_run_migration_playbook_complete signal' it_behaves_like 'doesn\'t allow shutdown_vm signal' it_behaves_like 'doesn\'t allow poll_shutdown_vm_complete signal' + it_behaves_like 'doesn\'t allow transform_vm signal' + it_behaves_like 'doesn\'t allow poll_transform_vm_complete signal' end end @@ -735,7 +772,7 @@ vm_vmware.update!(:raw_power_state => 'poweredOff') expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit) - expect(job).to receive(:queue_signal).with(:poll_automate_state_machine) + expect(job).to receive(:queue_signal).with(:transform_vm) job.signal(:shutdown_vm) expect(task.reload.options[:workflow_runner]).to eq('automate') end @@ -789,9 +826,106 @@ vm_vmware.update!(:raw_power_state => 'poweredOff') expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit) - expect(job).to receive(:queue_signal).with(:poll_automate_state_machine) + expect(job).to receive(:queue_signal).with(:transform_vm) job.signal(:poll_shutdown_vm_complete) - expect(task.reload.options[:workflow_runner]).to eq('automate') + end + end + + context '#transform_vm' do + before do + task.update_options(:migration_phase => 'pre') + job.state = 'shutting_down_vm' + end + + it 'sends run_conversion to migration task and exits' do + Timecop.freeze(2019, 2, 6) do + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit) + expect(job.migration_task).to receive(:run_conversion) + expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) + job.signal(:transform_vm) + end + end + end + + context '#poll_transform_vm_complete' do + before do + job.state = 'transforming_vm' + allow(job.migration_task).to receive(:get_conversion_state) + end + + it 'abort_conversion when shutting_down_vm times out' do + job.context[:retries_transforming_vm] = 5760 + expect(job).to receive(:abort_conversion).with('Converting disks timed out', 'error') + job.signal(:poll_transform_vm_complete) + end + + context 'virt-v2v has not started conversion' do + let(:virtv2v_disks) do + [ + { :path => '[datastore] test_vm/test_vm.vmdk', :size => 1_234_567, :percent => 0, :weight => 25 }, + { :path => '[datastore] test_vm/test_vm-2.vmdk', :size => 3_703_701, :percent => 0, :weight => 75 } + ] + end + + it 'returns a message stating conversion has not started' do + job.migration_task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) + job.migration_task.reload +# Timecop.freeze(2019, 2, 6) do + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Disk transformation is initializing.', :percent => 1) +# expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) + job.signal(:poll_transform_vm_complete) +# require 'byebug' ; byebug + expect(job.migration_task.options[:progress][:states][job.state.to_sym]).to include( + :message => 'Disk transformation is initializing.', + :percent => 1 + ) +# end + end + end + + context "conversion is still running" do + let(:virtv2v_disks) do + [ + { :path => '[datastore] test_vm/test_vm.vmdk', :size => 1_234_567, :percent => 100, :weight => 25 }, + { :path => '[datastore] test_vm/test_vm-2.vmdk', :size => 3_703_701, :percent => 25, :weight => 75 } + ] + end + + it "updates message and percentage, and retries if conversion is not finished" do + job.migration_task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) + Timecop.freeze(2019, 2, 6) do + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Converting disk 2 / 2 [43.75%].', :percent => 43.75) + expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) + job.signal(:poll_transform_vm_complete) + expect(job.migration_task.reload.options[:progress]).to have_attributes( + :message => 'Converting disk 2 / 2 [43.75%].', + :percent => 43.75 + ) + end + end + + it "aborts if conversion failed" do + job.migration_task.update_options(:virtv2v_status => 'failed') + task.reload + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) + expect(job).to receive(:abort_conversion).with('Converting disks timed out', 'error') + job.signal(:poll_transform_vm_complete) + end + + it "exits if conversion succeeded" do + job.migration_task.update_options(:virtv2v_status => 'succeeded') +# task.reload + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit) + expect(job).to receive(:queue_signal).with(:poll_automate_state_machine) + job.signal(:poll_transform_vm_complete) + job.migration_task.reload + expect(job.migration_task.options[:progress]).to have_attributes(:percent => 100.0) + expect(job.migration_task.options[:workflow_runner]).to eq('automate') + end end end From fc137cba1321f40925003bee48a2306ee8d5c00d Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 28 Aug 2019 23:31:57 +0200 Subject: [PATCH 2/3] Fix tests by using .and_call_original --- app/models/infra_conversion_job.rb | 8 ++--- spec/models/infra_conversion_job_spec.rb | 42 ++++++++++-------------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/app/models/infra_conversion_job.rb b/app/models/infra_conversion_job.rb index 49303f70ec8..7ebdf0303b1 100644 --- a/app/models/infra_conversion_job.rb +++ b/app/models/infra_conversion_job.rb @@ -134,8 +134,8 @@ def on_entry(state_hash, _) }.compact end - def on_retry(state_hash, state_progress = {}) - if state_progress.empty? + def on_retry(state_hash, state_progress = nil) + if state_progress.nil? state_hash[:percent] = context["retries_#{state}".to_sym].to_f / state_settings[state.to_sym][:max_retries].to_f * 100.0 else state_hash.merge!(state_progress) @@ -158,7 +158,7 @@ def on_error(state_hash, _) state_hash end - def update_migration_task_progress(state_phase, state_progress = {}) + def update_migration_task_progress(state_phase, state_progress = nil) progress = migration_task.options[:progress] || { :current_state => state, :percent => 0.0, :states => {} } state_hash = send(state_phase, progress[:states][state.to_sym], state_progress) progress[:states][state.to_sym] = state_hash @@ -386,7 +386,7 @@ def poll_transform_vm_complete else percent = 0 converted_disks.each { |disk| percent += (disk[:percent].to_f * disk[:weight].to_f / 100.0) } - message = "Converting disks #{converted_disks.length} / #{virtv2v_disks.length} [#{percent.round(2)}%]." + message = "Converting disk #{converted_disks.length} / #{virtv2v_disks.length} [#{percent.round(2)}%]." end update_migration_task_progress(:on_retry, :message => message, :percent => percent) queue_signal(:poll_transform_vm_complete, :deliver_on => Time.now.utc + state_retry_interval) diff --git a/spec/models/infra_conversion_job_spec.rb b/spec/models/infra_conversion_job_spec.rb index c15527bc117..160046c4c99 100644 --- a/spec/models/infra_conversion_job_spec.rb +++ b/spec/models/infra_conversion_job_spec.rb @@ -869,19 +869,17 @@ end it 'returns a message stating conversion has not started' do - job.migration_task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) - job.migration_task.reload -# Timecop.freeze(2019, 2, 6) do - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Disk transformation is initializing.', :percent => 1) -# expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) + task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) + Timecop.freeze(2019, 2, 6) do + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry).and_call_original + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Disk transformation is initializing.', :percent => 1).and_call_original + expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) job.signal(:poll_transform_vm_complete) -# require 'byebug' ; byebug - expect(job.migration_task.options[:progress][:states][job.state.to_sym]).to include( + expect(task.reload.options[:progress][:states][job.state.to_sym]).to include( :message => 'Disk transformation is initializing.', :percent => 1 ) -# end + end end end @@ -894,13 +892,13 @@ end it "updates message and percentage, and retries if conversion is not finished" do - job.migration_task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) + task.update_options(:virtv2v_status => 'active', :virtv2v_disks => virtv2v_disks) Timecop.freeze(2019, 2, 6) do - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Converting disk 2 / 2 [43.75%].', :percent => 43.75) + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry).and_call_original + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_retry, :message => 'Converting disk 2 / 2 [43.75%].', :percent => 43.75).and_call_original expect(job).to receive(:queue_signal).with(:poll_transform_vm_complete, :deliver_on => Time.now.utc + job.state_retry_interval) job.signal(:poll_transform_vm_complete) - expect(job.migration_task.reload.options[:progress]).to have_attributes( + expect(task.reload.options[:progress][:states][job.state.to_sym]).to include( :message => 'Converting disk 2 / 2 [43.75%].', :percent => 43.75 ) @@ -908,23 +906,19 @@ end it "aborts if conversion failed" do - job.migration_task.update_options(:virtv2v_status => 'failed') - task.reload - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) - expect(job).to receive(:abort_conversion).with('Converting disks timed out', 'error') + task.update_options(:virtv2v_status => 'failed', :virtv2v_message => 'virt-v2v failed for some reason') + expect(job).to receive(:abort_conversion).with('virt-v2v failed for some reason', 'error').and_call_original job.signal(:poll_transform_vm_complete) end it "exits if conversion succeeded" do - job.migration_task.update_options(:virtv2v_status => 'succeeded') -# task.reload - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry) - expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit) + task.update_options(:virtv2v_status => 'succeeded') + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_entry).and_call_original + expect(job).to receive(:update_migration_task_progress).once.ordered.with(:on_exit).and_call_original expect(job).to receive(:queue_signal).with(:poll_automate_state_machine) job.signal(:poll_transform_vm_complete) - job.migration_task.reload - expect(job.migration_task.options[:progress]).to have_attributes(:percent => 100.0) - expect(job.migration_task.options[:workflow_runner]).to eq('automate') + expect(task.reload.options[:progress][:states][job.state.to_sym]).to include(:percent => 100.0) + expect(task.options[:workflow_runner]).to eq('automate') end end end From 21f6c70a252cf66b5bd0dc02c496cff25946949f Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 29 Aug 2019 16:23:34 +0200 Subject: [PATCH 3/3] Address rubocop --- spec/models/infra_conversion_job_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/infra_conversion_job_spec.rb b/spec/models/infra_conversion_job_spec.rb index 160046c4c99..8c24e488073 100644 --- a/spec/models/infra_conversion_job_spec.rb +++ b/spec/models/infra_conversion_job_spec.rb @@ -503,7 +503,7 @@ it_behaves_like 'doesn\'t allow poll_automate_state_machine signal' end - context 'transforming_vm' do + context 'transforming_vm' do before do job.state = 'transforming_vm' end