From 289c53fc81c7ee520dc593e1a1148b29d53c4030 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 28 Sep 2018 09:27:24 -0400 Subject: [PATCH 01/13] Add methods to conversion_host to build virt-v2v wrapper options --- app/models/conversion_host.rb | 101 ++++++ ...rvice_template_transformation_plan_task.rb | 67 +++- spec/models/conversion_host_spec.rb | 288 ++++++++++++++++++ ..._template_transformation_plan_task_spec.rb | 145 ++++++++- 4 files changed, 589 insertions(+), 12 deletions(-) create mode 100644 spec/models/conversion_host_spec.rb diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index 493569ff804..b856c101174 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -4,4 +4,105 @@ class ConversionHost < ApplicationRecord acts_as_miq_taggable belongs_to :resource, :polymorphic => true + + def active_tasks + ServiceTemplateTransformationPlanTask.where(:state => 'active').select do |task| + task.conversion_host == self + end + end + + def eligible? + return true if concurrent_transformation_limit.nil? + active_tasks.size < concurrent_transformation_limit.to_i + end + + def source_transport_method + return 'vddk' if vddk_transport_supported + return 'ssh' if ssh_transport_supported + end + + def conversion_options(task) + source_vm = task.source + source_ems = source_vm.ext_management_system + source_cluster = source_vm.ems_cluster + source_storage = source_vm.hardware.disks.select { |d| d.device_type == 'disk' }.first.storage + + destination_cluster = task.transformation_destination(source_cluster) + destination_storage = task.transformation_destination(source_storage) + destination_ems = destination_cluster.ext_management_system + + source_provider_options = send( + "conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", + source_vm, + source_storage + ) + destination_provider_options = send( + "conversion_options_destination_provider_#{destination_ems.emstype}", + task, + destination_ems, + destination_cluster, + destination_storage + ) + options = { + :source_disks => task.source_disks.map { |disk| disk[:path] }, + :network_mappings => task.network_mappings + } + options.merge(source_provider_options).merge(destination_provider_options) + end + + def conversion_options_source_provider_vmwarews_vddk(vm, _storage) + { + :vm_name => vm.name, + :transport_method => 'vddk', + :vmware_fingerprint => vm.host.fingerprint, + :vmware_uri => URI::Generic.build( + :scheme => 'esx', + :userinfo => CGI.escape(vm.host.authentication_userid), + :host => vm.host.ipaddress, + :path => '/', + :query => { :no_verify => 1 }.to_query + ).to_s, + :vmware_password => vm.host.authentication_password + } + end + + def conversion_options_source_provider_vmwarews_ssh(vm, storage) + { + :vm_name => URI::Generic.build(:scheme => 'ssh', :userinfo => 'root', :host => vm.host.ipaddress, :path => "/vmfs/volumes").to_s + "/#{storage.name}/#{vm.location}", + :transport_method => 'ssh' + } + end + + def conversion_options_destination_provider_rhevm(_task, ems, cluster, storage) + { + :rhv_url => URI::Generic.build(:scheme => 'https', :host => ems.hostname, :path => '/ovirt-engine/api').to_s, + :rhv_cluster => cluster.name, + :rhv_storage => storage.name, + :rhv_password => ems.authentication_password, + :install_drivers => true, + :insecure_connection => true + } + end + + def conversion_options_destination_provider_openstack(task, ems, cluster, storage) + { + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => ems.hostname, + :port => ems.port, + :path => ems.api_version + ), + :os_user_domain_name => ems.uid_ems, + :os_username => ems.authentication_userid, + :os_password => ems.authentication_password, + :os_project_name => cluster.name + }, + :osp_destination_project_id => cluster.ems_ref, + :osp_volume_type_id => storage.ems_ref, + :osp_flavor_id => task.destination_flavor.ems_ref, + :osp_security_groups_ids => [task.destination_security_group.ems_ref] + } + end end diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index 000ba171a93..28a105c2ff4 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -47,7 +47,64 @@ def task_active end def conversion_host - Host.find_by(:id => options[:transformation_host_id]) + ConversionHost.find_by(:id => options[:transformation_host_id]) + end + + def source_ems + options[:source_ems_id] ||= source.ext_management_system.id + source.ext_management_system + end + + def destination_ems + options[:destination_ems_id] ||= transformation_destination(source.ems_cluster).ext_management_system.id + transformation_destination(source.ems_cluster).ext_management_system + end + + def source_disks + options[:source_disks] ||= source.hardware.disks.select { |d| d.device_type == 'disk' }.collect do |disk| + source_storage = disk.storage + destination_storage = transformation_destination(disk.storage) + raise "[#{source.name}] Disk #{disk.device_name} [#{source_storage.name}] has no mapping. Aborting." if destination_storage.nil? + { + :path => disk.filename, + :size => disk.size, + :percent => 0, + :weight => disk.size.to_f / source.allocated_disk_storage.to_f * 100 + } + end + end + + def network_mappings + options[:network_mappings] ||= source.hardware.nics.select { |n| n.device_type == 'ethernet' }.collect do |nic| + source_network = nic.lan + destination_network = transformation_destination(source_network) + raise "[#{source.name}] NIC #{nic.device_name} [#{source_network.name}] has no mapping. Aborting." if destination_network.nil? + { + :source => source_network.name, + :destination => destination_network_ref(destination_network), + :mac_address => nic.address + } + end + end + + def destination_network_ref(network) + send("destination_network_ref_#{destination_ems.emstype}", network) + end + + def destination_network_ref_rhevm(network) + network.name + end + + def destination_network_ref_openstack(network) + network.ems_ref + end + + def destination_flavor + Flavor.find_by(:id => miq_request.source.options[:config_info][:osp_flavor]) + end + + def destination_security_group + SecurityGroup.find_by(:id => miq_request.source.options[:config_info][:osp_security_group]) end def transformation_log @@ -58,9 +115,9 @@ def transformation_log raise MiqException::Error, msg end - userid, password = host.auth_user_pwd(:remote) + userid, password = host.resource.auth_user_pwd(:remote) if userid.blank? || password.blank? - msg = "Credential was not found for host #{host.name}. Download of transformation log aborted." + msg = "Credential was not found for host #{host.resource.name}. Download of transformation log aborted." _log.error(msg) raise MiqException::Error, msg end @@ -74,7 +131,7 @@ def transformation_log begin require 'net/scp' - Net::SCP.download!(host.ipaddress, userid, logfile, nil, :ssh => {:password => password}) + Net::SCP.download!(host.resource.ipaddress, userid, logfile, nil, :ssh => {:password => password}) rescue Net::SCP::Error => scp_err _log.error("Download of transformation log for #{description} with ID [#{id}] failed with error: #{scp_err.message}") raise scp_err @@ -98,7 +155,7 @@ def transformation_log_queue(userid = nil) :instance_id => id, :priority => MiqQueue::HIGH_PRIORITY, :args => [], - :zone => host.my_zone} + :zone => host.resource.my_zone} MiqTask.generic_action_with_callback(options, queue_options) end diff --git a/spec/models/conversion_host_spec.rb b/spec/models/conversion_host_spec.rb new file mode 100644 index 00000000000..7c5a00a063e --- /dev/null +++ b/spec/models/conversion_host_spec.rb @@ -0,0 +1,288 @@ +describe ConversionHost do + let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } + let(:conversion_host) { FactoryGirl.create(:conversion_host) } + + context "provider independent methods" do + let(:task_1) { FactoryGirl.create(:service_template_transformation_plan_task) } + let(:task_2) { FactoryGirl.create(:service_template_transformation_plan_task) } + let(:task_3) { FactoryGirl.create(:service_template_transformation_plan_task) } + + let(:conversion_host_1) { FactoryGirl.create(:conversion_host) } + let(:conversion_host_2) { FactoryGirl.create(:conversion_host) } + + before do + conversion_host_1.concurrent_transformation_limit = "2" + conversion_host_2.concurrent_transformation_limit = "1" + + task_1.options[:transformation_host_id] = conversion_host_1.id + task_2.options[:transformation_host_id] = conversion_host_1.id + task_3.options[:transformation_host_id] = conversion_host_2.id + + allow(ServiceTemplateTransformationPlanTask).to receive(:where).with(:state => 'active').and_return([task_1, task_3]) + end + + it "#active_tasks" do + expect(conversion_host_1.active_tasks).to eq([task_1]) + expect(conversion_host_2.active_tasks).to eq([task_3]) + end + + it "#eligible?" do + expect(conversion_host_1.eligible?).to eq(true) + expect(conversion_host_2.eligible?).to eq(false) + end + + context "#source_transport_method" do + it { expect(conversion_host_2.source_transport_method).to be_nil } + + context "ssh transport enabled" do + before { conversion_host_2.ssh_transport_supported = true } + it { expect(conversion_host_2.source_transport_method).to eq('ssh') } + + context "vddk transport enabled" do + before { conversion_host_2.vddk_transport_supported = true } + it { expect(conversion_host_2.source_transport_method).to eq('vddk') } + end + end + end + end + + context "source is vmwarews" do + let(:src_ems) { FactoryGirl.create(:ems_vmware, :zone => FactoryGirl.create(:zone)) } + let(:src_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => src_ems) } + let(:src_host) { FactoryGirl.create(:host, :ext_management_system => src_ems, :ipaddress => '10.0.0.1') } + let(:src_storage) { FactoryGirl.create(:storage, :ext_management_system => src_ems) } + + let(:src_lan_1) { FactoryGirl.create(:lan) } + let(:src_lan_2) { FactoryGirl.create(:lan) } + let(:src_nic_1) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_1) } + let(:src_nic_2) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_2) } + + let(:src_disk_1) { instance_double("disk", :device_name => "Hard disk 1", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm.vmdk", :size => 17_179_869_184) } + let(:src_disk_2) { instance_double("disk", :device_name => "Hard disk 2", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm-2.vmdk", :size => 17_179_869_184) } + + let(:src_hardware) { FactoryGirl.create(:hardware, :nics => [src_nic_1, src_nic_2]) } + + let(:src_vm) { FactoryGirl.create(:vm_vmware, :ext_management_system => src_ems, :ems_cluster => src_cluster, :host => src_host, :hardware => src_hardware) } + + let(:source_disks) do + [ + {:path => src_disk_1.filename, :size => src_disk_1.size, :percent => 0, :weight => 50.0 }, + {:path => src_disk_2.filename, :size => src_disk_2.size, :percent => 0, :weight => 50.0 } + ] + end + + before do + allow(src_hardware).to receive(:disks).and_return([src_disk_1, src_disk_2]) + allow(src_disk_1).to receive(:storage).and_return(src_storage) + allow(src_host).to receive(:fingerprint).and_return('01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67') + allow(src_host).to receive(:authentication_userid).and_return('esx_user') + allow(src_host).to receive(:authentication_password).and_return('esx_passwd') + end + + context "destination is rhevm" do + let(:dst_ems) { FactoryGirl.create(:ems_redhat, :zone => FactoryGirl.create(:zone)) } + let(:dst_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => dst_ems) } + let(:dst_storage) { FactoryGirl.create(:storage) } + let(:dst_lan_1) { FactoryGirl.create(:lan) } + let(:dst_lan_2) { FactoryGirl.create(:lan) } + + let(:mapping) do + FactoryGirl.create( + :transformation_mapping, + :transformation_mapping_items => [ + TransformationMappingItem.new(:source => src_cluster, :destination => dst_cluster), + TransformationMappingItem.new(:source => src_storage, :destination => dst_storage), + TransformationMappingItem.new(:source => src_lan_1, :destination => dst_lan_1), + TransformationMappingItem.new(:source => src_lan_2, :destination => dst_lan_2) + ] + ) + end + + let(:catalog_item_options) do + { + :name => 'Transformation Plan', + :description => 'a description', + :config_info => { + :transformation_mapping_id => mapping.id, + :pre_service_id => apst.id, + :post_service_id => apst.id, + :actions => [ + {:vm_id => src_vm.id.to_s, :pre_service => true, :post_service => true} + ], + } + } + end + + let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } + let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } + let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm) } + + before do + task.options[:transformation_host_id] = conversion_host.id + allow(task).to receive(:source_disks).and_return(source_disks) + end + + context "transport method is vddk" do + before do + conversion_host.vddk_transport_supported = true + end + + it "#conversion_options" do + expect(conversion_host.conversion_options(task)).to eq( + :vm_name => src_vm.name, + :transport_method => 'vddk', + :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', + :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", + :vmware_password => 'esx_passwd', + :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", + :rhv_cluster => dst_cluster.name, + :rhv_storage => dst_storage.name, + :rhv_password => dst_ems.authentication_password, + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task.network_mappings, + :install_drivers => true, + :insecure_connection => true + ) + end + end + + context "transport method is ssh" do + before do + conversion_host.vddk_transport_supported = false + conversion_host.ssh_transport_supported = true + end + + it "#conversion_options" do + expect(conversion_host.conversion_options(task)).to eq( + :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm.location}", + :transport_method => 'ssh', + :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", + :rhv_cluster => dst_cluster.name, + :rhv_storage => dst_storage.name, + :rhv_password => dst_ems.authentication_password, + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task.network_mappings, + :install_drivers => true, + :insecure_connection => true + ) + end + end + end + + context "destination is openstack" do + let(:dst_ems) { FactoryGirl.create(:ems_openstack, :zone => FactoryGirl.create(:zone)) } + let(:dst_cloud_tenant) { FactoryGirl.create(:cloud_tenant, :ext_management_system => dst_ems) } + let(:dst_cloud_volume_type) { FactoryGirl.create(:cloud_volume_type) } + let(:dst_cloud_network_1) { FactoryGirl.create(:cloud_network) } + let(:dst_cloud_network_2) { FactoryGirl.create(:cloud_network) } + let(:dst_flavor) { FactoryGirl.create(:flavor) } + let(:dst_security_group) { FactoryGirl.create(:security_group) } + + let(:mapping) do + FactoryGirl.create( + :transformation_mapping, + :transformation_mapping_items => [ + TransformationMappingItem.new(:source => src_cluster, :destination => dst_cloud_tenant), + TransformationMappingItem.new(:source => src_storage, :destination => dst_cloud_volume_type), + TransformationMappingItem.new(:source => src_lan_1, :destination => dst_cloud_network_1), + TransformationMappingItem.new(:source => src_lan_2, :destination => dst_cloud_network_2) + ] + ) + end + + let(:catalog_item_options) do + { + :name => 'Transformation Plan', + :description => 'a description', + :config_info => { + :transformation_mapping_id => mapping.id, + :pre_service_id => apst.id, + :post_service_id => apst.id, + :osp_flavor => dst_flavor.id, + :osp_security_group => dst_security_group.id, + :actions => [ + {:vm_id => src_vm.id.to_s, :pre_service => true, :post_service => true} + ], + } + } + end + + let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } + let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } + let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm) } + + before do + task.options[:transformation_host_id] = conversion_host.id + allow(task).to receive(:source_disks).and_return(source_disks) + end + + context "transport method is vddk" do + before do + conversion_host.vddk_transport_supported = true + end + + it "#conversion_options" do + expect(conversion_host.conversion_options(task)).to eq( + :vm_name => src_vm.name, + :transport_method => 'vddk', + :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', + :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", + :vmware_password => 'esx_passwd', + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => dst_ems.hostname, + :port => dst_ems.port, + :path => dst_ems.api_version + ), + :os_user_domain_name => dst_ems.uid_ems, + :os_username => dst_ems.authentication_userid, + :os_password => dst_ems.authentication_password, + :os_project_name => dst_cloud_tenant.name + }, + :osp_destination_project_id => dst_cloud_tenant.ems_ref, + :osp_volume_type_id => dst_cloud_volume_type.ems_ref, + :osp_flavor_id => dst_flavor.ems_ref, + :osp_security_groups_ids => [dst_security_group.ems_ref], + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task.network_mappings + ) + end + end + + context "transport method is ssh" do + before do + conversion_host.vddk_transport_supported = false + conversion_host.ssh_transport_supported = true + end + + it "#conversion_options" do + expect(conversion_host.conversion_options(task)).to eq( + :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm.location}", + :transport_method => 'ssh', + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => dst_ems.hostname, + :port => dst_ems.port, + :path => dst_ems.api_version + ), + :os_user_domain_name => dst_ems.uid_ems, + :os_username => dst_ems.authentication_userid, + :os_password => dst_ems.authentication_password, + :os_project_name => dst_cloud_tenant.name + }, + :osp_destination_project_id => dst_cloud_tenant.ems_ref, + :osp_volume_type_id => dst_cloud_volume_type.ems_ref, + :osp_flavor_id => dst_flavor.ems_ref, + :osp_security_groups_ids => [dst_security_group.ems_ref], + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task.network_mappings + ) + end + end + end + end +end diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index f379db74cfb..42913ff2711 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -12,9 +12,10 @@ end end - context 'populated request and task' do + context 'independent of provider' do let(:src) { FactoryGirl.create(:ems_cluster) } let(:dst) { FactoryGirl.create(:ems_cluster) } + let(:host) { FactoryGirl.create(:host, :ext_management_system => FactoryGirl.create(:ext_management_system, :zone => FactoryGirl.create(:zone))) } let(:vm) { FactoryGirl.create(:vm_or_template) } let(:vm2) { FactoryGirl.create(:vm_or_template) } let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } @@ -102,16 +103,16 @@ end describe '#transformation_log_queue' do - let(:host_id) { 22 } + let(:conversion_host_id) { 22 } before do - task.options[:transformation_host_id] = host_id + task.options[:transformation_host_id] = conversion_host_id task.save! end context 'when conversion host exists' do before do - FactoryGirl.create(:host, :id => host_id, :ext_management_system => FactoryGirl.create(:ext_management_system, :zone => FactoryGirl.create(:zone))) + FactoryGirl.create(:conversion_host, :id => conversion_host_id, :resource => host) allow(described_class).to receive(:find).and_return(task) @@ -146,7 +147,7 @@ it 'returns an error message' do taskid = task.transformation_log_queue('user') expect(MiqTask.find(taskid)).to have_attributes( - :message => "Conversion host was not found: ID [#{host_id}]. Cannot queue the download of transformation log.", + :message => "Conversion host was not found: ID [#{conversion_host_id}]. Cannot queue the download of transformation log.", :status => 'Error' ) end @@ -154,11 +155,11 @@ end describe '#transformation_log' do - let(:host) { FactoryGirl.create(:host, :id => 9) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :id => 9, :resource => host) } before do EvmSpecHelper.create_guid_miq_server_zone - task.options[:transformation_host_id] = host.id + task.options[:transformation_host_id] = conversion_host.id task.options.store_path(:virtv2v_wrapper, "v2v_log", "/path/to/log.file") task.save! @@ -205,4 +206,134 @@ end end end + + context 'populated request and task' do + let(:src_ems) { FactoryGirl.create(:ext_management_system, :zone => FactoryGirl.create(:zone)) } + let(:src_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => src_ems) } + let(:dst_ems) { FactoryGirl.create(:ext_management_system, :zone => FactoryGirl.create(:zone)) } + let(:dst_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => dst_ems) } + + let(:src_vm_1) { FactoryGirl.create(:vm_or_template, :ext_management_system => src_ems, :ems_cluster => src_cluster) } + let(:src_vm_2) { FactoryGirl.create(:vm_or_template, :ext_management_system => src_ems, :ems_cluster => src_cluster) } + let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } + + let(:mapping) do + FactoryGirl.create( + :transformation_mapping, + :transformation_mapping_items => [TransformationMappingItem.new(:source => src_cluster, :destination => dst_cluster)] + ) + end + + let(:catalog_item_options) do + { + :name => 'Transformation Plan', + :description => 'a description', + :config_info => { + :transformation_mapping_id => mapping.id, + :pre_service_id => apst.id, + :post_service_id => apst.id, + :actions => [ + {:vm_id => src_vm_1.id.to_s, :pre_service => true, :post_service => true}, + {:vm_id => src_vm_2.id.to_s, :pre_service => false, :post_service => false}, + ], + } + } + end + + let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } + let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } + let(:task_1) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm_1) } + let(:task_2) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm_2) } + + describe '#transformation_destination' do + it { expect(task_1.transformation_destination(src_cluster)).to eq(dst_cluster) } + end + + describe '#pre_ansible_playbook_service_template' do + it { expect(task_1.pre_ansible_playbook_service_template).to eq(apst) } + it { expect(task_2.pre_ansible_playbook_service_template).to be_nil } + end + + describe '#post_ansible_playbook_service_template' do + it { expect(task_1.post_ansible_playbook_service_template).to eq(apst) } + it { expect(task_2.post_ansible_playbook_service_template).to be_nil } + end + + context 'source is vmwarews' do + let(:src_ems) { FactoryGirl.create(:ems_vmware, :zone => FactoryGirl.create(:zone)) } + let(:src_host) { FactoryGirl.create(:host, :ext_management_system => src_ems) } + let(:src_storage) { FactoryGirl.create(:storage, :ext_management_system => src_ems) } + + let(:src_lan_1) { FactoryGirl.create(:lan) } + let(:src_lan_2) { FactoryGirl.create(:lan) } + let(:src_nic_1) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_1) } + let(:src_nic_2) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_2) } + + let(:src_disk_1) { instance_double("disk", :device_name => "Hard disk 1", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm.vmdk", :size => 17_179_869_184) } + let(:src_disk_2) { instance_double("disk", :device_name => "Hard disk 2", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm-2.vmdk", :size => 17_179_869_184) } + + let(:src_hardware) { FactoryGirl.create(:hardware, :nics => [src_nic_1, src_nic_2]) } + + let(:src_vm_1) { FactoryGirl.create(:vm_vmware, :ext_management_system => src_ems, :ems_cluster => src_cluster, :host => src_host, :hardware => src_hardware) } + let(:src_vm_2) { FactoryGirl.create(:vm_vmware, :ext_management_system => src_ems, :ems_cluster => src_cluster, :host => src_host) } + + let(:conversion_host) { FactoryGirl.create(:conversion_host) } + + # Disks have to be stubbed because there's no factory for Disk class + before do + allow(src_hardware).to receive(:disks).and_return([src_disk_1, src_disk_2]) + allow(src_disk_1).to receive(:storage).and_return(src_storage) + allow(src_disk_2).to receive(:storage).and_return(src_storage) + allow(src_vm_1).to receive(:allocated_disk_storage).and_return(34_359_738_368) + task_1.options[:transformation_host_id] = conversion_host.id + end + + it "has source_ems" do + expect(task_1.source_ems).to eq(src_ems) + expect(task_1.options[:source_ems_id]).to eq(src_ems.id) + end + + it 'has destination_ems' do + expect(task_1.destination_ems).to eq(dst_ems) + expect(task_1.options[:destination_ems_id]).to eq(dst_ems.id) + end + + context 'destination is rhevm' do + let(:dst_ems) { FactoryGirl.create(:ems_redhat, :zone => FactoryGirl.create(:zone)) } + let(:dst_storage) { FactoryGirl.create(:storage) } + let(:dst_lan_1) { FactoryGirl.create(:lan) } + let(:dst_lan_2) { FactoryGirl.create(:lan) } + + let(:mapping) do + FactoryGirl.create( + :transformation_mapping, + :transformation_mapping_items => [ + TransformationMappingItem.new(:source => src_cluster, :destination => dst_cluster), + TransformationMappingItem.new(:source => src_storage, :destination => dst_storage), + TransformationMappingItem.new(:source => src_lan_1, :destination => dst_lan_1), + TransformationMappingItem.new(:source => src_lan_2, :destination => dst_lan_2) + ] + ) + end + + it "has source_disks" do + expect(task_1.source_disks).to eq( + [ + { :path => src_disk_1.filename, :size => src_disk_1.size, :percent => 0, :weight => 50.0 }, + { :path => src_disk_2.filename, :size => src_disk_2.size, :percent => 0, :weight => 50.0 } + ] + ) + end + + it "has network_mappings" do + expect(task_1.network_mappings).to eq( + [ + { :source => src_lan_1.name, :destination => dst_lan_1.name, :mac_address => src_nic_1.address }, + { :source => src_lan_2.name, :destination => dst_lan_2.name, :mac_address => src_nic_2.address } + ] + ) + end + end + end + end end From 02b51cc39bdae3f506a77c074e21d36834abd26e Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 3 Oct 2018 10:25:03 -0400 Subject: [PATCH 02/13] Leverage task / conversion host relationship --- app/models/conversion_host.rb | 5 ++-- ...rvice_template_transformation_plan_task.rb | 10 +++---- spec/models/conversion_host_spec.rb | 27 +++++++++---------- ..._template_transformation_plan_task_spec.rb | 17 ++++-------- 4 files changed, 23 insertions(+), 36 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index b856c101174..1bf76a1e9c0 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -4,11 +4,10 @@ class ConversionHost < ApplicationRecord acts_as_miq_taggable belongs_to :resource, :polymorphic => true + has_many :service_template_transformation_plan_tasks, :dependent => :nullify def active_tasks - ServiceTemplateTransformationPlanTask.where(:state => 'active').select do |task| - task.conversion_host == self - end + service_template_transformation_plan_tasks { active } end def eligible? diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index 28a105c2ff4..d647531b274 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -1,4 +1,6 @@ class ServiceTemplateTransformationPlanTask < ServiceTemplateProvisionTask + belongs_to :conversion_host + def self.base_model ServiceTemplateTransformationPlanTask end @@ -46,10 +48,6 @@ def task_active vm_resource.update_attributes(:status => ServiceResource::STATUS_ACTIVE) end - def conversion_host - ConversionHost.find_by(:id => options[:transformation_host_id]) - end - def source_ems options[:source_ems_id] ||= source.ext_management_system.id source.ext_management_system @@ -110,7 +108,7 @@ def destination_security_group def transformation_log host = conversion_host if host.nil? - msg = "Conversion host was not found: ID [#{options[:transformation_host_id]}]. Download of transformation log aborted." + msg = "Conversion host was not found. Download of transformation log aborted." _log.error(msg) raise MiqException::Error, msg end @@ -144,7 +142,7 @@ def transformation_log_queue(userid = nil) userid ||= User.current_userid || 'system' host = conversion_host if host.nil? - msg = "Conversion host was not found: ID [#{options[:transformation_host_id]}]. Cannot queue the download of transformation log." + msg = "Conversion host was not found. Cannot queue the download of transformation log." return create_error_status_task(userid, msg).id end diff --git a/spec/models/conversion_host_spec.rb b/spec/models/conversion_host_spec.rb index 7c5a00a063e..d9d250fea90 100644 --- a/spec/models/conversion_host_spec.rb +++ b/spec/models/conversion_host_spec.rb @@ -1,23 +1,19 @@ describe ConversionHost do let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } - let(:conversion_host) { FactoryGirl.create(:conversion_host) } context "provider independent methods" do - let(:task_1) { FactoryGirl.create(:service_template_transformation_plan_task) } - let(:task_2) { FactoryGirl.create(:service_template_transformation_plan_task) } - let(:task_3) { FactoryGirl.create(:service_template_transformation_plan_task) } - - let(:conversion_host_1) { FactoryGirl.create(:conversion_host) } - let(:conversion_host_2) { FactoryGirl.create(:conversion_host) } + let(:host) { FactoryGirl.create(:host) } + let(:vm) { FactoryGirl.create(:vm_or_template) } + let(:conversion_host_1) { FactoryGirl.create(:conversion_host, :resource => host) } + let(:conversion_host_2) { FactoryGirl.create(:conversion_host, :resource => vm) } + let(:task_1) { FactoryGirl.create(:service_template_transformation_plan_task, :state => 'active', :conversion_host => conversion_host_1) } + let(:task_2) { FactoryGirl.create(:service_template_transformation_plan_task, :conversion_host => conversion_host_1) } + let(:task_3) { FactoryGirl.create(:service_template_transformation_plan_task, :state => 'active', :conversion_host => conversion_host_2) } before do conversion_host_1.concurrent_transformation_limit = "2" conversion_host_2.concurrent_transformation_limit = "1" - task_1.options[:transformation_host_id] = conversion_host_1.id - task_2.options[:transformation_host_id] = conversion_host_1.id - task_3.options[:transformation_host_id] = conversion_host_2.id - allow(ServiceTemplateTransformationPlanTask).to receive(:where).with(:state => 'active').and_return([task_1, task_3]) end @@ -85,6 +81,7 @@ let(:dst_storage) { FactoryGirl.create(:storage) } let(:dst_lan_1) { FactoryGirl.create(:lan) } let(:dst_lan_2) { FactoryGirl.create(:lan) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:host, :ext_management_system => dst_ems)) } let(:mapping) do FactoryGirl.create( @@ -115,11 +112,11 @@ let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } - let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm) } + let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm, :conversion_host => conversion_host) } before do - task.options[:transformation_host_id] = conversion_host.id allow(task).to receive(:source_disks).and_return(source_disks) + allow(conversion_host).to receive(:ipaddress).and_return('10.0.1.1') end context "transport method is vddk" do @@ -177,6 +174,7 @@ let(:dst_cloud_network_2) { FactoryGirl.create(:cloud_network) } let(:dst_flavor) { FactoryGirl.create(:flavor) } let(:dst_security_group) { FactoryGirl.create(:security_group) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:vm_or_template, :ext_management_system => dst_ems)) } let(:mapping) do FactoryGirl.create( @@ -209,10 +207,9 @@ let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } - let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm) } + let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm, :conversion_host => conversion_host) } before do - task.options[:transformation_host_id] = conversion_host.id allow(task).to receive(:source_disks).and_return(source_disks) end diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index 42913ff2711..13d9619b59e 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -19,6 +19,8 @@ let(:vm) { FactoryGirl.create(:vm_or_template) } let(:vm2) { FactoryGirl.create(:vm_or_template) } let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => host) } + let(:mapping) do FactoryGirl.create( :transformation_mapping, @@ -103,16 +105,9 @@ end describe '#transformation_log_queue' do - let(:conversion_host_id) { 22 } - - before do - task.options[:transformation_host_id] = conversion_host_id - task.save! - end - context 'when conversion host exists' do before do - FactoryGirl.create(:conversion_host, :id => conversion_host_id, :resource => host) + task.conversion_host = conversion_host allow(described_class).to receive(:find).and_return(task) @@ -147,7 +142,7 @@ it 'returns an error message' do taskid = task.transformation_log_queue('user') expect(MiqTask.find(taskid)).to have_attributes( - :message => "Conversion host was not found: ID [#{conversion_host_id}]. Cannot queue the download of transformation log.", + :message => "Conversion host was not found. Cannot queue the download of transformation log.", :status => 'Error' ) end @@ -155,11 +150,9 @@ end describe '#transformation_log' do - let(:conversion_host) { FactoryGirl.create(:conversion_host, :id => 9, :resource => host) } - before do EvmSpecHelper.create_guid_miq_server_zone - task.options[:transformation_host_id] = conversion_host.id + task.conversion_host = conversion_host task.options.store_path(:virtv2v_wrapper, "v2v_log", "/path/to/log.file") task.save! From 4e00a401e9cd82614a8154713303c06bf8eea826 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Wed, 3 Oct 2018 14:06:18 -0400 Subject: [PATCH 03/13] Move conversion_options logic to task rather than conversion_host. Add fingerprint for host --- app/models/conversion_host.rb | 85 ------ app/models/host.rb | 18 ++ ...rvice_template_transformation_plan_task.rb | 73 ++++++ spec/models/conversion_host_spec.rb | 241 ------------------ ..._template_transformation_plan_task_spec.rb | 182 ++++++++++++- 5 files changed, 268 insertions(+), 331 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index 1bf76a1e9c0..ee80c4c7dd3 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -19,89 +19,4 @@ def source_transport_method return 'vddk' if vddk_transport_supported return 'ssh' if ssh_transport_supported end - - def conversion_options(task) - source_vm = task.source - source_ems = source_vm.ext_management_system - source_cluster = source_vm.ems_cluster - source_storage = source_vm.hardware.disks.select { |d| d.device_type == 'disk' }.first.storage - - destination_cluster = task.transformation_destination(source_cluster) - destination_storage = task.transformation_destination(source_storage) - destination_ems = destination_cluster.ext_management_system - - source_provider_options = send( - "conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", - source_vm, - source_storage - ) - destination_provider_options = send( - "conversion_options_destination_provider_#{destination_ems.emstype}", - task, - destination_ems, - destination_cluster, - destination_storage - ) - options = { - :source_disks => task.source_disks.map { |disk| disk[:path] }, - :network_mappings => task.network_mappings - } - options.merge(source_provider_options).merge(destination_provider_options) - end - - def conversion_options_source_provider_vmwarews_vddk(vm, _storage) - { - :vm_name => vm.name, - :transport_method => 'vddk', - :vmware_fingerprint => vm.host.fingerprint, - :vmware_uri => URI::Generic.build( - :scheme => 'esx', - :userinfo => CGI.escape(vm.host.authentication_userid), - :host => vm.host.ipaddress, - :path => '/', - :query => { :no_verify => 1 }.to_query - ).to_s, - :vmware_password => vm.host.authentication_password - } - end - - def conversion_options_source_provider_vmwarews_ssh(vm, storage) - { - :vm_name => URI::Generic.build(:scheme => 'ssh', :userinfo => 'root', :host => vm.host.ipaddress, :path => "/vmfs/volumes").to_s + "/#{storage.name}/#{vm.location}", - :transport_method => 'ssh' - } - end - - def conversion_options_destination_provider_rhevm(_task, ems, cluster, storage) - { - :rhv_url => URI::Generic.build(:scheme => 'https', :host => ems.hostname, :path => '/ovirt-engine/api').to_s, - :rhv_cluster => cluster.name, - :rhv_storage => storage.name, - :rhv_password => ems.authentication_password, - :install_drivers => true, - :insecure_connection => true - } - end - - def conversion_options_destination_provider_openstack(task, ems, cluster, storage) - { - :osp_environment => { - :os_no_cache => true, - :os_auth_url => URI::Generic.build( - :scheme => ems.security_protocol == 'non-ssl' ? 'http' : 'https', - :host => ems.hostname, - :port => ems.port, - :path => ems.api_version - ), - :os_user_domain_name => ems.uid_ems, - :os_username => ems.authentication_userid, - :os_password => ems.authentication_password, - :os_project_name => cluster.name - }, - :osp_destination_project_id => cluster.ems_ref, - :osp_volume_type_id => storage.ems_ref, - :osp_flavor_id => task.destination_flavor.ems_ref, - :osp_security_groups_ids => [task.destination_security_group.ems_ref] - } - end end diff --git a/app/models/host.rb b/app/models/host.rb index 1b4d3a50b20..07f4176798c 100644 --- a/app/models/host.rb +++ b/app/models/host.rb @@ -500,6 +500,24 @@ def mac_addresses hardware.nil? ? [] : hardware.mac_addresses end + def fingerprint + require 'socket' + require 'openssl' + + tcp_client = TCPSocket.new(host.ipaddress, 443) + ssl_context = OpenSSL::SSL::SSLContext('SSLv23_client') + ssl_content.verify_mode = OpenSSL::SSL::VERIFY_NONE + ssl_client = OpenSSL::SSL::SSLSocker.new(tcp_client, ssl_context) + cert = OpenSSL::X509::Certificate.new(ssl_client.peer_cert) + + Digest::SHA1.hexdigest(cert.to_der).upcase.scan(/../).join(":") + rescue => err + _log.log_backtrace(err) + ensure + ssl_client.sysclose + tcp_client.close + end + def has_config_data? !operating_system.nil? && !hardware.nil? end diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index d647531b274..fb9c2df7c2a 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -1,5 +1,6 @@ class ServiceTemplateTransformationPlanTask < ServiceTemplateProvisionTask belongs_to :conversion_host + delegate :source_transport_method, :to => :conversion_host def self.base_model ServiceTemplateTransformationPlanTask @@ -169,6 +170,22 @@ def canceled update_attributes(:cancelation_status => MiqRequestTask::CANCEL_STATUS_FINISHED) end + def conversion_options + source_cluster = source.ems_cluster + source_storage = source.hardware.disks.select { |d| d.device_type == 'disk' }.first.storage + destination_cluster = transformation_destination(source_cluster) + destination_storage = transformation_destination(source_storage) + + [ + { + :source_disks => source_disks.map { |disk| disk[:path] }, + :network_mappings => network_mappings + }, + send("conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", source_storage), + send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage ) + ].inject(&:merge) + end + private def vm_resource @@ -184,4 +201,60 @@ def create_error_status_task(userid, msg) :message => msg ) end + + def conversion_options_source_provider_vmwarews_vddk(_storage) + { + :vm_name => source.name, + :transport_method => 'vddk', + :vmware_fingerprint => source.host.fingerprint, + :vmware_uri => URI::Generic.build( + :scheme => 'esx', + :userinfo => CGI.escape(source.host.authentication_userid), + :host => source.host.ipaddress, + :path => '/', + :query => { :no_verify => 1 }.to_query + ).to_s, + :vmware_password => source.host.authentication_password + } + end + + def conversion_options_source_provider_vmwarews_ssh(storage) + { + :vm_name => URI::Generic.build(:scheme => 'ssh', :userinfo => 'root', :host => source.host.ipaddress, :path => "/vmfs/volumes").to_s + "/#{storage.name}/#{source.location}", + :transport_method => 'ssh' + } + end + + def conversion_options_destination_provider_rhevm(cluster, storage) + { + :rhv_url => URI::Generic.build(:scheme => 'https', :host => destination_ems.hostname, :path => '/ovirt-engine/api').to_s, + :rhv_cluster => cluster.name, + :rhv_storage => storage.name, + :rhv_password => destination_ems.authentication_password, + :install_drivers => true, + :insecure_connection => true + } + end + + def conversion_options_destination_provider_openstack(cluster, storage) + { + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => destination_ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => destination_ems.hostname, + :port => destination_ems.port, + :path => destination_ems.api_version + ), + :os_user_domain_name => destination_ems.uid_ems, + :os_username => destination_ems.authentication_userid, + :os_password => destination_ems.authentication_password, + :os_project_name => cluster.name + }, + :osp_destination_project_id => cluster.ems_ref, + :osp_volume_type_id => storage.ems_ref, + :osp_flavor_id => destination_flavor.ems_ref, + :osp_security_groups_ids => [destination_security_group.ems_ref] + } + end end diff --git a/spec/models/conversion_host_spec.rb b/spec/models/conversion_host_spec.rb index d9d250fea90..1b1a4c744e6 100644 --- a/spec/models/conversion_host_spec.rb +++ b/spec/models/conversion_host_spec.rb @@ -41,245 +41,4 @@ end end end - - context "source is vmwarews" do - let(:src_ems) { FactoryGirl.create(:ems_vmware, :zone => FactoryGirl.create(:zone)) } - let(:src_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => src_ems) } - let(:src_host) { FactoryGirl.create(:host, :ext_management_system => src_ems, :ipaddress => '10.0.0.1') } - let(:src_storage) { FactoryGirl.create(:storage, :ext_management_system => src_ems) } - - let(:src_lan_1) { FactoryGirl.create(:lan) } - let(:src_lan_2) { FactoryGirl.create(:lan) } - let(:src_nic_1) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_1) } - let(:src_nic_2) { FactoryGirl.create(:guest_device_nic, :lan => src_lan_2) } - - let(:src_disk_1) { instance_double("disk", :device_name => "Hard disk 1", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm.vmdk", :size => 17_179_869_184) } - let(:src_disk_2) { instance_double("disk", :device_name => "Hard disk 2", :device_type => "disk", :filename => "[datastore12] test_vm/test_vm-2.vmdk", :size => 17_179_869_184) } - - let(:src_hardware) { FactoryGirl.create(:hardware, :nics => [src_nic_1, src_nic_2]) } - - let(:src_vm) { FactoryGirl.create(:vm_vmware, :ext_management_system => src_ems, :ems_cluster => src_cluster, :host => src_host, :hardware => src_hardware) } - - let(:source_disks) do - [ - {:path => src_disk_1.filename, :size => src_disk_1.size, :percent => 0, :weight => 50.0 }, - {:path => src_disk_2.filename, :size => src_disk_2.size, :percent => 0, :weight => 50.0 } - ] - end - - before do - allow(src_hardware).to receive(:disks).and_return([src_disk_1, src_disk_2]) - allow(src_disk_1).to receive(:storage).and_return(src_storage) - allow(src_host).to receive(:fingerprint).and_return('01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67') - allow(src_host).to receive(:authentication_userid).and_return('esx_user') - allow(src_host).to receive(:authentication_password).and_return('esx_passwd') - end - - context "destination is rhevm" do - let(:dst_ems) { FactoryGirl.create(:ems_redhat, :zone => FactoryGirl.create(:zone)) } - let(:dst_cluster) { FactoryGirl.create(:ems_cluster, :ext_management_system => dst_ems) } - let(:dst_storage) { FactoryGirl.create(:storage) } - let(:dst_lan_1) { FactoryGirl.create(:lan) } - let(:dst_lan_2) { FactoryGirl.create(:lan) } - let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:host, :ext_management_system => dst_ems)) } - - let(:mapping) do - FactoryGirl.create( - :transformation_mapping, - :transformation_mapping_items => [ - TransformationMappingItem.new(:source => src_cluster, :destination => dst_cluster), - TransformationMappingItem.new(:source => src_storage, :destination => dst_storage), - TransformationMappingItem.new(:source => src_lan_1, :destination => dst_lan_1), - TransformationMappingItem.new(:source => src_lan_2, :destination => dst_lan_2) - ] - ) - end - - let(:catalog_item_options) do - { - :name => 'Transformation Plan', - :description => 'a description', - :config_info => { - :transformation_mapping_id => mapping.id, - :pre_service_id => apst.id, - :post_service_id => apst.id, - :actions => [ - {:vm_id => src_vm.id.to_s, :pre_service => true, :post_service => true} - ], - } - } - end - - let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } - let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } - let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm, :conversion_host => conversion_host) } - - before do - allow(task).to receive(:source_disks).and_return(source_disks) - allow(conversion_host).to receive(:ipaddress).and_return('10.0.1.1') - end - - context "transport method is vddk" do - before do - conversion_host.vddk_transport_supported = true - end - - it "#conversion_options" do - expect(conversion_host.conversion_options(task)).to eq( - :vm_name => src_vm.name, - :transport_method => 'vddk', - :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', - :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", - :vmware_password => 'esx_passwd', - :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", - :rhv_cluster => dst_cluster.name, - :rhv_storage => dst_storage.name, - :rhv_password => dst_ems.authentication_password, - :source_disks => [src_disk_1.filename, src_disk_2.filename], - :network_mappings => task.network_mappings, - :install_drivers => true, - :insecure_connection => true - ) - end - end - - context "transport method is ssh" do - before do - conversion_host.vddk_transport_supported = false - conversion_host.ssh_transport_supported = true - end - - it "#conversion_options" do - expect(conversion_host.conversion_options(task)).to eq( - :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm.location}", - :transport_method => 'ssh', - :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", - :rhv_cluster => dst_cluster.name, - :rhv_storage => dst_storage.name, - :rhv_password => dst_ems.authentication_password, - :source_disks => [src_disk_1.filename, src_disk_2.filename], - :network_mappings => task.network_mappings, - :install_drivers => true, - :insecure_connection => true - ) - end - end - end - - context "destination is openstack" do - let(:dst_ems) { FactoryGirl.create(:ems_openstack, :zone => FactoryGirl.create(:zone)) } - let(:dst_cloud_tenant) { FactoryGirl.create(:cloud_tenant, :ext_management_system => dst_ems) } - let(:dst_cloud_volume_type) { FactoryGirl.create(:cloud_volume_type) } - let(:dst_cloud_network_1) { FactoryGirl.create(:cloud_network) } - let(:dst_cloud_network_2) { FactoryGirl.create(:cloud_network) } - let(:dst_flavor) { FactoryGirl.create(:flavor) } - let(:dst_security_group) { FactoryGirl.create(:security_group) } - let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:vm_or_template, :ext_management_system => dst_ems)) } - - let(:mapping) do - FactoryGirl.create( - :transformation_mapping, - :transformation_mapping_items => [ - TransformationMappingItem.new(:source => src_cluster, :destination => dst_cloud_tenant), - TransformationMappingItem.new(:source => src_storage, :destination => dst_cloud_volume_type), - TransformationMappingItem.new(:source => src_lan_1, :destination => dst_cloud_network_1), - TransformationMappingItem.new(:source => src_lan_2, :destination => dst_cloud_network_2) - ] - ) - end - - let(:catalog_item_options) do - { - :name => 'Transformation Plan', - :description => 'a description', - :config_info => { - :transformation_mapping_id => mapping.id, - :pre_service_id => apst.id, - :post_service_id => apst.id, - :osp_flavor => dst_flavor.id, - :osp_security_group => dst_security_group.id, - :actions => [ - {:vm_id => src_vm.id.to_s, :pre_service => true, :post_service => true} - ], - } - } - end - - let(:plan) { ServiceTemplateTransformationPlan.create_catalog_item(catalog_item_options) } - let(:request) { FactoryGirl.create(:service_template_transformation_plan_request, :source => plan) } - let(:task) { FactoryGirl.create(:service_template_transformation_plan_task, :miq_request => request, :request_type => 'transformation_plan', :source => src_vm, :conversion_host => conversion_host) } - - before do - allow(task).to receive(:source_disks).and_return(source_disks) - end - - context "transport method is vddk" do - before do - conversion_host.vddk_transport_supported = true - end - - it "#conversion_options" do - expect(conversion_host.conversion_options(task)).to eq( - :vm_name => src_vm.name, - :transport_method => 'vddk', - :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', - :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", - :vmware_password => 'esx_passwd', - :osp_environment => { - :os_no_cache => true, - :os_auth_url => URI::Generic.build( - :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', - :host => dst_ems.hostname, - :port => dst_ems.port, - :path => dst_ems.api_version - ), - :os_user_domain_name => dst_ems.uid_ems, - :os_username => dst_ems.authentication_userid, - :os_password => dst_ems.authentication_password, - :os_project_name => dst_cloud_tenant.name - }, - :osp_destination_project_id => dst_cloud_tenant.ems_ref, - :osp_volume_type_id => dst_cloud_volume_type.ems_ref, - :osp_flavor_id => dst_flavor.ems_ref, - :osp_security_groups_ids => [dst_security_group.ems_ref], - :source_disks => [src_disk_1.filename, src_disk_2.filename], - :network_mappings => task.network_mappings - ) - end - end - - context "transport method is ssh" do - before do - conversion_host.vddk_transport_supported = false - conversion_host.ssh_transport_supported = true - end - - it "#conversion_options" do - expect(conversion_host.conversion_options(task)).to eq( - :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm.location}", - :transport_method => 'ssh', - :osp_environment => { - :os_no_cache => true, - :os_auth_url => URI::Generic.build( - :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', - :host => dst_ems.hostname, - :port => dst_ems.port, - :path => dst_ems.api_version - ), - :os_user_domain_name => dst_ems.uid_ems, - :os_username => dst_ems.authentication_userid, - :os_password => dst_ems.authentication_password, - :os_project_name => dst_cloud_tenant.name - }, - :osp_destination_project_id => dst_cloud_tenant.ems_ref, - :osp_volume_type_id => dst_cloud_volume_type.ems_ref, - :osp_flavor_id => dst_flavor.ems_ref, - :osp_security_groups_ids => [dst_security_group.ems_ref], - :source_disks => [src_disk_1.filename, src_disk_2.filename], - :network_mappings => task.network_mappings - ) - end - end - end - end end diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index 13d9619b59e..4c6f1665241 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -210,6 +210,9 @@ let(:src_vm_2) { FactoryGirl.create(:vm_or_template, :ext_management_system => src_ems, :ems_cluster => src_cluster) } let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) } + let(:dst_flavor) { FactoryGirl.create(:flavor) } + let(:dst_security_group) { FactoryGirl.create(:security_group) } + let(:mapping) do FactoryGirl.create( :transformation_mapping, @@ -225,6 +228,8 @@ :transformation_mapping_id => mapping.id, :pre_service_id => apst.id, :post_service_id => apst.id, + :osp_flavor => dst_flavor.id, + :osp_security_group => dst_security_group.id, :actions => [ {:vm_id => src_vm_1.id.to_s, :pre_service => true, :post_service => true}, {:vm_id => src_vm_2.id.to_s, :pre_service => false, :post_service => false}, @@ -254,7 +259,7 @@ context 'source is vmwarews' do let(:src_ems) { FactoryGirl.create(:ems_vmware, :zone => FactoryGirl.create(:zone)) } - let(:src_host) { FactoryGirl.create(:host, :ext_management_system => src_ems) } + let(:src_host) { FactoryGirl.create(:host, :ext_management_system => src_ems, :ipaddress => '10.0.0.1') } let(:src_storage) { FactoryGirl.create(:storage, :ext_management_system => src_ems) } let(:src_lan_1) { FactoryGirl.create(:lan) } @@ -278,15 +283,18 @@ allow(src_disk_1).to receive(:storage).and_return(src_storage) allow(src_disk_2).to receive(:storage).and_return(src_storage) allow(src_vm_1).to receive(:allocated_disk_storage).and_return(34_359_738_368) + allow(src_host).to receive(:fingerprint).and_return('01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67') + allow(src_host).to receive(:authentication_userid).and_return('esx_user') + allow(src_host).to receive(:authentication_password).and_return('esx_passwd') task_1.options[:transformation_host_id] = conversion_host.id end - it "has source_ems" do + it "finds the source ems based on source vm" do expect(task_1.source_ems).to eq(src_ems) expect(task_1.options[:source_ems_id]).to eq(src_ems.id) end - it 'has destination_ems' do + it 'find the destination ems based on mapping' do expect(task_1.destination_ems).to eq(dst_ems) expect(task_1.options[:destination_ems_id]).to eq(dst_ems.id) end @@ -296,6 +304,7 @@ let(:dst_storage) { FactoryGirl.create(:storage) } let(:dst_lan_1) { FactoryGirl.create(:lan) } let(:dst_lan_2) { FactoryGirl.create(:lan) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:host, :ext_management_system => dst_ems)) } let(:mapping) do FactoryGirl.create( @@ -309,7 +318,11 @@ ) end - it "has source_disks" do + before do + task_1.conversion_host = conversion_host + end + + it "checks mapping and generates source_disks hash" do expect(task_1.source_disks).to eq( [ { :path => src_disk_1.filename, :size => src_disk_1.size, :percent => 0, :weight => 50.0 }, @@ -318,7 +331,7 @@ ) end - it "has network_mappings" do + it "checks network mappings and generates network_mappings hash" do expect(task_1.network_mappings).to eq( [ { :source => src_lan_1.name, :destination => dst_lan_1.name, :mac_address => src_nic_1.address }, @@ -326,6 +339,165 @@ ] ) end + + context "transport method is vddk" do + before do + conversion_host.vddk_transport_supported = true + end + + it "generates conversion options hash" do + expect(task_1.conversion_options).to eq( + :vm_name => src_vm_1.name, + :transport_method => 'vddk', + :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', + :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", + :vmware_password => 'esx_passwd', + :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", + :rhv_cluster => dst_cluster.name, + :rhv_storage => dst_storage.name, + :rhv_password => dst_ems.authentication_password, + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task_1.network_mappings, + :install_drivers => true, + :insecure_connection => true + ) + end + end + + context "transport method is ssh" do + before do + conversion_host.vddk_transport_supported = false + conversion_host.ssh_transport_supported = true + end + + it "generates conversion options hash" do + expect(task_1.conversion_options).to eq( + :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm_1.location}", + :transport_method => 'ssh', + :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", + :rhv_cluster => dst_cluster.name, + :rhv_storage => dst_storage.name, + :rhv_password => dst_ems.authentication_password, + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task_1.network_mappings, + :install_drivers => true, + :insecure_connection => true + ) + end + end + end + + context 'destination is openstack' do + let(:dst_ems) { FactoryGirl.create(:ems_openstack, :zone => FactoryGirl.create(:zone)) } + let(:dst_cloud_tenant) { FactoryGirl.create(:cloud_tenant, :ext_management_system => dst_ems) } + let(:dst_cloud_volume_type) { FactoryGirl.create(:cloud_volume_type) } + let(:dst_cloud_network_1) { FactoryGirl.create(:cloud_network) } + let(:dst_cloud_network_2) { FactoryGirl.create(:cloud_network) } + let(:dst_flavor) { FactoryGirl.create(:flavor) } + let(:dst_security_group) { FactoryGirl.create(:security_group) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => FactoryGirl.create(:vm, :ext_management_system => dst_ems)) } + + let(:mapping) do + FactoryGirl.create( + :transformation_mapping, + :transformation_mapping_items => [ + TransformationMappingItem.new(:source => src_cluster, :destination => dst_cloud_tenant), + TransformationMappingItem.new(:source => src_storage, :destination => dst_cloud_volume_type), + TransformationMappingItem.new(:source => src_lan_1, :destination => dst_cloud_network_1), + TransformationMappingItem.new(:source => src_lan_2, :destination => dst_cloud_network_2) + ] + ) + end + + before do + task_1.conversion_host = conversion_host + end + + it "checks mapping and generates source_disks hash" do + expect(task_1.source_disks).to eq( + [ + { :path => src_disk_1.filename, :size => src_disk_1.size, :percent => 0, :weight => 50.0 }, + { :path => src_disk_2.filename, :size => src_disk_2.size, :percent => 0, :weight => 50.0 } + ] + ) + end + + it "checks network mappings and generates network_mappings hash" do + expect(task_1.network_mappings).to eq( + [ + { :source => src_lan_1.name, :destination => dst_cloud_network_1.ems_ref, :mac_address => src_nic_1.address }, + { :source => src_lan_2.name, :destination => dst_cloud_network_2.ems_ref, :mac_address => src_nic_2.address } + ] + ) + end + + context "transport method is vddk" do + before do + conversion_host.vddk_transport_supported = true + end + + it "generates conversion options hash" do + expect(task_1.conversion_options).to eq( + :vm_name => src_vm_1.name, + :transport_method => 'vddk', + :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', + :vmware_uri => "esx://esx_user@10.0.0.1/?no_verify=1", + :vmware_password => 'esx_passwd', + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => dst_ems.hostname, + :port => dst_ems.port, + :path => dst_ems.api_version + ), + :os_user_domain_name => dst_ems.uid_ems, + :os_username => dst_ems.authentication_userid, + :os_password => dst_ems.authentication_password, + :os_project_name => dst_cloud_tenant.name + }, + :osp_destination_project_id => dst_cloud_tenant.ems_ref, + :osp_volume_type_id => dst_cloud_volume_type.ems_ref, + :osp_flavor_id => dst_flavor.ems_ref, + :osp_security_groups_ids => [dst_security_group.ems_ref], + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task_1.network_mappings + ) + end + end + + context "transport method is ssh" do + before do + conversion_host.vddk_transport_supported = false + conversion_host.ssh_transport_supported = true + end + + it "generates conversion options hash" do + expect(task_1.conversion_options).to eq( + :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm_1.location}", + :transport_method => 'ssh', + :osp_environment => { + :os_no_cache => true, + :os_auth_url => URI::Generic.build( + :scheme => dst_ems.security_protocol == 'non-ssl' ? 'http' : 'https', + :host => dst_ems.hostname, + :port => dst_ems.port, + :path => dst_ems.api_version + ), + :os_user_domain_name => dst_ems.uid_ems, + :os_username => dst_ems.authentication_userid, + :os_password => dst_ems.authentication_password, + :os_project_name => dst_cloud_tenant.name + }, + :osp_destination_project_id => dst_cloud_tenant.ems_ref, + :osp_volume_type_id => dst_cloud_volume_type.ems_ref, + :osp_flavor_id => dst_flavor.ems_ref, + :osp_security_groups_ids => [dst_security_group.ems_ref], + :source_disks => [src_disk_1.filename, src_disk_2.filename], + :network_mappings => task_1.network_mappings + ) + end + end end end end From 9adf7831e195dfc64e9de36779520ad87bf57a65 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 4 Oct 2018 07:54:41 -0400 Subject: [PATCH 04/13] Use vmware_web_service library to get ESX thumbprint --- app/models/host.rb | 19 +++---------------- ...rvice_template_transformation_plan_task.rb | 2 +- ..._template_transformation_plan_task_spec.rb | 2 +- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/app/models/host.rb b/app/models/host.rb index 07f4176798c..e833b7a0864 100644 --- a/app/models/host.rb +++ b/app/models/host.rb @@ -5,6 +5,7 @@ require 'metadata/linux/LinuxUsers' require 'metadata/linux/LinuxUtils' require 'metadata/ScanProfile/HostScanProfiles' +require 'VMwareWebService/esx_thumb_print' class Host < ApplicationRecord include SupportsFeatureMixin @@ -500,22 +501,8 @@ def mac_addresses hardware.nil? ? [] : hardware.mac_addresses end - def fingerprint - require 'socket' - require 'openssl' - - tcp_client = TCPSocket.new(host.ipaddress, 443) - ssl_context = OpenSSL::SSL::SSLContext('SSLv23_client') - ssl_content.verify_mode = OpenSSL::SSL::VERIFY_NONE - ssl_client = OpenSSL::SSL::SSLSocker.new(tcp_client, ssl_context) - cert = OpenSSL::X509::Certificate.new(ssl_client.peer_cert) - - Digest::SHA1.hexdigest(cert.to_der).upcase.scan(/../).join(":") - rescue => err - _log.log_backtrace(err) - ensure - ssl_client.sysclose - tcp_client.close + def thumbprint_sha1 + ESXThumbPrint.new(ipaddress, authentication_userid, authentication_password).to_sha1 end def has_config_data? diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index fb9c2df7c2a..8e463db8b21 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -206,7 +206,7 @@ def conversion_options_source_provider_vmwarews_vddk(_storage) { :vm_name => source.name, :transport_method => 'vddk', - :vmware_fingerprint => source.host.fingerprint, + :vmware_fingerprint => source.host.thumbprint_sha1, :vmware_uri => URI::Generic.build( :scheme => 'esx', :userinfo => CGI.escape(source.host.authentication_userid), diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index 4c6f1665241..2f6f5b3805a 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -283,7 +283,7 @@ allow(src_disk_1).to receive(:storage).and_return(src_storage) allow(src_disk_2).to receive(:storage).and_return(src_storage) allow(src_vm_1).to receive(:allocated_disk_storage).and_return(34_359_738_368) - allow(src_host).to receive(:fingerprint).and_return('01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67') + allow(src_host).to receive(:thumbprint_sha1).and_return('01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67') allow(src_host).to receive(:authentication_userid).and_return('esx_user') allow(src_host).to receive(:authentication_password).and_return('esx_passwd') task_1.options[:transformation_host_id] = conversion_host.id From b2e4f75d5843b1a9a6912688d3908d307ef63be1 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 4 Oct 2018 08:07:37 -0400 Subject: [PATCH 05/13] Fix rubocop --- ...rvice_template_transformation_plan_task.rb | 32 +++++++++---------- ..._template_transformation_plan_task_spec.rb | 8 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index 8e463db8b21..98d804547a2 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -177,14 +177,14 @@ def conversion_options destination_storage = transformation_destination(source_storage) [ - { + { :source_disks => source_disks.map { |disk| disk[:path] }, :network_mappings => network_mappings }, send("conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", source_storage), - send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage ) + send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage) ].inject(&:merge) - end + end private @@ -203,7 +203,7 @@ def create_error_status_task(userid, msg) end def conversion_options_source_provider_vmwarews_vddk(_storage) - { + { :vm_name => source.name, :transport_method => 'vddk', :vmware_fingerprint => source.host.thumbprint_sha1, @@ -215,29 +215,29 @@ def conversion_options_source_provider_vmwarews_vddk(_storage) :query => { :no_verify => 1 }.to_query ).to_s, :vmware_password => source.host.authentication_password - } - end + } + end def conversion_options_source_provider_vmwarews_ssh(storage) - { + { :vm_name => URI::Generic.build(:scheme => 'ssh', :userinfo => 'root', :host => source.host.ipaddress, :path => "/vmfs/volumes").to_s + "/#{storage.name}/#{source.location}", :transport_method => 'ssh' - } - end + } + end def conversion_options_destination_provider_rhevm(cluster, storage) - { + { :rhv_url => URI::Generic.build(:scheme => 'https', :host => destination_ems.hostname, :path => '/ovirt-engine/api').to_s, :rhv_cluster => cluster.name, :rhv_storage => storage.name, :rhv_password => destination_ems.authentication_password, :install_drivers => true, :insecure_connection => true - } - end + } + end def conversion_options_destination_provider_openstack(cluster, storage) - { + { :osp_environment => { :os_no_cache => true, :os_auth_url => URI::Generic.build( @@ -250,11 +250,11 @@ def conversion_options_destination_provider_openstack(cluster, storage) :os_username => destination_ems.authentication_userid, :os_password => destination_ems.authentication_password, :os_project_name => cluster.name - }, + }, :osp_destination_project_id => cluster.ems_ref, :osp_volume_type_id => storage.ems_ref, :osp_flavor_id => destination_flavor.ems_ref, :osp_security_groups_ids => [destination_security_group.ems_ref] - } - end + } + end end diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index 2f6f5b3805a..275cd30c22f 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -346,7 +346,7 @@ end it "generates conversion options hash" do - expect(task_1.conversion_options).to eq( + expect(task_1.conversion_options).to eq( :vm_name => src_vm_1.name, :transport_method => 'vddk', :vmware_fingerprint => '01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67', @@ -362,7 +362,7 @@ :insecure_connection => true ) end - end + end context "transport method is ssh" do before do @@ -371,7 +371,7 @@ end it "generates conversion options hash" do - expect(task_1.conversion_options).to eq( + expect(task_1.conversion_options).to eq( :vm_name => "ssh://root@10.0.0.1/vmfs/volumes/#{src_storage.name}/#{src_vm_1.location}", :transport_method => 'ssh', :rhv_url => "https://#{dst_ems.hostname}/ovirt-engine/api", @@ -384,7 +384,7 @@ :insecure_connection => true ) end - end + end end context 'destination is openstack' do From 4cc9a2a6e26a7d8d0c5b2b515778aa3667191bec Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 4 Oct 2018 11:10:33 -0400 Subject: [PATCH 06/13] Move thumprint to vmware provider. Simplify options hash build. Implemented failing scope --- app/models/conversion_host.rb | 5 +---- app/models/host.rb | 5 ----- app/models/miq_request_task.rb | 2 ++ ...service_template_transformation_plan_task.rb | 17 +++++++++-------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index ee80c4c7dd3..5d274b1ffa9 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -5,10 +5,7 @@ class ConversionHost < ApplicationRecord belongs_to :resource, :polymorphic => true has_many :service_template_transformation_plan_tasks, :dependent => :nullify - - def active_tasks - service_template_transformation_plan_tasks { active } - end + has_many :active_tasks, -> { active }, :class_name => ServiceTemplateTransformationPlanTask def eligible? return true if concurrent_transformation_limit.nil? diff --git a/app/models/host.rb b/app/models/host.rb index e833b7a0864..1b4d3a50b20 100644 --- a/app/models/host.rb +++ b/app/models/host.rb @@ -5,7 +5,6 @@ require 'metadata/linux/LinuxUsers' require 'metadata/linux/LinuxUtils' require 'metadata/ScanProfile/HostScanProfiles' -require 'VMwareWebService/esx_thumb_print' class Host < ApplicationRecord include SupportsFeatureMixin @@ -501,10 +500,6 @@ def mac_addresses hardware.nil? ? [] : hardware.mac_addresses end - def thumbprint_sha1 - ESXThumbPrint.new(ipaddress, authentication_userid, authentication_password).to_sha1 - end - def has_config_data? !operating_system.nil? && !hardware.nil? end diff --git a/app/models/miq_request_task.rb b/app/models/miq_request_task.rb index 9270c775e32..8a16044b333 100644 --- a/app/models/miq_request_task.rb +++ b/app/models/miq_request_task.rb @@ -22,6 +22,8 @@ class MiqRequestTask < ApplicationRecord validates_inclusion_of :status, :in => %w( Ok Warn Error Timeout ) + scope :active, -> { where(:state => "active") } + include MiqRequestMixin include TenancyMixin diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index 98d804547a2..d50ce173de0 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -176,14 +176,15 @@ def conversion_options destination_cluster = transformation_destination(source_cluster) destination_storage = transformation_destination(source_storage) - [ - { - :source_disks => source_disks.map { |disk| disk[:path] }, - :network_mappings => network_mappings - }, - send("conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", source_storage), - send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage) - ].inject(&:merge) + options = { + :source_disks => source_disks.map { |disk| disk[:path] }, + :network_mappings => network_mappings + } + + options.merge!(send("conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", source_storage)) + options.merge!(send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage)) + + options end private From fba16e2ad329b9976e6f3eb20d1438bdd6df1b5a Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 4 Oct 2018 12:15:31 -0400 Subject: [PATCH 07/13] Fix active_tasks association --- app/models/conversion_host.rb | 3 ++- app/models/miq_request_task.rb | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index 5d274b1ffa9..52902c7005d 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -5,10 +5,11 @@ class ConversionHost < ApplicationRecord belongs_to :resource, :polymorphic => true has_many :service_template_transformation_plan_tasks, :dependent => :nullify - has_many :active_tasks, -> { active }, :class_name => ServiceTemplateTransformationPlanTask + has_many :active_tasks, -> { where(:state => 'active') }, :class_name => ServiceTemplateTransformationPlanTask def eligible? return true if concurrent_transformation_limit.nil? + puts "Active tasks: #{active_tasks}" active_tasks.size < concurrent_transformation_limit.to_i end diff --git a/app/models/miq_request_task.rb b/app/models/miq_request_task.rb index 8a16044b333..9270c775e32 100644 --- a/app/models/miq_request_task.rb +++ b/app/models/miq_request_task.rb @@ -22,8 +22,6 @@ class MiqRequestTask < ApplicationRecord validates_inclusion_of :status, :in => %w( Ok Warn Error Timeout ) - scope :active, -> { where(:state => "active") } - include MiqRequestMixin include TenancyMixin From a94f7c5c45b19fe8b542df269576faf07c964ef1 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Thu, 4 Oct 2018 13:25:25 -0400 Subject: [PATCH 08/13] Remove dirty getter/setter. Handle default limit for max concurrent tasks. --- app/models/conversion_host.rb | 5 ++--- app/models/service_template_transformation_plan_task.rb | 2 -- config/settings.yml | 4 +++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index 52902c7005d..aac0baf0635 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -8,9 +8,8 @@ class ConversionHost < ApplicationRecord has_many :active_tasks, -> { where(:state => 'active') }, :class_name => ServiceTemplateTransformationPlanTask def eligible? - return true if concurrent_transformation_limit.nil? - puts "Active tasks: #{active_tasks}" - active_tasks.size < concurrent_transformation_limit.to_i + max_tasks = max_concurrent_tasks || Settings.transformation.limits.max_concurrent_tasks_per_host + active_tasks.size < max_tasks end def source_transport_method diff --git a/app/models/service_template_transformation_plan_task.rb b/app/models/service_template_transformation_plan_task.rb index d50ce173de0..ade30a8bb5a 100644 --- a/app/models/service_template_transformation_plan_task.rb +++ b/app/models/service_template_transformation_plan_task.rb @@ -50,12 +50,10 @@ def task_active end def source_ems - options[:source_ems_id] ||= source.ext_management_system.id source.ext_management_system end def destination_ems - options[:destination_ems_id] ||= transformation_destination(source.ems_cluster).ext_management_system.id transformation_destination(source.ems_cluster).ext_management_system end diff --git a/config/settings.yml b/config/settings.yml index b93bc505ff8..ccec2474aee 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1098,7 +1098,9 @@ :history: :keep_tasks: 1.week :purge_window_size: 1000 - +:transformation: + :limits: + :max_concurrent_tasks_per_host: 10 :ui: :mark_translated_strings: false :url: From 4e65f07672dc9007579b3ef43467a060f1ed3762 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 5 Oct 2018 06:58:05 -0400 Subject: [PATCH 09/13] Add criteria to eligibility --- app/models/conversion_host.rb | 28 +++++++++- spec/models/conversion_host_spec.rb | 79 +++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index aac0baf0635..de46502364b 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -5,15 +5,41 @@ class ConversionHost < ApplicationRecord belongs_to :resource, :polymorphic => true has_many :service_template_transformation_plan_tasks, :dependent => :nullify - has_many :active_tasks, -> { where(:state => 'active') }, :class_name => ServiceTemplateTransformationPlanTask + has_many :active_tasks, -> { where(:state => 'active') }, :class_name => ServiceTemplateTransformationPlanTask, :inverse_of => :conversion_host + # To be eligible, a conversion host must have the following properties + # - A transport mechanism is configured for source (set by 3rd party) + # - Credentials are set on the resource + # - The number of concurrent tasks has not reached the limit def eligible? + source_transport_method.present? && check_resource_credentials && check_concurrent_tasks + end + + def check_concurrent_tasks max_tasks = max_concurrent_tasks || Settings.transformation.limits.max_concurrent_tasks_per_host active_tasks.size < max_tasks end + def check_resource_credentials + send("check_resource_credentials_#{resource.ext_management_system.emstype}") + end + def source_transport_method return 'vddk' if vddk_transport_supported return 'ssh' if ssh_transport_supported end + + private + + def check_resource_credentials_rhevm + not(resource.authentication_userid.nil? || resource.authentication_password.nil?) + end + + def check_resource_credentials_openstack + ssh_authentications = resource.ext_management_system.authentications. + where(:authtype => 'ssh_keypair'). + where.not(:userid => nil, :auth_key => nil) + puts "Auth: #{ssh_authentications.inspect}" + not ssh_authentications.empty? + end end diff --git a/spec/models/conversion_host_spec.rb b/spec/models/conversion_host_spec.rb index 1b1a4c744e6..8245badfb23 100644 --- a/spec/models/conversion_host_spec.rb +++ b/spec/models/conversion_host_spec.rb @@ -11,20 +11,30 @@ let(:task_3) { FactoryGirl.create(:service_template_transformation_plan_task, :state => 'active', :conversion_host => conversion_host_2) } before do - conversion_host_1.concurrent_transformation_limit = "2" - conversion_host_2.concurrent_transformation_limit = "1" - - allow(ServiceTemplateTransformationPlanTask).to receive(:where).with(:state => 'active').and_return([task_1, task_3]) + allow(conversion_host_1).to receive(:active_tasks).and_return([task_1]) + allow(conversion_host_2).to receive(:active_tasks).and_return([task_3]) end - it "#active_tasks" do - expect(conversion_host_1.active_tasks).to eq([task_1]) - expect(conversion_host_2.active_tasks).to eq([task_3]) - end + describe "#check_concurrent_tasks" do + context "default max concurrent tasks is equal to current active tasks" do + before { stub_settings_merge(:transformation => {:limits => {:max_concurrent_tasks_per_host => 1}}) } + it { expect(conversion_host_1.check_concurrent_tasks).to eq(false) } + end + + context "default max concurrent tasks is greater than current active tasks" do + before { stub_settings_merge(:transformation => {:limits => {:max_concurrent_tasks_per_host => 10}}) } + it { expect(conversion_host_1.check_concurrent_tasks).to eq(true) } + end + + context "host's max concurrent tasks is equal to current active tasks" do + before { conversion_host_1.max_concurrent_tasks = "1" } + it { expect(conversion_host_1.check_concurrent_tasks).to eq(false) } + end - it "#eligible?" do - expect(conversion_host_1.eligible?).to eq(true) - expect(conversion_host_2.eligible?).to eq(false) + context "host's max concurrent tasks greater than current active tasks" do + before { conversion_host_2.max_concurrent_tasks = "2" } + it { expect(conversion_host_2.check_concurrent_tasks).to eq(true) } + end end context "#source_transport_method" do @@ -41,4 +51,51 @@ end end end + + context "resource provider is rhevm" do + let(:ems) { FactoryGirl.create(:ems_redhat, :zone => FactoryGirl.create(:zone)) } + let(:host) { FactoryGirl.create(:host, :ext_management_system => ems) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => host, :vddk_transport_supported => true) } + + context "host userid is nil" do + before { allow(host).to receive(:authentication_userid).and_return(nil) } + it { expect(conversion_host.check_resource_credentials).to eq(false) } + end + + context "host userid is set" do + before { allow(host).to receive(:authentication_userid).and_return('root') } + + context "and host password is nil" do + before { allow(host).to receive(:authentication_password).and_return(nil) } + it { expect(conversion_host.check_resource_credentials).to eq(false) } + end + + context "and host password is set" do + before { allow(host).to receive(:authentication_password).and_return('password') } + it { expect(conversion_host.check_resource_credentials).to eq(true) } + end + end + end + + context "resource provider is openstack" do + let(:ems) { FactoryGirl.create(:ems_openstack, :zone => FactoryGirl.create(:zone)) } + let(:vm) { FactoryGirl.create(:vm, :ext_management_system => ems) } + let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => vm, :vddk_transport_supported => true) } + + context "ems authentications is empty" do + it { expect(conversion_host.check_resource_credentials).to be(false) } + end + + context "ems authentications contains ssh_auth" do + let(:ssh_auth) { FactoryGirl.create(:authentication_ssh_keypair, :resource => ems) } + + it "with fake auth" do + allow(ems).to receive(:authentications).and_return(ssh_auth) + allow(ssh_auth).to receive(:where).with(:authype => 'ssh_keypair').and_return(ssh_auth) + allow(ssh_auth).to receive(:where).and_return(ssh_auth) + allow(ssh_auth).to receive(:not).with(:userid => nil, :auth_key => nil).and_return([ssh_auth]) + expect(conversion_host.check_resource_credentials).to be(true) + end + end + end end From 670f3dc6568d25518e24a160914956e1401b9224 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 5 Oct 2018 07:01:50 -0400 Subject: [PATCH 10/13] Fix rubocop --- app/models/conversion_host.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index de46502364b..ae36f96ab1d 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -39,7 +39,6 @@ def check_resource_credentials_openstack ssh_authentications = resource.ext_management_system.authentications. where(:authtype => 'ssh_keypair'). where.not(:userid => nil, :auth_key => nil) - puts "Auth: #{ssh_authentications.inspect}" not ssh_authentications.empty? end end From 531e98ff26dcfb079c8d7d9631ac9575e73c85a0 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 5 Oct 2018 07:06:09 -0400 Subject: [PATCH 11/13] Fix rubocop --- app/models/conversion_host.rb | 10 +++++----- spec/models/conversion_host_spec.rb | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index ae36f96ab1d..727d432c085 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -32,13 +32,13 @@ def source_transport_method private def check_resource_credentials_rhevm - not(resource.authentication_userid.nil? || resource.authentication_password.nil?) + !(resource.authentication_userid.nil? || resource.authentication_password.nil?) end def check_resource_credentials_openstack - ssh_authentications = resource.ext_management_system.authentications. - where(:authtype => 'ssh_keypair'). - where.not(:userid => nil, :auth_key => nil) - not ssh_authentications.empty? + ssh_authentications = resource.ext_management_system.authentications + .where(:authtype => 'ssh_keypair') + .where.not(:userid => nil, :auth_key => nil) + !ssh_authentications.empty? end end diff --git a/spec/models/conversion_host_spec.rb b/spec/models/conversion_host_spec.rb index 8245badfb23..5ccd29276c9 100644 --- a/spec/models/conversion_host_spec.rb +++ b/spec/models/conversion_host_spec.rb @@ -71,8 +71,8 @@ end context "and host password is set" do - before { allow(host).to receive(:authentication_password).and_return('password') } - it { expect(conversion_host.check_resource_credentials).to eq(true) } + before { allow(host).to receive(:authentication_password).and_return('password') } + it { expect(conversion_host.check_resource_credentials).to eq(true) } end end end From 3e5e83c7d610d320ac55b3306f8c520b6295fbd8 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 5 Oct 2018 08:09:57 -0400 Subject: [PATCH 12/13] Remove unneeded tests on pseudo-setter --- spec/models/service_template_transformation_plan_task_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/models/service_template_transformation_plan_task_spec.rb b/spec/models/service_template_transformation_plan_task_spec.rb index 275cd30c22f..61985907687 100644 --- a/spec/models/service_template_transformation_plan_task_spec.rb +++ b/spec/models/service_template_transformation_plan_task_spec.rb @@ -291,12 +291,10 @@ it "finds the source ems based on source vm" do expect(task_1.source_ems).to eq(src_ems) - expect(task_1.options[:source_ems_id]).to eq(src_ems.id) end it 'find the destination ems based on mapping' do expect(task_1.destination_ems).to eq(dst_ems) - expect(task_1.options[:destination_ems_id]).to eq(dst_ems.id) end context 'destination is rhevm' do From 5f683be548bf1113826dc78bab9f1177c4d46407 Mon Sep 17 00:00:00 2001 From: Fabien Dupont Date: Fri, 5 Oct 2018 08:12:07 -0400 Subject: [PATCH 13/13] Fix rubocop --- app/models/conversion_host.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/conversion_host.rb b/app/models/conversion_host.rb index 727d432c085..6556a4edf0c 100644 --- a/app/models/conversion_host.rb +++ b/app/models/conversion_host.rb @@ -37,8 +37,8 @@ def check_resource_credentials_rhevm def check_resource_credentials_openstack ssh_authentications = resource.ext_management_system.authentications - .where(:authtype => 'ssh_keypair') - .where.not(:userid => nil, :auth_key => nil) + .where(:authtype => 'ssh_keypair') + .where.not(:userid => nil, :auth_key => nil) !ssh_authentications.empty? end end