diff --git a/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb index f81f9848c..4283bafed 100644 --- a/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb @@ -4,6 +4,7 @@ class ManageIQ::Providers::Amazon::Inventory::Parser::CloudManager < ManageIQ::P def parse log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{collector.manager.name}] id: [#{collector.manager.id}]" $aws_log.info("#{log_header}...") + # The order of the below methods does matter, because they are searched using find instead of lazy_find flavors @@ -60,6 +61,7 @@ def images(images) image_hardware(persister_image, image) image_operating_system(persister_image, image) vm_and_template_labels(persister_image, image["tags"] || []) + vm_and_template_taggings(persister_image, map_labels("Image", image["tags"] || [])) end end @@ -95,6 +97,20 @@ def vm_and_template_labels(resource, tags) end end + # Returns array of InventoryObject. + def map_labels(model_name, labels) + label_hashes = labels.collect do |tag| + {:name => tag["key"], :value => tag["value"]} + end + persister.tag_mapper.map_labels(model_name, label_hashes) + end + + def vm_and_template_taggings(resource, tags_inventory_objects) + tags_inventory_objects.each do |tag| + persister.vm_and_template_taggings.build(:taggable => resource, :tag => tag) + end + end + def stacks collector.stacks.each do |stack| uid = stack['stack_id'].to_s @@ -202,6 +218,7 @@ def instances instance_hardware(persister_instance, instance, flavor) instance_operating_system(persister_instance, instance) vm_and_template_labels(persister_instance, instance["tags"] || []) + vm_and_template_taggings(persister_instance, map_labels("Vm", instance["tags"] || [])) end end diff --git a/app/models/manageiq/providers/amazon/inventory/persister.rb b/app/models/manageiq/providers/amazon/inventory/persister.rb index 92f9acd3b..ee087f6f5 100644 --- a/app/models/manageiq/providers/amazon/inventory/persister.rb +++ b/app/models/manageiq/providers/amazon/inventory/persister.rb @@ -5,6 +5,9 @@ class ManageIQ::Providers::Amazon::Inventory::Persister < ManagerRefresh::Invent # TODO(lsmola) figure out a way to pass collector info, probably via target, then remove the below attr_reader :collector + # Accessed by cloud parser. + attr_reader :tag_mapper + # @param manager [ManageIQ::Providers::BaseManager] A manager object # @param target [Object] A refresh Target object # @param target [ManagerRefresh::Inventory::Collector] A Collector object @@ -20,6 +23,13 @@ def initialize(manager, target = nil, collector = nil) protected + # TODO: this reads whole table ContainerLabelTagMapping.all. + # Is this expensive for each targeted refresh? + def initialize_tag_mapper + @tag_mapper = ContainerLabelTagMapping.mapper + collections[:tags_to_resolve] = @tag_mapper.tags_to_resolve_collection + end + def cloud ManageIQ::Providers::Amazon::InventoryCollectionDefault::CloudManager end diff --git a/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb index fbff77988..d2e892c85 100644 --- a/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb @@ -1,8 +1,11 @@ class ManageIQ::Providers::Amazon::Inventory::Persister::CloudManager < ManageIQ::Providers::Amazon::Inventory::Persister def initialize_inventory_collections + initialize_tag_mapper + add_inventory_collections( cloud, - %i(vms miq_templates hardwares operating_systems networks disks availability_zones vm_and_template_labels + %i(vms miq_templates hardwares operating_systems networks disks availability_zones + vm_and_template_labels vm_and_template_taggings flavors key_pairs orchestration_stacks orchestration_stacks_resources orchestration_stacks_outputs orchestration_stacks_parameters orchestration_templates) ) diff --git a/app/models/manageiq/providers/amazon/inventory/persister/target_collection.rb b/app/models/manageiq/providers/amazon/inventory/persister/target_collection.rb index 8ebfdfcd6..4cf3f0639 100644 --- a/app/models/manageiq/providers/amazon/inventory/persister/target_collection.rb +++ b/app/models/manageiq/providers/amazon/inventory/persister/target_collection.rb @@ -1,6 +1,8 @@ class ManageIQ::Providers::Amazon::Inventory::Persister::TargetCollection < ManageIQ::Providers::Amazon::Inventory::Persister def initialize_inventory_collections ######### Cloud ########## + initialize_tag_mapper + # Top level models with direct references for Cloud add_inventory_collections_with_references( cloud, @@ -16,8 +18,8 @@ def initialize_inventory_collections # Child models with references in the Parent InventoryCollections for Cloud add_inventory_collections( cloud, - %i(hardwares operating_systems networks disks vm_and_template_labels orchestration_stacks_resources - orchestration_stacks_outputs orchestration_stacks_parameters) + %i(hardwares operating_systems networks disks vm_and_template_labels vm_and_template_taggings + orchestration_stacks_resources orchestration_stacks_outputs orchestration_stacks_parameters) ) add_inventory_collection(cloud.orchestration_templates) diff --git a/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb index 73c268ca9..c1fc98733 100644 --- a/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb @@ -176,6 +176,30 @@ def vm_and_template_labels(extra_attributes = {}) attributes.merge!(extra_attributes) end + def vm_and_template_taggings(extra_attributes = {}) + # TODO: make a generic Taggings IC and move it to base class? + attributes = { + :model_class => Tagging, + :association => :vm_and_template_taggings, + :manager_ref => [:taggable, :tag], + :parent_inventory_collections => [:vms, :miq_templates], + :inventory_object_attributes => [ + :taggable, + :tag, + ] + } + + attributes[:targeted_arel] = lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).map(&:to_a).flatten + ems = inventory_collection.parent + ems.vm_and_template_taggings.where( + 'taggable_id' => ems.vms_and_templates.where(:ems_ref => manager_uuids) + ) + end + + attributes.merge!(extra_attributes) + end + def orchestration_stacks(extra_attributes = {}) attributes = { :model_class => ::ManageIQ::Providers::Amazon::CloudManager::OrchestrationStack, diff --git a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb index 169148eb6..c419d4697 100644 --- a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb +++ b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb @@ -81,7 +81,7 @@ def assert_common assert_specific_orchestration_stack assert_network_router assert_relationship_tree - # assert_specific_tags_on_vm + assert_specific_labels_on_vm end def assert_specific_flavor @@ -533,9 +533,7 @@ def assert_specific_vm_powered_on :product_name => "linux_generic", ) ) - expect(v.custom_attributes.size).to eq(2) expect(v.snapshots.size).to eq(0) - # expect(v.tags.size).to eq(0) expect(v.hardware).to have_attributes( :guest_os => "linux_generic", @@ -580,8 +578,10 @@ def assert_specific_vm_powered_on v.with_relationship_type("genealogy") do expect(v.parent).to eq(@template) end + expect(v.custom_attributes.size).to eq(2) expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-Basic3") expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN") + assert_mapped_tags_on_vm(v, :owner => "UNKNOWN") end def assert_specific_vm_powered_off @@ -633,6 +633,7 @@ def assert_specific_vm_powered_off expect(v.custom_attributes.size).to eq(2) expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOff") expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN") + assert_mapped_tags_on_vm(v, :owner => "UNKNOWN") expect(v.snapshots.size).to eq(0) expect(v.hardware).to have_attributes( @@ -744,6 +745,7 @@ def assert_specific_vm_on_cloud_network expect(v.custom_attributes.size).to eq(2) expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-VPC") expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN") + assert_mapped_tags_on_vm(v, :owner => "UNKNOWN") expect(v.snapshots.size).to eq(0) expect(v.hardware).to( @@ -898,6 +900,7 @@ def assert_specific_vm_on_cloud_network_public_ip expect(v.custom_attributes.size).to eq(2) expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-VPC1") expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN") + assert_mapped_tags_on_vm(v, :owner => "UNKNOWN") expect(v.snapshots.size).to eq(0) expect(v.hardware).to( @@ -1416,9 +1419,47 @@ def assert_relationship_tree expect(@ems.descendants_arranged).to match_relationship_tree({}) end - # TODO: Add some real specs here - # def assert_specific_tags_on_vm - # vm = ManageIQ::Providers::Amazon::CloudManager::Vm.where(:name => "EmsRefreshSpec-PoweredOn-Basic3").first - # expect(vm.tags).to be_empty - # end + def assert_specific_labels_on_vm + vm = ManageIQ::Providers::Amazon::CloudManager::Vm.find_by(:name => "ladas_test_5") + expect(vm.labels).to include( + an_object_having_attributes( + :name => "EmsRefreshSpecResourceGroupTag", + :value => "EmsRefreshSpecResourceGroupTagValue", + :source => "amazon" + ) + ) + end + + def create_tag_mapping + @all_tag_mapping = FactoryGirl.create(:tag_mapping_with_category, + :label_name => "owner") + @all_tag_mapping_category = @all_tag_mapping.tag.classification + + @image_tag_mapping = FactoryGirl.create(:tag_mapping_with_category, + :label_name => "Name", :labeled_resource_type => "Image") + @image_tag_mapping_category = @image_tag_mapping.tag.classification + end + + # Tests can assert this if they called create_tag_mapping before refresh. + def assert_mapped_tags_on_vm(vm, owner:) + expect(@all_tag_mapping_category.children.collect(&:description)).to include(owner) + + expect(vm.tags.count).to eq(1) + expect(vm.tags.first.category).to eq(@all_tag_mapping_category) + expect(vm.tags.first.classification.description).to eq("UNKNOWN") + end + + # Tests can assert this if they called create_tag_mapping before refresh. + def assert_mapped_tags_on_template + expect(@image_tag_mapping_category.children.collect(&:description)).to include( + "suse-11-server-64", + "ubuntu-12.04-server-32", + # and many more... + ) + + template = ManageIQ::Providers::Amazon::CloudManager::Template.find_by(:name => "suse-11-server-64") + expect(template.tags.count).to eq(1) + expect(template.tags.first.category).to eq(@image_tag_mapping_category) + expect(template.tags.first.classification.description).to eq("suse-11-server-64") + end end diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_inventory_object_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_inventory_object_spec.rb index 51047a9e5..5dcad1843 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_inventory_object_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_inventory_object_spec.rb @@ -17,6 +17,7 @@ context "with settings #{settings}" do before(:each) do stub_refresh_settings(settings) + create_tag_mapping end it "will perform a full refresh" do @@ -32,6 +33,7 @@ end assert_common + assert_mapped_tags_on_template end end end diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb index 36d7b3e8a..52e5bf242 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb @@ -17,6 +17,7 @@ context "with settings #{settings}" do before(:each) do stub_refresh_settings(settings) + create_tag_mapping end it "will perform a full refresh" do @@ -33,6 +34,7 @@ end assert_common + assert_mapped_tags_on_template end end end diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/targeted_refresh_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/targeted_refresh_spec.rb index 1c0182e04..63bc4d48d 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/targeted_refresh_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/targeted_refresh_spec.rb @@ -13,6 +13,7 @@ context "with settings #{settings}" do before(:each) do stub_refresh_settings(settings.merge(:allow_targeted_refresh => true)) + create_tag_mapping # The flavors are not fetched from the API, they can go in only by appliance update, so must be in place after # the full refresh, lets pre-create them in the DB. create_flavors