Merge pull request #382 from cben/tag-mapping

Implement tag mapping in graph refresh

app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb

@@ -4,6 +4,7 @@ class ManageIQ::Providers::Amazon::Inventory::Parser::CloudManager < ManageIQ::P
   def parse
     log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{collector.manager.name}] id: [#{collector.manager.id}]"
     $aws_log.info("#{log_header}...")
+
     # The order of the below methods does matter, because they are searched using find instead of lazy_find
     flavors
 
@@ -60,6 +61,7 @@ def images(images)
       image_hardware(persister_image, image)
       image_operating_system(persister_image, image)
       vm_and_template_labels(persister_image, image["tags"] || [])
+      vm_and_template_taggings(persister_image, map_labels("Image", image["tags"] || []))
     end
   end
 
@@ -95,6 +97,20 @@ def vm_and_template_labels(resource, tags)
     end
   end
 
+  # Returns array of InventoryObject<Tag>.
+  def map_labels(model_name, labels)
+    label_hashes = labels.collect do |tag|
+      {:name => tag["key"], :value => tag["value"]}
+    end
+    persister.tag_mapper.map_labels(model_name, label_hashes)
+  end
+
+  def vm_and_template_taggings(resource, tags_inventory_objects)
+    tags_inventory_objects.each do |tag|
+      persister.vm_and_template_taggings.build(:taggable => resource, :tag => tag)
+    end
+  end
+
   def stacks
     collector.stacks.each do |stack|
       uid = stack['stack_id'].to_s
@@ -202,6 +218,7 @@ def instances
       instance_hardware(persister_instance, instance, flavor)
       instance_operating_system(persister_instance, instance)
       vm_and_template_labels(persister_instance, instance["tags"] || [])
+      vm_and_template_taggings(persister_instance, map_labels("Vm", instance["tags"] || []))
     end
   end
 

app/models/manageiq/providers/amazon/inventory/persister.rb

@@ -5,6 +5,9 @@ class ManageIQ::Providers::Amazon::Inventory::Persister < ManagerRefresh::Invent
 
   # TODO(lsmola) figure out a way to pass collector info, probably via target, then remove the below
   attr_reader :collector
+  # Accessed by cloud parser.
+  attr_reader :tag_mapper
+
   # @param manager [ManageIQ::Providers::BaseManager] A manager object
   # @param target [Object] A refresh Target object
   # @param target [ManagerRefresh::Inventory::Collector] A Collector object
@@ -20,6 +23,13 @@ def initialize(manager, target = nil, collector = nil)
 
   protected
 
+  # TODO: this reads whole table ContainerLabelTagMapping.all.
+  #   Is this expensive for each targeted refresh?
+  def initialize_tag_mapper
+    @tag_mapper = ContainerLabelTagMapping.mapper
+    collections[:tags_to_resolve] = @tag_mapper.tags_to_resolve_collection
+  end
+
   def cloud
     ManageIQ::Providers::Amazon::InventoryCollectionDefault::CloudManager
   end

app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb

@@ -1,8 +1,11 @@
 class ManageIQ::Providers::Amazon::Inventory::Persister::CloudManager < ManageIQ::Providers::Amazon::Inventory::Persister
   def initialize_inventory_collections
+    initialize_tag_mapper
+
     add_inventory_collections(
       cloud,
-      %i(vms miq_templates hardwares operating_systems networks disks availability_zones vm_and_template_labels
+      %i(vms miq_templates hardwares operating_systems networks disks availability_zones
+         vm_and_template_labels vm_and_template_taggings
          flavors key_pairs orchestration_stacks orchestration_stacks_resources
          orchestration_stacks_outputs orchestration_stacks_parameters orchestration_templates)
     )

app/models/manageiq/providers/amazon/inventory/persister/target_collection.rb

@@ -1,6 +1,8 @@
 class ManageIQ::Providers::Amazon::Inventory::Persister::TargetCollection < ManageIQ::Providers::Amazon::Inventory::Persister
   def initialize_inventory_collections
     ######### Cloud ##########
+    initialize_tag_mapper
+
     # Top level models with direct references for Cloud
     add_inventory_collections_with_references(
       cloud,
@@ -16,8 +18,8 @@ def initialize_inventory_collections
     # Child models with references in the Parent InventoryCollections for Cloud
     add_inventory_collections(
       cloud,
-      %i(hardwares operating_systems networks disks vm_and_template_labels orchestration_stacks_resources
-         orchestration_stacks_outputs orchestration_stacks_parameters)
+      %i(hardwares operating_systems networks disks vm_and_template_labels vm_and_template_taggings
+         orchestration_stacks_resources orchestration_stacks_outputs orchestration_stacks_parameters)
     )
 
     add_inventory_collection(cloud.orchestration_templates)

app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb

@@ -176,6 +176,30 @@ def vm_and_template_labels(extra_attributes = {})
       attributes.merge!(extra_attributes)
     end
 
+    def vm_and_template_taggings(extra_attributes = {})
+      # TODO: make a generic Taggings IC and move it to base class?
+      attributes = {
+        :model_class                  => Tagging,
+        :association                  => :vm_and_template_taggings,
+        :manager_ref                  => [:taggable, :tag],
+        :parent_inventory_collections => [:vms, :miq_templates],
+        :inventory_object_attributes  => [
+          :taggable,
+          :tag,
+        ]
+      }
+
+      attributes[:targeted_arel] = lambda do |inventory_collection|
+        manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).map(&:to_a).flatten
+        ems = inventory_collection.parent
+        ems.vm_and_template_taggings.where(
+          'taggable_id'   => ems.vms_and_templates.where(:ems_ref => manager_uuids)
+        )
+      end
+
+      attributes.merge!(extra_attributes)
+    end
+
     def orchestration_stacks(extra_attributes = {})
       attributes = {
         :model_class                 => ::ManageIQ::Providers::Amazon::CloudManager::OrchestrationStack,

spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb

@@ -81,7 +81,7 @@ def assert_common
     assert_specific_orchestration_stack
     assert_network_router
     assert_relationship_tree
-    # assert_specific_tags_on_vm
+    assert_specific_labels_on_vm
   end
 
   def assert_specific_flavor
@@ -533,9 +533,7 @@ def assert_specific_vm_powered_on
         :product_name => "linux_generic",
       )
     )
-    expect(v.custom_attributes.size).to eq(2)
     expect(v.snapshots.size).to eq(0)
-    # expect(v.tags.size).to eq(0)
 
     expect(v.hardware).to have_attributes(
       :guest_os            => "linux_generic",
@@ -580,8 +578,10 @@ def assert_specific_vm_powered_on
     v.with_relationship_type("genealogy") do
       expect(v.parent).to eq(@template)
     end
+    expect(v.custom_attributes.size).to eq(2)
     expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-Basic3")
     expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN")
+    assert_mapped_tags_on_vm(v, :owner => "UNKNOWN")
   end
 
   def assert_specific_vm_powered_off
@@ -633,6 +633,7 @@ def assert_specific_vm_powered_off
     expect(v.custom_attributes.size).to eq(2)
     expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOff")
     expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN")
+    assert_mapped_tags_on_vm(v, :owner => "UNKNOWN")
     expect(v.snapshots.size).to eq(0)
 
     expect(v.hardware).to have_attributes(
@@ -744,6 +745,7 @@ def assert_specific_vm_on_cloud_network
     expect(v.custom_attributes.size).to eq(2)
     expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-VPC")
     expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN")
+    assert_mapped_tags_on_vm(v, :owner => "UNKNOWN")
     expect(v.snapshots.size).to eq(0)
 
     expect(v.hardware).to(
@@ -898,6 +900,7 @@ def assert_specific_vm_on_cloud_network_public_ip
     expect(v.custom_attributes.size).to eq(2)
     expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-VPC1")
     expect(v.custom_attributes.find_by(:name => "owner").value).to eq("UNKNOWN")
+    assert_mapped_tags_on_vm(v, :owner => "UNKNOWN")
     expect(v.snapshots.size).to eq(0)
 
     expect(v.hardware).to(
@@ -1416,9 +1419,47 @@ def assert_relationship_tree
     expect(@ems.descendants_arranged).to match_relationship_tree({})
   end
 
-  # TODO: Add some real specs here
-  # def assert_specific_tags_on_vm
-  #   vm = ManageIQ::Providers::Amazon::CloudManager::Vm.where(:name => "EmsRefreshSpec-PoweredOn-Basic3").first
-  #   expect(vm.tags).to be_empty
-  # end
+  def assert_specific_labels_on_vm
+    vm = ManageIQ::Providers::Amazon::CloudManager::Vm.find_by(:name => "ladas_test_5")
+    expect(vm.labels).to include(
+      an_object_having_attributes(
+        :name   => "EmsRefreshSpecResourceGroupTag",
+        :value  => "EmsRefreshSpecResourceGroupTagValue",
+        :source => "amazon"
+      )
+    )
+  end
+
+  def create_tag_mapping
+    @all_tag_mapping = FactoryGirl.create(:tag_mapping_with_category,
+                                          :label_name => "owner")
+    @all_tag_mapping_category = @all_tag_mapping.tag.classification
+
+    @image_tag_mapping = FactoryGirl.create(:tag_mapping_with_category,
+                                            :label_name => "Name", :labeled_resource_type => "Image")
+    @image_tag_mapping_category = @image_tag_mapping.tag.classification
+  end
+
+  # Tests can assert this if they called create_tag_mapping before refresh.
+  def assert_mapped_tags_on_vm(vm, owner:)
+    expect(@all_tag_mapping_category.children.collect(&:description)).to include(owner)
+
+    expect(vm.tags.count).to eq(1)
+    expect(vm.tags.first.category).to eq(@all_tag_mapping_category)
+    expect(vm.tags.first.classification.description).to eq("UNKNOWN")
+  end
+
+  # Tests can assert this if they called create_tag_mapping before refresh.
+  def assert_mapped_tags_on_template
+    expect(@image_tag_mapping_category.children.collect(&:description)).to include(
+      "suse-11-server-64",
+      "ubuntu-12.04-server-32",
+      # and many more...
+    )
+
+    template = ManageIQ::Providers::Amazon::CloudManager::Template.find_by(:name => "suse-11-server-64")
+    expect(template.tags.count).to eq(1)
+    expect(template.tags.first.category).to eq(@image_tag_mapping_category)
+    expect(template.tags.first.classification.description).to eq("suse-11-server-64")
+  end
 end

spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_inventory_object_spec.rb

@@ -17,6 +17,7 @@
     context "with settings #{settings}" do
       before(:each) do
         stub_refresh_settings(settings)
+        create_tag_mapping
       end
 
       it "will perform a full refresh" do
@@ -32,6 +33,7 @@
           end
 
           assert_common
+          assert_mapped_tags_on_template
         end
       end
     end

spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb

@@ -17,6 +17,7 @@
     context "with settings #{settings}" do
       before(:each) do
         stub_refresh_settings(settings)
+        create_tag_mapping
       end
 
       it "will perform a full refresh" do
@@ -33,6 +34,7 @@
           end
 
           assert_common
+          assert_mapped_tags_on_template
         end
       end
     end

spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/targeted_refresh_spec.rb

@@ -13,6 +13,7 @@
     context "with settings #{settings}" do
       before(:each) do
         stub_refresh_settings(settings.merge(:allow_targeted_refresh => true))
+        create_tag_mapping
         # The flavors are not fetched from the API, they can go in only by appliance update, so must be in place after
         # the full refresh, lets pre-create them in the DB.
         create_flavors