Merge pull request ManageIQ#74 from mansam/targeted-refresh

Targeted Refresh for Cloud VMs

app/models/manageiq/providers/openstack/builder.rb

@@ -0,0 +1,54 @@
+class ManageIQ::Providers::Openstack::Builder
+  class << self
+    def build_inventory(ems, target)
+      case target
+      when ManageIQ::Providers::Openstack::CloudManager
+        cloud_manager_inventory(ems, target)
+      when ManageIQ::Providers::Openstack::NetworkManager
+        inventory(
+          ems,
+          target,
+          ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager,
+          ManageIQ::Providers::Openstack::Inventory::Persister::NetworkManager,
+          [ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager]
+        )
+      when ManagerRefresh::TargetCollection
+        inventory(
+          ems,
+          target,
+          ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection,
+          ManageIQ::Providers::Openstack::Inventory::Persister::TargetCollection,
+          [ManageIQ::Providers::Openstack::Inventory::Parser::CloudManager,
+           ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager]
+        )
+      else
+        # Fallback to ems refresh
+        cloud_manager_inventory(ems, target)
+      end
+    end
+
+    private
+
+    def cloud_manager_inventory(ems, target)
+      inventory(
+        ems,
+        target,
+        ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager,
+        ManageIQ::Providers::Openstack::Inventory::Persister::CloudManager,
+        [ManageIQ::Providers::Openstack::Inventory::Parser::CloudManager]
+      )
+    end
+
+    def inventory(manager, raw_target, collector_class, persister_class, parsers_classes)
+      collector = collector_class.new(manager, raw_target)
+      # TODO(lsmola) figure out a way to pass collector info, probably via target
+      persister = persister_class.new(manager, raw_target, collector)
+
+      ::ManageIQ::Providers::Openstack::Inventory.new(
+        persister,
+        collector,
+        parsers_classes.map(&:new)
+      )
+    end
+  end
+end

app/models/manageiq/providers/openstack/cloud_manager/refresher.rb

@@ -2,29 +2,80 @@ module ManageIQ::Providers
   class Openstack::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
     include ::EmsRefresh::Refreshers::EmsRefresherMixin
 
+    def collect_inventory_for_targets(ems, targets)
+      targets_with_data = targets.collect do |target|
+        target_name = target.try(:name) || target.try(:event_type)
+
+        _log.info("Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]...")
+        if ::Settings.ems.ems_openstack.try(:refresh).try(:inventory_object_refresh)
+          inventory = ManageIQ::Providers::Openstack::Builder.build_inventory(ems, target)
+        end
+
+        _log.info("Filtering inventory...Complete")
+        [target, inventory]
+      end
+
+      targets_with_data
+    end
+
     def parse_legacy_inventory(ems)
       ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
     end
 
-    # TODO(lsmola) NetworkManager, remove this once we have a full representation of the NetworkManager.
-    # NetworkManager should refresh base on its own conditions
-    def save_inventory(ems, target, hashes)
+    def save_inventory(ems, target, inventory_collections)
       super
-      EmsRefresh.queue_refresh(ems.network_manager)
-      EmsRefresh.queue_refresh(ems.cinder_manager)
-      EmsRefresh.queue_refresh(ems.swift_manager)
+      EmsRefresh.queue_refresh(ems.network_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
+      EmsRefresh.queue_refresh(ems.cinder_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
+      EmsRefresh.queue_refresh(ems.swift_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
     end
 
-    def parse_targeted_inventory(ems, target, collector)
-      if ::Settings.ems.ems_openstack.refresh.inventory_object_refresh
-        super(ems, target, collector)
-      else
-        super(ems, target, nil)
+    def parse_targeted_inventory(ems, _target, inventory)
+      log_header = format_ems_for_logging(ems)
+      _log.debug("#{log_header} Parsing inventory...")
+      hashes, = Benchmark.realtime_block(:parse_inventory) do
+        if ::Settings.ems.ems_openstack.try(:refresh).try(:inventory_object_refresh)
+          inventory.inventory_collections
+        else
+          ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+        end
       end
+      _log.debug("#{log_header} Parsing inventory...Complete")
+
+      hashes
+    end
+
+    def preprocess_targets
+      @targets_by_ems_id.each do |ems_id, targets|
+        if targets.any? { |t| t.kind_of?(ExtManagementSystem) }
+          ems             = @ems_by_ems_id[ems_id]
+          targets_for_log = targets.map { |t| "#{t.class} [#{t.name}] id [#{t.id}] " }
+          _log.info("Defaulting to full refresh for EMS: [#{ems.name}], id: [#{ems.id}], from targets: #{targets_for_log}") if targets.length > 1
+        end
+
+        # We want all targets of class EmsEvent to be merged into one target,
+        # so they can be refreshed together, otherwise we could be missing some
+        # crosslinks in the refreshed data
+        all_targets, sub_ems_targets = targets.partition { |x| x.kind_of?(ExtManagementSystem) }
+
+        unless sub_ems_targets.blank?
+          if ::Settings.ems.ems_openstack.try(:refresh).try(:event_targeted_refresh)
+            # We can disable targeted refresh with a setting, then we will just do full ems refresh on any event
+            ems_event_collection = ManagerRefresh::TargetCollection.new(:targets    => sub_ems_targets,
+                                                                        :manager_id => ems_id)
+            all_targets << ems_event_collection
+          else
+            all_targets << @ems_by_ems_id[ems_id]
+          end
+        end
+
+        @targets_by_ems_id[ems_id] = all_targets
+      end
+
+      super
     end
 
     def post_process_refresh_classes
-      [Vm, CloudTenant]
+      [::Vm, CloudTenant]
     end
   end
 end

app/models/manageiq/providers/openstack/inventory.rb

@@ -0,0 +1,5 @@
+class ManageIQ::Providers::Openstack::Inventory < ManagerRefresh::Inventory
+  require_nested :Collector
+  require_nested :Parser
+  require_nested :Persister
+end

app/models/manageiq/providers/openstack/inventory/collector.rb

@@ -0,0 +1,87 @@
+class ManageIQ::Providers::Openstack::Inventory::Collector < ManagerRefresh::Inventory::Collector
+  include ManageIQ::Providers::Openstack::RefreshParserCommon::HelperMethods
+  include Vmdb::Logging
+
+  require_nested :CloudManager
+  require_nested :NetworkManager
+  require_nested :TargetCollection
+
+  attr_reader :availability_zones
+  attr_reader :cloud_services
+  attr_reader :tenants
+  attr_accessor :flavors
+  attr_reader :host_aggregates
+  attr_reader :key_pairs
+  attr_reader :miq_templates
+  attr_reader :orchestration_stacks
+  attr_reader :quotas
+  attr_reader :vms
+  attr_reader :vnfs
+  attr_reader :vnfds
+  attr_reader :cloud_networks
+  attr_reader :floating_ips
+  attr_reader :network_ports
+  attr_reader :network_routers
+  attr_reader :security_groups
+
+  def initialize(_manager, _target)
+    super
+
+    initialize_inventory_sources
+  end
+
+  def initialize_inventory_sources
+    # cloud
+    @availability_zones   = []
+    @cloud_services       = []
+    @tenants              = []
+    @flavors              = []
+    @host_aggregates      = []
+    @key_pairs            = []
+    @images               = []
+    @orchestration_stacks = []
+    @quotas               = []
+    @vms                  = []
+    @vnfs                 = []
+    @vnfds                = []
+    # network
+    @cloud_networks       = []
+    @floating_ips         = []
+    @network_ports        = []
+    @network_routers      = []
+    @security_groups      = []
+  end
+
+  def connection
+    @os_handle ||= manager.openstack_handle
+    @connection ||= manager.connect
+  end
+
+  def compute_service
+    connection
+  end
+
+  def identity_service
+    @identity_service ||= manager.openstack_handle.identity_service
+  end
+
+  def image_service
+    @image_service ||= manager.openstack_handle.detect_image_service
+  end
+
+  def network_service
+    @network_service ||= manager.openstack_handle.detect_network_service
+  end
+
+  def nfv_service
+    @nfv_service ||= manager.openstack_handle.detect_nfv_service
+  end
+
+  def volume_service
+    @volume_service ||= manager.openstack_handle.detect_volume_service
+  end
+
+  def orchestration_service
+    @orchestration_service ||= manager.openstack_handle.detect_orchestration_service
+  end
+end