InventoryCollection Builder

Persister's add_collection() interface

app/models/manageiq/providers/redhat/inventory/persister.rb

@@ -10,13 +10,23 @@ def initialize(manager, target, collector)
     @collector = collector
 
     @collections = {}
+    @collection_group = nil
 
     initialize_inventory_collections
   end
 
   protected
 
-  def infra
-    ManageIQ::Providers::Redhat::InventoryCollectionDefault::InfraManager
+  # should be overriden by subclasses
+  def strategy
+    nil
+  end
+
+  def shared_options
+    {:strategy => strategy}
+  end
+
+  def manager_refs
+    references(@collection_group)
   end
 end

app/models/manageiq/providers/redhat/inventory/persister/infra_manager.rb

@@ -1,65 +1,11 @@
 class ManageIQ::Providers::Redhat::Inventory::Persister::InfraManager < ManageIQ::Providers::Redhat::Inventory::Persister
-  def initialize_inventory_collections
-    add_inventory_collections(
-      infra,
-      %i(ems_clusters hosts resource_pools vms miq_templates
-         storages vm_and_template_ems_custom_fields disks guest_devices hardwares
-         host_hardwares host_networks host_operating_systems host_storages
-         host_switches lans networks operating_systems snapshots switches)
-    )
-
-    add_inventory_collection(
-      infra.datacenters(
-        :arel     => manager.ems_folders.where(:type => 'Datacenter'),
-        :strategy => :local_db_find_missing_references
-      )
-    )
-
-    add_inventory_collection(
-      infra.ems_folder_children(
-        :dependency_attributes => {
-          :clusters    => [collections[:ems_clusters]],
-          :datacenters => [collections[:datacenters]],
-          :vms         => [collections[:vms]],
-          :templates   => [collections[:miq_templates]]
-        }
-      )
-    )
+  include ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraCollections
 
-    add_inventory_collection(
-      infra.ems_clusters_children(
-        :dependency_attributes => {
-          :vms      => [collections[:vms]],
-          :clusters => [collections[:ems_clusters]]
-        }
-      )
-    )
-
-    add_inventory_collection(
-      infra.snapshot_parent(
-        :dependency_attributes => {
-          :snapshots => [collections[:snapshots]]
-        }
-      )
-    )
+  def initialize_inventory_collections
+    initialize_infra_inventory_collections
+  end
 
-    add_inventory_collection(
-      infra.vm_folders(
-        :arel     => manager.ems_folders.where(:name => 'vm'),
-        :strategy => :local_db_find_missing_references
-      )
-    )
-    add_inventory_collection(
-      infra.host_folders(
-        :arel     => manager.ems_folders.where(:name => 'host'),
-        :strategy => :local_db_find_missing_references
-      )
-    )
-    add_inventory_collection(
-      infra.root_folders(
-        :arel     => manager.ems_folders.where(:uid_ems => 'root_dc'),
-        :strategy => :local_db_find_missing_references
-      )
-    )
+  def strategy
+    @collection_group == :datacenters ? :local_db_find_missing_references : nil
   end
 end

app/models/manageiq/providers/redhat/inventory/persister/network_manager.rb

@@ -1,14 +1,9 @@
+# !! Inherited from OpenStack
 class ManageIQ::Providers::Redhat::Inventory::Persister::NetworkManager < ManageIQ::Providers::Openstack::Inventory::Persister::NetworkManager
-  def network
-    ManageIQ::Providers::Redhat::InventoryCollectionDefault::NetworkManager
-  end
+  include ManageIQ::Providers::Redhat::Inventory::Persister::Shared::NetworkCollections
 
   def initialize_inventory_collections
     super
-    add_inventory_collections(
-      network,
-      %i(cloud_tenants),
-      :builder_params => {:ext_management_system => manager.parent_manager}
-    )
+    add_cloud_tenants
   end
 end

app/models/manageiq/providers/redhat/inventory/persister/shared/infra_collections.rb

@@ -0,0 +1,97 @@
+module ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraCollections
+  extend ActiveSupport::Concern
+
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::ClusterCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::VmsCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::HostsCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::DatacentersCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::StoragedomainsCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::NetworksCollections
+  include ::ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::VmsDependencyCollections
+
+  def infra
+    ::ManagerRefresh::InventoryCollection::Builder::InfraManager
+  end
+
+  def initialize_infra_inventory_collections
+    @collection_group = nil
+
+    add_clusters_group
+    add_vms_group
+    add_hosts_group
+    add_datacenters_group
+    add_storagedomains_group
+    add_networks_group
+    add_vms_dependency_collections_group
+    add_other_collections
+  end
+
+  # --- IC groups definitions ---
+
+  def add_clusters_group
+    @collection_group = :ems_clusters
+
+    add_ems_clusters
+    add_resource_pools
+  end
+
+  def add_vms_group
+    @collection_group = :vms
+
+    add_vms
+    add_miq_templates
+    add_disks
+    add_networks
+    add_hardwares
+    add_guest_devices
+    add_snapshots
+    add_operating_systems
+    add_vm_and_template_ems_custom_fields
+  end
+
+  def add_vms_dependency_collections_group
+    @collection_group = :vms_dependency
+
+    add_ems_folder_children
+    add_ems_cluster_children
+    add_snapshot_parent
+  end
+
+  def add_datacenters_group
+    @collection_group = :datacenters
+
+    add_datacenters
+    add_vm_folders
+    add_host_folders
+    add_root_folders
+  end
+
+  def add_hosts_group
+    @collection_group = :hosts
+
+    add_hosts
+    add_host_hardwares
+    add_host_networks
+    add_host_operating_systems
+    add_host_storages
+    add_host_switches
+  end
+
+  def add_storagedomains_group
+    @collection_group = :storagedomains
+
+    add_storages
+  end
+
+  def add_networks_group
+    @collection_group = :networks
+
+    add_switches
+  end
+
+  def add_other_collections
+    @collection_group = nil
+
+    add_collection(infra, :lans)
+  end
+end

app/models/manageiq/providers/redhat/inventory/persister/shared/infra_group/cluster_collections.rb

@@ -0,0 +1,19 @@
+module ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::ClusterCollections
+  extend ActiveSupport::Concern
+
+  # group :ems_clusters
+  def add_ems_clusters
+    add_collection(infra, :ems_clusters) do |builder|
+      if targeted?
+        builder.add_properties(:arel => manager.ems_clusters.where(:ems_ref => manager_refs))
+      end
+    end
+  end
+
+  # group :ems_clusters
+  def add_resource_pools
+    add_collection(infra, :resource_pools) do |builder|
+      builder.add_properties(:arel => manager.resource_pools.where(:uid_ems => manager_refs.collect { |ref| "#{URI(ref).path.split('/').last}_respool" })) if targeted?
+    end
+  end
+end

app/models/manageiq/providers/redhat/inventory/persister/shared/infra_group/datacenters_collections.rb

@@ -0,0 +1,69 @@
+module ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::DatacentersCollections
+  extend ActiveSupport::Concern
+
+  # group :datacenters
+  def add_datacenters
+    add_collection(infra, :datacenters) do |builder|
+      arel = if targeted?
+               manager.ems_folders.where(:type => 'Datacenter').where(:ems_ref => manager_refs) if manager_refs.present?
+             else
+               manager.ems_folders.where(:type => 'Datacenter')
+             end
+
+      builder.add_properties(:arel => arel) unless arel.nil?
+    end
+  end
+
+  # group :datacenters
+  def add_vm_folders
+    add_collection(infra, :vm_folders, {}, {:auto_inventory_attributes => false}) do |builder|
+      builder.ems_folders
+
+      builder.add_properties(:model_class => ::EmsFolder)
+
+      arel = if targeted?
+               manager.ems_folders.where(:uid_ems => manager_refs.collect { |ref| "#{URI(ref).path.split('/').last}_vm" }) if manager_refs.present?
+             else
+               manager.ems_folders.where(:name => 'vm')
+             end
+
+      builder.add_properties(:arel => arel) unless arel.nil?
+
+      builder.add_inventory_attributes(%i(name type uid_ems hidden))
+    end
+  end
+
+  # group :datacenters
+  def add_host_folders
+    add_collection(infra, :host_folders, {}, {:auto_inventory_attributes => false}) do |builder|
+      builder.ems_folders
+
+      builder.add_properties(:model_class => ::EmsFolder)
+
+      arel = if targeted?
+               manager.ems_folders.where(:uid_ems => manager_refs.collect { |ref| "#{URI(ref).path.split('/').last}_host" }) if manager_refs.present?
+             else
+               manager.ems_folders.where(:name => 'host')
+             end
+
+      builder.add_properties(:arel => arel) unless arel.nil?
+
+      builder.add_inventory_attributes(%i(name type uid_ems hidden))
+    end
+  end
+
+  # group :datacenters
+  def add_root_folders
+    add_collection(infra, :root_folders, {}, {:auto_inventory_attributes => false}) do |builder|
+      builder.ems_folders
+
+      builder.add_properties(:model_class => ::EmsFolder)
+
+      arel = manager.ems_folders.where(:uid_ems => 'root_dc') if !targeted? || manager_refs.present?
+
+      builder.add_properties(:arel => arel) unless arel.nil?
+
+      builder.add_inventory_attributes(%i(name type uid_ems hidden))
+    end
+  end
+end

app/models/manageiq/providers/redhat/inventory/persister/shared/infra_group/hosts_collections.rb

@@ -0,0 +1,94 @@
+module ManageIQ::Providers::Redhat::Inventory::Persister::Shared::InfraGroup::HostsCollections
+  extend ActiveSupport::Concern
+
+  # group :hosts
+  def add_hosts
+    add_collection(infra, :hosts) do |builder|
+      builder.add_properties(
+        :manager_ref            => %i(uid_ems),
+        :custom_reconnect_block => hosts_custom_reconnect_block
+      )
+
+      if targeted?
+        builder.add_properties(:arel => manager.hosts.where(:ems_ref => manager_refs)) if manager_refs.present?
+      end
+
+      builder.add_builder_params(:ems_id => ->(persister) { persister.manager.id })
+    end
+  end
+
+  # group :hosts
+  def add_host_hardwares
+    add_collection(infra, :host_hardwares) do |builder|
+      if targeted? && manager_refs.present?
+        builder.add_properties(:arel => manager.host_hardwares.joins(:host).where('hosts' => {:ems_ref => manager_refs}))
+      end
+    end
+  end
+
+  # group :hosts
+  def add_host_networks
+    add_collection(infra, :host_networks) do |builder|
+      if targeted? && manager_refs.present?
+        builder.add_properties(:arel => manager.networks.joins(:hardware => :host).where(:hardware => {'hosts' => {:ems_ref => manager_refs}}))
+      end
+    end
+  end
+
+  # group :hosts
+  def add_host_operating_systems
+    add_collection(infra, :host_operating_systems) do |builder|
+      if targeted? && manager_refs.present?
+        builder.add_properties(:arel => ::OperatingSystem.joins(:host).where('hosts' => {:ems_ref => manager_refs}))
+      end
+    end
+  end
+
+  # group :hosts
+  def add_host_storages
+    add_collection(infra, :host_storages) do |builder|
+      if targeted? && manager_refs.present?
+        builder.add_properties(:arel => manager.host_storages.where(:ems_ref => manager_refs))
+      end
+    end
+  end
+
+  # group :hosts
+  def add_host_switches
+    add_collection(infra, :host_switches) do |builder|
+      if targeted? && manager_refs.present?
+        builder.add_properties(:arel => HostSwitch.joins(:host).where('hosts' => {:ems_ref => manager_refs}))
+      end
+    end
+  end
+
+  # ---
+
+  # Custom reconnect block for Hosts IC
+  def hosts_custom_reconnect_block
+    lambda do |inventory_collection, inventory_objects_index, attributes_index|
+      relation = inventory_collection.model_class.where(:ems_id => nil)
+
+      return if relation.count <= 0
+
+      inventory_objects_index.each_slice(100) do |batch|
+        relation.where(inventory_collection.manager_ref.first => batch.map(&:first)).each do |record|
+          index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record)
+
+          # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it
+          # would be sent for create.
+          inventory_object = inventory_objects_index.delete(index)
+          hash             = attributes_index.delete(index)
+
+          record.assign_attributes(hash.except(:id, :type))
+          if !inventory_collection.check_changed? || record.changed?
+            record.save!
+            inventory_collection.store_updated_records(record)
+          end
+
+          inventory_object.id = record.id
+        end
+      end
+    end
+  end
+end