diff --git a/app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb b/app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb index d404348376..ced0141318 100644 --- a/app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb +++ b/app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb @@ -1,247 +1,324 @@ module ManageIQ::Providers::Kubernetes::ContainerManager::InventoryCollections - def initialize_inventory_collections(ems, _options) + def targeted + false + end + + def strategy + nil + end + + def shared_options + settings_options = options[:inventory_collections].try(:to_hash) || {} + + settings_options.merge( + :strategy => strategy, + :targeted => targeted, + ) + end + + def initialize_inventory_collections # TODO: Targeted refreshes will require adjusting the associations / arels. (duh) - @inv_collections = {} - @inv_collections[:container_projects] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerProject, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_projects, - :secondary_refs => {:by_name => [:name]}, - :delete_method => :disconnect_inv, + @collections[:container_projects] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerProject, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_projects, + :secondary_refs => {:by_name => [:name]}, + :delete_method => :disconnect_inv, + ) ) - initialize_custom_attributes_collections(ems.container_projects, %w(labels additional_attributes)) + initialize_custom_attributes_collections(manager.container_projects, %w(labels additional_attributes)) - @inv_collections[:container_quotas] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerQuota, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_quotas, - :attributes_blacklist => [:namespace], + @collections[:container_quotas] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerQuota, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_quotas, + :attributes_blacklist => [:namespace], + ) ) - @inv_collections[:container_quota_items] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerQuotaItem, - :parent => ems, - :association => :container_quota_items, - :manager_ref => [:container_quota, :resource], + @collections[:container_quota_items] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerQuotaItem, + :parent => manager, + :association => :container_quota_items, + :manager_ref => [:container_quota, :resource], + ) ) - @inv_collections[:container_limits] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerLimit, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_limits, - :attributes_blacklist => [:namespace], + @collections[:container_limits] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerLimit, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_limits, + :attributes_blacklist => [:namespace], + ) ) - @inv_collections[:container_limit_items] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerLimitItem, - :parent => ems, - :association => :container_limit_items, - :manager_ref => [:container_limit, :resource, :item_type], + @collections[:container_limit_items] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerLimitItem, + :parent => manager, + :association => :container_limit_items, + :manager_ref => [:container_limit, :resource, :item_type], + ) ) - @inv_collections[:container_nodes] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerNode, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_nodes, - :secondary_refs => {:by_name => [:name]}, + @collections[:container_nodes] = ::ManagerRefresh::InventoryCollection.new( + shared_options.merge( + :model_class => ContainerNode, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_nodes, + :secondary_refs => {:by_name => [:name]}, + ) ) - initialize_container_conditions_collection(ems.container_nodes) - initialize_custom_attributes_collections(ems.container_nodes, %w(labels additional_attributes)) + initialize_container_conditions_collection(manager.container_nodes) + initialize_custom_attributes_collections(manager.container_nodes, %w(labels additional_attributes)) # polymorphic child of ContainerNode & ContainerImage, # but refresh only sets it on nodes. - @inv_collections[:computer_systems] = + @collections[:computer_systems] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ComputerSystem, - :parent => ems, - :association => :computer_systems, - :manager_ref => [:managed_entity], + shared_options.merge( + :model_class => ComputerSystem, + :parent => manager, + :association => :computer_systems, + :manager_ref => [:managed_entity], + ) ) - @inv_collections[:computer_system_hardwares] = + @collections[:computer_system_hardwares] = ::ManagerRefresh::InventoryCollection.new( - :model_class => Hardware, - :parent => ems, - :association => :computer_system_hardwares, - :manager_ref => [:computer_system], + shared_options.merge( + :model_class => Hardware, + :parent => manager, + :association => :computer_system_hardwares, + :manager_ref => [:computer_system], + ) ) - @inv_collections[:computer_system_operating_systems] = + @collections[:computer_system_operating_systems] = ::ManagerRefresh::InventoryCollection.new( - :model_class => OperatingSystem, - :parent => ems, - :association => :computer_system_operating_systems, - :manager_ref => [:computer_system], + shared_options.merge( + :model_class => OperatingSystem, + :parent => manager, + :association => :computer_system_operating_systems, + :manager_ref => [:computer_system], + ) ) - @inv_collections[:container_image_registries] = + @collections[:container_image_registries] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerImageRegistry, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_image_registries, - :manager_ref => [:host, :port], + shared_options.merge( + :model_class => ContainerImageRegistry, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_image_registries, + :manager_ref => [:host, :port], + ) ) - @inv_collections[:container_images] = + @collections[:container_images] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerImage, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_images, - # TODO: old save matches on [:image_ref, :container_image_registry_id] - # TODO: should match on digest when available - :manager_ref => [:image_ref], - :delete_method => :disconnect_inv, + shared_options.merge( + :model_class => ContainerImage, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_images, + # TODO: old save matches on [:image_ref, :container_image_registry_id] + # TODO: should match on digest when available + :manager_ref => [:image_ref], + :delete_method => :disconnect_inv, + :use_ar_object => true, # several serialized attributes + ) ) # images have custom_attributes but that's done conditionally in openshift parser - @inv_collections[:container_groups] = + @collections[:container_groups] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerGroup, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_groups, - :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, - :attributes_blacklist => [:namespace], - :delete_method => :disconnect_inv, + shared_options.merge( + :model_class => ContainerGroup, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_groups, + :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + :attributes_blacklist => [:namespace], + :delete_method => :disconnect_inv, + ) ) - initialize_container_conditions_collection(ems.container_groups) - initialize_custom_attributes_collections(ems.container_groups, %w(labels node_selectors)) - @inv_collections[:container_volumes] = + initialize_container_conditions_collection(manager.container_groups) + initialize_custom_attributes_collections(manager.container_groups, %w(labels node_selectors)) + @collections[:container_volumes] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerVolume, - :parent => ems, - :association => :container_volumes, - :manager_ref => [:parent, :name], + shared_options.merge( + :model_class => ContainerVolume, + :parent => manager, + :association => :container_volumes, + :manager_ref => [:parent, :name], + ) ) - @inv_collections[:containers] = + @collections[:containers] = ::ManagerRefresh::InventoryCollection.new( - :model_class => Container, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :containers, - # parser sets :ems_ref => "#{pod_id}_#{container.name}_#{container.image}" - :delete_method => :disconnect_inv, + shared_options.merge( + :model_class => Container, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :containers, + # parser sets :ems_ref => "#{pod_id}_#{container.name}_#{container.image}" + :delete_method => :disconnect_inv, + ) ) - @inv_collections[:container_port_configs] = + @collections[:container_port_configs] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerPortConfig, - :parent => ems, - :association => :container_port_configs, - # parser sets :ems_ref => "#{pod_id}_#{container_name}_#{port_config.containerPort}_#{port_config.hostPort}_#{port_config.protocol}" + shared_options.merge( + :model_class => ContainerPortConfig, + :parent => manager, + :association => :container_port_configs, + # parser sets :ems_ref => "#{pod_id}_#{container_name}_#{port_config.containerPort}_#{port_config.hostPort}_#{port_config.protocol}" + ) ) - @inv_collections[:container_env_vars] = + @collections[:container_env_vars] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerEnvVar, - :parent => ems, - :association => :container_env_vars, - # TODO: old save matches on all :name, :value, :field_path - does this matter? - :manager_ref => [:container, :name], + shared_options.merge( + :model_class => ContainerEnvVar, + :parent => manager, + :association => :container_env_vars, + # TODO: old save matches on all :name, :value, :field_path - does this matter? + :manager_ref => [:container, :name], + ) ) - @inv_collections[:security_contexts] = + @collections[:security_contexts] = ::ManagerRefresh::InventoryCollection.new( - :model_class => SecurityContext, - :parent => ems, - :association => :security_contexts, - :manager_ref => [:resource], + shared_options.merge( + :model_class => SecurityContext, + :parent => manager, + :association => :security_contexts, + :manager_ref => [:resource], + ) ) - @inv_collections[:container_replicators] = + @collections[:container_replicators] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerReplicator, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_replicators, - :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, - :attributes_blacklist => [:namespace], + shared_options.merge( + :model_class => ContainerReplicator, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_replicators, + :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + :attributes_blacklist => [:namespace], + ) ) - initialize_custom_attributes_collections(ems.container_replicators, %w(labels selectors)) + initialize_custom_attributes_collections(manager.container_replicators, %w(labels selectors)) - @inv_collections[:container_services] = + @collections[:container_services] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerService, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_services, - :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, - :attributes_blacklist => [:namespace], + shared_options.merge( + :model_class => ContainerService, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_services, + :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + :attributes_blacklist => [:namespace], + :saver_strategy => :default # TODO(perf) Can't use batch strategy because of usage of M:N container_groups relation + ) ) - initialize_custom_attributes_collections(ems.container_services, %w(labels selectors)) - @inv_collections[:container_service_port_configs] = + initialize_custom_attributes_collections(manager.container_services, %w(labels selectors)) + @collections[:container_service_port_configs] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerServicePortConfig, - :parent => ems, - :association => :container_service_port_configs, + shared_options.merge( + :model_class => ContainerServicePortConfig, + :parent => manager, + :association => :container_service_port_configs, + :manager_ref => [:ems_ref, :protocol] # TODO(lsmola) make protocol part of the ems_ref? + ) ) - @inv_collections[:container_routes] = + @collections[:container_routes] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerRoute, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_routes, - :attributes_blacklist => [:namespace], + shared_options.merge( + :model_class => ContainerRoute, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_routes, + :attributes_blacklist => [:namespace, :tags], + ) ) - initialize_custom_attributes_collections(ems.container_routes, %w(labels)) + initialize_custom_attributes_collections(manager.container_routes, %w(labels)) - @inv_collections[:container_component_statuses] = + @collections[:container_component_statuses] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerComponentStatus, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_component_statuses, - :manager_ref => [:name], + shared_options.merge( + :model_class => ContainerComponentStatus, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_component_statuses, + :manager_ref => [:name], + :use_ar_object => true, # TODO(perf) :message has # stringified + ) ) - @inv_collections[:container_templates] = + @collections[:container_templates] = ::ManagerRefresh::InventoryCollection.new( :model_class => ContainerTemplate, - :parent => ems, - :builder_params => {:ems_id => ems.id}, + :parent => manager, + :builder_params => {:ems_id => manager.id}, :association => :container_templates, :attributes_blacklist => [:namespace], ) - initialize_custom_attributes_collections(ems.container_templates, %w(labels)) - @inv_collections[:container_template_parameters] = + initialize_custom_attributes_collections(manager.container_templates, %w(labels)) + @collections[:container_template_parameters] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerTemplateParameter, - :parent => ems, - :association => :container_template_parameters, - :manager_ref => [:container_template, :name], + shared_options.merge( + :model_class => ContainerTemplateParameter, + :parent => manager, + :association => :container_template_parameters, + :manager_ref => [:container_template, :name], + ) ) - @inv_collections[:container_builds] = + @collections[:container_builds] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerBuild, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_builds, - :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + shared_options.merge( + :model_class => ContainerBuild, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_builds, + :attributes_blacklist => [:tags], + :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + ) ) - initialize_custom_attributes_collections(ems.container_builds, %w(labels)) - @inv_collections[:container_build_pods] = + initialize_custom_attributes_collections(manager.container_builds, %w(labels)) + @collections[:container_build_pods] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerBuildPod, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :container_build_pods, - # TODO: convert namespace column -> container_project_id? - :manager_ref => [:namespace, :name], - :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + shared_options.merge( + :model_class => ContainerBuildPod, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :container_build_pods, + # TODO: convert namespace column -> container_project_id? + :manager_ref => [:namespace, :name], + :secondary_refs => {:by_namespace_and_name => [:namespace, :name]}, + ) ) - initialize_custom_attributes_collections(ems.container_build_pods, %w(labels)) + initialize_custom_attributes_collections(manager.container_build_pods, %w(labels)) - @inv_collections[:persistent_volumes] = + @collections[:persistent_volumes] = ::ManagerRefresh::InventoryCollection.new( - :model_class => PersistentVolume, - :parent => ems, - :builder_params => {:parent => ems}, - :association => :persistent_volumes, + shared_options.merge( + :model_class => PersistentVolume, + :parent => manager, + :builder_params => {:parent => manager}, + :association => :persistent_volumes, + :use_ar_object => true # serialized :storage attr + ) ) - @inv_collections[:persistent_volume_claims] = + @collections[:persistent_volume_claims] = ::ManagerRefresh::InventoryCollection.new( - :model_class => PersistentVolumeClaim, - :parent => ems, - :builder_params => {:ems_id => ems.id}, - :association => :persistent_volume_claims, + shared_options.merge( + :model_class => PersistentVolumeClaim, + :parent => manager, + :builder_params => {:ems_id => manager.id}, + :association => :persistent_volume_claims, + :use_ar_object => true # serialized :capacity attr + ) ) end @@ -251,25 +328,31 @@ def initialize_container_conditions_collection(relation) :container_entity_type => relation.model.name, :container_entity_id => relation, # nested SELECT. TODO: compare to a JOIN. ) - @inv_collections[[:container_conditions_for, relation.model.name]] = + @collections[[:container_conditions_for, relation.model.name]] = ::ManagerRefresh::InventoryCollection.new( - :model_class => ContainerCondition, - :arel => query, - :manager_ref => [:container_entity, :name], + shared_options.merge( + :model_class => ContainerCondition, + :arel => query, + :manager_ref => [:container_entity, :name], + ) ) end # CustomAttribute is polymorphic child of many models def initialize_custom_attributes_collections(relation, sections) sections.each do |section| - query = CustomAttribute.where(:resource_type => relation.model.name, - :resource_id => relation, - :section => section.to_s) - @inv_collections[[:custom_attributes_for, relation.model.name, section.to_s]] = + query = CustomAttribute.where( + :resource_type => relation.model.name, + :resource_id => relation, + :section => section.to_s + ) + @collections[[:custom_attributes_for, relation.model.name, section.to_s]] = ::ManagerRefresh::InventoryCollection.new( - :model_class => CustomAttribute, - :arel => query, - :manager_ref => [:resource, :section, :name], + shared_options.merge( + :model_class => CustomAttribute, + :arel => query, + :manager_ref => [:resource, :section, :name], + ) ) end end diff --git a/app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb b/app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb index 27a4b6272a..da2495240c 100644 --- a/app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb +++ b/app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb @@ -4,7 +4,6 @@ module ManageIQ::Providers::Kubernetes class ContainerManager::RefreshParser include Vmdb::Logging include ContainerManager::EntitiesMapping - include ContainerManager::InventoryCollections def self.ems_inv_to_hashes(inventory, options = Config::Options.new) new(options).ems_inv_to_hashes(inventory, options) @@ -41,8 +40,14 @@ def ems_inv_to_hashes(inventory, _options = Config::Options.new) @data end + def persister_class + ManageIQ::Providers::Kubernetes::Inventory::Persister::ContainerManager + end + def ems_inv_to_inv_collections(ems, inventory, options = Config::Options.new) - initialize_inventory_collections(ems, options) + persister = persister_class.new(ems) + # TODO expose Persistor and use that + @inv_collections = persister.collections ems_inv_populate_collections(inventory, options) @@ -52,7 +57,7 @@ def ems_inv_to_inv_collections(ems, inventory, options = Config::Options.new) get_container_image_registries_graph # Returning an array triggers ManagerRefresh::SaveInventory code path. - @inv_collections.values + persister.inventory_collections end def ems_inv_populate_collections(inventory, _options) diff --git a/app/models/manageiq/providers/kubernetes/inventory.rb b/app/models/manageiq/providers/kubernetes/inventory.rb new file mode 100644 index 0000000000..512f68cd52 --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/inventory.rb @@ -0,0 +1,3 @@ +class ManageIQ::Providers::Kubernetes::Inventory < ManagerRefresh::Inventory + require_nested :Persister +end diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister.rb b/app/models/manageiq/providers/kubernetes/inventory/persister.rb new file mode 100644 index 0000000000..5cf7002ec9 --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/inventory/persister.rb @@ -0,0 +1,3 @@ +class ManageIQ::Providers::Kubernetes::Inventory::Persister < ManagerRefresh::Inventory::Persister + require_nested :ContainerManager +end diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/persister/container_manager.rb new file mode 100644 index 0000000000..9d0f44277a --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/inventory/persister/container_manager.rb @@ -0,0 +1,3 @@ +class ManageIQ::Providers::Kubernetes::Inventory::Persister::ContainerManager < ManageIQ::Providers::Kubernetes::Inventory::Persister + include ManageIQ::Providers::Kubernetes::ContainerManager::InventoryCollections +end diff --git a/spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb b/spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb index 115acf63cc..d733bfc33e 100644 --- a/spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb +++ b/spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb @@ -526,7 +526,26 @@ def assert_disconnected(object) expect(ManageIQ::Providers::Kubernetes::ContainerManager::RefreshParser).not_to receive(:ems_inv_to_hashes) end - # TODO: pending graph tag mapping implementation - include_examples "kubernetes refresher VCR tests", :check_tag_mapping => false + context "with :default saver" do + before(:each) do + stub_settings_merge( + :ems_refresh => {:kubernetes => {:inventory_collections => {:saver_strategy => :default}}} + ) + end + + # TODO: pending graph tag mapping implementation + include_examples "kubernetes refresher VCR tests", :check_tag_mapping => false + end + + context "with :batch saver" do + before(:each) do + stub_settings_merge( + :ems_refresh => {:kubernetes => {:inventory_collections => {:saver_strategy => :batch}}} + ) + end + + # TODO: pending graph tag mapping implementation + include_examples "kubernetes refresher VCR tests", :check_tag_mapping => false + end end end