From 3021bfc9fcfa9061885b26b4b90c691db621d897 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:17:46 +0100 Subject: [PATCH 01/15] AWS Inventory base class AWS Inventory base class, serving for defining connections to AWS API --- .../manageiq/providers/amazon/inventory.rb | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory.rb diff --git a/app/models/manageiq/providers/amazon/inventory.rb b/app/models/manageiq/providers/amazon/inventory.rb new file mode 100644 index 000000000..1153d950f --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory.rb @@ -0,0 +1,102 @@ +class ManageIQ::Providers::Amazon::Inventory + require_nested :Factory + require_nested :HashCollection + require_nested :Targets + + attr_reader :ems, :target, :inventory_collections, :options + + def initialize(ems, target) + @ems = ems + @target = target + @options = Settings.ems_refresh[ems.class.ems_type] + @inventory_collections = {:_inventory_collection => true} + + @known_flavors = Set.new + + initialize_inventory_collections + end + + def aws_ec2 + @aws_ec2 ||= ems.connect + end + + def aws_cloud_formation + @aws_cloud_formation ||= ems.connect(:service => :CloudFormation) + end + + def aws_elb + @aws_elb ||= ems.connect(:service => :ElasticLoadBalancing) + end + + def instances + [] + end + + def flavors + [] + end + + def availability_zones + [] + end + + def key_pairs + [] + end + + def private_images + [] + end + + def shared_images + [] + end + + def public_images + [] + end + + def stacks + [] + end + + def stack_resources(_stack_name) + [] + end + + def stack_template(_stack_name) + [] + end + + def cloud_networks + [] + end + + def cloud_subnets + [] + end + + def security_groups + [] + end + + def network_ports + [] + end + + def load_balancers + [] + end + + def health_check_members(load_balancer_name) + [] + end + + def floating_ips + [] + end + + def instances + [] + end +end From 2ec244f53c6e3d80569d63c06e70be6f62db4dc8 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:22:19 +0100 Subject: [PATCH 02/15] AWS inventory Factory AWS inventory Factory serving for getting a correct Target Inventory based on a target or EmsEvent. --- .../providers/amazon/inventory/factory.rb | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/factory.rb diff --git a/app/models/manageiq/providers/amazon/inventory/factory.rb b/app/models/manageiq/providers/amazon/inventory/factory.rb new file mode 100644 index 000000000..8e1002d16 --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/factory.rb @@ -0,0 +1,29 @@ +class ManageIQ::Providers::Amazon::Inventory::Factory + class << self + def inventory(ems, target) + if target.kind_of?(EmsEvent) + event_target(ems, target) + else + target(ems, target) + end + end + + def target(ems, target) + case target + when ManageIQ::Providers::Amazon::CloudManager + ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target) + when ManageIQ::Providers::Amazon::NetworkManager + ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager.new(ems, target) + when Vm + ManageIQ::Providers::Amazon::Inventory::Targets::Vm.new(ems, target) + end + end + + def event_target(ems, target) + case target[:full_data]["configurationItem"]["resourceType"] + when "AWS::EC2::Instance" + ManageIQ::Providers::Amazon::Inventory::Targets::EventPayloadVm.new(ems, target) + end + end + end +end From ed17c0fda690153f7b017800c98e3c5502543f8a Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:24:03 +0100 Subject: [PATCH 03/15] HashCollection wrapper for AWS APi results HashCollection wrapper for AWS API results allowing us to convert API result into general hash format, we use also when parsing event's payload. --- .../amazon/inventory/hash_collection.rb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/hash_collection.rb diff --git a/app/models/manageiq/providers/amazon/inventory/hash_collection.rb b/app/models/manageiq/providers/amazon/inventory/hash_collection.rb new file mode 100644 index 000000000..70950342b --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/hash_collection.rb @@ -0,0 +1,25 @@ +class ManageIQ::Providers::Amazon::Inventory::HashCollection + attr_reader :collection + + def initialize(collection) + @collection = collection + end + + def each + collection.each do |item| + item_data = item.respond_to?(:to_hash) ? item.to_hash : item.data.to_hash + yield(transform_keys(item_data)) + end + end + + def transform_keys(value) + case value + when Array + value.map { |x| transform_keys(x) } + when Hash + Hash[value.map { |k, v| [k.to_s.underscore, transform_keys(v)] }] + else + value + end + end +end From 2c4587552bea60da3978edac7668853416041fa6 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:25:55 +0100 Subject: [PATCH 04/15] A helper for default InventoryCollection attributes A helper for default InventoryCollection attributes, in most cases it's enough to change one attribute when creating InventoryCollection with a different source. --- .../inventory_collection_default_init_data.rb | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb diff --git a/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb new file mode 100644 index 000000000..fcee3f864 --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb @@ -0,0 +1,270 @@ +module ManageIQ::Providers::Amazon::Inventory::InventoryCollectionDefaultInitData + def init_data(model_class, attributes, extra_attributes) + init_data = { + :delete_method => model_class.new.respond_to?(:disconnect_inv) ? :disconnect_inv : nil + } + + init_data.merge!(attributes).merge!(extra_attributes) + return model_class, init_data + end + + def vms_init_data(extra_attributes = {}) + attributes = { + :association => :vms, + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::Vm, attributes, extra_attributes) + end + + def miq_templates_init_data(extra_attributes = {}) + attributes = { + :association => :miq_templates, + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::Template, attributes, extra_attributes) + end + + def availability_zones_init_data(extra_attributes = {}) + attributes = { + :association => :availability_zones, + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::AvailabilityZone, attributes, extra_attributes) + end + + def flavors_init_data(extra_attributes = {}) + attributes = { + :association => :flavors, + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::Flavor, attributes, extra_attributes) + end + + def key_pairs_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:name], + :association => :key_pairs + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::AuthKeyPair, attributes, extra_attributes) + end + + def hardwares_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:vm_or_template], + :association => :hardwares + } + + if extra_attributes[:strategy] == :local_db_cache_all + attributes[:custom_manager_uuid] = lambda do |hardware| + [hardware.vm_or_template.ems_ref] + end + end + + init_data(::Hardware, attributes, extra_attributes) + end + + def disks_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:hardware, :device_name], + :association => :disks + } + + if extra_attributes[:strategy] == :local_db_cache_all + attributes[:custom_manager_uuid] = lambda do |disk| + [disk.hardware.vm_or_template.ems_ref, disk.device_name] + end + end + + init_data(::Disk, attributes, extra_attributes) + end + + def networks_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:hardware, :description], + :association => :networks + } + + if extra_attributes[:strategy] == :local_db_cache_all + attributes[:custom_manager_uuid] = lambda do |network| + [network.hardware.vm_or_template.ems_ref, network.description] + end + end + + init_data(::Network, attributes, extra_attributes) + end + + def orchestration_stacks_init_data(extra_attributes = {}) + attributes = { + :association => :orchestration_stacks, + } + + init_data(::ManageIQ::Providers::Amazon::CloudManager::OrchestrationStack, attributes, extra_attributes) + end + + def orchestration_stacks_resources_init_data(extra_attributes = {}) + attributes = { + :association => :orchestration_stacks_resources, + } + + init_data(::OrchestrationStackResource, attributes, extra_attributes) + end + + def orchestration_stacks_outputs_init_data(extra_attributes = {}) + attributes = { + :association => :orchestration_stacks_outputs, + } + + init_data(::OrchestrationStackOutput, attributes, extra_attributes) + end + + def orchestration_stacks_parameters_init_data(extra_attributes = {}) + attributes = { + :association => :orchestration_stacks_parameters, + } + + init_data(::OrchestrationStackParameter, attributes, extra_attributes) + end + + def orchestration_templates_init_data(extra_attributes = {}) + # TODO(lsmola) do refactoring, we shouldn't need this custom saving block + orchestration_template_save_block = lambda do |_ems, inventory_collection| + hashes = inventory_collection.data.map(&:attributes) + + templates = ::OrchestrationTemplate.find_or_create_by_contents(hashes) + inventory_collection.data.zip(templates).each { |inventory_object, template| inventory_object.object = template } + end + + attributes = { + :association => :orchestration_templates, + :custom_save_block => orchestration_template_save_block + } + + init_data(::OrchestrationTemplateCfn, attributes, extra_attributes) + end + + def cloud_subnet_network_ports_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:address, :cloud_subnet, :network_port], + :association => :cloud_subnet_network_ports, + } + + init_data(::CloudSubnetNetworkPort, attributes, extra_attributes) + end + + def network_ports_init_data(extra_attributes = {}) + attributes = { + :association => :network_ports, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::NetworkPort, attributes, extra_attributes) + end + + def floating_ips_init_data(extra_attributes = {}) + attributes = { + :association => :floating_ips, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::FloatingIp, attributes, extra_attributes) + end + + def cloud_subnets_init_data(extra_attributes = {}) + attributes = { + :association => :cloud_subnets, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::CloudSubnet, attributes, extra_attributes) + end + + def cloud_networks_init_data(extra_attributes = {}) + attributes = { + :association => :cloud_networks, + } + + init_data(ManageIQ::Providers::Amazon::NetworkManager::CloudNetwork, attributes, extra_attributes) + end + + def security_groups_init_data(extra_attributes = {}) + attributes = { + :association => :security_groups, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::SecurityGroup, attributes, extra_attributes) + end + + def firewall_rules_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:resource, :source_security_group, :direction, :host_protocol, :port, :end_port, :source_ip_range], + :association => :firewall_rules, + } + + init_data(::FirewallRule, attributes, extra_attributes) + end + + def load_balancers_init_data(extra_attributes = {}) + attributes = { + :association => :load_balancers, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::LoadBalancer, attributes, extra_attributes) + end + + def load_balancer_pools_init_data(extra_attributes = {}) + attributes = { + :association => :load_balancer_pools, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerPool, attributes, extra_attributes) + end + + def load_balancer_pool_members_init_data(extra_attributes = {}) + attributes = { + :association => :load_balancer_pool_members, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerPoolMember, attributes, extra_attributes) + end + + def load_balancer_pool_member_pools_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:load_balancer_pool, :load_balancer_pool_member], + :association => :load_balancer_pool_member_pools, + } + + init_data(::LoadBalancerPoolMemberPool, attributes, extra_attributes) + end + + def load_balancer_listeners_init_data(extra_attributes = {}) + attributes = { + :association => :load_balancer_listeners, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerListener, attributes, extra_attributes) + end + + def load_balancer_listener_pools_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:load_balancer_listener, :load_balancer_pool], + :association => :load_balancer_listener_pools, + } + + init_data(::LoadBalancerListenerPool, attributes, extra_attributes) + end + + def load_balancer_health_checks_init_data(extra_attributes = {}) + attributes = { + :association => :load_balancer_health_checks, + } + + init_data(::ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerHealthCheck, attributes, extra_attributes) + end + + def load_balancer_health_check_members_init_data(extra_attributes = {}) + attributes = { + :manager_ref => [:load_balancer_health_check, :load_balancer_pool_member], + :association => :load_balancer_health_check_members, + } + + init_data(::LoadBalancerHealthCheckMember, attributes, extra_attributes) + end +end From f8df524c8973d1c3bc7f84a732cd6224fb93898d Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:27:35 +0100 Subject: [PATCH 05/15] A targets base class Contains a helper for adding an InventoryCollection into inventory. And a helper for parsing AWS event payload. --- .../providers/amazon/inventory/targets.rb | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/targets.rb diff --git a/app/models/manageiq/providers/amazon/inventory/targets.rb b/app/models/manageiq/providers/amazon/inventory/targets.rb new file mode 100644 index 000000000..93a9d4c7f --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/targets.rb @@ -0,0 +1,32 @@ +class ManageIQ::Providers::Amazon::Inventory::Targets < ManageIQ::Providers::Amazon::Inventory + include ManageIQ::Providers::Amazon::Inventory::InventoryCollectionDefaultInitData + + protected + + def initialize_inventory_collections + raise "initialize_inventory_collections must be defined in a subclass" + end + + def add_inventory_collection(inventory_collection_data, key = nil) + model_class, data = inventory_collection_data + data[:parent] ||= ems + + key ||= data[:association] + inventory_collections[key] = ::ManagerRefresh::InventoryCollection.new(model_class, data) + end + + def event_payload(event) + transform_keys(event[:full_data]["configurationItem"]["configuration"]) + end + + def transform_keys(value) + case value + when Array + value.map { |x| transform_keys(x) } + when Hash + Hash[value.map { |k, v| [k.to_s.underscore, transform_keys(v)] }] + else + value + end + end +end From 76814a2d1cfe7f7620628c463927ea42621e63cb Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:28:50 +0100 Subject: [PATCH 06/15] Inventory::Targets::CloudManager A target for refreshing all objects under a cloud manager, formerly known as a cloud manager full refresh. --- .../amazon/inventory/targets/cloud_manager.rb | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb diff --git a/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb new file mode 100644 index 000000000..f308188f9 --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb @@ -0,0 +1,65 @@ +class ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager < ManageIQ::Providers::Amazon::Inventory::Targets + def initialize_inventory_collections + add_inventory_collection(vms_init_data) + add_inventory_collection(miq_templates_init_data) + add_inventory_collection(hardwares_init_data) + add_inventory_collection(networks_init_data) + add_inventory_collection(disks_init_data) + add_inventory_collection(availability_zones_init_data) + add_inventory_collection(flavors_init_data) + add_inventory_collection(key_pairs_init_data) + add_inventory_collection(orchestration_stacks_init_data) + add_inventory_collection(orchestration_stacks_resources_init_data) + add_inventory_collection(orchestration_stacks_outputs_init_data) + add_inventory_collection(orchestration_stacks_parameters_init_data) + add_inventory_collection(orchestration_templates_init_data) + end + + def instances + HashCollection.new(aws_ec2.instances) + end + + def flavors + ManageIQ::Providers::Amazon::InstanceTypes.all + end + + def availability_zones + HashCollection.new(aws_ec2.client.describe_availability_zones[:availability_zones]) + end + + def key_pairs + HashCollection.new(aws_ec2.client.describe_key_pairs[:key_pairs]) + end + + def private_images + HashCollection.new(aws_ec2.client.describe_images(:owners => [:self], + :filters => [{:name => "image-type", + :values => ["machine"]}])[:images]) + end + + def shared_images + HashCollection.new(aws_ec2.client.describe_images(:executable_users => [:self], + :filters => [{:name => "image-type", + :values => ["machine"]}])[:images]) + end + + def public_images + filters = options.public_images_filters + HashCollection.new(aws_ec2.client.describe_images(:executable_users => [:all], + :filters => filters)[:images]) + end + + def stacks + HashCollection.new(aws_cloud_formation.client.describe_stacks[:stacks]) + end + + def stack_resources(stack_name) + stack_resources = aws_cloud_formation.client.list_stack_resources(:stack_name => stack_name).try(:stack_resource_summaries) + + HashCollection.new(stack_resources || []) + end + + def stack_template(stack_name) + aws_cloud_formation.client.get_template(:stack_name => stack_name).template_body + end +end From bf062d4ef01ca37f9c100e4f99c8a52ffe1e9ca4 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:31:31 +0100 Subject: [PATCH 07/15] Inventory::Targets::NetworkManager A target for refreshing all objects under a network manager, formerly known as a network manager full refresh. --- .../inventory/targets/network_manager.rb | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb diff --git a/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb new file mode 100644 index 000000000..d850dc4cf --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb @@ -0,0 +1,60 @@ +class ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager < ManageIQ::Providers::Amazon::Inventory::Targets + def initialize_inventory_collections + add_inventory_collection(cloud_subnet_network_ports_init_data) + add_inventory_collection(network_ports_init_data) + add_inventory_collection(floating_ips_init_data) + add_inventory_collection(cloud_subnets_init_data) + add_inventory_collection(cloud_networks_init_data) + add_inventory_collection(security_groups_init_data) + add_inventory_collection(firewall_rules_init_data) + add_inventory_collection(load_balancers_init_data) + add_inventory_collection(load_balancer_pools_init_data) + add_inventory_collection(load_balancer_pool_members_init_data) + add_inventory_collection(load_balancer_pool_member_pools_init_data) + add_inventory_collection(load_balancer_listeners_init_data) + add_inventory_collection(load_balancer_listener_pools_init_data) + add_inventory_collection(load_balancer_health_checks_init_data) + add_inventory_collection(load_balancer_health_check_members_init_data) + + add_inventory_collection(vms_init_data(:parent => ems.parent_manager, + :strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_stacks_init_data(:parent => ems.parent_manager, + :strategy => :local_db_cache_all)) + add_inventory_collection(availability_zones_init_data(:parent => ems.parent_manager, + :strategy => :local_db_cache_all)) + end + + def cloud_networks + HashCollection.new(aws_ec2.client.describe_vpcs[:vpcs]) + end + + def cloud_subnets + HashCollection.new(aws_ec2.client.describe_subnets[:subnets]) + end + + def security_groups + HashCollection.new(aws_ec2.security_groups) + end + + def network_ports + HashCollection.new(aws_ec2.client.describe_network_interfaces.network_interfaces) + end + + def load_balancers + HashCollection.new(aws_elb.client.describe_load_balancers.load_balancer_descriptions) + end + + def health_check_members(load_balancer_name) + HashCollection.new(aws_elb.client.describe_instance_health( + :load_balancer_name => load_balancer_name).instance_states) + end + + def floating_ips + HashCollection.new(aws_ec2.client.describe_addresses.addresses) + end + + def instances + # TODO(lsmola) do the filtering on the API side + HashCollection.new(aws_ec2.instances.select { |instance| instance.network_interfaces.blank? }) + end +end From 7b56b17f18ab3b6e13dd543026a0a1429fa47b61 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 6 Jan 2017 16:32:51 +0100 Subject: [PATCH 08/15] Inventory::Targets::EventPayloadVm EventPayloadVm server for refreshing of a VM based on AWS event payload. --- .../inventory/targets/event_payload_vm.rb | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb diff --git a/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb new file mode 100644 index 000000000..a514306c5 --- /dev/null +++ b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb @@ -0,0 +1,32 @@ +class ManageIQ::Providers::Amazon::Inventory::Targets::EventPayloadVm < ManageIQ::Providers::Amazon::Inventory::Targets + def initialize_inventory_collections + instance_ems_ref = event_payload(target)["instance_id"] + + add_inventory_collection( + vms_init_data( + :arel => ems.vms.where(:ems_ref => instance_ems_ref))) + add_inventory_collection( + hardwares_init_data( + :arel => ems.hardwares.joins(:vm_or_template).where(:vms => {:ems_ref => instance_ems_ref}))) + add_inventory_collection( + disks_init_data( + :arel => ems.disks.joins(:hardware => :vm_or_template).where(:hardware => {:vms => {:ems_ref => instance_ems_ref}}))) + add_inventory_collection( + networks_init_data( + :arel => ems.networks.joins(:hardware => :vm_or_template).where(:hardware => {:vms => {:ems_ref => instance_ems_ref}}))) + + add_inventory_collection(flavors_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(miq_templates_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(availability_zones_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(key_pairs_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_stacks_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_stacks_resources_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_stacks_outputs_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_stacks_parameters_init_data(:strategy => :local_db_cache_all)) + add_inventory_collection(orchestration_templates_init_data(:strategy => :local_db_cache_all)) + end + + def instances + [event_payload(target)] + end +end From 853faeeb230464a32e025d5c3c05a2f978dcd9f3 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 9 Jan 2017 21:28:23 +0100 Subject: [PATCH 09/15] Helpers for adding multiple inventory collections Helpers for adding multiple inventory collections --- .../providers/amazon/inventory/targets.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/models/manageiq/providers/amazon/inventory/targets.rb b/app/models/manageiq/providers/amazon/inventory/targets.rb index 93a9d4c7f..98d3420c5 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets.rb @@ -15,6 +15,25 @@ def add_inventory_collection(inventory_collection_data, key = nil) inventory_collections[key] = ::ManagerRefresh::InventoryCollection.new(model_class, data) end + def add_inventory_collections(inventory_collections, inventory_collections_data = {}) + inventory_collections.each do |inventory_collection| + add_inventory_collection(send("#{inventory_collection}_init_data", inventory_collections_data)) + end + end + + def add_remaining_inventory_collections(inventory_collections_data) + # Get names of all inventory collections defined in InventoryCollectionDefaultInitData + all_inventory_collections = ManageIQ::Providers::Amazon::Inventory::InventoryCollectionDefaultInitData. + public_instance_methods.grep(/.+_init_data/).map { |x| x.to_s.gsub("_init_data", "") } + + # Get names of all defined inventory_collections + defined_inventory_collections = inventory_collections.keys.map(&:to_s) + + # Add all missing inventory_collections with defined init_data + add_inventory_collections(all_inventory_collections - defined_inventory_collections, + inventory_collections_data) + end + def event_payload(event) transform_keys(event[:full_data]["configurationItem"]["configuration"]) end From a5746d7f4d0924a0658482dd7b9019f060ac7a44 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 9 Jan 2017 21:30:16 +0100 Subject: [PATCH 10/15] Use helper for adding multiple inventory collections for targets Use helper for adding multiple inventory collections for targets --- .../amazon/inventory/targets/cloud_manager.rb | 16 ++-------- .../inventory/targets/event_payload_vm.rb | 13 ++------ .../inventory/targets/network_manager.rb | 31 ++++++------------- 3 files changed, 15 insertions(+), 45 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb index f308188f9..29ef627c5 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets/cloud_manager.rb @@ -1,18 +1,8 @@ class ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager < ManageIQ::Providers::Amazon::Inventory::Targets def initialize_inventory_collections - add_inventory_collection(vms_init_data) - add_inventory_collection(miq_templates_init_data) - add_inventory_collection(hardwares_init_data) - add_inventory_collection(networks_init_data) - add_inventory_collection(disks_init_data) - add_inventory_collection(availability_zones_init_data) - add_inventory_collection(flavors_init_data) - add_inventory_collection(key_pairs_init_data) - add_inventory_collection(orchestration_stacks_init_data) - add_inventory_collection(orchestration_stacks_resources_init_data) - add_inventory_collection(orchestration_stacks_outputs_init_data) - add_inventory_collection(orchestration_stacks_parameters_init_data) - add_inventory_collection(orchestration_templates_init_data) + add_inventory_collections(%i(vms miq_templates hardwares networks disks availability_zones availability_zones + flavors key_pairs orchestration_stacks orchestration_stacks_resources + orchestration_stacks_outputs orchestration_stacks_parameters orchestration_templates)) end def instances diff --git a/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb index a514306c5..111014ce9 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb @@ -7,7 +7,8 @@ def initialize_inventory_collections :arel => ems.vms.where(:ems_ref => instance_ems_ref))) add_inventory_collection( hardwares_init_data( - :arel => ems.hardwares.joins(:vm_or_template).where(:vms => {:ems_ref => instance_ems_ref}))) + :arel => ems.hardwares.joins(:vm_or_template).where(:vms => {:ems_ref => instance_ems_ref}), + :strategy => :find_missing_in_local_db)) add_inventory_collection( disks_init_data( :arel => ems.disks.joins(:hardware => :vm_or_template).where(:hardware => {:vms => {:ems_ref => instance_ems_ref}}))) @@ -15,15 +16,7 @@ def initialize_inventory_collections networks_init_data( :arel => ems.networks.joins(:hardware => :vm_or_template).where(:hardware => {:vms => {:ems_ref => instance_ems_ref}}))) - add_inventory_collection(flavors_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(miq_templates_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(availability_zones_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(key_pairs_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_stacks_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_stacks_resources_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_stacks_outputs_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_stacks_parameters_init_data(:strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_templates_init_data(:strategy => :local_db_cache_all)) + add_remaining_inventory_collections(:strategy => :local_db_find_one) end def instances diff --git a/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb index d850dc4cf..cd2315446 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb @@ -1,27 +1,14 @@ class ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager < ManageIQ::Providers::Amazon::Inventory::Targets def initialize_inventory_collections - add_inventory_collection(cloud_subnet_network_ports_init_data) - add_inventory_collection(network_ports_init_data) - add_inventory_collection(floating_ips_init_data) - add_inventory_collection(cloud_subnets_init_data) - add_inventory_collection(cloud_networks_init_data) - add_inventory_collection(security_groups_init_data) - add_inventory_collection(firewall_rules_init_data) - add_inventory_collection(load_balancers_init_data) - add_inventory_collection(load_balancer_pools_init_data) - add_inventory_collection(load_balancer_pool_members_init_data) - add_inventory_collection(load_balancer_pool_member_pools_init_data) - add_inventory_collection(load_balancer_listeners_init_data) - add_inventory_collection(load_balancer_listener_pools_init_data) - add_inventory_collection(load_balancer_health_checks_init_data) - add_inventory_collection(load_balancer_health_check_members_init_data) - - add_inventory_collection(vms_init_data(:parent => ems.parent_manager, - :strategy => :local_db_cache_all)) - add_inventory_collection(orchestration_stacks_init_data(:parent => ems.parent_manager, - :strategy => :local_db_cache_all)) - add_inventory_collection(availability_zones_init_data(:parent => ems.parent_manager, - :strategy => :local_db_cache_all)) + add_inventory_collections( + %i(cloud_subnet_network_ports network_ports floating_ips cloud_subnets cloud_networks security_groups + firewall_rules load_balancers load_balancer_pools load_balancer_pool_members load_balancer_pool_member_pools + load_balancer_listeners load_balancer_listener_pools load_balancer_health_checks + load_balancer_health_check_members)) + + add_inventory_collections(%i(vms orchestration_stack availability_zones), + :parent => ems.parent_manager, + :strategy => :local_db_cache_all) end def cloud_networks From b6ce29e867a136e00443a4899148bb4b22a0703c Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 9 Jan 2017 21:31:22 +0100 Subject: [PATCH 11/15] Add hardware's custom finder and vm's check_changed Add hardware's custom finder and vm's check_changed --- .../inventory_collection_default_init_data.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb index fcee3f864..cded29204 100644 --- a/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb +++ b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb @@ -10,7 +10,13 @@ def init_data(model_class, attributes, extra_attributes) def vms_init_data(extra_attributes = {}) attributes = { - :association => :vms, + :association => :vms, + # TODO(lsmola) replace genealogy_parent with db relation and do computing of genealogyparent in a separate queued + # job. Then we can check for changed? again, otherwise the after save hood on vmOrTemplate will not fire and + # the association to genealogy_parent will not populate, if list of templates changed (like allowing collection + # of public images ) + :check_changed => false + } init_data(::ManageIQ::Providers::Amazon::CloudManager::Vm, attributes, extra_attributes) @@ -55,10 +61,16 @@ def hardwares_init_data(extra_attributes = {}) :association => :hardwares } - if extra_attributes[:strategy] == :local_db_cache_all + case extra_attributes[:strategy] + when :local_db_cache_all attributes[:custom_manager_uuid] = lambda do |hardware| [hardware.vm_or_template.ems_ref] end + when :find_missing_in_local_db + attributes[:custom_db_finder] = lambda do |inventory_collection, hash_uuid_by_ref| + inventory_collection.parent.send(inventory_collection.association).references(:vm_or_template).where( + :vms => {:ems_ref => hash_uuid_by_ref[:vm_or_template]}).first + end end init_data(::Hardware, attributes, extra_attributes) From 656eb0250b1309376426ecd550e474538e41848e Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 9 Jan 2017 21:41:14 +0100 Subject: [PATCH 12/15] Fix rubocop issues Fix rubocop issues --- app/models/manageiq/providers/amazon/inventory.rb | 6 +----- .../inventory/inventory_collection_default_init_data.rb | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory.rb b/app/models/manageiq/providers/amazon/inventory.rb index 1153d950f..9ac0b527a 100644 --- a/app/models/manageiq/providers/amazon/inventory.rb +++ b/app/models/manageiq/providers/amazon/inventory.rb @@ -88,15 +88,11 @@ def load_balancers [] end - def health_check_members(load_balancer_name) + def health_check_members(_load_balancer_name) [] end def floating_ips [] end - - def instances - [] - end end diff --git a/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb index cded29204..4ff562ed8 100644 --- a/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb +++ b/app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb @@ -16,7 +16,6 @@ def vms_init_data(extra_attributes = {}) # the association to genealogy_parent will not populate, if list of templates changed (like allowing collection # of public images ) :check_changed => false - } init_data(::ManageIQ::Providers::Amazon::CloudManager::Vm, attributes, extra_attributes) @@ -196,7 +195,7 @@ def cloud_networks_init_data(extra_attributes = {}) init_data(ManageIQ::Providers::Amazon::NetworkManager::CloudNetwork, attributes, extra_attributes) end - def security_groups_init_data(extra_attributes = {}) + def security_groups_init_data(extra_attributes = {}) attributes = { :association => :security_groups, } From cf018628018f5340bf2507a7988e7d41d3f2814d Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Tue, 10 Jan 2017 09:24:53 +0100 Subject: [PATCH 13/15] NetworkManager target fix, typo in stacks name NetworkManager target fix, typo in stacks name --- .../providers/amazon/inventory/targets/network_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb index cd2315446..19bc29afb 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets/network_manager.rb @@ -6,7 +6,7 @@ def initialize_inventory_collections load_balancer_listeners load_balancer_listener_pools load_balancer_health_checks load_balancer_health_check_members)) - add_inventory_collections(%i(vms orchestration_stack availability_zones), + add_inventory_collections(%i(vms orchestration_stacks availability_zones), :parent => ems.parent_manager, :strategy => :local_db_cache_all) end From 488ac7c374b7983709bce8598b9a8525dd09f097 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Wed, 11 Jan 2017 13:05:10 +0100 Subject: [PATCH 14/15] Use resourceId so the code works also for instance delete Use resourceId so the code works also for instance delete --- .../providers/amazon/inventory/targets/event_payload_vm.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb index 111014ce9..034e59ad5 100644 --- a/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb +++ b/app/models/manageiq/providers/amazon/inventory/targets/event_payload_vm.rb @@ -1,6 +1,6 @@ class ManageIQ::Providers::Amazon::Inventory::Targets::EventPayloadVm < ManageIQ::Providers::Amazon::Inventory::Targets def initialize_inventory_collections - instance_ems_ref = event_payload(target)["instance_id"] + instance_ems_ref = target[:full_data]["configurationItem"]["resourceId"] add_inventory_collection( vms_init_data( @@ -20,6 +20,6 @@ def initialize_inventory_collections end def instances - [event_payload(target)] + [event_payload(target)].compact end end From f1d228c86e5a8cb58d1e8818b1eef9c8c57ea176 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Wed, 11 Jan 2017 13:05:48 +0100 Subject: [PATCH 15/15] Add default targets into the factory Add default targets into the factory, in the case that target is noit recognized. --- app/models/manageiq/providers/amazon/inventory/factory.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory/factory.rb b/app/models/manageiq/providers/amazon/inventory/factory.rb index 8e1002d16..400b360d5 100644 --- a/app/models/manageiq/providers/amazon/inventory/factory.rb +++ b/app/models/manageiq/providers/amazon/inventory/factory.rb @@ -2,7 +2,7 @@ class ManageIQ::Providers::Amazon::Inventory::Factory class << self def inventory(ems, target) if target.kind_of?(EmsEvent) - event_target(ems, target) + event_target(ems, target) || target(ems, ems) else target(ems, target) end @@ -14,8 +14,8 @@ def target(ems, target) ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target) when ManageIQ::Providers::Amazon::NetworkManager ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager.new(ems, target) - when Vm - ManageIQ::Providers::Amazon::Inventory::Targets::Vm.new(ems, target) + else + ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target) end end