From 1ceaf4e49efe7135b0fb8678e39fac049748a7cb Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 25 Jul 2019 14:59:42 -0400 Subject: [PATCH] Cleanup legacy refresh parsers and settings Remove the config to select graph or legacy refresh, only allow graph refresh, delete old code. --- .../providers/amazon/cloud_manager.rb | 9 +- .../amazon/cloud_manager/refresh_parser.rb | 425 -------------- .../providers/amazon/network_manager.rb | 5 +- .../amazon/network_manager/refresh_parser.rb | 538 ------------------ .../providers/amazon/storage_manager/ebs.rb | 5 +- .../storage_manager/ebs/refresh_parser.rb | 147 ----- .../amazon/storage_manager/ebs/refresher.rb | 7 - .../providers/amazon/storage_manager/s3.rb | 5 +- .../storage_manager/s3/refresh_parser.rb | 124 ---- .../amazon/storage_manager/s3/refresher.rb | 10 - config/settings.yml | 5 - .../amazon/aws_refresher_spec_common.rb | 6 - .../cloud_manager/refresh_parser_spec.rb | 52 -- .../specific_stubbed_refresher_spec.rb | 3 +- .../cloud_manager/stubbed_refresher_spec.rb | 3 +- .../vcr_specs/refresher_other_region_spec.rb | 266 --------- .../cloud_manager/vcr_specs/refresher_spec.rb | 50 -- .../network_manager/refresh_parser_spec.rb | 65 --- .../network_manager/stubbed_refresher_spec.rb | 3 +- .../ebs/refresh_parser_spec.rb | 31 - .../ebs/stubbed_refresher_spec.rb | 3 +- .../s3/stubbed_refresher_spec.rb | 3 +- 22 files changed, 25 insertions(+), 1740 deletions(-) delete mode 100644 app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb delete mode 100644 app/models/manageiq/providers/amazon/network_manager/refresh_parser.rb delete mode 100644 app/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser.rb delete mode 100644 app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser.rb delete mode 100644 spec/models/manageiq/providers/amazon/cloud_manager/refresh_parser_spec.rb delete mode 100644 spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_other_region_spec.rb delete mode 100644 spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb delete mode 100644 spec/models/manageiq/providers/amazon/network_manager/refresh_parser_spec.rb delete mode 100644 spec/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser_spec.rb diff --git a/app/models/manageiq/providers/amazon/cloud_manager.rb b/app/models/manageiq/providers/amazon/cloud_manager.rb index 130f5db57..073527fdd 100644 --- a/app/models/manageiq/providers/amazon/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/cloud_manager.rb @@ -11,7 +11,6 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana require_nested :OrchestrationTemplate require_nested :Provision require_nested :ProvisionWorkflow - require_nested :RefreshParser require_nested :RefreshWorker require_nested :Refresher require_nested :Template @@ -131,6 +130,14 @@ def supported_catalog_types %w(amazon).freeze end + def inventory_object_refresh? + true + end + + def allow_targeted_refresh? + true + end + # # Operations # diff --git a/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb b/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb deleted file mode 100644 index fe80bdbf5..000000000 --- a/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb +++ /dev/null @@ -1,425 +0,0 @@ -# TODO: Separate collection from parsing (perhaps collecting in parallel a la RHEVM) - -class ManageIQ::Providers::Amazon::CloudManager::RefreshParser < ManageIQ::Providers::CloudManager::RefreshParser - include ManageIQ::Providers::Amazon::RefreshHelperMethods - - def initialize(ems, options = Config::Options.new) - @ems = ems - @aws_ec2 = ems.connect - @aws_cloud_formation = ems.connect(:service => :CloudFormation) - @data = {} - @data_index = {} - @known_flavors = Set.new - @options = options - @tag_mapper = ContainerLabelTagMapping.mapper - @data[:tag_mapper] = @tag_mapper - end - - def ems_inv_to_hashes - log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]" - - $aws_log.info("#{log_header}...") - # The order of the below methods does matter, because there are inner dependencies of the data! - get_flavors - get_availability_zones - get_key_pairs - get_stacks - get_private_images if @options.get_private_images - get_shared_images if @options.get_shared_images - get_public_images if @options.get_public_images - get_instances - $aws_log.info("#{log_header}...Complete") - - filter_unused_disabled_flavors - - @data - end - - private - - def map_labels(model_name, labels) - @tag_mapper.map_labels(model_name, labels) - end - - def get_flavors - process_collection(ManageIQ::Providers::Amazon::InstanceTypes.all, :flavors) { |flavor| parse_flavor(flavor) } - end - - def get_availability_zones - azs = @aws_ec2.client.describe_availability_zones[:availability_zones] - process_collection(azs, :availability_zones) { |az| parse_availability_zone(az) } - end - - def get_key_pairs - kps = @aws_ec2.client.describe_key_pairs[:key_pairs] - process_collection(kps, :key_pairs) { |kp| parse_key_pair(kp) } - end - - def get_private_images - get_images( - @aws_ec2.client.describe_images(:owners => [:self], - :filters => [{:name => "image-type", - :values => ["machine"]}])[:images]) - end - - def get_shared_images - get_images( - @aws_ec2.client.describe_images(:executable_users => [:self], - :filters => [{:name => "image-type", - :values => ["machine"]}])[:images]) - end - - def get_public_images - filters = @options.to_hash[:public_images_filters] - get_images( - @aws_ec2.client.describe_images(:executable_users => [:all], - :filters => filters)[:images], true) - end - - def get_images(images, is_public = false) - process_collection(images, :vms) { |image| parse_image(image, is_public) } - end - - def get_stacks - stacks = @aws_cloud_formation.stacks - process_collection(stacks, :orchestration_stacks) { |stack| parse_stack(stack) } - update_nested_stack_relations - end - - def get_stack_parameters(stack_id, parameters) - process_collection(parameters, :orchestration_stack_parameters) do |param_key, param_val| - parse_stack_parameter(param_key, param_val, stack_id) - end - end - - def get_stack_outputs(stack_id, outputs) - process_collection(outputs, :orchestration_stack_outputs) do |output| - parse_stack_output(output, stack_id) - end - end - - def get_stack_resources(resources) - process_collection(resources, :orchestration_stack_resources) { |resource| parse_stack_resource(resource) } - end - - def get_stack_template(stack) - process_collection([stack], :orchestration_templates) { |the_stack| parse_stack_template(the_stack) } - end - - def get_instances - instances = @aws_ec2.instances - process_collection(instances, :vms) { |instance| parse_instance(instance) } - end - - def parse_flavor(flavor) - name = uid = flavor[:name] - - cpus = flavor[:vcpu] - - new_result = { - :type => ManageIQ::Providers::Amazon::CloudManager::Flavor.name, - :ems_ref => uid, - :name => name, - :description => flavor[:description], - :enabled => !flavor[:disabled], - :cpus => cpus, - :cpu_cores => 1, - :memory => flavor[:memory], - :supports_32_bit => flavor[:architecture].include?(:i386), - :supports_64_bit => flavor[:architecture].include?(:x86_64), - :supports_hvm => flavor[:virtualization_type].include?(:hvm), - :supports_paravirtual => flavor[:virtualization_type].include?(:paravirtual), - :block_storage_based_only => flavor[:ebs_only], - :cloud_subnet_required => flavor[:vpc_only], - :ephemeral_disk_size => flavor[:instance_store_size], - :ephemeral_disk_count => flavor[:instance_store_volumes] - } - - return uid, new_result - end - - def parse_availability_zone(az) - name = uid = az.zone_name - - # power_state = (az.state == :available) ? "on" : "off", - - new_result = { - :type => ManageIQ::Providers::Amazon::CloudManager::AvailabilityZone.name, - :ems_ref => uid, - :name => name, - } - - return uid, new_result - end - - def parse_key_pair(kp) - name = uid = kp.key_name - - new_result = { - :type => self.class.key_pair_type, - :name => name, - :fingerprint => kp.key_fingerprint - } - - return uid, new_result - end - - def self.key_pair_type - ManageIQ::Providers::Amazon::CloudManager::AuthKeyPair.name - end - - def parse_image(image, is_public) - uid = image.image_id - location = image.image_location - guest_os = image.platform == "windows" ? "windows_generic" : "linux_generic" - if guest_os == "linux_generic" - guest_os = OperatingSystem.normalize_os_name(location) - guest_os = "linux_generic" if guest_os == "unknown" - end - - name = get_from_tags(image, :name) - name = image.name if name.blank? - name = $1 if name.blank? && location =~ /^(.+?)(\.(image|img))?\.manifest\.xml$/ - name = uid if name.blank? - - labels = parse_labels(image.tags) - - new_result = { - :type => ManageIQ::Providers::Amazon::CloudManager::Template.name, - :uid_ems => uid, - :ems_ref => uid, - :name => name, - :location => location, - :vendor => "amazon", - :connection_state => "connected", - :raw_power_state => "never", - :template => true, - :labels => labels, - :tags => map_labels('Image', labels), - # the is_public flag here avoids having to make an additional API call - # per image, since we already know whether it's a public image - :publicly_available => is_public, - :operating_system => { - :product_name => guest_os # FIXME: duplicated information used by some default reports - }, - :hardware => { - :guest_os => guest_os, - :bitness => architecture_to_bitness(image.architecture), - :virtualization_type => image.virtualization_type, - :root_device_type => image.root_device_type, - }, - } - - return uid, new_result - end - - def parse_instance(instance) - status = instance.state.name - return if @options.ignore_terminated_instances && status.to_sym == :terminated - - uid = instance.id - name = get_from_tags(instance, :name) - name = name.blank? ? uid : name - - flavor_uid = instance.instance_type - @known_flavors << flavor_uid - flavor = @data_index.fetch_path(:flavors, flavor_uid) || - @data_index.fetch_path(:flavors, "unknown") - - private_network = { - :ipaddress => instance.private_ip_address.presence, - :hostname => instance.private_dns_name.presence - }.delete_nils - - public_network = { - :ipaddress => instance.public_ip_address.presence, - :hostname => instance.public_dns_name.presence - }.delete_nils - - labels = parse_labels(instance.tags) - - new_result = { - :type => ManageIQ::Providers::Amazon::CloudManager::Vm.name, - :uid_ems => uid, - :ems_ref => uid, - :name => name, - :vendor => "amazon", - :connection_state => "connected", - :raw_power_state => status, - :boot_time => instance.launch_time, - :labels => labels, - :tags => map_labels('Vm', labels), - :hardware => { - :bitness => architecture_to_bitness(instance.architecture), - :virtualization_type => instance.virtualization_type, - :root_device_type => instance.root_device_type, - :cpu_sockets => flavor[:cpus], - :cpu_cores_per_socket => 1, - :cpu_total_cores => flavor[:cpus], - :memory_mb => flavor[:memory] / 1.megabyte, - :disk_capacity => flavor[:ephemeral_disk_size], - :disks => [], # Filled in later conditionally on flavor - :networks => [], # Filled in later conditionally on what's available - }, - - :availability_zone => @data_index.fetch_path(:availability_zones, instance.placement.availability_zone), - :flavor => flavor, - :cloud_network => @data_index.fetch_path(:cloud_networks, instance.vpc_id), - :cloud_subnet => @data_index.fetch_path(:cloud_subnets, instance.subnet_id), - :key_pairs => [@data_index.fetch_path(:key_pairs, instance.key_name)].compact, - :orchestration_stack => @data_index.fetch_path(:orchestration_stacks, - get_from_tags(instance, "aws:cloudformation:stack-id")), - } - new_result[:location] = public_network[:hostname] if public_network[:hostname] - new_result[:hardware][:networks] << private_network.merge(:description => "private") unless private_network.blank? - new_result[:hardware][:networks] << public_network.merge(:description => "public") unless public_network.blank? - - parent_image = @data_index.fetch_path(:vms, instance.image_id) - if parent_image - new_result[:parent_vm] = parent_image - guest_os = parent_image.fetch_path(:hardware, :guest_os) - new_result.store_path(:hardware, :guest_os, guest_os) - new_result.store_path(:operating_system, :product_name, guest_os) # FIXME: duplicated information used by some default reports - end - - if flavor[:ephemeral_disk_count] > 0 - disks = new_result[:hardware][:disks] - single_disk_size = flavor[:ephemeral_disk_size] / flavor[:ephemeral_disk_count] - flavor[:ephemeral_disk_count].times do |i| - add_instance_disk(disks, single_disk_size, i, "Disk #{i}") - end - end - - instance.block_device_mappings.each do |blk_map| - disks = new_result[:hardware][:disks] - device = File.basename(blk_map.device_name) - add_block_device_disk(disks, device, device) - end - - return uid, new_result - end - - def parse_stack(stack) - uid = stack.stack_id.to_s - child_stacks, resources = find_stack_resources(stack) - new_result = { - :type => ManageIQ::Providers::Amazon::CloudManager::OrchestrationStack.name, - :ems_ref => uid, - :name => stack.name.presence || uid, - :description => stack.description, - :status => stack.stack_status, - :status_reason => stack.stack_status_reason, - :children => child_stacks, - :resources => resources, - :outputs => find_stack_outputs(stack), - :parameters => find_stack_parameters(stack), - - :orchestration_template => find_stack_template(stack) - } - return uid, new_result - end - - def find_stack_template(stack) - get_stack_template(stack) - @data_index.fetch_path(:orchestration_templates, stack.stack_id) - end - - def find_stack_parameters(stack) - raw_parameters = stack.parameters.each_with_object({}) { |sp, hash| hash[sp.parameter_key] = sp.parameter_value } - get_stack_parameters(stack.stack_id, raw_parameters) - raw_parameters.collect do |parameter| - @data_index.fetch_path(:orchestration_stack_parameters, compose_ems_ref(stack.stack_id, parameter[0])) - end - end - - def find_stack_outputs(stack) - raw_outputs = stack.outputs - get_stack_outputs(stack.stack_id, raw_outputs) - raw_outputs.collect do |output| - @data_index.fetch_path(:orchestration_stack_outputs, compose_ems_ref(stack.stack_id, output.output_key)) - end - end - - def find_stack_resources(stack) - raw_resources = stack.resource_summaries.entries - - # physical_resource_id can be empty if the resource was not successfully created; ignore such - raw_resources.reject! { |r| r.physical_resource_id.nil? } - - get_stack_resources(raw_resources) - - child_stacks = [] - resources = raw_resources.collect do |resource| - physical_id = resource.physical_resource_id - child_stacks << physical_id if resource.resource_type == "AWS::CloudFormation::Stack" - @data_index.fetch_path(:orchestration_stack_resources, physical_id) - end - - return child_stacks, resources - end - - def parse_stack_template(stack) - # Only need a temporary unique identifier for the template. Using the stack id is the cheapest way. - uid = stack.stack_id - new_result = { - :type => "ManageIQ::Providers::Amazon::CloudManager::OrchestrationTemplate", - :name => stack.name.presence || uid, - :description => stack.description, - :content => stack.client.get_template(:stack_name => stack.name).template_body, - :orderable => false - } - return uid, new_result - end - - def parse_stack_parameter(param_key, param_val, stack_id) - uid = compose_ems_ref(stack_id, param_key) - new_result = { - :ems_ref => uid, - :name => param_key, - :value => param_val - } - return uid, new_result - end - - def parse_stack_output(output, stack_id) - uid = compose_ems_ref(stack_id, output.output_key) - new_result = { - :ems_ref => uid, - :key => output.output_key, - :value => output.output_value, - :description => output.description - } - return uid, new_result - end - - def parse_stack_resource(resource) - uid = resource.physical_resource_id - new_result = { - :ems_ref => uid, - :name => resource.logical_resource_id, - :logical_resource => resource.logical_resource_id, - :physical_resource => uid, - :resource_category => resource.resource_type, - :resource_status => resource.resource_status, - :resource_status_reason => resource.resource_status_reason, - :last_updated => resource.last_updated_timestamp - } - return uid, new_result - end - - def parse_labels(tags) - result = [] - return result if tags.empty? - tags.each do |tag| - custom_attr = { - :section => 'labels', - :name => tag[:key], - :value => tag[:value], - :source => 'amazon' - } - result << custom_attr - end - - result - end -end diff --git a/app/models/manageiq/providers/amazon/network_manager.rb b/app/models/manageiq/providers/amazon/network_manager.rb index 3a5ae10de..ffe482c66 100644 --- a/app/models/manageiq/providers/amazon/network_manager.rb +++ b/app/models/manageiq/providers/amazon/network_manager.rb @@ -9,7 +9,6 @@ class ManageIQ::Providers::Amazon::NetworkManager < ManageIQ::Providers::Network require_nested :LoadBalancerPoolMember require_nested :NetworkPort require_nested :NetworkRouter - require_nested :RefreshParser require_nested :RefreshWorker require_nested :Refresher require_nested :SecurityGroup @@ -59,4 +58,8 @@ def self.default_blacklisted_event_names def self.display_name(number = 1) n_('Network Provider (Amazon)', 'Network Providers (Amazon)', number) end + + def inventory_object_refresh? + true + end end diff --git a/app/models/manageiq/providers/amazon/network_manager/refresh_parser.rb b/app/models/manageiq/providers/amazon/network_manager/refresh_parser.rb deleted file mode 100644 index a85e4098e..000000000 --- a/app/models/manageiq/providers/amazon/network_manager/refresh_parser.rb +++ /dev/null @@ -1,538 +0,0 @@ -# TODO: Separate collection from parsing (perhaps collecting in parallel a la RHEVM) - -class ManageIQ::Providers::Amazon::NetworkManager::RefreshParser - include ManageIQ::Providers::Amazon::RefreshHelperMethods - - def initialize(ems, options = nil) - @ems = ems - @aws_ec2 = ems.connect - @aws_elb = ems.connect(:service => :ElasticLoadBalancing) - @data = {} - @data_index = {} - @options = options || {} - end - - def ems_inv_to_hashes - log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]" - - $aws_log.info("#{log_header}...") - # The order of the below methods does matter, because there are inner dependencies of the data! - get_cloud_networks - get_cloud_subnets - get_security_groups - get_network_ports - get_load_balancers - get_load_balancer_pools - get_load_balancer_listeners - get_load_balancer_health_checks - get_ec2_floating_ips_and_ports - get_floating_ips - get_public_ips - get_load_balancers_ports_and_floating_ips - $aws_log.info("#{log_header}...Complete") - - @data - end - - private - - def security_groups - @security_groups ||= @aws_ec2.security_groups - end - - def load_balancers - @load_balancers ||= @aws_elb.client.describe_load_balancers.load_balancer_descriptions - end - - def network_ports - @network_ports ||= @aws_ec2.client.describe_network_interfaces.network_interfaces - end - - def get_cloud_networks - vpcs = @aws_ec2.client.describe_vpcs[:vpcs] - process_collection(vpcs, :cloud_networks) { |vpc| parse_cloud_network(vpc) } - end - - def get_cloud_subnets - cloud_subnets = @aws_ec2.client.describe_subnets[:subnets] - process_collection(cloud_subnets, :cloud_subnets) { |s| parse_cloud_subnet(s) } - end - - def get_security_groups - process_collection(security_groups, :security_groups) { |sg| parse_security_group(sg) } - get_firewall_rules - end - - def get_firewall_rules - security_groups.each do |sg| - new_sg = @data_index.fetch_path(:security_groups, sg.group_id) - new_sg[:firewall_rules] = get_inbound_firewall_rules(sg) + get_outbound_firewall_rules(sg) - end - end - - def get_inbound_firewall_rules(sg) - sg.ip_permissions.collect { |perm| parse_firewall_rule(perm, "inbound") }.flatten - end - - def get_outbound_firewall_rules(sg) - sg.ip_permissions_egress.collect { |perm| parse_firewall_rule(perm, "outbound") }.flatten - end - - def lb_cloud_subnets(subnet_id) - { - :address => nil, - :cloud_subnet => @data_index[:cloud_subnets][subnet_id] - } - end - - def get_load_balancers - process_collection(load_balancers, :load_balancers) { |lb| parse_load_balancer(lb) } - end - - def get_load_balancers_ports_and_floating_ips - load_balancers.each do |lb| - uid = lb.load_balancer_name - network_port = { - :ems_ref => uid, - :name => uid, - :status => nil, - :mac_address => nil, - :device_owner => uid, - :device_ref => uid, - :device => @data_index[:load_balancers][uid], - :cloud_subnet_network_ports => lb.subnets.to_a.map { |subnet_id| lb_cloud_subnets(subnet_id) }, - :security_groups => lb.security_groups.to_a.collect do |security_group_id| - @data_index[:security_groups][security_group_id] - end.compact - } - - @data[:network_ports] << network_port - @data_index.store_path(:network_ports, uid, network_port) - - floating_ip = { - :ems_ref => uid, - :address => lb.dns_name, - :fixed_ip_address => nil, - :cloud_network_only => lb.vpc_id.present?, - :network_port => network_port, - :cloud_network => lb.vpc_id.present? ? @data_index[:cloud_networks][lb.vpc_id] : nil, - :status => nil, - :vm => nil - } - - @data[:floating_ips] << floating_ip - @data_index.store_path(:floating_ips, uid, floating_ip) - end - end - - def get_load_balancer_pools - process_collection(load_balancers, :load_balancer_pools) { |lb| parse_load_balancer_pool(lb) } - get_load_balancer_pool_members - end - - def get_load_balancer_pool_members - @data[:load_balancer_pool_members] = [] - - load_balancers.each do |lb| - new_lb = @data_index.fetch_path(:load_balancer_pools, lb.load_balancer_name) - load_balancer_pool_members = lb.instances.collect { |m| parse_load_balancer_pool_member(m) } - load_balancer_pool_members.each do |member| - # Store all unique pool members - if @data_index.fetch_path(:load_balancer_pool_members, member[:ems_ref]).blank? - @data_index.store_path(:load_balancer_pool_members, member[:ems_ref], member) - @data[:load_balancer_pool_members] << member - end - end - - # fill M:N relation of pool to pool members - new_lb[:load_balancer_pool_member_pools] = load_balancer_pool_members.collect do |x| - {:load_balancer_pool_member => @data_index.fetch_path(:load_balancer_pool_members, x[:ems_ref])} - end - end - end - - def get_load_balancer_listeners - # Initialize the data, otherwise the saving code will not catch this when empty - @data[:load_balancer_listeners] ||= [] - load_balancers.each do |lb| - process_collection(lb.listener_descriptions, :load_balancer_listeners) do|listener| - parse_load_balancer_listener(lb, listener) - end - end - end - - def get_load_balancer_health_checks - process_collection(load_balancers, :load_balancer_health_checks) { |lb| parse_load_balancer_health_check(lb) } - end - - def get_floating_ips - ips = @aws_ec2.client.describe_addresses.addresses - # Take only floating ips that are not already in stored by ec2 flaoting_ips - ips = ips.select do |floating_ip| - floating_ip_id = floating_ip.allocation_id.blank? ? floating_ip.public_ip : floating_ip.allocation_id - @data_index.fetch_path(:floating_ips, floating_ip_id).nil? - end - process_collection(ips, :floating_ips) { |ip| parse_floating_ip(ip) } - end - - def get_public_ips - public_ips = [] - network_ports.each do |network_port| - network_port.private_ip_addresses.each do |private_address| - if private_address.association && !(public_ip = private_address.association.public_ip).blank? - allocation_id = private_address.association.allocation_id - floating_ip_id = allocation_id.blank? ? public_ip : allocation_id - unless @data_index.fetch_path(:floating_ips, floating_ip_id) - public_ips << { - :network_port_id => network_port.network_interface_id, - :private_ip_address => private_address.private_ip_address, - :public_ip_address => public_ip - } - end - end - end - end - process_collection(public_ips, :floating_ips) { |public_ip| parse_public_ip(public_ip) } - end - - def get_network_ports - process_collection(network_ports, :network_ports) { |n| parse_network_port(n) } - end - - def get_ec2_floating_ips_and_ports - instances = @aws_ec2.instances.select { |instance| instance.network_interfaces.blank? } - process_collection(instances, :network_ports) { |instance| parse_network_port_inferred_from_instance(instance) } - process_collection(instances, :floating_ips) { |instance| parse_floating_ip_inferred_from_instance(instance) } - end - - def parse_cloud_network(vpc) - uid = vpc.vpc_id - name = get_from_tags(vpc, :name) || uid - - status = (vpc.state == "available") ? "active" : "inactive" - - new_result = { - :type => self.class.cloud_network_type, - :ems_ref => uid, - :name => name, - :cidr => vpc.cidr_block, - :status => status, - :enabled => true, - :orchestration_stack => parent_manager_fetch_path(:orchestration_stacks, - get_from_tags(vpc, "aws:cloudformation:stack-id")), - } - return uid, new_result - end - - def parse_cloud_subnet(subnet) - uid = subnet.subnet_id - name = get_from_tags(subnet, :name) || uid - - new_result = { - :type => self.class.cloud_subnet_type, - :ems_ref => uid, - :name => name, - :cidr => subnet.cidr_block, - :status => subnet.state.try(:to_s), - :availability_zone => parent_manager_fetch_path(:availability_zones, subnet.availability_zone), - :cloud_network => @data_index.fetch_path(:cloud_networks, subnet.vpc_id), - } - - return uid, new_result - end - - def parse_security_group(sg) - uid = sg.group_id - - new_result = { - :type => self.class.security_group_type, - :ems_ref => uid, - :name => sg.group_name.presence || uid, - :description => sg.description.try(:truncate, 255), - :cloud_network => @data_index.fetch_path(:cloud_networks, sg.vpc_id), - :orchestration_stack => parent_manager_fetch_path(:orchestration_stacks, - get_from_tags(sg, "aws:cloudformation:stack-id")), - } - return uid, new_result - end - - # TODO: Should ICMP protocol values have their own 2 columns, or - # should they override port and end_port like the Amazon API. - def parse_firewall_rule(perm, direction) - ret = [] - - common = { - :direction => direction, - :host_protocol => perm.ip_protocol.to_s == "-1" ? _("All") : perm.ip_protocol.to_s.upcase, - :port => perm.from_port, - :end_port => perm.to_port, - } - - perm.user_id_group_pairs.each do |g| - new_result = common.dup - new_result[:source_security_group] = @data_index.fetch_path(:security_groups, g.group_id) - ret << new_result - end - perm.ip_ranges.each do |r| - new_result = common.dup - new_result[:source_ip_range] = r.cidr_ip - ret << new_result - end - perm.ipv_6_ranges.each do |r| - new_result = common.dup - new_result[:source_ip_range] = r.cidr_ipv_6 - ret << new_result - end - - ret - end - - def parse_load_balancer(lb) - uid = lb.load_balancer_name - - new_result = { - :type => self.class.load_balancer_type, - :ems_ref => uid, - :name => uid, - } - - return uid, new_result - end - - def parse_load_balancer_pool(lb) - uid = name = lb.load_balancer_name - - new_result = { - :type => self.class.load_balancer_pool_type, - :ems_ref => uid, - :name => name, - } - - return uid, new_result - end - - def parse_load_balancer_pool_member(member) - uid = member.instance_id - - { - :type => self.class.load_balancer_pool_member_type, - :ems_ref => uid, - # TODO(lsmola) AWS always associates to eth0 of the instances, we do not collect that info now, we need to do that - # :network_port => get eth0 network_port - :vm => parent_manager_fetch_path(:vms, uid) - } - end - - def parse_load_balancer_listener(lb, listener_struct) - listener = listener_struct.listener - - uid = "#{lb.load_balancer_name}__#{listener.protocol}__#{listener.load_balancer_port}__"\ - "#{listener.instance_protocol}__#{listener.instance_port}__#{listener.ssl_certificate_id}" - - new_result = { - :type => self.class.load_balancer_listener_type, - :ems_ref => uid, - :load_balancer_protocol => listener.protocol, - :load_balancer_port_range => (listener.load_balancer_port.to_i..listener.load_balancer_port.to_i), - :instance_protocol => listener.instance_protocol, - :instance_port_range => (listener.instance_port.to_i..listener.instance_port.to_i), - :load_balancer => @data_index.fetch_path(:load_balancers, lb.load_balancer_name), - :load_balancer_listener_pools => [ - {:load_balancer_pool => @data_index.fetch_path(:load_balancer_pools, lb.load_balancer_name)}] - } - - return uid, new_result - end - - def parse_load_balancer_health_check(lb) - uid = lb.load_balancer_name - - health_check_members = @aws_elb.client.describe_instance_health(:load_balancer_name => lb.load_balancer_name) - health_check_members = health_check_members.instance_states.collect do |m| - parse_load_balancer_health_check_member(m) - end - - health_check = lb.health_check - target_match = health_check.target.match(/^(\w+)\:(\d+)\/?(.*?)$/) - protocol = target_match[1] - port = target_match[2].to_i - url_path = target_match[3] - - new_result = { - :type => self.class.load_balancer_health_check_type, - :ems_ref => uid, - :protocol => protocol, - :port => port, - :url_path => url_path, - :interval => health_check.interval, - :timeout => health_check.timeout, - :unhealthy_threshold => health_check.unhealthy_threshold, - :healthy_threshold => health_check.healthy_threshold, - :load_balancer => @data_index.fetch_path(:load_balancers, lb.load_balancer_name), - :load_balancer_health_check_members => health_check_members - } - - return uid, new_result - end - - def parse_load_balancer_health_check_member(member) - { - :load_balancer_pool_member => @data_index.fetch_path(:load_balancer_pool_members, member.instance_id), - :status => member.state, - :status_reason => member.description - } - end - - def parse_floating_ip(ip) - cloud_network_only = ip.domain == "vpc" ? true : false - address = ip.public_ip - uid = cloud_network_only ? ip.allocation_id : ip.public_ip - - new_result = { - :type => self.class.floating_ip_type, - :ems_ref => uid, - :address => address, - :fixed_ip_address => ip.private_ip_address, - :cloud_network_only => cloud_network_only, - :network_port => @data_index.fetch_path(:network_ports, ip.network_interface_id), - :vm => parent_manager_fetch_path(:vms, ip.instance_id) - } - - return uid, new_result - end - - def parse_floating_ip_inferred_from_instance(instance) - address = uid = instance.public_ip_address - - new_result = { - :type => self.class.floating_ip_type, - :ems_ref => uid, - :address => address, - :fixed_ip_address => instance.private_ip_address, - :cloud_network_only => false, - :network_port => @data_index.fetch_path(:network_ports, instance.id), - :vm => parent_manager_fetch_path(:vms, instance.id) - } - - return uid, new_result - end - - def parse_public_ip(public_ip) - address = uid = public_ip[:public_ip_address] - new_result = { - :type => self.class.floating_ip_type, - :ems_ref => uid, - :address => address, - :fixed_ip_address => public_ip[:private_ip_address], - :cloud_network_only => true, - :network_port => @data_index.fetch_path(:network_ports, public_ip[:network_port_id]), - :vm => @data_index.fetch_path(:network_ports, public_ip[:network_port_id], :device) - } - - return uid, new_result - end - - def parse_cloud_subnet_network_port(cloud_subnet_network_port, subnet_id) - { - :address => cloud_subnet_network_port.private_ip_address, - :cloud_subnet => @data_index.fetch_path(:cloud_subnets, subnet_id) - } - end - - def parse_network_port(network_port) - uid = network_port.network_interface_id - cloud_subnet_network_ports = network_port.private_ip_addresses.map do |x| - parse_cloud_subnet_network_port(x, network_port.subnet_id) - end - device = parent_manager_fetch_path(:vms, network_port.try(:attachment).try(:instance_id)) - security_groups = network_port.groups.blank? ? [] : network_port.groups.map do |x| - @data_index.fetch_path(:security_groups, x.group_id) - end - - new_result = { - :type => self.class.network_port_type, - :name => uid, - :ems_ref => uid, - :status => network_port.status, - :mac_address => network_port.mac_address, - :device_owner => network_port.try(:attachment).try(:instance_owner_id), - :device_ref => network_port.try(:attachment).try(:instance_id), - :device => device, - :cloud_subnet_network_ports => cloud_subnet_network_ports, - :security_groups => security_groups, - } - return uid, new_result - end - - def parse_network_port_inferred_from_instance(instance) - # Create network_port placeholder for old EC2 instances, those do not have interface nor subnet nor VPC - cloud_subnet_network_ports = [parse_cloud_subnet_network_port(instance, nil)] - - uid = instance.id - name = get_from_tags(instance, :name) || uid - - device = parent_manager_fetch_path(:vms, uid) - - new_result = { - :type => self.class.network_port_type, - :name => name, - :ems_ref => uid, - :status => nil, - :mac_address => nil, - :device_owner => nil, - :device_ref => nil, - :device => device, - :cloud_subnet_network_ports => cloud_subnet_network_ports, - :security_groups => instance.security_groups.to_a.collect do |sg| - @data_index.fetch_path(:security_groups, sg.group_id) - end.compact, - } - return uid, new_result - end - - class << self - def load_balancer_type - ManageIQ::Providers::Amazon::NetworkManager::LoadBalancer.name - end - - def load_balancer_listener_type - ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerListener.name - end - - def load_balancer_health_check_type - ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerHealthCheck.name - end - - def load_balancer_pool_type - ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerPool.name - end - - def load_balancer_pool_member_type - ManageIQ::Providers::Amazon::NetworkManager::LoadBalancerPoolMember.name - end - - def security_group_type - ManageIQ::Providers::Amazon::NetworkManager::SecurityGroup.name - end - - def network_router_type - ManageIQ::Providers::Amazon::NetworkManager::NetworkRouter.name - end - - def cloud_network_type - ManageIQ::Providers::Amazon::NetworkManager::CloudNetwork.name - end - - def cloud_subnet_type - ManageIQ::Providers::Amazon::NetworkManager::CloudSubnet.name - end - - def floating_ip_type - ManageIQ::Providers::Amazon::NetworkManager::FloatingIp.name - end - - def network_port_type - ManageIQ::Providers::Amazon::NetworkManager::NetworkPort.name - end - end -end diff --git a/app/models/manageiq/providers/amazon/storage_manager/ebs.rb b/app/models/manageiq/providers/amazon/storage_manager/ebs.rb index d10fc2321..4818e63b7 100644 --- a/app/models/manageiq/providers/amazon/storage_manager/ebs.rb +++ b/app/models/manageiq/providers/amazon/storage_manager/ebs.rb @@ -1,7 +1,6 @@ class ManageIQ::Providers::Amazon::StorageManager::Ebs < ManageIQ::Providers::StorageManager require_nested :CloudVolume require_nested :CloudVolumeSnapshot - require_nested :RefreshParser require_nested :RefreshWorker require_nested :Refresher @@ -36,4 +35,8 @@ def self.description def self.hostname_required? false end + + def inventory_object_refresh? + true + end end diff --git a/app/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser.rb b/app/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser.rb deleted file mode 100644 index 81773b117..000000000 --- a/app/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser.rb +++ /dev/null @@ -1,147 +0,0 @@ -class ManageIQ::Providers::Amazon::StorageManager::Ebs::RefreshParser - include ManageIQ::Providers::Amazon::RefreshHelperMethods - - def initialize(ems, options = nil) - @ems = ems - @aws_ec2 = ems.connect - @data = {} - @data_index = {} - @options = options || {} - end - - def ems_inv_to_hashes - log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]" - - $aws_log.info("#{log_header}...") - get_volumes - get_snapshots - $aws_log.info("#{log_header}...Complete") - - link_storage_associations - - @data - end - - private - - def get_volumes - volumes = @aws_ec2.client.describe_volumes[:volumes] - process_collection(volumes, :cloud_volumes) { |volume| parse_volume(volume) } - end - - def get_snapshots - snapshots = @aws_ec2.client.describe_snapshots(:owner_ids => [:self])[:snapshots] - process_collection(snapshots, :cloud_volume_snapshots) { |snap| parse_snapshot(snap) } - end - - def parse_volume(volume) - uid = volume.volume_id - - new_result = { - :type => self.class.volume_type, - :ems_ref => uid, - :name => get_from_tags(volume, :name) || uid, - :status => volume.state, - :creation_time => volume.create_time, - :volume_type => volume.volume_type, - :size => volume.size.to_i.gigabytes, - :snapshot_uid => volume.snapshot_id, - :availability_zone => parent_manager_fetch_path(:availability_zones, volume.availability_zone), - :encrypted => volume.encrypted, - :iops => volume.iops - } - - link_volume_to_disk(new_result, volume.attachments) - - return uid, new_result - end - - def parse_snapshot(snap) - uid = snap.snapshot_id - - new_result = { - :ems_ref => uid, - :type => self.class.volume_snapshot_type, - :name => get_from_tags(snap, :name) || uid, - :status => snap.state, - :creation_time => snap.start_time, - :description => snap.description, - :size => snap.volume_size.to_i.gigabytes, - :volume => @data_index.fetch_path(:cloud_volumes, snap.volume_id), - :encrypted => snap.encrypted, - } - - return uid, new_result - end - - def link_storage_associations - @data[:cloud_volumes].each do |cv| - base_snapshot_uid = cv.delete(:snapshot_uid) - base_snapshot = @data_index.fetch_path(:cloud_volume_snapshots, base_snapshot_uid) - cv[:base_snapshot] = base_snapshot unless base_snapshot.nil? - end if @data[:cloud_volumes] - end - - def link_volume_to_disk(volume_hash, attachments) - log_header = "MIQ(#{self.class.name}.#{__method__})" - - uid = volume_hash[:ems_ref] - - attachments.each do |a| - if a['device'].blank? - $aws_log.warn "#{log_header}: Volume: #{uid}, is missing a mountpoint, skipping the volume processing" - $aws_log.warn "#{log_header}: EMS: #{@ems.name}, Instance: #{a['instance_id']}" - next - end - - dev = File.basename(a['device']) - - vm = @ems.parent_manager.vms.find_by(:ems_ref => a['instance_id']) - unless vm - $aws_log.warn "VM referenced by attachment (#{a['instance_id']} not found." - next - end - - hardware = vm.hardware - disks = hardware.disks - unless disks - $aws_log.warn "#{log_header}: Volume: #{uid}, attached to instance not visible in the scope of this EMS" - $aws_log.warn "#{log_header}: EMS: #{@ems.name}, Instance: #{a['instance_id']}" - next - end - - disk_hash = { - :size => volume_hash[:size], - :backing_volume => volume_hash - } - - if (disk = disks.detect { |d| d.location == dev }) - # Disk exists: save id. - disk_hash[:id] = disk.id - else - # New disk. - disk_hash[:hardware_id] = hardware.id - disk_hash[:device_name] = dev - disk_hash[:device_type] = "disk" - disk_hash[:controller_type] = "EBS Volume" - disk_hash[:location] = dev - end - - backing_links << disk_hash - end - end - - def backing_links - @data[:backing_links] ||= [] - end - - class << self - def volume_type - "ManageIQ::Providers::Amazon::StorageManager::Ebs::CloudVolume" - end - - def volume_snapshot_type - "ManageIQ::Providers::Amazon::StorageManager::Ebs::CloudVolumeSnapshot" - end - end -end diff --git a/app/models/manageiq/providers/amazon/storage_manager/ebs/refresher.rb b/app/models/manageiq/providers/amazon/storage_manager/ebs/refresher.rb index c4a966c02..8b233f109 100644 --- a/app/models/manageiq/providers/amazon/storage_manager/ebs/refresher.rb +++ b/app/models/manageiq/providers/amazon/storage_manager/ebs/refresher.rb @@ -1,11 +1,4 @@ module ManageIQ::Providers class Amazon::StorageManager::Ebs::Refresher < ManageIQ::Providers::BaseManager::Refresher - def parse_legacy_inventory(ems) - ::ManageIQ::Providers::Amazon::StorageManager::Ebs::RefreshParser.ems_inv_to_hashes(ems) - end - - def post_process_refresh_classes - [] - end end end diff --git a/app/models/manageiq/providers/amazon/storage_manager/s3.rb b/app/models/manageiq/providers/amazon/storage_manager/s3.rb index e8e6a6151..f7db3fb8c 100644 --- a/app/models/manageiq/providers/amazon/storage_manager/s3.rb +++ b/app/models/manageiq/providers/amazon/storage_manager/s3.rb @@ -1,7 +1,6 @@ class ManageIQ::Providers::Amazon::StorageManager::S3 < ManageIQ::Providers::StorageManager require_nested :CloudObjectStoreContainer require_nested :CloudObjectStoreObject - require_nested :RefreshParser require_nested :RefreshWorker require_nested :Refresher @@ -36,6 +35,10 @@ def self.hostname_required? false end + def inventory_object_refresh? + true + end + def connect(options = {}) options[:service] = :S3 parent_manager.connect options diff --git a/app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser.rb b/app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser.rb deleted file mode 100644 index 7e7abf753..000000000 --- a/app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser.rb +++ /dev/null @@ -1,124 +0,0 @@ -class ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParser - include ManageIQ::Providers::Amazon::RefreshHelperMethods - - def initialize(ems, options = nil) - @ems = ems - @aws_s3 = ems.connect(:service => :S3) - @data = {} - @data_index = {} - @options = options || {} - end - - def ems_inv_to_hashes - log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]" - - $aws_log.info("#{log_header}...") - object_store - - $aws_log.info("#{log_header}...Complete") - - @data - end - - def object_store - process_collection( - @aws_s3.client.list_buckets.buckets, - :cloud_object_store_containers - ) { |c| parse_container(c) } - - process_containers_content - end - - def process_containers_content - containers = @data[:cloud_object_store_containers] - if containers.empty? - process_collection([], :cloud_object_store_objects) - return - end - - containers.each do |bucket| - bucket_id = bucket[:ems_ref] - - # S3 bucket accessible only for API client with same region - region = @aws_s3.client.get_bucket_location(:bucket => bucket_id).location_constraint - api_client = regional_client(region) - - # AWS SDK doesn't show information about overall size and object count. - # We need to collect it manually. - bytes = 0 - object_count = 0 - # API returns up to 1000 objects per request - token = nil - proceed = true - while proceed - begin - response = api_client.list_objects_v2( - :bucket => bucket_id, - :continuation_token => token - ) - rescue => e - log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]" - $aws_log.warn("#{log_header}: Unable to collect S3 objects in a bucket #{bucket_id}, Message: #{e.message}") - $aws_log.warn(e.backtrace.join("\n")) - break - end - process_collection(response.contents, :cloud_object_store_objects) do |o| - uid, new_result = parse_object(o, bucket_id) - bytes += new_result[:content_length] - object_count += 1 - [uid, new_result] - end - token = response.next_continuation_token - - proceed = token.present? - end - bucket[:bytes] = bytes - bucket[:object_count] = object_count - end - end - - def regional_client(region) - if !region || region == @ems.provider_region - @aws_s3 - else - @regional_resources ||= {} - @regional_resources[region] ||= @ems.connect(:service => :S3, :region => region) - end.client - end - - def parse_container(bucket) - uid = bucket.name - - new_result = { - :type => self.class.container_type, - :ems_ref => uid, - :key => bucket.name - } - return uid, new_result - end - - def parse_object(object, bucket_id) - uid = object.key - - new_result = { - :type => self.class.object_type, - :ems_ref => "#{bucket_id}_#{uid}", - :etag => object.etag, - :last_modified => object.last_modified, - :content_length => object.size, - :key => object.key, - :cloud_object_store_container_id => @data_index.fetch_path(:cloud_object_store_containers, bucket_id) - } - return uid, new_result - end - - class << self - def container_type - ManageIQ::Providers::Amazon::StorageManager::S3::CloudObjectStoreContainer.name - end - - def object_type - ManageIQ::Providers::Amazon::StorageManager::S3::CloudObjectStoreObject.name - end - end -end diff --git a/app/models/manageiq/providers/amazon/storage_manager/s3/refresher.rb b/app/models/manageiq/providers/amazon/storage_manager/s3/refresher.rb index e7ce049aa..deb911ff3 100644 --- a/app/models/manageiq/providers/amazon/storage_manager/s3/refresher.rb +++ b/app/models/manageiq/providers/amazon/storage_manager/s3/refresher.rb @@ -1,14 +1,4 @@ module ManageIQ::Providers class Amazon::StorageManager::S3::Refresher < ManageIQ::Providers::BaseManager::Refresher - # Legacy parse - # - # @param ems [ManageIQ::Providers::BaseManager] - def parse_legacy_inventory(ems) - ::ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParser.ems_inv_to_hashes(ems) - end - - def post_process_refresh_classes - [] - end end end diff --git a/config/settings.yml b/config/settings.yml index c04bb6279..8a0f7898b 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -82,20 +82,15 @@ - machine :ignore_terminated_instances: true - :inventory_object_refresh: true - :allow_targeted_refresh: true :inventory_collections: :saver_strategy: batch :ec2_network: - :inventory_object_refresh: true :inventory_collections: :saver_strategy: batch :s3: - :inventory_object_refresh: true :inventory_collections: :saver_strategy: batch :ec2_ebs_storage: - :inventory_object_refresh: true :inventory_collections: :saver_strategy: batch :http_proxy: diff --git a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb index b9e574745..d3ea05ed1 100644 --- a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb +++ b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb @@ -34,12 +34,6 @@ module AwsRefresherSpecCommon } ].freeze - ALL_OLD_REFRESH_SETTINGS = [ - { - :inventory_object_refresh => false - } - ].freeze - def stub_refresh_settings(settings) # TODO(lsmola) extract the batch sizes to the settings and stub the settings instead # Lower batch sizes to test multiple batches for each collection diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/refresh_parser_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/refresh_parser_spec.rb deleted file mode 100644 index 105d574dc..000000000 --- a/spec/models/manageiq/providers/amazon/cloud_manager/refresh_parser_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -describe ManageIQ::Providers::Amazon::CloudManager::RefreshParser do - let(:ems) { FactoryBot.create(:ems_amazon_with_authentication) } - let(:parser) { described_class.new(ems, Settings.ems_refresh.ec2) } - let(:client) { parser.instance_variable_get(:@aws_ec2).client } - - describe "#get_public_images" do - subject { parser.send(:get_public_images) } - - context "default filter" do - let(:default_filter) { Settings.ems_refresh[ems.class.ems_type].to_hash[:public_images_filters] } - - require 'aws-sdk' - before do - parser.instance_variable_set(:@aws_ec2, Aws::EC2::Resource.new(:stub_responses => true)) - end - - it "gets applied" do - expect(client).to receive(:describe_images) - .with(hash_including(:filters => default_filter)) - .and_return(client.stub_data(:describe_images)) - - subject - end - - it "validated by SDK" do - is_expected.to eq([]) - end - end - end - - describe 'empty names replaced with ids' do - let(:cf_resource) { Aws::CloudFormation::Resource.new(:stub_responses => true) } - let(:cf_stack) do - Aws::CloudFormation::Stack.new( - :client => cf_resource.client, - :name => " \t ", - :data => { :outputs => [], :parameters => [], :stack_id => 'abc' }, - ) - end - - before do - parser.instance_variable_set(:@aws_cloud_formation, cf_resource) - allow(cf_resource).to receive(:stacks).and_return([cf_stack]) - parser.send(:get_stacks) - end - - let(:parser_data) { parser.instance_variable_get(:@data) } - subject { parser_data[:orchestration_stacks].first[:name] } - - it { is_expected.to eq('abc') } - end -end diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb index 6c9b71d78..659874640 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb @@ -14,8 +14,7 @@ EvmSpecHelper.local_miq_server(:zone => Zone.seed) end - (AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS + AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS - ).each do |settings| + AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS.each do |settings| context "with settings #{settings}" do before :each do stub_refresh_settings( diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb index 244c1cf1d..c99914954 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb @@ -14,8 +14,7 @@ EvmSpecHelper.local_miq_server(:zone => Zone.seed) end - (AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS + AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS - ).each do |settings| + AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS.each do |settings| context "with settings #{settings}" do before :each do stub_refresh_settings( diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_other_region_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_other_region_spec.rb deleted file mode 100644 index 5180f4cf2..000000000 --- a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_other_region_spec.rb +++ /dev/null @@ -1,266 +0,0 @@ -require_relative "../../aws_refresher_spec_common" -require_relative "../../aws_refresher_spec_counts" - -describe ManageIQ::Providers::Amazon::CloudManager::Refresher do - include AwsRefresherSpecCommon - include AwsRefresherSpecCounts - - before(:each) do - @ems = FactoryBot.create(:ems_amazon_with_vcr_authentication, :provider_region => "us-west-1") - end - - AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS.each do |settings| - context "with settings #{settings}" do - before(:each) do - stub_refresh_settings(settings) - end - - it "will perform a full refresh on another region" do - 2.times do # Run twice to verify that a second run with existing data does not change anything - @ems.reload - - VCR.use_cassette("#{described_class.name.underscore}_other_region") do - EmsRefresh.refresh(@ems) - EmsRefresh.refresh(@ems.network_manager) - EmsRefresh.refresh(@ems.ebs_storage_manager) - - @ems.reload - assert_counts(table_counts_from_api) - end - - assert_specific_flavor - assert_specific_az - assert_specific_floating_ip - assert_specific_key_pair - assert_specific_security_group - assert_specific_template - assert_specific_vm_powered_on - assert_specific_vm_in_other_region - assert_relationship_tree - assert_subnet_required - end - end - end - - def table_counts_from_api - counts = super - counts[:network_router] = 0 # We do not collect NetworkRouters in old refresh - # Old refresh can't fetch some images, those will have missing operating_system - counts[:operating_system] = counts[:operating_system] - Vm.all.select { |x| x.genealogy_parent.nil? }.count - counts - end - end - - def assert_specific_flavor - @flavor = ManageIQ::Providers::Amazon::CloudManager::Flavor.where(:name => "t1.micro").first - expect(@flavor).to have_attributes( - :name => "t1.micro", - :description => "T1 Micro", - :enabled => true, - :cpus => 1, - :cpu_cores => 1, - :memory => 0.613.gigabytes.to_i, - :supports_32_bit => true, - :supports_64_bit => true, - :supports_hvm => false, - :supports_paravirtual => true - ) - - expect(@flavor.ext_management_system).to eq(@ems) - end - - def assert_specific_az - @az = ManageIQ::Providers::Amazon::CloudManager::AvailabilityZone.where(:name => "us-west-1a").first - expect(@az).to have_attributes( - :name => "us-west-1a", - ) - end - - def assert_specific_floating_ip - ip = ManageIQ::Providers::Amazon::NetworkManager::FloatingIp.where(:address => "54.215.0.230").first - expect(ip).to have_attributes( - :address => "54.215.0.230", - :ems_ref => "54.215.0.230", - :cloud_network_only => false - ) - - @ip = ManageIQ::Providers::Amazon::NetworkManager::FloatingIp.where(:address => "204.236.137.154").first - expect(@ip).to have_attributes( - :address => "204.236.137.154", - :ems_ref => "204.236.137.154", - :fixed_ip_address => "10.191.129.95", - :cloud_network_only => false - ) - end - - def assert_specific_key_pair - @kp = ManageIQ::Providers::Amazon::CloudManager::AuthKeyPair.where(:name => "EmsRefreshSpec-KeyPair-OtherRegion").first - expect(@kp).to have_attributes( - :name => "EmsRefreshSpec-KeyPair-OtherRegion", - :fingerprint => "fc:53:30:aa:d2:23:c7:8d:e2:e8:05:95:a0:d2:90:fb:15:30:a2:51" - ) - end - - def assert_specific_security_group - @sg = ManageIQ::Providers::Amazon::NetworkManager::SecurityGroup.where(:name => "EmsRefreshSpec-SecurityGroup-OtherRegion").first - expect(@sg).to have_attributes( - :name => "EmsRefreshSpec-SecurityGroup-OtherRegion", - :description => "EmsRefreshSpec-SecurityGroup-OtherRegion", - :ems_ref => "sg-2b87746f" - ) - - expect(@sg.firewall_rules.size).to eq(1) - expect(@sg.firewall_rules.first).to have_attributes( - :host_protocol => "TCP", - :direction => "inbound", - :port => 0, - :end_port => 65535, - :source_security_group_id => nil, - :source_ip_range => "0.0.0.0/0" - ) - end - - def assert_specific_template - @template = ManageIQ::Providers::Amazon::CloudManager::Template.where(:name => "EmsRefreshSpec-Image-OtherRegion").first - expect(@template).to have_attributes( - :template => true, - :ems_ref => "ami-183e175d", - :ems_ref_obj => nil, - :uid_ems => "ami-183e175d", - :vendor => "amazon", - :power_state => "never", - :location => "200278856672/EmsRefreshSpec-Image-OtherRegion", - :tools_status => nil, - :boot_time => nil, - :standby_action => nil, - :connection_state => "connected", - :cpu_affinity => nil, - :memory_reserve => nil, - :memory_reserve_expand => nil, - :memory_limit => nil, - :memory_shares => nil, - :memory_shares_level => nil, - :cpu_reserve => nil, - :cpu_reserve_expand => nil, - :cpu_limit => nil, - :cpu_shares => nil, - :cpu_shares_level => nil - ) - - expect(@template.ext_management_system).to eq(@ems) - expect(@template.operating_system).to( - have_attributes( - :product_name => "linux_generic", - ) - ) - expect(@template.custom_attributes.size).to eq(0) - expect(@template.snapshots.size).to eq(0) - - expect(@template.hardware).to have_attributes( - :guest_os => "linux_generic", - :guest_os_full_name => nil, - :bios => nil, - :annotation => nil, - :cpu_sockets => 1, # wtf - :memory_mb => nil, - :disk_capacity => nil, - :bitness => 64 - ) - - expect(@template.hardware.disks.size).to eq(0) - expect(@template.hardware.guest_devices.size).to eq(0) - expect(@template.hardware.nics.size).to eq(0) - expect(@template.hardware.networks.size).to eq(0) - end - - def assert_specific_vm_powered_on - v = ManageIQ::Providers::Amazon::CloudManager::Vm.where(:name => "EmsRefreshSpec-PoweredOn-OtherRegion", :raw_power_state => "running").first - expect(v).to have_attributes( - :template => false, - :ems_ref => "i-dc1ee486", - :ems_ref_obj => nil, - :uid_ems => "i-dc1ee486", - :vendor => "amazon", - :power_state => "on", - :location => "ec2-204-236-137-154.us-west-1.compute.amazonaws.com", - :tools_status => nil, - :boot_time => Time.zone.parse("2013-08-31T00:12:43.000"), - :standby_action => nil, - :connection_state => "connected", - :cpu_affinity => nil, - :memory_reserve => nil, - :memory_reserve_expand => nil, - :memory_limit => nil, - :memory_shares => nil, - :memory_shares_level => nil, - :cpu_reserve => nil, - :cpu_reserve_expand => nil, - :cpu_limit => nil, - :cpu_shares => nil, - :cpu_shares_level => nil - ) - - expect(v.ext_management_system).to eq(@ems) - expect(v.availability_zone).to eq(@az) - expect(v.floating_ip).to eq(@ip) - expect(v.flavor).to eq(@flavor) - expect(v.cloud_network).to be_nil - expect(v.cloud_subnet).to be_nil - expect(v.security_groups).to eq([@sg]) - expect(v.key_pairs).to eq([@kp]) - expect(v.operating_system).to( - have_attributes( - :product_name => "linux_generic", - ) - ) - expect(v.custom_attributes.size).to eq(1) - expect(v.snapshots.size).to eq(0) - - expect(v.hardware).to have_attributes( - :guest_os => "linux_generic", - :guest_os_full_name => nil, - :bios => nil, - :annotation => nil, - :cpu_sockets => 1, - :memory_mb => 627, - :disk_capacity => 0, # TODO: Change to a flavor that has disks - :bitness => 64 - ) - - expect(v.hardware.disks.size).to eq(1) # TODO: Change to a flavor that has disks - expect(v.hardware.guest_devices.size).to eq(0) - expect(v.hardware.nics.size).to eq(0) - - expect(v.hardware.networks.size).to eq(2) - network = v.hardware.networks.where(:description => "public").first - expect(network).to have_attributes( - :description => "public", - :ipaddress => "204.236.137.154", - :hostname => "ec2-204-236-137-154.us-west-1.compute.amazonaws.com" - ) - network = v.hardware.networks.where(:description => "private").first - expect(network).to have_attributes( - :description => "private", - :ipaddress => "10.191.129.95", - :hostname => "ip-10-191-129-95.us-west-1.compute.internal" - ) - - v.with_relationship_type("genealogy") do - expect(v.parent).to eq(@template) - end - end - - def assert_specific_vm_in_other_region - v = ManageIQ::Providers::Amazon::CloudManager::Vm.where(:name => "EmsRefreshSpec-PoweredOn-Basic").first - expect(v).to be_nil - end - - def assert_relationship_tree - expect(@ems.descendants_arranged).to match_relationship_tree({}) - end - - def assert_subnet_required - @flavor = ManageIQ::Providers::Amazon::CloudManager::Flavor.where(:name => "t2.small").first - expect(@flavor).to have_attributes(:cloud_subnet_required => true) - end -end diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb deleted file mode 100644 index 70cd0ee7c..000000000 --- a/spec/models/manageiq/providers/amazon/cloud_manager/vcr_specs/refresher_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -require_relative "../../aws_refresher_spec_common" -require_relative "../../aws_refresher_spec_counts" - -describe ManageIQ::Providers::Amazon::CloudManager::Refresher do - include AwsRefresherSpecCommon - include AwsRefresherSpecCounts - - before(:each) do - @ems = FactoryBot.create(:ems_amazon_with_vcr_authentication) - end - - it ".ems_type" do - expect(described_class.ems_type).to eq(:ec2) - end - - AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS.each do |settings| - context "with settings #{settings}" do - before(:each) do - stub_refresh_settings(settings) - create_tag_mapping - end - - it "will perform a full refresh" do - 2.times do # Run twice to verify that a second run with existing data does not change anything - @ems.reload - - VCR.use_cassette(described_class.name.underscore) do - EmsRefresh.refresh(@ems) - EmsRefresh.refresh(@ems.network_manager) - EmsRefresh.refresh(@ems.ebs_storage_manager) - - @ems.reload - assert_counts(table_counts_from_api) - end - - assert_common - assert_mapped_tags_on_template - end - end - end - end - - def table_counts_from_api - counts = super - counts[:network_router] = 0 # We do not collect NetworkRouters in old refresh - # Old refresh can't fetch some images, those will have missing operating_system - counts[:operating_system] = counts[:operating_system] - Vm.all.select { |x| x.genealogy_parent.nil? }.count - counts - end -end diff --git a/spec/models/manageiq/providers/amazon/network_manager/refresh_parser_spec.rb b/spec/models/manageiq/providers/amazon/network_manager/refresh_parser_spec.rb deleted file mode 100644 index 7c4878c96..000000000 --- a/spec/models/manageiq/providers/amazon/network_manager/refresh_parser_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -describe ManageIQ::Providers::Amazon::NetworkManager::RefreshParser do - require 'aws-sdk' - let(:ems) { FactoryBot.create(:ems_amazon_with_authentication) } - let(:parser) { described_class.new(ems.network_manager, Settings.ems_refresh.ec2_network) } - - describe "#parse_firewall_rule" do - let(:perm) { Aws::EC2::Types::IpPermission.new(rule_options) } - let(:ip_protocol) { "icmp" } - let(:ip_ranges) { [] } - let(:ipv_6_ranges) { [] } - let(:rule_options) do - { - :ip_protocol => ip_protocol, - :ip_ranges => ip_ranges.map { |ip| Aws::EC2::Types::IpRange.new(:cidr_ip => ip) }, - :ipv_6_ranges => ipv_6_ranges.map { |ip| Aws::EC2::Types::Ipv6Range.new(:cidr_ipv_6 => ip) }, - :user_id_group_pairs => [] - } - end - subject { parser.send(:parse_firewall_rule, perm, 'inbound') } - - context "all ip_protocols" do - let(:ip_protocol) { -1 } - let(:ip_ranges) { ["1.1.1.0/24"] } - - it { is_expected.to all(include(:host_protocol => "All")) } - end - - context "ipv6 ranges" do - let(:ipv_6_ranges) { ["2001:DB8::0/120", "2001:DB8::80/122"] } - - it { expect(subject.length).to eq(2) } - it { expect(subject.collect { |i| i[:source_ip_range] }).to eq(ipv_6_ranges) } - end - - context "ipv4 ranges" do - let(:ip_ranges) { ["10.0.0.0/24", "10.0.1.0/24"] } - - it { expect(subject.length).to eq(2) } - it { expect(subject.collect { |i| i[:source_ip_range] }).to eq(ip_ranges) } - end - end - - describe 'empty names replaced with ids' do - let(:ec2_resource) { Aws::EC2::Resource.new(:stub_responses => true) } - let(:vpc) do - Aws::EC2::Vpc.new( - 'asd', - :client => ec2_resource.client, - :name => " \t \n ", - :data => { :tags => [], :vpc_id => 'sdf' }, - ) - end - - before do - parser.instance_variable_set(:@aws_ec2, ec2_resource) - allow(ec2_resource.client).to receive(:describe_vpcs).and_return(:vpcs => [vpc]) - parser.send(:get_cloud_networks) - end - - let(:parser_data) { parser.instance_variable_get(:@data) } - subject { parser_data[:cloud_networks].first[:name] } - - it { is_expected.to eq('sdf') } - end -end diff --git a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb index a6cbd6ec3..03cab1e5c 100644 --- a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb @@ -15,8 +15,7 @@ end # Test all kinds of refreshes, DTO refresh, DTO with batch saving and the original refresh - (AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS + AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS - ).each do |settings| + AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS.each do |settings| context "with settings #{settings}" do before :each do stub_refresh_settings(settings) diff --git a/spec/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser_spec.rb b/spec/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser_spec.rb deleted file mode 100644 index 954457d1f..000000000 --- a/spec/models/manageiq/providers/amazon/storage_manager/ebs/refresh_parser_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -describe ManageIQ::Providers::Amazon::StorageManager::Ebs::RefreshParser do - let(:ems) { FactoryBot.create(:ems_amazon_with_authentication) } - let(:parser) { described_class.new(ems.ebs_storage_manager, Settings.ems_refresh.ec2) } - - let(:ec2_resource) { Aws::EC2::Resource.new(:stub_responses => true) } - let(:ec2_client) { ec2_resource.client } - - before { parser.instance_variable_set(:@aws_ec2, ec2_resource) } - - describe 'empty names replaced with ids' do - let(:ebs_volume) do - Aws::EC2::Volume.new( - 'def', - :client => ec2_client, - :name => " \n\t ", - :data => { :tags => [], :attachments => [], :volume_id => 'fed' }, - ) - end - let(:ebs_volumes) { { :volumes => [ebs_volume] } } - - before do - allow(ec2_client).to receive(:describe_volumes).and_return(ebs_volumes) - parser.send(:get_volumes) - end - - let(:parser_data) { parser.instance_variable_get(:@data) } - subject { parser_data[:cloud_volumes].first[:name] } - - it { is_expected.to eq('fed') } - end -end diff --git a/spec/models/manageiq/providers/amazon/storage_manager/ebs/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/storage_manager/ebs/stubbed_refresher_spec.rb index 87873c9b4..c7d60d964 100644 --- a/spec/models/manageiq/providers/amazon/storage_manager/ebs/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/storage_manager/ebs/stubbed_refresher_spec.rb @@ -18,8 +18,7 @@ FactoryBot.create(:vm_amazon, :ext_management_system => @ems, :ems_ref => "instance_0", :hardware => hardware) end - (AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS + AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS - ).each do |settings| + AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS.each do |settings| context "with settings #{settings}" do before :each do stub_refresh_settings(settings) diff --git a/spec/models/manageiq/providers/amazon/storage_manager/s3/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/storage_manager/s3/stubbed_refresher_spec.rb index 6ff42be1d..c78ce27d6 100644 --- a/spec/models/manageiq/providers/amazon/storage_manager/s3/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/storage_manager/s3/stubbed_refresher_spec.rb @@ -21,8 +21,7 @@ end describe "refresh" do - (AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS + AwsRefresherSpecCommon::ALL_OLD_REFRESH_SETTINGS - ).each do |settings| + AwsRefresherSpecCommon::ALL_GRAPH_REFRESH_SETTINGS.each do |settings| context "with settings #{settings}" do before :each do stub_refresh_settings(settings)