From c95b796fffda6cecdd15684f1ca0eee9142e8667 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Thu, 9 Mar 2017 17:53:00 +0100 Subject: [PATCH] Use ManagerRefresh::TargetCollection collection for TargetCollection Collector Use ManagerRefresh::TargetCollection collection for TargetCollection Collector --- .../providers/amazon/inventory/collector.rb | 60 +++------- .../inventory/collector/target_collection.rb | 103 +++++++----------- 2 files changed, 57 insertions(+), 106 deletions(-) diff --git a/app/models/manageiq/providers/amazon/inventory/collector.rb b/app/models/manageiq/providers/amazon/inventory/collector.rb index 112220d3a..e0db51470 100644 --- a/app/models/manageiq/providers/amazon/inventory/collector.rb +++ b/app/models/manageiq/providers/amazon/inventory/collector.rb @@ -3,22 +3,22 @@ class ManageIQ::Providers::Amazon::Inventory::Collector < ManagerRefresh::Invent require_nested :NetworkManager require_nested :TargetCollection - attr_reader :instances, :instances_refs, :instances_deleted - attr_reader :flavors, :flavors_refs, :flavors_deleted - attr_reader :availability_zones, :availability_zones_refs, :availability_zones_deleted - attr_reader :key_pairs, :key_pairs_refs, :key_pairs_deleted - attr_reader :private_images, :private_images_refs, :private_images_deleted - attr_reader :shared_images, :shared_images_refs, :shared_images_deleted - attr_reader :public_images, :public_images_refs, :public_images_deleted - attr_reader :cloud_networks, :cloud_networks_refs, :cloud_networks_deleted - attr_reader :cloud_subnets, :cloud_subnets_refs, :cloud_subnets_deleted - attr_reader :security_groups, :security_groups_refs, :security_groups_deleted - attr_reader :floating_ips, :floating_ips_refs, :floating_ips_deleted - attr_reader :network_ports, :network_ports_refs, :network_ports_deleted - attr_reader :load_balancers, :load_balancers_refs, :load_balancers_deleted - attr_reader :stacks, :stacks_refs, :stacks_deleted - attr_reader :cloud_volumes, :cloud_volumes_refs - attr_reader :cloud_volume_snapshots, :cloud_volume_snapshots_refs + attr_reader :instances + attr_reader :flavors + attr_reader :availability_zones + attr_reader :key_pairs + attr_reader :private_images + attr_reader :shared_images + attr_reader :public_images + attr_reader :cloud_networks + attr_reader :cloud_subnets + attr_reader :security_groups + attr_reader :floating_ips + attr_reader :network_ports + attr_reader :load_balancers + attr_reader :stacks + attr_reader :cloud_volumes + attr_reader :cloud_volume_snapshots attr_reader :cloud_objects_store_containers attr_reader :cloud_objects_store_objects @@ -30,49 +30,21 @@ def initialize(_manager, _target) def initialize_inventory_sources @instances = [] - @instances_refs = Set.new - @instances_deleted = [] @flavors = [] - @flavors_refs = Set.new - @flavors_deleted = [] @availability_zones = [] - @availability_zones_refs = Set.new - @availability_zones_deleted = [] @key_pairs = [] - @key_pairs_refs = Set.new - @key_pairs_deleted = [] @private_images = [] - @private_images_refs = Set.new - @private_images_deleted = [] @shared_images = [] - @shared_images_refs = Set.new - @shared_images_deleted = [] @public_images = [] - @public_images_refs = Set.new - @public_images_deleted = [] @cloud_networks = [] - @cloud_networks_refs = Set.new - @cloud_networks_deleted = [] @cloud_subnets = [] - @cloud_subnets_refs = Set.new - @cloud_subnets_deleted = [] @security_groups = [] - @security_groups_refs = Set.new - @security_groups_deleted = [] @floating_ips = [] - @floating_ips_refs = Set.new - @floating_ips_deleted = [] @network_ports = [] - @network_ports_refs = Set.new - @network_ports_deleted = [] @load_balancers = [] - @load_balancers_refs = Set.new @stacks = [] - @stacks_refs = Set.new @cloud_volumes = [] - @cloud_volumes_refs = Set.new @cloud_volume_snapshots = [] - @cloud_volume_snapshots_refs = Set.new # Nested resources @stack_resources = {} @stack_resources_refs = {} diff --git a/app/models/manageiq/providers/amazon/inventory/collector/target_collection.rb b/app/models/manageiq/providers/amazon/inventory/collector/target_collection.rb index b94d76b21..ed7a715a6 100644 --- a/app/models/manageiq/providers/amazon/inventory/collector/target_collection.rb +++ b/app/models/manageiq/providers/amazon/inventory/collector/target_collection.rb @@ -3,24 +3,30 @@ def initialize(_manager, _target) super parse_targets! infer_related_ems_refs! + + target.manager_refs_by_association_reset + end + + def references(collection) + target.manager_refs_by_association.try(:[], collection).try(:[], :ems_ref).try(:to_a) || [] end def instances hash_collection.new( - aws_ec2.instances(:filters => [{:name => 'instance-id', :values => instances_refs.to_a}]) + aws_ec2.instances(:filters => [{:name => 'instance-id', :values => references(:vms)}]) ) end def private_images hash_collection.new( - aws_ec2.client.describe_images(:filters => [{:name => 'image-id', :values => private_images_refs.to_a}]).images + aws_ec2.client.describe_images(:filters => [{:name => 'image-id', :values => references(:miq_templates)}]).images ) end def stacks # TODO(lsmola) we can filter only one stack, so that means too many requests, lets try to figure out why # CLoudFormations API doesn't support a standard filter - result = stacks_refs.to_a.map do |stack_ref| + result = references(:orchestrations_stacks).map do |stack_ref| begin aws_cloud_formation.client.describe_stacks(:stack_name => stack_ref)[:stacks] rescue Aws::CloudFormation::Errors::ValidationError => _e @@ -33,33 +39,33 @@ def stacks def cloud_networks hash_collection.new( - aws_ec2.client.describe_vpcs(:filters => [{:name => 'vpc-id', :values => cloud_networks_refs.to_a}]).vpcs + aws_ec2.client.describe_vpcs(:filters => [{:name => 'vpc-id', :values => references(:cloud_networks)}]).vpcs ) end def cloud_subnets hash_collection.new( - aws_ec2.client.describe_subnets(:filters => [{:name => 'subnet-id', :values => cloud_subnets_refs.to_a}]).subnets + aws_ec2.client.describe_subnets(:filters => [{:name => 'subnet-id', :values => references(:cloud_subnets)}]).subnets ) end def security_groups hash_collection.new( - aws_ec2.security_groups(:filters => [{:name => 'group-id', :values => security_groups_refs.to_a}]) + aws_ec2.security_groups(:filters => [{:name => 'group-id', :values => references(:security_groups).to_a}]) ) end def network_ports hash_collection.new(aws_ec2.client.describe_network_interfaces( - :filters => [{:name => 'network-interface-id', :values => network_ports_refs.to_a}] + :filters => [{:name => 'network-interface-id', :values => references(:network_ports).to_a}] ).network_interfaces) end def load_balancers - return [] if load_balancers_refs.blank? + return [] if references(:load_balancers).blank? result = [] - load_balancers_refs.to_a.each do |load_balancers_ref| + references(:load_balancers).each do |load_balancers_ref| begin result += aws_elb.client.describe_load_balancers( :load_balancer_names => [load_balancers_ref] @@ -75,20 +81,20 @@ def load_balancers def floating_ips hash_collection.new( - aws_ec2.client.describe_addresses(:filters => [{:name => 'allocation-id', :values => floating_ips_refs.to_a}]).addresses + aws_ec2.client.describe_addresses(:filters => [{:name => 'allocation-id', :values => references(:floating_ips)}]).addresses ) end def cloud_volumes hash_collection.new( - aws_ec2.client.describe_volumes(:filters => [{:name => 'volume-id', :values => cloud_volumes_refs.to_a}]).volumes + aws_ec2.client.describe_volumes(:filters => [{:name => 'volume-id', :values => references(:cloud_volumes)}]).volumes ) end def cloud_volume_snapshots hash_collection.new( aws_ec2.client.describe_snapshots( - :filters => [{:name => 'snapshot-id', :values => cloud_volume_snapshots_refs.to_a}] + :filters => [{:name => 'snapshot-id', :values => references(:cloud_volumes_snapshots)}] ).snapshots ) end @@ -131,56 +137,15 @@ def stack_template(stack_name) def parse_targets! target.targets.each do |t| - if t.kind_of?(::EmsEvent) - parse_parse_ems_event_target!(t) - elsif t.kind_of?(::Vm) + case t + when Vm parse_vm_target!(t) end end end - def parse_parse_ems_event_target!(t) - collect_references!(t.full_data.fetch_path("detail", "requestParameters") || {}) - collect_references!(t.full_data.fetch_path("detail", "responseElements") || {}) - - instance_id = t.full_data.fetch_path("detail", "instance-id") - instances_refs << instance_id if instance_id - end - - def collect_references!(hash) - instances_refs << hash["instanceId"] if hash["instanceId"] - private_images_refs << hash["imageId"] if hash["imageId"] - key_pairs_refs << hash["keyName"] if hash["keyName"] - stacks_refs << hash["stackId"] if hash["stackId"] - stacks_refs << hash["stackName"] if hash["stackName"] - cloud_networks_refs << hash["vpcId"] if hash["vpcId"] - cloud_subnets_refs << hash["subnetId"] if hash["subnetId"] - network_ports_refs << hash["networkInterfaceId"] if hash["networkInterfaceId"] - security_groups_refs << hash["groupId"] if hash["groupId"] - floating_ips_refs << hash["allocationId"] if hash["allocationId"] - load_balancers_refs << hash["loadBalancerName"] if hash["loadBalancerName"] - cloud_volumes_refs << hash["volumeId"] if hash["volumeId"] - cloud_volume_snapshots_refs << hash["snapshotId"] if hash["snapshotId"] - - # TODO(lsmola) how to handle tagging? Tagging affects e.g. a name of any resource, but contains only a generic - # resourceID - # "requestParameters"=> - # {"resourcesSet"=>{"items"=>[{"resourceId"=>"vol-07ad036724e3175a5"}]}, - # "tagSet"=>{"items"=>[{"key"=>"Name", "value"=>"ladas_volue_2"}]}}, - # I think we can parse the resource id, so guess where it belongs. - # TODO(lsmola) RegisterImage, by creating image from a volume snapshot, should we track the block device - # mapping and refresh also snapshot? - - collect_references!(hash["networkInterface"]) if hash["networkInterface"] - - (hash.fetch_path("groupSet", "items") || []).each { |x| collect_references!(x) } - (hash.fetch_path("instancesSet", "items") || []).each { |x| collect_references!(x) } - (hash.fetch_path("instances") || []).each { |x| collect_references!(x) } - (hash.fetch_path("networkInterfaceSet", "items") || []).each { |x| collect_references!(x) } - end - def parse_vm_target!(t) - instances_refs << t.ems_ref if t.ems_ref + target.add_target!(:association => :vms, :manager_ref => {:ems_ref => t.ems_ref}) if t.ems_ref end def infer_related_ems_refs! @@ -188,15 +153,29 @@ def infer_related_ems_refs! # ems_refs of every related object. Now this is not very nice fro ma design point of view, but we really want # to see changes in VM's associated objects, so the VM view is always consistent and have fresh data. The partial # reason for this is, that AWS doesn't send all the objects state change, - changed_vms = manager.vms.where(:ems_ref => instances_refs.to_a).includes(:key_pairs, :network_ports, :floating_ips, - :orchestration_stack) + byebug + changed_vms = manager.vms.where(:ems_ref => references(:vms)).includes(:key_pairs, :network_ports, :floating_ips, + :orchestration_stack) changed_vms.each do |vm| stack = vm.orchestration_stack all_stacks = ([stack] + (stack.try(:ancestors) || [])).compact - stacks_refs.merge all_stacks.collect(&:ems_ref).compact - key_pairs_refs.merge vm.key_pairs.collect(&:name).compact - network_ports_refs.merge vm.network_ports.collect(&:ems_ref).compact - floating_ips_refs.merge vm.floating_ips.collect(&:ems_ref).compact + + all_stacks.collect(&:ems_ref).compact.each do |ems_ref| + target.add_target!(:association => :orchestration_stacks, :manager_ref => {:ems_ref => ems_ref}) + end + + vm.key_pairs.collect(&:name).compact.each do |ems_ref| + target.add_target!(:association => :key_pairs, :manager_ref => {:ems_ref => ems_ref}) + end + + vm.network_ports.collect(&:ems_ref).compact.each do |ems_ref| + target.add_target!(:association => :network_ports, :manager_ref => {:ems_ref => ems_ref}) + end + + vm.floating_ips.collect(&:ems_ref).compact.each do |ems_ref| + target.add_target!(:association => :floating_ips, :manager_ref => {:ems_ref => ems_ref}) + end end + byebug end end