Use ManagerRefresh::TargetCollection collection for TargetCollection …

…Collector

Use ManagerRefresh::TargetCollection collection for TargetCollection Collector

app/models/manageiq/providers/amazon/inventory/collector.rb

@@ -3,22 +3,22 @@ class ManageIQ::Providers::Amazon::Inventory::Collector < ManagerRefresh::Invent
   require_nested :NetworkManager
   require_nested :TargetCollection
 
-  attr_reader :instances, :instances_refs, :instances_deleted
-  attr_reader :flavors, :flavors_refs, :flavors_deleted
-  attr_reader :availability_zones, :availability_zones_refs, :availability_zones_deleted
-  attr_reader :key_pairs, :key_pairs_refs, :key_pairs_deleted
-  attr_reader :private_images, :private_images_refs, :private_images_deleted
-  attr_reader :shared_images, :shared_images_refs, :shared_images_deleted
-  attr_reader :public_images, :public_images_refs, :public_images_deleted
-  attr_reader :cloud_networks, :cloud_networks_refs, :cloud_networks_deleted
-  attr_reader :cloud_subnets, :cloud_subnets_refs, :cloud_subnets_deleted
-  attr_reader :security_groups, :security_groups_refs, :security_groups_deleted
-  attr_reader :floating_ips, :floating_ips_refs, :floating_ips_deleted
-  attr_reader :network_ports, :network_ports_refs, :network_ports_deleted
-  attr_reader :load_balancers, :load_balancers_refs, :load_balancers_deleted
-  attr_reader :stacks, :stacks_refs, :stacks_deleted
-  attr_reader :cloud_volumes, :cloud_volumes_refs
-  attr_reader :cloud_volume_snapshots, :cloud_volume_snapshots_refs
+  attr_reader :instances
+  attr_reader :flavors
+  attr_reader :availability_zones
+  attr_reader :key_pairs
+  attr_reader :private_images
+  attr_reader :shared_images
+  attr_reader :public_images
+  attr_reader :cloud_networks
+  attr_reader :cloud_subnets
+  attr_reader :security_groups
+  attr_reader :floating_ips
+  attr_reader :network_ports
+  attr_reader :load_balancers
+  attr_reader :stacks
+  attr_reader :cloud_volumes
+  attr_reader :cloud_volume_snapshots
   attr_reader :cloud_objects_store_containers
   attr_reader :cloud_objects_store_objects
 
@@ -30,49 +30,21 @@ def initialize(_manager, _target)
 
   def initialize_inventory_sources
     @instances                   = []
-    @instances_refs              = Set.new
-    @instances_deleted           = []
     @flavors                     = []
-    @flavors_refs                = Set.new
-    @flavors_deleted             = []
     @availability_zones          = []
-    @availability_zones_refs     = Set.new
-    @availability_zones_deleted  = []
     @key_pairs                   = []
-    @key_pairs_refs              = Set.new
-    @key_pairs_deleted           = []
     @private_images              = []
-    @private_images_refs         = Set.new
-    @private_images_deleted      = []
     @shared_images               = []
-    @shared_images_refs          = Set.new
-    @shared_images_deleted       = []
     @public_images               = []
-    @public_images_refs          = Set.new
-    @public_images_deleted       = []
     @cloud_networks              = []
-    @cloud_networks_refs         = Set.new
-    @cloud_networks_deleted      = []
     @cloud_subnets               = []
-    @cloud_subnets_refs          = Set.new
-    @cloud_subnets_deleted       = []
     @security_groups             = []
-    @security_groups_refs        = Set.new
-    @security_groups_deleted     = []
     @floating_ips                = []
-    @floating_ips_refs           = Set.new
-    @floating_ips_deleted        = []
     @network_ports               = []
-    @network_ports_refs          = Set.new
-    @network_ports_deleted       = []
     @load_balancers              = []
-    @load_balancers_refs         = Set.new
     @stacks                      = []
-    @stacks_refs                 = Set.new
     @cloud_volumes               = []
-    @cloud_volumes_refs          = Set.new
     @cloud_volume_snapshots      = []
-    @cloud_volume_snapshots_refs = Set.new
     # Nested resources
     @stack_resources             = {}
     @stack_resources_refs        = {}

app/models/manageiq/providers/amazon/inventory/collector/target_collection.rb

@@ -3,24 +3,30 @@ def initialize(_manager, _target)
     super
     parse_targets!
     infer_related_ems_refs!
+
+    target.manager_refs_by_association_reset
+  end
+
+  def references(collection)
+    target.manager_refs_by_association.try(:[], collection).try(:[], :ems_ref).try(:to_a) || []
   end
 
   def instances
     hash_collection.new(
-      aws_ec2.instances(:filters => [{:name => 'instance-id', :values => instances_refs.to_a}])
+      aws_ec2.instances(:filters => [{:name => 'instance-id', :values => references(:vms)}])
     )
   end
 
   def private_images
     hash_collection.new(
-      aws_ec2.client.describe_images(:filters => [{:name => 'image-id', :values => private_images_refs.to_a}]).images
+      aws_ec2.client.describe_images(:filters => [{:name => 'image-id', :values => references(:miq_templates)}]).images
     )
   end
 
   def stacks
     # TODO(lsmola) we can filter only one stack, so that means too many requests, lets try to figure out why
     # CLoudFormations API doesn't support a standard filter
-    result = stacks_refs.to_a.map do |stack_ref|
+    result = references(:orchestrations_stacks).map do |stack_ref|
       begin
         aws_cloud_formation.client.describe_stacks(:stack_name => stack_ref)[:stacks]
       rescue Aws::CloudFormation::Errors::ValidationError => _e
@@ -33,33 +39,33 @@ def stacks
 
   def cloud_networks
     hash_collection.new(
-      aws_ec2.client.describe_vpcs(:filters => [{:name => 'vpc-id', :values => cloud_networks_refs.to_a}]).vpcs
+      aws_ec2.client.describe_vpcs(:filters => [{:name => 'vpc-id', :values => references(:cloud_networks)}]).vpcs
     )
   end
 
   def cloud_subnets
     hash_collection.new(
-      aws_ec2.client.describe_subnets(:filters => [{:name => 'subnet-id', :values => cloud_subnets_refs.to_a}]).subnets
+      aws_ec2.client.describe_subnets(:filters => [{:name => 'subnet-id', :values => references(:cloud_subnets)}]).subnets
     )
   end
 
   def security_groups
     hash_collection.new(
-      aws_ec2.security_groups(:filters => [{:name => 'group-id', :values => security_groups_refs.to_a}])
+      aws_ec2.security_groups(:filters => [{:name => 'group-id', :values => references(:security_groups).to_a}])
     )
   end
 
   def network_ports
     hash_collection.new(aws_ec2.client.describe_network_interfaces(
-      :filters => [{:name => 'network-interface-id', :values => network_ports_refs.to_a}]
+      :filters => [{:name => 'network-interface-id', :values => references(:network_ports).to_a}]
     ).network_interfaces)
   end
 
   def load_balancers
-    return [] if load_balancers_refs.blank?
+    return [] if references(:load_balancers).blank?
 
     result = []
-    load_balancers_refs.to_a.each do |load_balancers_ref|
+    references(:load_balancers).each do |load_balancers_ref|
       begin
         result += aws_elb.client.describe_load_balancers(
           :load_balancer_names => [load_balancers_ref]
@@ -75,20 +81,20 @@ def load_balancers
 
   def floating_ips
     hash_collection.new(
-      aws_ec2.client.describe_addresses(:filters => [{:name => 'allocation-id', :values => floating_ips_refs.to_a}]).addresses
+      aws_ec2.client.describe_addresses(:filters => [{:name => 'allocation-id', :values => references(:floating_ips)}]).addresses
     )
   end
 
   def cloud_volumes
     hash_collection.new(
-      aws_ec2.client.describe_volumes(:filters => [{:name => 'volume-id', :values => cloud_volumes_refs.to_a}]).volumes
+      aws_ec2.client.describe_volumes(:filters => [{:name => 'volume-id', :values => references(:cloud_volumes)}]).volumes
     )
   end
 
   def cloud_volume_snapshots
     hash_collection.new(
       aws_ec2.client.describe_snapshots(
-        :filters => [{:name => 'snapshot-id', :values => cloud_volume_snapshots_refs.to_a}]
+        :filters => [{:name => 'snapshot-id', :values => references(:cloud_volumes_snapshots)}]
       ).snapshots
     )
   end
@@ -131,72 +137,45 @@ def stack_template(stack_name)
 
   def parse_targets!
     target.targets.each do |t|
-      if t.kind_of?(::EmsEvent)
-        parse_parse_ems_event_target!(t)
-      elsif t.kind_of?(::Vm)
+      case t
+      when Vm
         parse_vm_target!(t)
       end
     end
   end
 
-  def parse_parse_ems_event_target!(t)
-    collect_references!(t.full_data.fetch_path("detail", "requestParameters") || {})
-    collect_references!(t.full_data.fetch_path("detail", "responseElements") || {})
-
-    instance_id = t.full_data.fetch_path("detail", "instance-id")
-    instances_refs << instance_id if instance_id
-  end
-
-  def collect_references!(hash)
-    instances_refs << hash["instanceId"] if hash["instanceId"]
-    private_images_refs << hash["imageId"] if hash["imageId"]
-    key_pairs_refs << hash["keyName"] if hash["keyName"]
-    stacks_refs << hash["stackId"] if hash["stackId"]
-    stacks_refs << hash["stackName"] if hash["stackName"]
-    cloud_networks_refs << hash["vpcId"] if hash["vpcId"]
-    cloud_subnets_refs << hash["subnetId"] if hash["subnetId"]
-    network_ports_refs << hash["networkInterfaceId"] if hash["networkInterfaceId"]
-    security_groups_refs << hash["groupId"] if hash["groupId"]
-    floating_ips_refs << hash["allocationId"] if hash["allocationId"]
-    load_balancers_refs << hash["loadBalancerName"] if hash["loadBalancerName"]
-    cloud_volumes_refs << hash["volumeId"] if hash["volumeId"]
-    cloud_volume_snapshots_refs << hash["snapshotId"] if hash["snapshotId"]
-
-    # TODO(lsmola) how to handle tagging? Tagging affects e.g. a name of any resource, but contains only a generic
-    # resourceID
-    # "requestParameters"=>
-    #   {"resourcesSet"=>{"items"=>[{"resourceId"=>"vol-07ad036724e3175a5"}]},
-    #    "tagSet"=>{"items"=>[{"key"=>"Name", "value"=>"ladas_volue_2"}]}},
-    # I think we can parse the resource id, so guess where it belongs.
-    # TODO(lsmola) RegisterImage, by creating image from a volume snapshot, should we track the block device
-    # mapping and refresh also snapshot?
-
-    collect_references!(hash["networkInterface"]) if hash["networkInterface"]
-
-    (hash.fetch_path("groupSet", "items") || []).each { |x| collect_references!(x) }
-    (hash.fetch_path("instancesSet", "items") || []).each { |x| collect_references!(x) }
-    (hash.fetch_path("instances") || []).each { |x| collect_references!(x) }
-    (hash.fetch_path("networkInterfaceSet", "items") || []).each { |x| collect_references!(x) }
-  end
-
   def parse_vm_target!(t)
-    instances_refs << t.ems_ref if t.ems_ref
+    target.add_target!(:association => :vms, :manager_ref => {:ems_ref => t.ems_ref}) if t.ems_ref
   end
 
   def infer_related_ems_refs!
     # We have a list of instances_refs collected from events. Now we want to look into our DB and API, and collect
     # ems_refs of every related object. Now this is not very nice fro ma design point of view, but we really want
     # to see changes in VM's associated objects, so the VM view is always consistent and have fresh data. The partial
     # reason for this is, that AWS doesn't send all the objects state change,
-    changed_vms = manager.vms.where(:ems_ref => instances_refs.to_a).includes(:key_pairs, :network_ports, :floating_ips,
-                                                                          :orchestration_stack)
+    byebug
+    changed_vms = manager.vms.where(:ems_ref => references(:vms)).includes(:key_pairs, :network_ports, :floating_ips,
+                                                                           :orchestration_stack)
     changed_vms.each do |vm|
       stack      = vm.orchestration_stack
       all_stacks = ([stack] + (stack.try(:ancestors) || [])).compact
-      stacks_refs.merge all_stacks.collect(&:ems_ref).compact
-      key_pairs_refs.merge vm.key_pairs.collect(&:name).compact
-      network_ports_refs.merge vm.network_ports.collect(&:ems_ref).compact
-      floating_ips_refs.merge vm.floating_ips.collect(&:ems_ref).compact
+
+      all_stacks.collect(&:ems_ref).compact.each do |ems_ref|
+        target.add_target!(:association => :orchestration_stacks, :manager_ref => {:ems_ref => ems_ref})
+      end
+
+      vm.key_pairs.collect(&:name).compact.each do |ems_ref|
+        target.add_target!(:association => :key_pairs, :manager_ref => {:ems_ref => ems_ref})
+      end
+
+      vm.network_ports.collect(&:ems_ref).compact.each do |ems_ref|
+        target.add_target!(:association => :network_ports, :manager_ref => {:ems_ref => ems_ref})
+      end
+
+      vm.floating_ips.collect(&:ems_ref).compact.each do |ems_ref|
+        target.add_target!(:association => :floating_ips, :manager_ref => {:ems_ref => ems_ref})
+      end
     end
+    byebug
   end
 end