Merge pull request ManageIQ#14673 from Ladas/do_not_store_ar_object_f…

…or_old_refresh_euwe

[EUWE] Do not store ar object for old refresh

app/models/ems_refresh/save_inventory.rb

@@ -53,6 +53,8 @@ def save_vms_inventory(ems, hashes, target = nil)
     _log.info "#{log_header} Duplicate unique values found: #{dup_vms_uids.inspect}" unless dup_vms_uids.empty?
 
     invalids_found = false
+    # Clear vms, so GC can clean them
+    vms = nil
 
     ActiveRecord::Base.transaction do
       hashes.each do |h|
@@ -69,7 +71,7 @@ def save_vms_inventory(ems, hashes, target = nil)
         h[:cloud_network_id]       = key_backup.fetch_path(:cloud_network, :id)
         h[:cloud_subnet_id]        = key_backup.fetch_path(:cloud_subnet, :id)
         h[:cloud_tenant_id]        = key_backup.fetch_path(:cloud_tenant, :id)
-        h[:cloud_tenants]          = key_backup.fetch_path(:cloud_tenants).compact.map { |x| x[:_object] } if key_backup.fetch_path(:cloud_tenants, 0, :_object)
+        h[:cloud_tenant_ids]       = key_backup.fetch_path(:cloud_tenants).compact.map { |x| x[:id] } if key_backup.fetch_path(:cloud_tenants, 0, :id)
         h[:orchestration_stack_id] = key_backup.fetch_path(:orchestration_stack, :id)
         begin
           raise MiqException::MiqIncompleteData if h[:invalid]
@@ -96,7 +98,7 @@ def save_vms_inventory(ems, hashes, target = nil)
             h[:location] = "unknown" if h[:location].blank?
 
             # build a type-specific vm or template
-            found = ems.vms_and_templates.build(h)
+            found = ems.vms_and_templates.klass.new(h)
           else
             vms_by_uid_ems[h[:uid_ems]].delete(found)
             h.delete(:type)
@@ -115,8 +117,6 @@ def save_vms_inventory(ems, hashes, target = nil)
 
           found.save!
           h[:id] = found.id
-          found.reload
-          h[:_object] = found
         rescue => err
           # If a vm failed to process, mark it as invalid and log an error
           h[:invalid] = invalids_found = true

app/models/ems_refresh/save_inventory_cloud.rb

@@ -90,7 +90,7 @@ def save_flavors_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      h[:cloud_tenants] = (h.fetch_path(:cloud_tenants) || []).compact.map { |x| x[:_object] }.uniq
+      h[:cloud_tenant_ids] = (h.delete(:cloud_tenants) || []).compact.map { |x| x[:id] }.uniq
     end
 
     save_inventory_multi(ems.flavors, hashes, deletes, [:ems_ref])

app/models/ems_refresh/save_inventory_helper.rb

@@ -43,6 +43,8 @@ def save_inventory_multi(association, hashes, deletes, find_key, child_keys = []
     end
     deletes = deletes.to_a
     deletes_index = deletes.index_by { |x| x }
+    # Alow GC to clean the AR objects as they are removed from deletes_index
+    deletes = nil
 
     child_keys = Array.wrap(child_keys)
     remove_keys = Array.wrap(extra_keys) + child_keys

app/models/ems_refresh/save_inventory_infra.rb

@@ -193,7 +193,6 @@ def save_hosts_inventory(ems, hashes, target = nil)
 
         found.save!
         h[:id] = found.id
-        h[:_object] = found
       rescue => err
         # If a host failed to process, mark it as invalid and log an error
         h[:invalid] = invalids_found = true

app/models/ems_refresh/save_inventory_network.rb

@@ -72,12 +72,6 @@ def save_cloud_networks_inventory(ems, hashes, target = nil)
                 []
               end
 
-    hashes.each do |h|
-      %i(cloud_tenant orchestration_stack).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
-    end
-
     save_inventory_multi(ems.cloud_networks,
                          hashes,
                          deletes,
@@ -96,10 +90,6 @@ def save_network_groups_inventory(ems, hashes, target = nil)
                 []
               end
 
-    hashes.each do |h|
-      h[:orchestration_stack_id] = h.fetch_path(:orchestration_stack, :id)
-    end
-
     save_inventory_multi(ems.network_groups,
                          hashes,
                          deletes,
@@ -111,14 +101,11 @@ def save_network_groups_inventory(ems, hashes, target = nil)
 
   def save_cloud_subnets_inventory(network, hashes)
     hashes.each do |h|
-      %i(availability_zone parent_cloud_subnet).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
-
+      h[:cloud_network_id] = h.fetch_path(:cloud_network, :id) if h.key?(:cloud_network)
       h[:ems_id] = network.ems_id
     end
 
-    save_inventory_multi(network.cloud_subnets, hashes, :use_association, [:ems_ref], nil, [:network_router])
+    save_inventory_multi(network.cloud_subnets, hashes, :use_association, [:ems_ref], nil, [:network_router, :cloud_network])
 
     network.save!
     store_ids_for_new_records(network.cloud_subnets, hashes, :ems_ref)
@@ -135,15 +122,16 @@ def save_security_groups_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      %i(cloud_tenant cloud_network orchestration_stack network_group).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:cloud_network_id] = h.fetch_path(:cloud_network, :id)
+      h[:network_group_id] = h.fetch_path(:network_group, :id)
     end
 
-    save_inventory_multi(ems.security_groups, hashes,
+    save_inventory_multi(ems.security_groups,
+                         hashes,
                          deletes,
                          [:ems_ref],
-                         :firewall_rules)
+                         :firewall_rules,
+                         [:cloud_network, :network_group])
     store_ids_for_new_records(ems.security_groups, hashes, :ems_ref)
 
     # Reset the source_security_group_id for the firewall rules after all
@@ -168,12 +156,11 @@ def save_floating_ips_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      %i(vm cloud_tenant cloud_network network_port).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:cloud_network_id] = h.fetch_path(:cloud_network, :id)
+      h[:network_port_id]  = h.fetch_path(:network_port, :id)
     end
 
-    save_inventory_multi(ems.floating_ips, hashes, deletes, [:ems_ref])
+    save_inventory_multi(ems.floating_ips, hashes, deletes, [:ems_ref], nil, [:cloud_network, :network_port])
     store_ids_for_new_records(ems.floating_ips, hashes, :ems_ref)
   end
 
@@ -211,15 +198,16 @@ def save_network_routers_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      %i(cloud_tenant cloud_network network_group).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:cloud_network_id] = h.fetch_path(:cloud_network, :id)
+      h[:network_group_id] = h.fetch_path(:network_group, :id)
     end
 
     save_inventory_multi(ems.network_routers,
                          hashes,
                          deletes,
-                         [:ems_ref])
+                         [:ems_ref],
+                         nil,
+                         [:cloud_network, :network_group])
     store_ids_for_new_records(ems.network_routers, hashes, :ems_ref)
   end
 
@@ -251,15 +239,24 @@ def save_network_ports_inventory(ems, hashes, target = nil, mode = :refresh)
     hashes.compact!
 
     hashes.each do |h|
-      %i(cloud_tenant device cloud_subnet).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
+      device = h.fetch_path(:device)
+      if device.kind_of?(Hash)
+        h.delete(:device)
+
+        h[:device_id]   = device[:id]
+        h[:device_type] = device[:type].constantize.base_class.name
       end
 
-      h[:security_groups] = (h.fetch_path(:security_groups) || []).map { |x| x.try(:[], :_object) }.compact.uniq
+      h[:security_group_ids] = (h.delete(:security_groups) || []).map { |x| x.try(:[], :id) }.compact.uniq
       h[:source] = mode
     end
 
-    save_inventory_multi(ems.network_ports, hashes, deletes, [:ems_ref], :cloud_subnet_network_ports)
+    save_inventory_multi(ems.network_ports,
+                         hashes,
+                         deletes,
+                         [:ems_ref],
+                         :cloud_subnet_network_ports,
+                         [:cloud_subnet])
 
     store_ids_for_new_records(ems.network_ports, hashes, :ems_ref)
   end
@@ -268,12 +265,15 @@ def save_cloud_subnet_network_ports_inventory(network_port, hashes)
     deletes = network_port.cloud_subnet_network_ports.reload.dup
 
     hashes.each do |h|
-      %i(cloud_subnet).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:cloud_subnet_id] = h.fetch_path(:cloud_subnet, :id)
     end
 
-    save_inventory_multi(network_port.cloud_subnet_network_ports, hashes, deletes, [:cloud_subnet, :address])
+    save_inventory_multi(network_port.cloud_subnet_network_ports,
+                         hashes,
+                         deletes,
+                         [:cloud_subnet_id, :address],
+                         nil,
+                         [:cloud_subnet])
   end
 
   def save_load_balancer_pool_members_inventory(ems, hashes, target = nil)
@@ -286,12 +286,6 @@ def save_load_balancer_pool_members_inventory(ems, hashes, target = nil)
                 []
               end
 
-    hashes.each do |h|
-      %i(vm).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
-    end
-
     save_inventory_multi(ems.load_balancer_pool_members,
                          hashes,
                          deletes,
@@ -321,12 +315,13 @@ def save_load_balancer_pool_member_pools_inventory(load_balancer_pool, hashes)
     deletes = load_balancer_pool.load_balancer_pool_member_pools.reload.dup
 
     hashes.each do |h|
-      %i(load_balancer_pool_member).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:load_balancer_pool_member_id] = h.fetch_path(:load_balancer_pool_member, :id)
     end
 
-    save_inventory_multi(load_balancer_pool.load_balancer_pool_member_pools, hashes, deletes,
+    save_inventory_multi(load_balancer_pool.load_balancer_pool_member_pools,
+                         hashes, deletes,
+                         [:load_balancer_pool_member_id],
+                         nil,
                          [:load_balancer_pool_member])
   end
 
@@ -341,29 +336,31 @@ def save_load_balancer_listeners_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      %i(load_balancer).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:load_balancer_id] = h.fetch_path(:load_balancer, :id)
     end
 
     save_inventory_multi(ems.load_balancer_listeners,
                          hashes,
                          deletes,
                          [:ems_ref],
-                         :load_balancer_listener_pools)
+                         :load_balancer_listener_pools,
+                         [:load_balancer])
     store_ids_for_new_records(ems.load_balancer_listeners, hashes, :ems_ref)
   end
 
   def save_load_balancer_listener_pools_inventory(load_balancer_listener, hashes)
     deletes = load_balancer_listener.load_balancer_listener_pools.reload.dup
 
     hashes.each do |h|
-      %i(load_balancer_pool).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:load_balancer_pool_id] = h.fetch_path(:load_balancer_pool, :id)
     end
 
-    save_inventory_multi(load_balancer_listener.load_balancer_listener_pools, hashes, deletes, [:load_balancer_pool])
+    save_inventory_multi(load_balancer_listener.load_balancer_listener_pools,
+                         hashes,
+                         deletes,
+                         [:load_balancer_pool_id],
+                         nil,
+                         [:load_balancer_pool])
   end
 
   def save_load_balancer_health_checks_inventory(ems, hashes, target = nil)
@@ -377,36 +374,39 @@ def save_load_balancer_health_checks_inventory(ems, hashes, target = nil)
               end
 
     hashes.each do |h|
-      %i(load_balancer load_balancer_listener).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:load_balancer_id] = h.fetch_path(:load_balancer, :id)
+      h[:load_balancer_listener_id] = h.fetch_path(:load_balancer_listener, :id)
     end
 
     save_inventory_multi(ems.load_balancer_health_checks,
                          hashes,
                          deletes,
                          [:ems_ref],
-                         :load_balancer_health_check_members)
+                         :load_balancer_health_check_members,
+                         [:load_balancer, :load_balancer_listener])
     store_ids_for_new_records(ems.load_balancer_health_checks, hashes, :ems_ref)
   end
 
   def save_load_balancer_health_check_members_inventory(load_balancer_health_check, hashes)
     deletes = load_balancer_health_check.load_balancer_health_check_members.reload.dup
 
     hashes.each do |h|
-      %i(load_balancer_pool_member).each do |relation|
-        h[relation] = h.fetch_path(relation, :_object) if h.fetch_path(relation, :_object)
-      end
+      h[:load_balancer_pool_member_id] = h.fetch_path(:load_balancer_pool_member, :id)
     end
 
-    save_inventory_multi(load_balancer_health_check.load_balancer_health_check_members, hashes, deletes,
+    save_inventory_multi(load_balancer_health_check.load_balancer_health_check_members,
+                         hashes,
+                         deletes,
+                         [:load_balancer_pool_member_id],
+                         nil,
                          [:load_balancer_pool_member])
   end
 
   def link_cloud_subnets_to_network_routers(hashes)
+    return if hashes.blank?
+
     hashes.each do |hash|
-      network_router = hash.fetch_path(:network_router, :_object)
-      hash[:_object].update_attributes(:network_router => network_router)
+      CloudSubnet.where(:id => hash[:id]).update_all(:network_router_id => hash.fetch_path(:network_router, :id))
     end
   end
 end