[Gaprindashvili] Improve network manager refresh speed

app/models/manageiq/providers/openstack/inventory/collector.rb

@@ -50,6 +50,7 @@ def initialize_inventory_sources
     @volume_snapshot_templates = []
     # network
     @cloud_networks            = []
+    @cloud_subnets             = []
     @floating_ips              = []
     @network_ports             = []
     @network_routers           = []

app/models/manageiq/providers/openstack/inventory/collector/network_manager.rb

@@ -8,7 +8,12 @@ def floating_ips
 
   def cloud_networks
     return @cloud_networks if @cloud_networks.any?
-    @cloud_networks = network_service.handled_list(:networks, {}, openstack_network_admin?)
+    @cloud_networks = safe_list { network_service.list_networks.body["networks"] }
+  end
+
+  def cloud_subnets
+    return @cloud_subnets if @cloud_subnets.any?
+    @cloud_subnets = network_service.handled_list(:subnets, {}, openstack_network_admin?)
   end
 
   def network_ports

app/models/manageiq/providers/openstack/inventory/collector/target_collection.rb

@@ -34,10 +34,15 @@ def cloud_networks
     return [] if references(:cloud_networks).blank?
     return @cloud_networks if @cloud_networks.any?
     @cloud_networks = references(:cloud_networks).collect do |network_id|
-      safe_get { network_service.networks.get(network_id) }
+      safe_get { network_service.get_network(network_id).body["network"] }
     end.compact
   end
 
+  def cloud_subnets
+    return @cloud_subnets if @cloud_subnets.any?
+    @cloud_subnets = network_service.handled_list(:subnets, {}, openstack_network_admin?)
+  end
+
   def network_ports
     return [] if references(:network_ports).blank?
     return @network_ports if @network_ports.any?

app/models/manageiq/providers/openstack/inventory/parser/network_manager.rb

@@ -3,6 +3,7 @@ class ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager < Manage
 
   def parse
     cloud_networks
+    cloud_subnets
     floating_ips
     network_ports
     network_routers
@@ -11,44 +12,47 @@ def parse
 
   def cloud_networks
     collector.cloud_networks.each do |n|
-      status = status = n.status.to_s.downcase == "active" ? "active" : "inactive"
-      network_type_suffix = n.router_external ? "::Public" : "::Private"
+      status = n["status"].to_s.downcase == "active" ? "active" : "inactive"
+      network_type_suffix = n["router:external"] ? "::Public" : "::Private"
 
-      network = persister.cloud_networks.find_or_build(n.id)
+      network = persister.cloud_networks.find_or_build(n["id"])
       network.type = "ManageIQ::Providers::Openstack::NetworkManager::CloudNetwork" + network_type_suffix
-      network.name = n.name
-      network.shared = n.shared
+      network.name = n["name"]
+      network.shared = n["shared"]
       network.status = status
-      network.enabled = n.admin_state_up
-      network.external_facing = n.router_external
-      network.provider_physical_network = n.provider_physical_network
-      network.provider_network_type = n.provider_network_type
-      network.provider_segmentation_id = n.provider_segmentation_id
-      network.port_security_enabled = n.attributes["port_security_enabled"]
-      network.qos_policy_id = n.attributes["qos_policy_id"]
-      network.vlan_transparent = n.attributes["vlan_transparent"]
-      network.maximum_transmission_unit = n.attributes["mtu"]
-      network.cloud_tenant = persister.cloud_tenants.lazy_find(n.tenant_id)
-      network.orchestration_stack = persister.orchestration_stacks.lazy_find(collector.orchestration_stack_by_resource_id(n.id))
-      n.subnets.each do |s|
-        subnet = persister.cloud_subnets.find_or_build(s.id)
-        subnet.type = "ManageIQ::Providers::Openstack::NetworkManager::CloudSubnet"
-        subnet.name = s.name
-        subnet.cidr = s.cidr
-        subnet.status = status
-        subnet.network_protocol = "ipv#{s.ip_version}"
-        subnet.gateway = s.gateway_ip
-        subnet.dhcp_enabled = s.enable_dhcp
-        subnet.dns_nameservers = s.dns_nameservers
-        subnet.ipv6_router_advertisement_mode = s.attributes["ipv6_ra_mode"]
-        subnet.ipv6_address_mode = s.attributes["ipv6_address_mode"]
-        subnet.allocation_pools = s.allocation_pools
-        subnet.host_routes = s.host_routes
-        subnet.ip_version = s.ip_version
-        subnet.parent_cloud_subnet = persister.cloud_subnets.lazy_find(s.attributes["vsd_managed"])
-        subnet.cloud_tenant = persister.cloud_tenants.lazy_find(s.tenant_id)
-        subnet.cloud_network = network
-      end
+      network.enabled = n["admin_state_up"]
+      network.external_facing = n["router:external"]
+      network.provider_physical_network = n["provider:physical_network"]
+      network.provider_network_type = n["provider:network_type"]
+      network.provider_segmentation_id = n["provider:segmentation_id"]
+      network.port_security_enabled = n["port_security_enabled"]
+      network.qos_policy_id = n["qos_policy_id"]
+      network.vlan_transparent = n["vlan_transparent"]
+      network.maximum_transmission_unit = n["mtu"]
+      network.cloud_tenant = persister.cloud_tenants.lazy_find(n["tenant_id"])
+      network.orchestration_stack = persister.orchestration_stacks.lazy_find(collector.orchestration_stack_by_resource_id(n["id"]))
+    end
+  end
+
+  def cloud_subnets
+    collector.cloud_subnets.each do |s|
+      subnet = persister.cloud_subnets.find_or_build(s.id)
+      subnet.type = "ManageIQ::Providers::Openstack::NetworkManager::CloudSubnet"
+      subnet.name = s.name
+      subnet.cidr = s.cidr
+      subnet.network_protocol = "ipv#{s.ip_version}"
+      subnet.gateway = s.gateway_ip
+      subnet.dhcp_enabled = s.enable_dhcp
+      subnet.dns_nameservers = s.dns_nameservers
+      subnet.ipv6_router_advertisement_mode = s.attributes["ipv6_ra_mode"]
+      subnet.ipv6_address_mode = s.attributes["ipv6_address_mode"]
+      subnet.allocation_pools = s.allocation_pools
+      subnet.host_routes = s.host_routes
+      subnet.ip_version = s.ip_version
+      subnet.parent_cloud_subnet = persister.cloud_subnets.lazy_find(s.attributes["vsd_managed"])
+      subnet.cloud_tenant = persister.cloud_tenants.lazy_find(s.tenant_id)
+      subnet.cloud_network = persister.cloud_networks.lazy_find(s.network_id)
+      subnet.status = persister.cloud_networks.lazy_find(s.network_id, :key => :status)
     end
   end
 

app/models/manageiq/providers/openstack/network_manager/event_target_parser.rb

@@ -32,6 +32,8 @@ def parse_ems_event_targets(ems_event)
                     :ports
                   elsif ems_event.event_type.start_with?("network.")
                     :networks
+                  elsif ems_event.event_type.start_with?("subnet.")
+                    :subnets
                   elsif ems_event.event_type.start_with?("security_group.")
                     :security_groups
                   end

app/models/manageiq/providers/openstack/network_manager/refresh_parser.rb

@@ -68,7 +68,13 @@ def security_groups
     end
 
     def networks
-      @networks ||= uniques(@network_service.handled_list(:networks, {}, openstack_network_admin?))
+      @networks ||= uniques(@network_service.list_networks.body["networks"])
+    rescue
+      []
+    end
+
+    def subnets
+      @subnets ||= uniques(@network_service.handled_list(:subnets, {}, openstack_network_admin?))
     end
 
     def network_ports
@@ -105,14 +111,11 @@ def get_network_ports
     def get_subnets
       return unless @network_service.name == :neutron
 
-      networks.each do |n|
-        new_net = @data_index.fetch_path(:cloud_networks, n.id)
-        new_net[:cloud_subnets] = n.subnets.collect { |s| parse_subnet(s, n) }
+      process_collection(subnets, :cloud_subnets) { |s| parse_subnet(s) }
 
-        # Lets store also subnets into indexed data, so we can reference them elsewhere
-        new_net[:cloud_subnets].each do |x|
-          @data_index.store_path(:cloud_subnets, x[:ems_ref], x)
-        end
+      networks.each do |n|
+        new_net = @data_index.fetch_path(:cloud_networks, n["id"])
+        new_net[:cloud_subnets] = n["subnets"].collect { |s| @data_index.fetch_path(:cloud_subnets, s) }
       end
     end
 
@@ -154,30 +157,30 @@ def parse_firewall_rule(rule)
     end
 
     def parse_network(network)
-      uid     = network.id
-      status  = (network.status.to_s.downcase == "active") ? "active" : "inactive"
+      uid     = network["id"]
+      status  = network["status"].to_s.downcase == "active" ? "active" : "inactive"
 
-      network_type_suffix = network.router_external ? "::Public" : "::Private"
+      network_type_suffix = network["router:external"] ? "::Public" : "::Private"
 
       new_result = {
         :type                      => self.class.cloud_network_type + network_type_suffix,
-        :name                      => network.name,
+        :name                      => network["name"],
         :ems_ref                   => uid,
-        :shared                    => network.shared,
+        :shared                    => network["shared"],
         :status                    => status,
-        :enabled                   => network.admin_state_up,
-        :external_facing           => network.router_external,
-        :cloud_tenant              => parent_manager_fetch_path(:cloud_tenants, network.tenant_id),
+        :enabled                   => network["admin_state_up"],
+        :external_facing           => network["router:external"],
+        :cloud_tenant              => parent_manager_fetch_path(:cloud_tenants, network["tenant_id"]),
         :orchestration_stack       => parent_manager_fetch_path(:orchestration_stacks, @resource_to_stack[uid]),
-        :provider_physical_network => network.provider_physical_network,
-        :provider_network_type     => network.provider_network_type,
-        :provider_segmentation_id  => network.provider_segmentation_id,
-        :port_security_enabled     => network.attributes["port_security_enabled"],
-        :qos_policy_id             => network.attributes["qos_policy_id"],
-        :vlan_transparent          => network.attributes["vlan_transparent"],
+        :provider_physical_network => network["provider:physical_network"],
+        :provider_network_type     => network["provider:network_type"],
+        :provider_segmentation_id  => network["provider:segmentation_id"],
+        :port_security_enabled     => network["port_security_enabled"],
+        :qos_policy_id             => network["qos_policy_id"],
+        :vlan_transparent          => network["vlan_transparent"],
 
         # TODO(lsmola) expose attributes in FOG
-        :maximum_transmission_unit => network.attributes["mtu"],
+        :maximum_transmission_unit => network["mtu"],
       }
       return uid, new_result
     end
@@ -265,13 +268,15 @@ def parse_network_router(network_router)
       return uid, new_result
     end
 
-    def parse_subnet(subnet, network)
-      {
+    def parse_subnet(subnet)
+      network = @data_index.fetch_path(:cloud_networks, subnet.network_id)
+      network[:cloud_subnets] ||= []
+      new_result = {
         :type                           => self.class.cloud_subnet_type,
         :name                           => subnet.name,
         :ems_ref                        => subnet.id,
         :cidr                           => subnet.cidr,
-        :status                         => (network.status.to_s.downcase == "active") ? "active" : "inactive",
+        :status                         => network["status"].to_s.downcase == "active" ? "active" : "inactive",
         :network_protocol               => "ipv#{subnet.ip_version}",
         :gateway                        => subnet.gateway_ip,
         :dhcp_enabled                   => subnet.enable_dhcp,
@@ -284,6 +289,8 @@ def parse_subnet(subnet, network)
         :ip_version                     => subnet.ip_version,
         :parent_cloud_subnet            => subnet.attributes["vsd_managed"] ? CloudSubnet.find_by(:ems_ref => subnet.attributes["vsd_id"]) : nil,
       }
+      network[:cloud_subnets] << new_result
+      return subnet.id, new_result
     end
 
     def parse_firewall_rule_neutron(rule)

lib/vcr_recorder.rb

@@ -60,6 +60,9 @@ def load_credentials
 
       file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_tenant_targeted_refresh.yml")
       change_file(file_name, OBFUSCATED_PASSWORD, env["password"], OBFUSCATED_IP, env["ip"])
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_network_targeted_refresh.yml")
+      change_file(file_name, OBFUSCATED_PASSWORD, env["password"], OBFUSCATED_IP, env["ip"])
     end
   end
 
@@ -93,6 +96,9 @@ def obfuscate_credentials
 
       file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_tenant_targeted_refresh.yml")
       change_file(file_name, env["password"], OBFUSCATED_PASSWORD, env["ip"], OBFUSCATED_IP)
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_network_targeted_refresh.yml")
+      change_file(file_name, env["password"], OBFUSCATED_PASSWORD, env["ip"], OBFUSCATED_IP)
     end
   end
 

spec/models/manageiq/providers/openstack/cloud_manager/refresher_rhos_liberty_spec.rb

@@ -113,5 +113,27 @@
         end
       end
     end
+
+    it "will not wipe out subnet relationships when performing a targeted network refresh against RHOS #{@environment}" do
+      with_cassette("#{@environment}_network_targeted_refresh", @ems) do
+        EmsRefresh.refresh(@ems)
+        EmsRefresh.refresh(@ems.network_manager)
+        EmsRefresh.refresh(@ems.cinder_manager)
+        EmsRefresh.refresh(@ems.swift_manager)
+
+        @ems.cloud_subnets.each do |subnet|
+          expect(subnet.cloud_network_id).to_not be(nil)
+        end
+
+        network = CloudNetwork.find_by(:name => "EmsRefreshSpec-NetworkPublic")
+        network_target = ManagerRefresh::Target.new(:manager     => @ems,
+                                                    :association => :cloud_networks,
+                                                    :manager_ref => {:ems_ref => network.ems_ref})
+        EmsRefresh.refresh(network_target)
+        @ems.cloud_subnets.each do |subnet|
+          expect(subnet.cloud_network_id).to_not be(nil)
+        end
+      end
+    end
   end
 end

spec/models/manageiq/providers/openstack/network_manager/cloud_subnet_refresh_spec.rb

@@ -98,15 +98,18 @@ def mocked_network_ports
     end
 
     def mocked_cloud_networks
+      [{
+        "id"      => "cloud_network_1",
+        "name"    => "cloud_network_1_name",
+        "subnets" => ["cloud_subnet_1"]
+      }]
+    end
+
+    def mocked_cloud_subnets
       [OpenStruct.new(
-        :id         => "cloud_network_1",
-        :name       => "cloud_network_1_name",
+        :id         => "cloud_subnet_1",
+        :name       => "cloud_subnet_1_name",
         :attributes => {},
-        :subnets    => [OpenStruct.new(
-          :id         => "cloud_subnet_1",
-          :name       => "cloud_subnet_1_name",
-          :attributes => {}
-        )],
       )]
     end
 
@@ -125,6 +128,7 @@ def setup_mocked_collector
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:tenants).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:orchestration_stacks).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:cloud_networks).and_return(mocked_cloud_networks)
+      allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:cloud_subnets).and_return(mocked_cloud_subnets)
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:floating_ips).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:network_ports).and_return(mocked_network_ports)
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager).to receive(:network_routers).and_return(mocked_network_routers)
@@ -138,6 +142,7 @@ def setup_mocked_targeted_collector
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:tenants).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:orchestration_stacks).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:cloud_networks).and_return(mocked_cloud_networks)
+      allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:cloud_subnets).and_return(mocked_cloud_subnets)
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:floating_ips).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:network_ports).and_return([])
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection).to receive(:network_routers).and_return([])

spec/models/manageiq/providers/openstack/network_manager/event_target_parser_spec.rb

@@ -21,6 +21,19 @@
       )
     end
 
+    it "parses subnet events" do
+      ems_event = create_ems_event("subnet.create.end", "resource_id" => "subnet_id_test",)
+      parsed_targets = described_class.new(ems_event).parse
+      expect(parsed_targets.size).to eq(1)
+      expect(target_references(parsed_targets)).to(
+        match_array(
+          [
+            [:subnets, {:ems_ref => "subnet_id_test"}]
+          ]
+        )
+      )
+    end
+
     it "parses router events" do
       ems_event = create_ems_event("router.create.end", "resource_id" => "router_id_test",)
       parsed_targets = described_class.new(ems_event).parse