Skip to content

Commit

Permalink
Merge pull request #175 from mansam/event-target-parser-network-manager
Browse files Browse the repository at this point in the history
Improve Targeted Refresh for Cloud and Network managers
(cherry picked from commit 92aed61)

https://bugzilla.redhat.com/show_bug.cgi?id=1533225
  • Loading branch information
aufi authored and simaishi committed Jan 10, 2018
1 parent b13816c commit 1c1ba1a
Show file tree
Hide file tree
Showing 37 changed files with 22,686 additions and 22,523 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,18 @@ def parse
# @return [Array] Array of ManagerRefresh::Target objects
def parse_ems_event_targets(ems_event)
target_collection = ManagerRefresh::TargetCollection.new(:manager => ems_event.ext_management_system, :event => ems_event)

# there's almost always a tenant id regardless of event type
collect_identity_tenant_references!(target_collection, ems_event)

if ems_event.event_type.start_with?("compute.instance")
collect_compute_instance_references!(target_collection, ems_event)
elsif ems_event.event_type.start_with?("identity.project")
collect_identity_tenant_references!(target_collection, ems_event)
elsif ems_event.event_type.start_with?("orchestration.stack")
collect_orchestration_stack_references!(target_collection, ems_event)
elsif ems_event.event_type.start_with?("image.")
collect_image_references!(target_collection, ems_event)
elsif ems_event.event_type.start_with?("aggregate.")
collect_host_aggregate_references!(target_collection, ems_event)
end

target_collection.targets
Expand All @@ -44,8 +50,13 @@ def collect_compute_instance_references!(target_collection, ems_event)
add_target(target_collection, :vms, instance_id) if instance_id
end

def collect_image_references!(target_collection, ems_event)
resource_id = ems_event.full_data.fetch_path(:payload, 'resource_id')
add_target(target_collection, :images, resource_id) if resource_id
end

def collect_identity_tenant_references!(target_collection, ems_event)
tenant_id = ems_event.full_data.fetch_path(:payload, 'project_id') || ems_event.full_data.fetch_path(:payload, 'initiator', 'project_id')
tenant_id = ems_event.full_data.fetch_path(:payload, 'tenant_id') || ems_event.full_data.fetch_path(:payload, 'project_id') || ems_event.full_data.fetch_path(:payload, 'initiator', 'project_id')
add_target(target_collection, :cloud_tenants, tenant_id) if tenant_id
end

Expand All @@ -54,4 +65,12 @@ def collect_orchestration_stack_references!(target_collection, ems_event)
tenant_id = ems_event.full_data.fetch_path(:payload, 'tenant_id')
target_collection.add_target(:association => :orchestration_stacks, :manager_ref => {:ems_ref => stack_id}, :options => {:tenant_id => tenant_id})
end

def collect_host_aggregate_references!(target_collection, ems_event)
# aggregate events from ceilometer don't have an id field for the aggregate,
# but they do have a "service" field in the form of "aggregate.<id>"
aggregate_id = ems_event.full_data.fetch_path(:payload, 'service')
aggregate_id&.sub!('aggregate.', '')
add_target(target_collection, :host_aggregates, aggregate_id) if aggregate_id
end
end
67 changes: 2 additions & 65 deletions app/models/manageiq/providers/openstack/cloud_manager/refresher.rb
Original file line number Diff line number Diff line change
@@ -1,75 +1,14 @@
module ManageIQ::Providers
class Openstack::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
include ::EmsRefresh::Refreshers::EmsRefresherMixin

def collect_inventory_for_targets(ems, targets)
targets_with_data = targets.collect do |target|
target_name = target.try(:name) || target.try(:event_type)

_log.info("Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]...")
if ::Settings.ems.ems_refresh.openstack.try(:inventory_object_refresh)
inventory = ManageIQ::Providers::Openstack::Builder.build_inventory(ems, target)
end

_log.info("Filtering inventory...Complete")
[target, inventory]
end

targets_with_data
end

def parse_legacy_inventory(ems)
ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
end

class Openstack::CloudManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
def save_inventory(ems, target, inventory_collections)
super
EmsRefresh.queue_refresh(ems.network_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
EmsRefresh.queue_refresh(ems.cinder_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
EmsRefresh.queue_refresh(ems.swift_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
end

def parse_targeted_inventory(ems, _target, inventory)
log_header = format_ems_for_logging(ems)
_log.debug("#{log_header} Parsing inventory...")
hashes, = Benchmark.realtime_block(:parse_inventory) do
if ::Settings.ems.ems_refresh.openstack.try(:inventory_object_refresh)
inventory.inventory_collections
else
ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
end
end
_log.debug("#{log_header} Parsing inventory...Complete")

hashes
end

def preprocess_targets
@targets_by_ems_id.each do |ems_id, targets|
if targets.any? { |t| t.kind_of?(ExtManagementSystem) }
ems = @ems_by_ems_id[ems_id]
targets_for_log = targets.map { |t| "#{t.class} [#{t.name}] id [#{t.id}] " }
_log.info("Defaulting to full refresh for EMS: [#{ems.name}], id: [#{ems.id}], from targets: #{targets_for_log}") if targets.length > 1
end

# We want all targets of class EmsEvent to be merged into one target,
# so they can be refreshed together, otherwise we could be missing some
# crosslinks in the refreshed data
all_targets, sub_ems_targets = targets.partition { |x| x.kind_of?(ExtManagementSystem) }

unless sub_ems_targets.blank?
if ::Settings.ems.ems_refresh.openstack.try(:allow_targeted_refresh)
# We can disable targeted refresh with a setting, then we will just do full ems refresh on any event
ems_event_collection = ManagerRefresh::TargetCollection.new(:targets => sub_ems_targets,
:manager_id => ems_id)
all_targets << ems_event_collection
else
all_targets << @ems_by_ems_id[ems_id]
end
end

@targets_by_ems_id[ems_id] = all_targets
end
super

# sort the EMSes to be refreshed with cloud managers before other EMSes.
# since @targets_by_ems_id is a hash, we have to insert the items into a new
Expand All @@ -82,8 +21,6 @@ def preprocess_targets
# over to the new hash and then replace @targets_by_ems_id.
@targets_by_ems_id.keys.each { |ems_id| sorted_ems_targets[ems_id] = @targets_by_ems_id.delete(ems_id) }
@targets_by_ems_id = sorted_ems_targets

super
end

def post_process_refresh_classes
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
class ManageIQ::Providers::Openstack::NetworkManager::EventTargetParser
attr_reader :ems_event

# @param ems_event [EmsEvent] EmsEvent object to be parsed to derive an object to be refreshed
def initialize(ems_event)
@ems_event = ems_event
end

# Parses all targets present in the EmsEvent givin in the initializer
# @return [Array] Array of ManagerRefresh::Target objects
def parse
parse_ems_event_targets(ems_event)
end

private

# Parses list of ManagerRefresh::Target(s) out of the given EmsEvent
#
# @param ems_event [EmsEvent] EmsEvent object
# @return [Array] Array of ManagerRefresh::Target objects
def parse_ems_event_targets(ems_event)
target_collection = ManagerRefresh::TargetCollection.new(:manager => ems_event.ext_management_system.parent_manager, :event => ems_event)

# there's almost always a tenant id regardless of event type
collect_identity_tenant_references!(target_collection, ems_event)

target_type = if ems_event.event_type.start_with?("floatingip.")
:floating_ips
elsif ems_event.event_type.start_with?("router.")
:routers
elsif ems_event.event_type.start_with?("port.")
:ports
elsif ems_event.event_type.start_with?("network.")
:networks
elsif ems_event.event_type.start_with?("security_group.")
:security_groups
end

resource_id = ems_event.full_data.fetch_path(:payload, 'resource_id')
add_target(target_collection, target_type, resource_id) if resource_id

target_collection.targets
end

def collect_identity_tenant_references!(target_collection, ems_event)
tenant_id = ems_event.full_data.fetch_path(:payload, 'tenant_id') || ems_event.full_data.fetch_path(:payload, 'project_id') || ems_event.full_data.fetch_path(:payload, 'initiator', 'project_id')
add_target(target_collection, :cloud_tenants, tenant_id) if tenant_id
end

def parsed_targets(target_collection = {})
target_collection.select { |_target_class, references| references[:manager_ref].present? }
end

def add_target(target_collection, association, ref)
target_collection.add_target(:association => association, :manager_ref => {:ems_ref => ref})
end
end
Original file line number Diff line number Diff line change
@@ -1,43 +1,5 @@
module ManageIQ::Providers
class Openstack::NetworkManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
include ::EmsRefresh::Refreshers::EmsRefresherMixin

def collect_inventory_for_targets(ems, targets)
targets_with_data = targets.collect do |target|
target_name = target.try(:name) || target.try(:event_type)

_log.info("Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]...")

if ::Settings.ems.ems_refresh.openstack_network.try(:inventory_object_refresh)
inventory = ManageIQ::Providers::Openstack::Builder.build_inventory(ems, target)
end

_log.info("Filtering inventory...Complete")
[target, inventory]
end

targets_with_data
end

def parse_legacy_inventory(ems)
ManageIQ::Providers::Openstack::NetworkManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
end

def parse_targeted_inventory(ems, _target, inventory)
log_header = format_ems_for_logging(ems)
_log.debug("#{log_header} Parsing inventory...")
hashes, = Benchmark.realtime_block(:parse_inventory) do
if ::Settings.ems.ems_refresh.openstack_network.try(:inventory_object_refresh)
inventory.inventory_collections
else
ManageIQ::Providers::Openstack::NetworkManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
end
end
_log.debug("#{log_header} Parsing inventory...Complete")

hashes
end

class Openstack::NetworkManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
def post_process_refresh_classes
[]
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ module Openstack
module RefreshParserCommon
module HelperMethods
def openstack_admin?
::Settings.ems.ems_refresh.openstack.try(:is_admin)
::Settings.ems_refresh.openstack.try(:is_admin)
end

def openstack_network_admin?
::Settings.ems.ems_refresh.openstack_network.try(:is_admin)
::Settings.ems_refresh.openstack_network.try(:is_admin)
end

def openstack_heat_global_admin?
::Settings.ems.ems_refresh.openstack.try(:heat).try(:is_global_admin)
::Settings.ems_refresh.openstack.try(:heat).try(:is_global_admin)
end

def process_collection(collection, key, &block)
Expand Down
20 changes: 10 additions & 10 deletions config/settings.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
---
:ems_refresh:
:openstack:
:is_admin: false
:heat:
:is_global_admin: false
:inventory_object_refresh: true
:allow_targeted_refresh: true
:openstack_network:
:is_admin: false
:inventory_object_refresh: true
:ems:
:ems_refresh:
:openstack:
:is_admin: false
:heat:
:is_global_admin: false
:inventory_object_refresh: true
:allow_targeted_refresh: true
:openstack_network:
:is_admin: false
:inventory_object_refresh: true
:ems_openstack:
:excon:
:omit_default_port: true
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
describe ManageIQ::Providers::Openstack::CloudManager::EventTargetParser do
before :each do
_guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
@ems = FactoryGirl.create(:ems_openstack, :zone => zone)

allow_any_instance_of(EmsEvent).to receive(:handle_event)
allow(EmsEvent).to receive(:create_completed_event)
end

context "Openstack Event Parsing" do
it "parses compute.instance events" do
ems_event = create_ems_event("compute.instance.create.end", "instance_id" => "instance_id_test",
"tenant_id" => "tenant_id_test")
parsed_targets = described_class.new(ems_event).parse
expect(parsed_targets.size).to eq(2)
expect(target_references(parsed_targets)).to(
match_array(
[
[:vms, {:ems_ref => "instance_id_test"}],
[:cloud_tenants, {:ems_ref => "tenant_id_test"}]
]
)
)
end

it "parses identity.project events" do
ems_event = create_ems_event("identity.project.create.end", "project_id" => "tenant_id_test")
parsed_targets = described_class.new(ems_event).parse
expect(parsed_targets.size).to eq(1)
expect(target_references(parsed_targets)).to(
match_array(
[
[:cloud_tenants, {:ems_ref => "tenant_id_test"}]
]
)
)
end

it "parses orchestration.stack events" do
ems_event = create_ems_event("orchestration.stack.create.end", "stack_id" => "stack_id_test")
parsed_targets = described_class.new(ems_event).parse
expect(parsed_targets.size).to eq(1)
expect(target_references(parsed_targets)).to(
match_array(
[
[:orchestration_stacks, {:ems_ref => "stack_id_test"}]
]
)
)
end

it "parses image events" do
ems_event = create_ems_event("image.create.end", "resource_id" => "image_id_test")
parsed_targets = described_class.new(ems_event).parse
expect(parsed_targets.size).to eq(1)
expect(target_references(parsed_targets)).to(
match_array(
[
[:images, {:ems_ref => "image_id_test"}]
]
)
)
end

it "parses host aggregate events" do
ems_event = create_ems_event("aggregate.create.end", "service" => "aggregate.id_test")
parsed_targets = described_class.new(ems_event).parse
expect(parsed_targets.size).to eq(1)
expect(target_references(parsed_targets)).to(
match_array(
[
[:host_aggregates, {:ems_ref => "id_test"}]
]
)
)
end
end

def target_references(parsed_targets)
parsed_targets.map { |x| [x.association, x.manager_ref] }.uniq
end

def create_ems_event(event_type, payload)
event_hash = {
:event_type => event_type,
:source => "OPENSTACK",
:message => payload,
:timestamp => "2016-03-13T16:59:01.760000",
:username => "",
:full_data => {:payload => payload },
:ems_id => @ems.id
}
EmsEvent.add(@ems.id, event_hash)
end
end
Loading

0 comments on commit 1c1ba1a

Please sign in to comment.