Skip to content

Commit

Permalink
Flatten worker settings
Browse files Browse the repository at this point in the history
  • Loading branch information
@Fryguy
Fryguy committed Feb 9, 2021
1 parent e906b1d commit 13044af
Show file tree
Hide file tree
Showing 9 changed files with 272 additions and 255 deletions.
82 changes: 35 additions & 47 deletions app/models/miq_worker.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,63 +152,59 @@ def self.sync_workers
result
end

# Convert the Models name from MiqGenericWorker to :generic_worker
# Determine the settings key from a worker's class name
def self.settings_name
@settings_name ||=
if self == MiqWorker
:worker_base
elsif module_parent.try(:short_token)
# :generic_worker_infra, :generic_worker_vmware
# ManageIQ::Providers::Vmware::InfraManager::RefreshWorker => :ems_refresh_worker_vmware
:"#{normalized_type}_#{module_parent.short_token.underscore}"
else
# :generic_worker
# MiqGenericWorker => :generic_worker
normalized_type.to_sym
end
end

# Grab all the classes in the hierarchy below ActiveRecord::Base
def self.path_to_my_worker_settings
@path_to_my_worker_settings ||=
ancestors.grep(Class).select { |c| c <= MiqWorker }.reverse.collect(&:settings_name)
def self.fetch_worker_settings_from_server(miq_server, options = {})
return {} if miq_server.nil?

config = (options[:config] || miq_server.settings)[:workers]
return {} if config.nil?

flatten_worker_settings(config, options[:raw])
end

def self.fetch_worker_settings_from_server(miq_server, options = {})
settings = {}

unless miq_server.nil?
server_config = options[:config] || miq_server.settings
# Get the configuration values
section = server_config[:workers]
unless section.nil?
classes = path_to_my_worker_settings
classes.each do |c|
section = section[c]
raise _("Missing config section %{section_name}") % {:section_name => c} if section.nil?
defaults = section[:defaults]
settings.merge!(defaults) unless defaults.nil?
end
def self.flatten_worker_settings(config, raw = false)
layers = []
key = settings_name
while key
layer = config[key.to_sym]
layers.unshift(layer)
key = layer[:inherits_from]
end

settings.merge!(section)
{}.tap do |settings|
layers.each { |l| settings.merge!(l) }
resolve_worker_settings_values!(settings) if raw != true
end
end

# If not specified, provide the worker_settings cleaned up in fixnums, etc. instead of 1.seconds, 10.megabytes
raw = options[:raw] == true
# Resolve the settings values stored as strings or in method format into a number
#
# "1.megabyte" => 1_048_576
# "1_048_576" => 1_048_576
# "1_048_576.0" => 1_048_576
private_class_method def self.resolve_worker_settings_values!(settings)
settings.keys.each do |k|
next unless settings[k].kind_of?(String)

# Clean up the configuration values in a format like "30.seconds"
unless raw
settings.keys.each do |k|
if settings[k].kind_of?(String)
if settings[k].number_with_method?
settings[k] = settings[k].to_i_with_method
elsif settings[k] =~ /\A\d+(.\d+)?\z/ # case where int/float saved as string
settings[k] = settings[k].to_i
end
end
end
end
if settings[k].number_with_method?
settings[k] = settings[k].to_i_with_method
elsif settings[k] =~ /\A\d+(.\d+)?\z/ # case where int/float saved as string
settings[k] = settings[k].to_i
end
end

settings
end

def worker_settings(options = {})
Expand Down Expand Up @@ -528,14 +524,6 @@ def update_heartbeat
update_attribute(:last_heartbeat, Time.now.utc)
end

def self.config_settings_path
@config_settings_path ||= [:workers] + path_to_my_worker_settings
end

class << self
attr_writer :config_settings_path
end

def update_spid(spid = ActiveRecord::Base.connection.spid)
self.sql_spid = spid
end
Expand Down
262 changes: 139 additions & 123 deletions config/settings.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1048,127 +1048,143 @@
:authentication_timeout: 30.seconds
:url:
:workers:
:cockpit_ws_worker:
:count: 1
:inherits_from: worker_base
:ems_refresh_core_worker:
:inherits_from: worker_base
:nice_delta: 1
:poll: 1.seconds
:thread_shutdown_timeout: 10.seconds
:ems_metrics_collector_worker:
:count: 2
:inherits_from: queue_worker_base
:nice_delta: 3
:poll_method: escalate
:ems_metrics_processor_worker:
:count: 2
:inherits_from: queue_worker_base
:memory_threshold: 800.megabytes
:nice_delta: 7
:poll_method: escalate
:ems_operations_worker:
:inherits_from: queue_worker_base
:ems_refresh_worker:
:dequeue_method: sql
:inherits_from: queue_worker_base
:memory_threshold: 2.gigabytes
:nice_delta: 7
:poll: 10.seconds
:poll_method: normal
:queue_timeout: 120.minutes
:event_catcher:
:ems_event_page_size: 100
:ems_event_thread_shutdown_timeout: 10.seconds
:flooding_events_per_minute: 30
:flooding_monitor_enabled: false
:inherits_from: worker_base
:memory_threshold: 2.gigabytes
:nice_delta: 1
:poll: 1.seconds
:event_handler:
:inherits_from: queue_worker_base
:nice_delta: 7
:generic_worker:
:count: 2
:inherits_from: queue_worker_base
:priority_worker:
:count: 2
:inherits_from: queue_worker_base
:nice_delta: 1
:poll: 1.seconds
:queue_worker_base:
:dequeue_method: drb
:inherits_from: worker_base
:poll_method: normal
:queue_timeout: 10.minutes
:remote_console_worker:
:connection_pool_size: 14
:inherits_from: worker_base
:memory_threshold: 1.gigabytes
:nice_delta: 1
:reporting_worker:
:count: 2
:inherits_from: queue_worker_base
:nice_delta: 7
:schedule_worker:
:audit_managed_resources: 1.days
:authentication_check_interval: 1.hour
:binary_blob_purge_interval: 1.hour
:chargeback_generation_interval: 1.day
:chargeback_generation_time_utc: 01:00:00
:compliance_purge_interval: 1.day
:container_entities_purge_interval: 1.day
:db_diagnostics_interval: 30.minutes
:drift_state_purge_interval: 1.day
:event_streams_purge_interval: 1.day
:evm_snapshot_delete_delay_for_job_not_found: 1.hour
:evm_snapshot_interval: 1.hour
:inherits_from: worker_base
:job_proxy_dispatcher_interval: 15.seconds
:job_proxy_dispatcher_stale_message_check_interval: 60.seconds
:job_proxy_dispatcher_stale_message_timeout: 2.minutes
:job_timeout_interval: 60.seconds
:log_active_configuration_interval: 1.days
:memory_threshold: 500.megabytes
:nice_delta: 3
:notifications_purge_interval: 1.day
:orchestration_stack_retired_interval: 10.minutes
:performance_collection_interval: 3.minutes
:performance_collection_start_delay: 5.minutes
:performance_realtime_purging_interval: 21.minutes
:performance_realtime_purging_start_delay: 5.minutes
:performance_rollup_purging_interval: 4.hours
:performance_rollup_purging_start_delay: 5.minutes
:policy_events_purge_interval: 1.day
:poll: 15.seconds
:queue_timeout_interval: 15.seconds
:report_result_purge_interval: 1.week
:server_log_stats_interval: 5.minutes
:server_stats_interval: 60.seconds
:service_retired_interval: 10.minutes
:session_timeout_interval: 30.seconds
:storage_file_collection_interval: 1.days
:storage_file_collection_time_utc: 21600
:task_purge_interval: 1.day
:task_timeout_check_frequency: 1.hour
:vim_performance_states_purge_interval: 1.day
:vm_retired_interval: 10.minutes
:yum_update_check: 12.hours
:smart_proxy_worker:
:count: 2
:heartbeat_thread_shutdown_timeout: 10.seconds
:inherits_from: queue_worker_base
:memory_threshold: 2.gigabytes
:queue_timeout: 20.minutes
:ui_worker:
:connection_pool_size: 8
:inherits_from: worker_base
:memory_threshold: 1.gigabytes
:nice_delta: 1
:web_service_worker:
:connection_pool_size: 8
:inherits_from: worker_base
:memory_threshold: 1.gigabytes
:nice_delta: 1
:worker_base:
:defaults:
:count: 1
:cpu_request_percent: 15
:cpu_threshold_percent: 100
:gc_interval: 15.minutes
:heartbeat_freq: 10.seconds
:heartbeat_timeout: 2.minutes
:memory_request: 500.megabytes
:memory_threshold: 600.megabytes
:nice_delta: 10
:parent_time_threshold: 3.minutes
:poll: 3.seconds
:poll_escalate_max: 30.seconds
:poll_method: normal
:starting_timeout: 10.minutes
:stopping_timeout: 10.minutes
:systemd_enabled: true
:event_catcher:
:defaults:
:flooding_events_per_minute: 30
:flooding_monitor_enabled: false
:ems_event_page_size: 100
:ems_event_thread_shutdown_timeout: 10.seconds
:memory_threshold: 2.gigabytes
:nice_delta: 1
:poll: 1.seconds
:queue_worker_base:
:defaults:
:dequeue_method: drb
:poll_method: normal
:queue_timeout: 10.minutes
:ems_metrics_collector_worker:
:defaults:
:count: 2
:nice_delta: 3
:poll_method: escalate
:ems_metrics_processor_worker:
:count: 2
:memory_threshold: 800.megabytes
:nice_delta: 7
:poll_method: escalate
:ems_operations_worker: {}
:ems_refresh_worker:
:defaults:
:memory_threshold: 2.gigabytes
:nice_delta: 7
:poll: 10.seconds
:poll_method: normal
:queue_timeout: 120.minutes
:dequeue_method: sql
:event_handler:
:nice_delta: 7
:generic_worker:
:count: 2
:priority_worker:
:count: 2
:nice_delta: 1
:poll: 1.seconds
:reporting_worker:
:count: 2
:nice_delta: 7
:smart_proxy_worker:
:count: 2
:memory_threshold: 2.gigabytes
:queue_timeout: 20.minutes
:heartbeat_thread_shutdown_timeout: 10.seconds
:schedule_worker:
:audit_managed_resources: 1.days
:container_entities_purge_interval: 1.day
:binary_blob_purge_interval: 1.hour
:authentication_check_interval: 1.hour
:chargeback_generation_interval: 1.day
:chargeback_generation_time_utc: 01:00:00
:compliance_purge_interval: 1.day
:db_diagnostics_interval: 30.minutes
:drift_state_purge_interval: 1.day
:event_streams_purge_interval: 1.day
:evm_snapshot_delete_delay_for_job_not_found: 1.hour
:evm_snapshot_interval: 1.hour
:job_proxy_dispatcher_interval: 15.seconds
:job_proxy_dispatcher_stale_message_check_interval: 60.seconds
:job_proxy_dispatcher_stale_message_timeout: 2.minutes
:job_timeout_interval: 60.seconds
:log_active_configuration_interval: 1.days
:memory_threshold: 500.megabytes
:nice_delta: 3
:notifications_purge_interval: 1.day
:orchestration_stack_retired_interval: 10.minutes
:performance_collection_interval: 3.minutes
:performance_collection_start_delay: 5.minutes
:performance_realtime_purging_interval: 21.minutes
:performance_realtime_purging_start_delay: 5.minutes
:performance_rollup_purging_interval: 4.hours
:performance_rollup_purging_start_delay: 5.minutes
:policy_events_purge_interval: 1.day
:poll: 15.seconds
:queue_timeout_interval: 15.seconds
:report_result_purge_interval: 1.week
:server_log_stats_interval: 5.minutes
:server_stats_interval: 60.seconds
:service_retired_interval: 10.minutes
:session_timeout_interval: 30.seconds
:storage_file_collection_interval: 1.days
:storage_file_collection_time_utc: 21600
:task_purge_interval: 1.day
:task_timeout_check_frequency: 1.hour
:vim_performance_states_purge_interval: 1.day
:vm_retired_interval: 10.minutes
:yum_update_check: 12.hours
:ui_worker:
:connection_pool_size: 8
:memory_threshold: 1.gigabytes
:nice_delta: 1
:web_service_worker:
:connection_pool_size: 8
:memory_threshold: 1.gigabytes
:nice_delta: 1
:remote_console_worker:
:connection_pool_size: 14
:memory_threshold: 1.gigabytes
:nice_delta: 1
:cockpit_ws_worker:
:count: 1
:count: 1
:cpu_request_percent: 15
:cpu_threshold_percent: 100
:gc_interval: 15.minutes
:heartbeat_freq: 10.seconds
:heartbeat_timeout: 2.minutes
:memory_request: 500.megabytes
:memory_threshold: 600.megabytes
:nice_delta: 10
:parent_time_threshold: 3.minutes
:poll: 3.seconds
:poll_escalate_max: 30.seconds
:poll_method: :normal
:starting_timeout: 10.minutes
:stopping_timeout: 10.minutes
:systemd_enabled: true
Loading

0 comments on commit 13044af

Please sign in to comment.