From 1e8242f0e0e6c7bb7bc27a4fe07aa8bb961983c8 Mon Sep 17 00:00:00 2001 From: Jason Frey Date: Wed, 20 Jan 2021 16:37:02 -0500 Subject: [PATCH] Flatten worker settings --- app/models/miq_worker.rb | 60 ++-- config/settings.yml | 288 ++++++++++-------- .../provider/templates/config/settings.yml | 15 +- lib/vmdb/settings/validator.rb | 13 + spec/lib/vmdb/settings/validator_spec.rb | 4 + 5 files changed, 212 insertions(+), 168 deletions(-) diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 6348fe6d5390..befa610ad4aa 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -152,58 +152,46 @@ def self.sync_workers result end - # Convert the Models name from MiqGenericWorker to :generic_worker + # Determine the settings key from a worker's class name def self.settings_name @settings_name ||= if self == MiqWorker :worker_base elsif module_parent.try(:short_token) - # :generic_worker_infra, :generic_worker_vmware + # ManageIQ::Providers::Vmware::InfraManager::RefreshWorker => :ems_refresh_worker_vmware :"#{normalized_type}_#{module_parent.short_token.underscore}" else - # :generic_worker + # MiqGenericWorker => :generic_worker normalized_type.to_sym end end - # Grab all the classes in the hierarchy below ActiveRecord::Base - def self.path_to_my_worker_settings - @path_to_my_worker_settings ||= - ancestors.grep(Class).select { |c| c <= MiqWorker }.reverse.collect(&:settings_name) - end - def self.fetch_worker_settings_from_server(miq_server, options = {}) - settings = {} + return {} if miq_server.nil? - unless miq_server.nil? - server_config = options[:config] || miq_server.settings - # Get the configuration values - section = server_config[:workers] - unless section.nil? - classes = path_to_my_worker_settings - classes.each do |c| - section = section[c] - raise _("Missing config section %{section_name}") % {:section_name => c} if section.nil? - defaults = section[:defaults] - settings.merge!(defaults) unless defaults.nil? - end + config = (options[:config] || miq_server.settings)[:workers] + return {} if config.nil? - settings.merge!(section) + settings = {} + settings_key = settings_name + while settings_key + section = config[settings_key] + setting.merge!(section) + settings_key = section[:inherits_from] + end - # If not specified, provide the worker_settings cleaned up in fixnums, etc. instead of 1.seconds, 10.megabytes - raw = options[:raw] == true + # If not specified, provide the worker_settings cleaned up in fixnums, etc. instead of 1.seconds, 10.megabytes + raw = options[:raw] == true - # Clean up the configuration values in a format like "30.seconds" - unless raw - settings.keys.each do |k| - if settings[k].kind_of?(String) - if settings[k].number_with_method? - settings[k] = settings[k].to_i_with_method - elsif settings[k] =~ /\A\d+(.\d+)?\z/ # case where int/float saved as string - settings[k] = settings[k].to_i - end - end - end + # Clean up the configuration values in a format like "30.seconds" + unless raw + settings.keys.each do |k| + next unless settings[k].kind_of?(String) + + if settings[k].number_with_method? + settings[k] = settings[k].to_i_with_method + elsif settings[k] =~ /\A\d+(.\d+)?\z/ # case where int/float saved as string + settings[k] = settings[k].to_i end end end diff --git a/config/settings.yml b/config/settings.yml index 97f136d30970..bb834752ca06 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1048,127 +1048,169 @@ :authentication_timeout: 30.seconds :url: :workers: + :agent_coordinator_worker: + :heartbeat_timeout: 30.minutes + :inherits_from: worker_base + :poll: 30.seconds + :cockpit_ws_worker: + :count: 1 + :inherits_from: worker_base + :ems_refresh_core_worker: + :inherits_from: worker_base + :nice_delta: 1 + :poll: 1.seconds + :thread_shutdown_timeout: 10.seconds + :ems_metrics_collector_worker: + :count: 2 + :inherits_from: queue_worker_base + :nice_delta: 3 + :poll_method: escalate + :ems_metrics_collector_worker_google: + :inherits_from: ems_metrics_collector_worker + :ems_metrics_processor_worker: + :count: 2 + :inherits_from: queue_worker_base + :memory_threshold: 800.megabytes + :nice_delta: 7 + :poll_method: escalate + :ems_operations_worker: + :inherits_from: queue_worker_base + :ems_refresh_worker: + :dequeue_method: sql + :inherits_from: queue_worker_base + :memory_threshold: 2.gigabytes + :nice_delta: 7 + :poll: 10.seconds + :poll_method: normal + :queue_timeout: 120.minutes + :ems_refresh_worker_ansible_tower_automation: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_foreman_configuration: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_foreman_provisioning: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_google: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_google_network: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_microsoft: + :inherits_from: ems_refresh_worker + :ems_refresh_worker_nuage_network: + :inherits_from: ems_refresh_worker + :event_catcher: + :ems_event_page_size: 100 + :ems_event_thread_shutdown_timeout: 10.seconds + :flooding_events_per_minute: 30 + :flooding_monitor_enabled: false + :inherits_from: worker_base + :memory_threshold: 2.gigabytes + :nice_delta: 1 + :poll: 1.seconds + :event_catcher_ansible_tower: + :inherits_from: event_catcher + :poll: 20.seconds + :event_catcher_google: + :inherits_from: event_catcher + :poll: 15.seconds + :event_handler: + :inherits_from: queue_worker_base + :nice_delta: 7 + :generic_worker: + :count: 2 + :inherits_from: queue_worker_base + :priority_worker: + :count: 2 + :inherits_from: queue_worker_base + :nice_delta: 1 + :poll: 1.seconds + :queue_worker_base: + :dequeue_method: drb + :inherits_from: worker_base + :poll_method: normal + :queue_timeout: 10.minutes + :remote_console_worker: + :connection_pool_size: 14 + :inherits_from: worker_base + :memory_threshold: 1.gigabytes + :nice_delta: 1 + :reporting_worker: + :count: 2 + :inherits_from: queue_worker_base + :nice_delta: 7 + :schedule_worker: + :audit_managed_resources: 1.days + :authentication_check_interval: 1.hour + :binary_blob_purge_interval: 1.hour + :chargeback_generation_interval: 1.day + :chargeback_generation_time_utc: 01:00:00 + :compliance_purge_interval: 1.day + :container_entities_purge_interval: 1.day + :db_diagnostics_interval: 30.minutes + :drift_state_purge_interval: 1.day + :event_streams_purge_interval: 1.day + :evm_snapshot_delete_delay_for_job_not_found: 1.hour + :evm_snapshot_interval: 1.hour + :inherits_from: worker_base + :job_proxy_dispatcher_interval: 15.seconds + :job_proxy_dispatcher_stale_message_check_interval: 60.seconds + :job_proxy_dispatcher_stale_message_timeout: 2.minutes + :job_timeout_interval: 60.seconds + :log_active_configuration_interval: 1.days + :memory_threshold: 500.megabytes + :nice_delta: 3 + :notifications_purge_interval: 1.day + :orchestration_stack_retired_interval: 10.minutes + :performance_collection_interval: 3.minutes + :performance_collection_start_delay: 5.minutes + :performance_realtime_purging_interval: 21.minutes + :performance_realtime_purging_start_delay: 5.minutes + :performance_rollup_purging_interval: 4.hours + :performance_rollup_purging_start_delay: 5.minutes + :policy_events_purge_interval: 1.day + :poll: 15.seconds + :queue_timeout_interval: 15.seconds + :report_result_purge_interval: 1.week + :server_log_stats_interval: 5.minutes + :server_stats_interval: 60.seconds + :service_retired_interval: 10.minutes + :session_timeout_interval: 30.seconds + :storage_file_collection_interval: 1.days + :storage_file_collection_time_utc: 21600 + :task_purge_interval: 1.day + :task_timeout_check_frequency: 1.hour + :vim_performance_states_purge_interval: 1.day + :vm_retired_interval: 10.minutes + :yum_update_check: 12.hours + :smart_proxy_worker: + :count: 2 + :heartbeat_thread_shutdown_timeout: 10.seconds + :inherits_from: queue_worker_base + :memory_threshold: 2.gigabytes + :queue_timeout: 20.minutes + :ui_worker: + :connection_pool_size: 8 + :inherits_from: worker_base + :memory_threshold: 1.gigabytes + :nice_delta: 1 + :web_service_worker: + :connection_pool_size: 8 + :inherits_from: worker_base + :memory_threshold: 1.gigabytes + :nice_delta: 1 :worker_base: - :defaults: - :count: 1 - :cpu_request_percent: 15 - :cpu_threshold_percent: 100 - :gc_interval: 15.minutes - :heartbeat_freq: 10.seconds - :heartbeat_timeout: 2.minutes - :memory_request: 500.megabytes - :memory_threshold: 600.megabytes - :nice_delta: 10 - :parent_time_threshold: 3.minutes - :poll: 3.seconds - :poll_escalate_max: 30.seconds - :poll_method: normal - :starting_timeout: 10.minutes - :stopping_timeout: 10.minutes - :systemd_enabled: true - :event_catcher: - :defaults: - :flooding_events_per_minute: 30 - :flooding_monitor_enabled: false - :ems_event_page_size: 100 - :ems_event_thread_shutdown_timeout: 10.seconds - :memory_threshold: 2.gigabytes - :nice_delta: 1 - :poll: 1.seconds - :queue_worker_base: - :defaults: - :dequeue_method: drb - :poll_method: normal - :queue_timeout: 10.minutes - :ems_metrics_collector_worker: - :defaults: - :count: 2 - :nice_delta: 3 - :poll_method: escalate - :ems_metrics_processor_worker: - :count: 2 - :memory_threshold: 800.megabytes - :nice_delta: 7 - :poll_method: escalate - :ems_operations_worker: {} - :ems_refresh_worker: - :defaults: - :memory_threshold: 2.gigabytes - :nice_delta: 7 - :poll: 10.seconds - :poll_method: normal - :queue_timeout: 120.minutes - :dequeue_method: sql - :event_handler: - :nice_delta: 7 - :generic_worker: - :count: 2 - :priority_worker: - :count: 2 - :nice_delta: 1 - :poll: 1.seconds - :reporting_worker: - :count: 2 - :nice_delta: 7 - :smart_proxy_worker: - :count: 2 - :memory_threshold: 2.gigabytes - :queue_timeout: 20.minutes - :heartbeat_thread_shutdown_timeout: 10.seconds - :schedule_worker: - :audit_managed_resources: 1.days - :container_entities_purge_interval: 1.day - :binary_blob_purge_interval: 1.hour - :authentication_check_interval: 1.hour - :chargeback_generation_interval: 1.day - :chargeback_generation_time_utc: 01:00:00 - :compliance_purge_interval: 1.day - :db_diagnostics_interval: 30.minutes - :drift_state_purge_interval: 1.day - :event_streams_purge_interval: 1.day - :evm_snapshot_delete_delay_for_job_not_found: 1.hour - :evm_snapshot_interval: 1.hour - :job_proxy_dispatcher_interval: 15.seconds - :job_proxy_dispatcher_stale_message_check_interval: 60.seconds - :job_proxy_dispatcher_stale_message_timeout: 2.minutes - :job_timeout_interval: 60.seconds - :log_active_configuration_interval: 1.days - :memory_threshold: 500.megabytes - :nice_delta: 3 - :notifications_purge_interval: 1.day - :orchestration_stack_retired_interval: 10.minutes - :performance_collection_interval: 3.minutes - :performance_collection_start_delay: 5.minutes - :performance_realtime_purging_interval: 21.minutes - :performance_realtime_purging_start_delay: 5.minutes - :performance_rollup_purging_interval: 4.hours - :performance_rollup_purging_start_delay: 5.minutes - :policy_events_purge_interval: 1.day - :poll: 15.seconds - :queue_timeout_interval: 15.seconds - :report_result_purge_interval: 1.week - :server_log_stats_interval: 5.minutes - :server_stats_interval: 60.seconds - :service_retired_interval: 10.minutes - :session_timeout_interval: 30.seconds - :storage_file_collection_interval: 1.days - :storage_file_collection_time_utc: 21600 - :task_purge_interval: 1.day - :task_timeout_check_frequency: 1.hour - :vim_performance_states_purge_interval: 1.day - :vm_retired_interval: 10.minutes - :yum_update_check: 12.hours - :ui_worker: - :connection_pool_size: 8 - :memory_threshold: 1.gigabytes - :nice_delta: 1 - :web_service_worker: - :connection_pool_size: 8 - :memory_threshold: 1.gigabytes - :nice_delta: 1 - :remote_console_worker: - :connection_pool_size: 14 - :memory_threshold: 1.gigabytes - :nice_delta: 1 - :cockpit_ws_worker: - :count: 1 + :count: 1 + :cpu_request_percent: 15 + :cpu_threshold_percent: 100 + :gc_interval: 15.minutes + :heartbeat_freq: 10.seconds + :heartbeat_timeout: 2.minutes + :memory_request: 500.megabytes + :memory_threshold: 600.megabytes + :nice_delta: 10 + :parent_time_threshold: 3.minutes + :poll: 3.seconds + :poll_escalate_max: 30.seconds + :poll_method: :normal + :starting_timeout: 10.minutes + :stopping_timeout: 10.minutes + :systemd_enabled: true diff --git a/lib/generators/manageiq/provider/templates/config/settings.yml b/lib/generators/manageiq/provider/templates/config/settings.yml index a5edd6269454..7566e911c5bd 100644 --- a/lib/generators/manageiq/provider/templates/config/settings.yml +++ b/lib/generators/manageiq/provider/templates/config/settings.yml @@ -22,13 +22,10 @@ :level_<%= provider_name %>: info <% if options[:scaffolding] -%> :workers: - :worker_base: - :event_catcher: - :event_catcher_<%= provider_name %>: - :poll: 20.seconds - :queue_worker_base: - :ems_metrics_collector_worker: - :ems_metrics_collector_worker_<%= provider_name %>: {} - :ems_refresh_worker: - :ems_refresh_worker_<%= provider_name %>: {} + :event_catcher_<%= provider_name %>: + :inherits_from: event_catcher + :ems_metrics_collector_worker_<%= provider_name %>: + :inherits_from: ems_metrics_collector_worker + :ems_refresh_worker_<%= provider_name %>: + :inherits_from: ems_refresh_worker <% end -%> diff --git a/lib/vmdb/settings/validator.rb b/lib/vmdb/settings/validator.rb index 84d805a8bff4..3e275111c7e9 100644 --- a/lib/vmdb/settings/validator.rb +++ b/lib/vmdb/settings/validator.rb @@ -167,6 +167,19 @@ def smtp(data) return valid, errors end + + def workers(data) + valid, errors = true, [] + + data.each_pair do |worker, worker_settings| + if worker_settings.include?(:inherits_from) && worker_settings[:inherits_from] != ::Settings.workers[worker].inherits_from + valid = false + errors << ["#{worker}-inherits_from", "cannot be modified"] + end + end + + return valid, errors + end end end end diff --git a/spec/lib/vmdb/settings/validator_spec.rb b/spec/lib/vmdb/settings/validator_spec.rb index d1afd98b7f3d..4313f2730bd5 100644 --- a/spec/lib/vmdb/settings/validator_spec.rb +++ b/spec/lib/vmdb/settings/validator_spec.rb @@ -67,6 +67,10 @@ {:smtp => {:from => "a@example.com"}}, true, {:smtp => {:from => "xxx"}}, false, + + {:workers => {:base => {:inherits_from => "xxx"}}}, false, + {:workers => {:ui_worker => {:inherits_from => "base"}}}, true, + {:workers => {:ui_worker => {:inherits_from => "xxx"}}}, false, ].freeze VALIDATOR_CASES.each_slice(2) do |c, expected|