Merge pull request #19734 from carbonin/monitor_multiple_servers

Monitor multiple servers when we're running in pods

app/models/miq_worker.rb

@@ -272,7 +272,7 @@ def self.init_worker_object(*params)
   end
 
   def self.create_worker_record(*params)
-    init_worker_object(*params).tap(&:save)
+    init_worker_object(*params).tap(&:save!)
   end
 
   def self.start_worker(*params)
@@ -414,6 +414,11 @@ def kill
   end
 
   def kill_process
+    if containerized_worker?
+      delete_container_objects
+      return
+    end
+
     unless pid.nil?
       begin
         _log.info("Killing worker: ID [#{id}], PID [#{pid}], GUID [#{guid}], status [#{status}]")

app/models/miq_worker/service_worker.rb

@@ -27,7 +27,6 @@ def create_container_objects
     def delete_container_objects
       orch = ContainerOrchestrator.new
       orch.delete_deployment(worker_deployment_name)
-      orch.delete_service(service_name)
     end
 
     def stop_container

lib/vmdb/settings.rb

@@ -107,6 +107,16 @@ def self.dump_to_log_directory(settings)
       DUMP_LOG_FILE.write(mask_passwords!(settings.to_hash).to_yaml)
     end
 
+    # This is a near copy of Config.load_and_set_settings, but we can't use that
+    # method as it also calls Config.load_files, which enforces specific file
+    # sources and doesn't allow you insert new sources into the middle of the
+    # stack.
+    def self.reset_settings_constant(settings)
+      name = ::Config.const_name
+      Object.send(:remove_const, name) if Object.const_defined?(name)
+      Object.const_set(name, settings)
+    end
+
     def self.build_template
       ::Config::Options.new.tap do |settings|
         template_sources.each { |s| settings.add_source!(s) }
@@ -171,17 +181,6 @@ def self.local_sources
     end
     private_class_method :local_sources
 
-    # This is a near copy of Config.load_and_set_settings, but we can't use that
-    # method as it also calls Config.load_files, which enforces specific file
-    # sources and doesn't allow you insert new sources into the middle of the
-    # stack.
-    def self.reset_settings_constant(settings)
-      name = ::Config.const_name
-      Object.send(:remove_const, name) if Object.const_defined?(name)
-      Object.const_set(name, settings)
-    end
-    private_class_method :reset_settings_constant
-
     def self.replace_magic_values!(settings, resource)
       parent_settings = nil
 

lib/workers/evm_server.rb

@@ -4,12 +4,13 @@
 class EvmServer
   include Vmdb::Logging
 
-  ##
-  # String used as a title for a linux process. Visible in ps, htop, ...
   SERVER_PROCESS_TITLE = 'MIQ Server'.freeze
 
+  attr_accessor :servers_to_monitor
+
   def initialize
     $log ||= Rails.logger
+    @servers_to_monitor = servers_from_db
   end
 
   def start
@@ -24,33 +25,78 @@ def start
     EvmDatabase.seed_primordial
     check_migrations_up_to_date
 
-    @server = MiqServer.my_server(true)
-
-    start_server
-    loop do
-      monitor
-      sleep ::Settings.server.monitor_poll.to_i_with_method
-    end
+    start_servers
+    monitor_servers
   rescue Interrupt => e
     _log.info("Received #{e.message} signal, killing server")
-    MiqServer.kill
+    kill_servers
     exit 1
   rescue SignalException => e
     _log.info("Received #{e.message} signal, shutting down server")
-    @server.shutdown_and_exit
+    stop_servers
+    exit 0
   end
 
-  ##
-  # Sets the server process' name if it is possible.
-  #
-  def set_process_title
-    Process.setproctitle(SERVER_PROCESS_TITLE) if Process.respond_to?(:setproctitle)
+  def start_servers
+    refresh_servers_to_monitor
+    as_each_server { start_server }
+  end
+
+  def monitor_servers
+    loop do
+      refresh_servers_to_monitor
+      as_each_server { monitor }
+      sleep ::Settings.server.monitor_poll.to_i_with_method
+    end
+  end
+
+  def stop_servers
+    as_each_server { @current_server.shutdown }
+  end
+
+  def kill_servers
+    as_each_server do
+      @current_server.kill_all_workers
+      @current_server.update(:stopped_on => Time.now.utc, :status => "killed", :is_master => false)
+    end
+  end
+
+  def refresh_servers_to_monitor
+    servers_to_start    = servers_from_db    - servers_to_monitor
+    servers_to_shutdown = servers_to_monitor - servers_from_db
+
+    servers_to_start.each do |s|
+      servers_to_monitor << s
+      impersonate_server(s)
+      start_server
+    end
+
+    servers_to_shutdown.each do |s|
+      servers_to_monitor.delete(s)
+      s.shutdown
+    end
   end
 
   def self.start(*args)
     new.start
   end
 
+  private
+
+  def monitoring_server?(server)
+    servers_to_monitor.any? do |monitor_server|
+      monitor_server.id == server.id
+    end
+  end
+
+  def servers_from_db
+    MiqEnvironment::Command.is_podified? ? MiqServer.in_my_region.to_a : [MiqServer.my_server(true)].compact
+  end
+
+  def set_process_title
+    Process.setproctitle(SERVER_PROCESS_TITLE) if Process.respond_to?(:setproctitle)
+  end
+
   def start_server
     Vmdb::Settings.activate
 
@@ -61,21 +107,21 @@ def start_server
 
     Vmdb::Appliance.log_config_on_startup
 
-    @server.ntp_reload
-    @server.set_database_application_name
+    @current_server.ntp_reload
+    @current_server.set_database_application_name
 
     EvmDatabase.seed_rest
 
     MiqServer.start_memcached
     MiqApache::Control.restart if MiqEnvironment::Command.supports_apache?
 
-    MiqEvent.raise_evm_event(@server, "evm_server_start")
+    MiqEvent.raise_evm_event(@current_server, "evm_server_start")
 
     msg = "Server starting in #{MiqServer.startup_mode} mode."
     _log.info(msg)
     puts "** #{msg}"
 
-    @server.starting_server_record
+    @current_server.starting_server_record
 
     configure_server_roles
     clear_queue
@@ -85,16 +131,16 @@ def start_server
     MiqServer.clean_dequeued_messages
     MiqServer.purge_report_results
 
-    @server.delete_active_log_collections_queue
+    @current_server.delete_active_log_collections_queue
 
     start_workers
 
-    @server.update(:status => "started")
+    @current_server.update(:status => "started")
     _log.info("Server starting complete")
   end
 
   def monitor
-    _dummy, timings = Benchmark.realtime_block(:total_time) { @server.monitor }
+    _dummy, timings = Benchmark.realtime_block(:total_time) { @current_server.monitor }
     _log.info("Server Monitoring Complete - Timings: #{timings.inspect}") unless timings[:total_time] < ::Settings.server.server_log_timings_threshold.to_i_with_method
   end
 
@@ -135,26 +181,26 @@ def save_local_network_info
     end
 
     if config_hash.any?
-      Vmdb::Settings.save!(@server, :server => config_hash)
+      Vmdb::Settings.save!(@current_server, :server => config_hash)
       ::Settings.reload!
     end
 
-    @server.update(server_hash)
+    @current_server.update(server_hash)
   end
 
   def set_local_server_vm
-    if @server.vm_id.nil?
-      vms = Vm.find_all_by_mac_address_and_hostname_and_ipaddress(@server.mac_address, @server.hostname, @server.ipaddress)
+    if @current_server.vm_id.nil?
+      vms = Vm.find_all_by_mac_address_and_hostname_and_ipaddress(@current_server.mac_address, @current_server.hostname, @current_server.ipaddress)
       if vms.length > 1
         _log.warn("Found multiple Vms that may represent this MiqServer: #{vms.collect(&:id).sort.inspect}")
       elsif vms.length == 1
-        @server.update(:vm_id => vms.first.id)
+        @current_server.update(:vm_id => vms.first.id)
       end
     end
   end
 
   def reset_server_runtime_info
-    @server.update(
+    @current_server.update(
       :drb_uri        => nil,
       :last_heartbeat => nil,
       :memory_usage   => nil,
@@ -166,9 +212,9 @@ def reset_server_runtime_info
   end
 
   def log_server_info
-    _log.info("Server IP Address: #{@server.ipaddress}")    unless @server.ipaddress.blank?
-    _log.info("Server Hostname: #{@server.hostname}")       unless @server.hostname.blank?
-    _log.info("Server MAC Address: #{@server.mac_address}") unless @server.mac_address.blank?
+    _log.info("Server IP Address: #{@current_server.ipaddress}")    unless @current_server.ipaddress.blank?
+    _log.info("Server Hostname: #{@current_server.hostname}")       unless @current_server.hostname.blank?
+    _log.info("Server MAC Address: #{@current_server.mac_address}") unless @current_server.mac_address.blank?
     _log.info("Server GUID: #{MiqServer.my_guid}")
     _log.info("Server Zone: #{MiqServer.my_zone}")
     _log.info("Server Role: #{MiqServer.my_role}")
@@ -178,7 +224,7 @@ def log_server_info
   end
 
   def configure_server_roles
-    @server.ensure_default_roles
+    @current_server.ensure_default_roles
 
     #############################################################
     # Server Role Assignment
@@ -188,30 +234,72 @@ def configure_server_roles
     # - Role activation should happen inside monitor_servers
     # - Synchronize active roles to monitor for role changes
     #############################################################
-    @server.deactivate_all_roles
-    @server.set_database_owner_role(EvmDatabase.local?)
-    @server.monitor_servers
-    @server.monitor_server_roles if @server.is_master?
-    @server.sync_active_roles
-    @server.set_active_role_flags
+    @current_server.deactivate_all_roles
+    @current_server.set_database_owner_role(EvmDatabase.local?)
+    @current_server.monitor_servers
+    @current_server.monitor_server_roles if @current_server.is_master?
+    @current_server.sync_active_roles
+    @current_server.set_active_role_flags
   end
 
   def clear_queue
     #############################################################
     # Clear the MiqQueue for server and its workers
     #############################################################
-    @server.clean_stop_worker_queue_items
-    @server.clear_miq_queue_for_this_server
+    @current_server.clean_stop_worker_queue_items
+    @current_server.clear_miq_queue_for_this_server
   end
 
   def start_workers
     #############################################################
     # Start all the configured workers
     #############################################################
-    @server.clean_heartbeat_files
-    @server.sync_config
-    @server.start_drb_server
-    @server.sync_workers
-    @server.wait_for_started_workers
+    @current_server.clean_heartbeat_files
+    @current_server.sync_config
+    @current_server.start_drb_server
+    @current_server.sync_workers
+    @current_server.wait_for_started_workers
+  end
+
+  ######################################################################
+  # Warning:
+  #
+  # The following methods can lead to unexpected (and likely unpleasant)
+  # behavior if used out of the scope of the orchestrator process.
+  #
+  # They change the global state which is used to determine the current
+  # server's identity. This intentionally will alter the values of calls
+  # such as MiqServer.my_server and MiqServer.my_guid, and also the
+  # contents of the global ::Settings constant.
+  ######################################################################
+  def as_each_server
+    initial_server = @current_server
+    servers_to_monitor.each do |s|
+      impersonate_server(s)
+      yield
+    end
+  ensure
+    clear_server_caches if @current_server != initial_server
+  end
+
+  def impersonate_server(s)
+    return if s == @current_server
+
+    _log.info("Impersonating server - id: #{s.id}, guid: #{s.guid}")
+
+    MiqServer.my_server_clear_cache
+    MiqServer.my_guid = s.guid
+
+    # It is important that we continue to use the same server instance here.
+    # A lot of "global" state is stored in instance variables on the server.
+    @current_server = s
+    Vmdb::Settings.reset_settings_constant(s.settings_for_resource)
+  end
+
+  def clear_server_caches
+    MiqServer.my_guid = nil
+    MiqServer.my_server_clear_cache
+    # Use Vmdb::Settings.for_resource(:my_server) here as MiqServer.my_server might be nil
+    Vmdb::Settings.reset_settings_constant(Vmdb::Settings.for_resource(:my_server))
   end
 end