Use systemd-notify for worker heartbeating

Notify the systemd supervisor when a worker is finished starting up,
when it is shutting down, and poke the watchdog for heartbeating

Gemfile

@@ -161,6 +161,7 @@ end
 
 group :systemd, :optional => true do
   gem "dbus-systemd",    "~>1.1.0", :require => false
+  gem "sd_notify",       "~>0.1.0", :require => false
   gem "systemd-journal", "~>1.4.2", :require => false
 end
 

app/models/miq_worker/runner.rb

@@ -142,6 +142,7 @@ def starting_worker_record
 
   def started_worker_record
     reload_worker_record
+    @worker.sd_notify_started if @worker.systemd_worker?
     @worker.status         = "started"
     @worker.last_heartbeat = Time.now.utc
     @worker.update_spid
@@ -191,6 +192,7 @@ def update_worker_record_at_exit(exit_code)
     @worker.stopped_on = Time.now.utc
     @worker.save
 
+    @worker.sd_notify_stopping if @worker.systemd_worker?
     @worker.status_update
     @worker.log_status
   end
@@ -289,6 +291,7 @@ def heartbeat
     return if @last_hb.kind_of?(Time) && (@last_hb + worker_settings[:heartbeat_freq]) >= now
 
     heartbeat_to_file
+    @worker.sd_notify_watchdog if @worker.systemd_worker?
 
     if config_out_of_date?
       _log.info("#{log_prefix} Synchronizing configuration...")

app/models/miq_worker/systemd_common.rb

@@ -115,6 +115,21 @@ def stop_systemd_unit(mode: "replace")
       systemd.StopUnit(unit_name, mode)
     end
 
+    def sd_notify_started
+      require "sd_notify"
+      SdNotify.ready
+    end
+
+    def sd_notify_stopping
+      require "sd_notify"
+      SdNotify.stopping
+    end
+
+    def sd_notify_watchdog
+      require "sd_notify"
+      SdNotify.watchdog
+    end
+
     private
 
     def systemd
@@ -166,6 +181,7 @@ def unit_config_file
         MemoryHigh=#{worker_settings[:memory_threshold].bytes}
         TimeoutStartSec=#{worker_settings[:starting_timeout]}
         TimeoutStopSec=#{worker_settings[:stopping_timeout]}
+        WatchdogSec=#{worker_settings[:heartbeat_timeout]}
         #{unit_environment_variables.map { |env_var| "Environment=#{env_var}" }.join("\n")}
       UNIT_CONFIG_FILE
     end