feat(prometheus): Add shutdown timeout variable

In order to allow Prometheus more time to shutdown, especially if `memory-snapshot-on-shutdown` is used, override the default 90s systemd shutdown timeout. Otherwise systemd will SIGKILL Prometheus. * Use 10min shutdown timeout by default. Signed-off-by: SuperQ <[email protected]>
prometheus-community · Sep 27, 2023 · e557c20 · e557c20
1 parent 643d16c
commit e557c20
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 0 deletions.
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
@@ -226,3 +226,5 @@ prometheus_alert_rules:  # noqa yaml[line-length]  # noqa line-length
     labels:
       severity: warning
 # yamllint enable rule:line-length
+
+prometheus_stop_timeout: '600s'
diff --git a/roles/prometheus/meta/argument_specs.yml b/roles/prometheus/meta/argument_specs.yml
@@ -150,3 +150,8 @@ argument_specs:
         default:
           - "prometheus/targets/*.yml"
           - "prometheus/targets/*.json"
+      prometheus_stop_timeout:
+        description:
+          - "How long to wait for Prometheus to shutdown. This is passed as a systemd TimeoutStopSec time spec."
+        type: "string"
+        default: "600s"
diff --git a/roles/prometheus/molecule/alternative/molecule.yml b/roles/prometheus/molecule/alternative/molecule.yml
@@ -87,3 +87,4 @@ provisioner:
               - target_label: __address__
                 replacement: 127.0.0.1:9115  # Blackbox exporter.
         version: 2.25.2
+        prometheus_stop_timeout: 1min
diff --git a/roles/prometheus/molecule/alternative/tests/test_alternative.py b/roles/prometheus/molecule/alternative/tests/test_alternative.py
@@ -42,6 +42,8 @@ def test_files(host, files):
      "enable-feature=promql-at-modifier"),
     ("/etc/systemd/system/prometheus.service",
      "enable-feature=remote-write-receiver"),
+    ("/etc/systemd/system/prometheus.service",
+     "TimeoutStopSec=1min"),
 ])
 def test_file_contents(host, file, content):
     f = host.file(file)

diff --git a/roles/prometheus/templates/prometheus.service.j2 b/roles/prometheus/templates/prometheus.service.j2
@@ -85,6 +85,7 @@ Environment="HTTP_PROXY={{ http_proxy }}"{% if https_proxy is defined %} "HTTPS_
 
 SyslogIdentifier=prometheus
 Restart=always
+TimeoutStopSec={{ prometheus_stop_timeout }}
 
 [Install]
 WantedBy=multi-user.target