#2700 upgrade Prometheus to v2.31.1 + AlertManager to v0.23.0

ansible/playbooks/roles/prometheus/defaults/main.yml

@@ -3,11 +3,13 @@ prometheus_external_labels:
   environment: "{{ ansible_fqdn }}"
 
 prometheus:
+  version: 2.31.1
   file_name:
-    x86_64: "prometheus-2.10.0.linux-amd64.tar.gz"
-    aarch64: "prometheus-2.10.0.linux-arm64.tar.gz"
+    x86_64: "prometheus-2.31.1.linux-amd64.tar.gz"
+    aarch64: "prometheus-2.31.1.linux-arm64.tar.gz"
 
 alertmanager:
+  version: 0.23.0
   file_name:
-    x86_64: "alertmanager-0.17.0.linux-amd64.tar.gz"
-    aarch64: "alertmanager-0.17.0.linux-arm64.tar.gz"
+    x86_64: "alertmanager-0.23.0.linux-amd64.tar.gz"
+    aarch64: "alertmanager-0.23.0.linux-arm64.tar.gz"

ansible/playbooks/roles/prometheus/tasks/upgrade/main.yml

@@ -0,0 +1,122 @@
+---
+# General things needed 
+- name: Prometheus | Include new version default variables
+  include_vars:
+    file: defaults/main.yml
+    name: prometheus_defaults
+
+- name: Prometheus | Populate service facts
+  service_facts:
+  when: ansible_facts.service is undefined
+  no_log: true
+
+
+# Prometheus upgrade
+- name: Prometheus upgrade | Set prometheus_will_upgrade fact
+  set_fact:
+    prometheus_will_upgrade: false
+
+- name: Prometheus upgrade | Check binary existance
+  stat:
+    path: /usr/local/bin/prometheus
+  register: prometheus_stat
+
+- name: Prometheus upgrade | Start upgrade
+  when:
+    - ansible_facts.services["prometheus.service"] is defined
+    - prometheus_stat.stat.exists
+  block:
+    - name: Prometheus upgrade | Get version
+      command: prometheus --version
+      changed_when: false
+      check_mode: false
+      register: __prometheus_version
+
+    # The following issue is also present in prometheus itself and fixed in newer versions:
+    # https://github.com/prometheus/node_exporter/issues/1271
+    # So we check both stderr and stdout just in case.
+    - name: Prometheus upgrade | Set version fact (stderr)
+      set_fact:
+        prometheus_version: "{{ __prometheus_version.stderr_lines[0].split(' ')[2] }}"
+      when: __prometheus_version.stderr_lines is defined and (__prometheus_version.stderr_lines|length>0)
+
+    - name: Prometheus upgrade | Set version fact (stdout)
+      set_fact:
+        prometheus_version: "{{ __prometheus_version.stdout_lines[0].split(' ')[2] }}"
+      when: __prometheus_version.stdout_lines is defined and (__prometheus_version.stdout_lines|length>0)
+
+    - name: Prometheus upgrade | Print versions
+      debug:
+        msg:
+          - "Installed version: {{ prometheus_version }}"
+          - "Target version: {{ prometheus_defaults.prometheus.version }}"
+
+    - name: Prometheus upgrade |  Check versions and set prometheus_will_upgrade
+      set_fact:
+        prometheus_will_upgrade: "{{ prometheus_defaults.prometheus.version is version(prometheus_version, '>') }}"
+
+    - name: Prometheus upgrade | Stop services
+      include_tasks: upgrade/stop-services.yml
+      when: prometheus_will_upgrade
+
+    - name: Prometheus upgrade | Upgrading Prometheus
+      include_tasks: upgrade/upgrade-prometheus.yml
+      when: prometheus_will_upgrade
+
+
+# AlertManager upgrade
+- name: AlertManager upgrade | Set alertmanager_will_upgrade fact
+  set_fact:
+    alertmanager_will_upgrade: false
+
+- name: AlertManager upgrade | Check binary existance
+  stat:
+    path: /usr/local/bin/alertmanager
+  register: alertmanager_stat
+
+- name: AlertManager upgrade | Start upgrade
+  when:
+    - ansible_facts.services["alertmanager.service"] is defined
+    - alertmanager_stat.stat.exists
+  block:
+    - name: AlertManager upgrade | Get version
+      command: alertmanager --version
+      changed_when: false
+      check_mode: false
+      register: __alertmanager_version
+
+    # The following issue is also present in prometheus itself and fixed in newer versions: 
+    # https://github.com/prometheus/node_exporter/issues/1271
+    # So we check both stderr and stdout just in case.
+    - name: AlertManager upgrade | Set version fact (stderr)
+      set_fact:
+        alertmanager_version: "{{ __alertmanager_version.stderr_lines[0].split(' ')[2] }}"
+      when: __alertmanager_version.stderr_lines is defined and (__alertmanager_version.stderr_lines|length>0)
+
+    - name: AlertManager upgrade | Set version fact (stdout)
+      set_fact:
+        alertmanager_version: "{{ __alertmanager_version.stdout_lines[0].split(' ')[2] }}"
+      when: __alertmanager_version.stdout_lines is defined and (__alertmanager_version.stdout_lines|length>0)
+
+    - name: AlertManager upgrade | Print versions
+      debug:
+        msg:
+          - "Installed version: {{ alertmanager_version }}"
+          - "Target version: {{ prometheus_defaults.alertmanager.version }}"
+
+    - name: AlertManager upgrade | Check versions and set alertmanager_will_upgrade
+      set_fact:
+        alertmanager_will_upgrade: "{{ prometheus_defaults.alertmanager.version is version(alertmanager_version, '>') }}"
+
+    - name: AlertManager upgrade | Stop services
+      include_tasks: upgrade/stop-services.yml
+      when: alertmanager_will_upgrade and not prometheus_will_upgrade
+
+    - name: AlertManager upgrade | Upgrading AlertManager
+      include_tasks: upgrade/upgrade-alertmanager.yml
+      when: alertmanager_will_upgrade
+
+# Start services again if needed
+- name: Prometheus upgrade | Start services
+  include_tasks: upgrade/start-services.yml
+  when: prometheus_will_upgrade or alertmanager_will_upgrade

ansible/playbooks/roles/prometheus/tasks/upgrade/start-services.yml

@@ -0,0 +1,21 @@
+---
+- name: Prometheus upgrade | Start Prometheus service
+  systemd:
+    state: started
+    name: prometheus
+  when:
+    - ansible_facts.services["prometheus.service"] is defined
+
+- name: Prometheus upgrade | Start AlertManager service
+  systemd:
+    state: started
+    name: alertmanager
+  when:
+    - ansible_facts.services["alertmanager.service"] is defined
+
+- name: Prometheus upgrade | Start Grafana service
+  systemd:
+    state: started
+    name: grafana-server
+  when:
+    - ansible_facts.services["grafana-server.service"] is defined

ansible/playbooks/roles/prometheus/tasks/upgrade/stop-services.yml

@@ -0,0 +1,21 @@
+---
+- name: Prometheus upgrade | Stop Prometheus service
+  systemd:
+    state: stopped
+    name: prometheus
+  when:
+    - ansible_facts.services["prometheus.service"] is defined
+
+- name: Prometheus upgrade | Stop AlertManager service
+  systemd:
+    state: stopped
+    name: alertmanager
+  when:
+    - ansible_facts.services["alertmanager.service"] is defined
+
+- name: Prometheus upgrade | Stop Grafana service
+  systemd:
+    state: stopped
+    name: grafana-server
+  when:
+    - ansible_facts.services["grafana-server.service"] is defined

ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-alertmanager.yml

@@ -0,0 +1,29 @@
+---
+- name: AlertManager upgrade | Remove old versions ({{alertmanager_version}})
+  file:
+    path: /usr/local/bin/alertmanager
+    state: absent
+
+- name: AlertManager upgrade | Set AlertManager file name to install
+  set_fact:
+    binary_file_name: "{{ prometheus_defaults.alertmanager.file_name[ansible_architecture] }}"
+
+- name: AlertManager upgrade | Download AlertManager binaries
+  include_role:
+    name: download
+    tasks_from: download_file
+  vars:
+    file_name: "{{ binary_file_name }}"
+
+- name: AlertManager upgrade | Unpack AlertManager binary
+  become: true
+  unarchive:
+    src: "{{ download_directory }}/{{ binary_file_name }}"
+    remote_src: yes
+    dest: "/usr/local/bin"
+    creates: "/usr/local/bin/alertmanager"
+    extra_opts: [--strip-components=1]
+    mode: u=rwx,go=rx
+    owner: prometheus
+    group: prometheus
+  check_mode: false

ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-prometheus.yml

@@ -0,0 +1,29 @@
+---
+- name: Prometheus upgrade | Remove old versions ({{prometheus_version}})
+  file:
+    path: /usr/local/bin/prometheus
+    state: absent
+
+- name: Prometheus upgrade | Set Prometheus file name to install
+  set_fact:
+    binary_file_name: "{{ prometheus_defaults.prometheus.file_name[ansible_architecture] }}"
+
+- name: Prometheus upgrade | Download Prometheus binaries
+  include_role:
+    name: download
+    tasks_from: download_file
+  vars:
+    file_name: "{{ binary_file_name }}"
+
+- name: Prometheus upgrade | Unpack Prometheus binary
+  become: true
+  unarchive:
+    src: "{{ download_directory }}/{{ binary_file_name }}"
+    remote_src: yes
+    dest: "/usr/local/bin"
+    creates: "/usr/local/bin/prometheus"
+    extra_opts: [--strip-components=1]
+    mode: u=rwx,go=rx
+    owner: prometheus
+    group: prometheus
+  check_mode: false

ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt

@@ -154,8 +154,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export
 https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-arm64.tar.gz
 # --- Misc ---
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz
-https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-arm64.tar.gz
-https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz
+https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-arm64.tar.gz
+https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz
 https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip
 https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip

ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt

@@ -156,8 +156,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export
 https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-amd64.tar.gz
 # --- Misc ---
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz
-https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
-https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
+https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz
+https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz
 https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip
 https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip

ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt

@@ -152,8 +152,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export
 https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-amd64.tar.gz
 # --- Misc ---
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz
-https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
-https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
+https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz
+https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz
 https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip
 https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip

ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt

@@ -215,8 +215,8 @@ https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9
 # --- Misc ---
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz
 https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
-https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
-https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
+https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz
+https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz
 https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip
 https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip
 https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz

ansible/playbooks/upgrade.yml

@@ -276,3 +276,12 @@
         name: postgresql
         tasks_from: upgrade/main
       when: "'postgresql' in upgrade_components or upgrade_components|length == 0"
+
+- hosts: prometheus
+  become: true
+  become_method: sudo
+  tasks:
+    - include_role:
+        name: prometheus
+        tasks_from: upgrade/main
+      when: "'prometheus' in upgrade_components or upgrade_components|length == 0"

cli/epicli.py

@@ -260,6 +260,7 @@ def upgrade_parser(subparsers):
         'node_exporter',
         'opendistro_for_elasticsearch',
         'postgresql',
+        'prometheus',
         'rabbitmq',
         'zookeeper',
         ])

docs/changelogs/CHANGELOG-1.3.md

@@ -41,6 +41,7 @@
 - [#2537](https://github.com/epiphany-platform/epiphany/issues/2537) - [PostgreSQL] [upgrade] Do not remove new packages automatically in rollback
 - [#2180](https://github.com/epiphany-platform/epiphany/issues/2180) - [documentation] Missing clear information about supported CNI plugins
 - [#2755](https://github.com/epiphany-platform/epiphany/issues/2755) - Upgrade Python dependencies to the latest
+- [#2700](https://github.com/epiphany-platform/epiphany/issues/2700) - Upgrade Prometheus to 2.31.1 and AlertManager to 0.23.0
 
 ### Deprecated
 

docs/home/COMPONENTS.md

@@ -25,7 +25,7 @@ Note that versions are default versions and can be changed in certain cases thro
 | Filebeat                   | 7.9.2    | https://github.com/elastic/beats                      | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Filebeat Helm Chart        | 7.9.2    | https://github.com/elastic/helm-charts                | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Logstash OSS               | 7.12.0   | https://github.com/elastic/logstash                   | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
-| Prometheus                 | 2.10.0   | https://github.com/prometheus/prometheus              | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
+| Prometheus                 | 2.31.1   | https://github.com/prometheus/prometheus              | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Grafana                    | 7.3.5    | https://github.com/grafana/grafana                    | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Node Exporter              | 1.0.1    | https://github.com/prometheus/node_exporter           | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Bitnami Node Exporter Helm Chart      | 1.1.2    | https://github.com/bitnami/charts          | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
@@ -39,7 +39,7 @@ Note that versions are default versions and can be changed in certain cases thro
 | PgBouncer                  | 1.16.0   | https://github.com/pgbouncer/pgbouncer                | [ISC License](https://opensource.org/licenses/isc)                |
 | repmgr                     | 5.2.1    | https://github.com/EnterpriseDB/repmgr                | [GNU General Public License 3.0](https://github.com/EnterpriseDB/repmgr/blob/master/LICENSE) |
 | Pgpool                     | 4.2.4    | https://www.pgpool.net/                               | [License](https://www.pgpool.net/mediawiki/index.php/pgpool-II_License) |
-| Alertmanager               | 0.17.0   | https://github.com/prometheus/alertmanager            | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) |
+| Alertmanager               | 0.23.0   | https://github.com/prometheus/alertmanager            | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) |
 | Apache Ignite              | 2.9.1    | https://github.com/apache/ignite                      | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Apache2                    | 2.4.29   | https://httpd.apache.org/                             | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Hashicorp Vault            | 1.7.0    | https://github.com/hashicorp/vault                    | [Mozilla Public License 2.0](https://github.com/hashicorp/vault/blob/master/LICENSE) |