diff --git a/ansible/playbooks/roles/prometheus/defaults/main.yml b/ansible/playbooks/roles/prometheus/defaults/main.yml index 70c4d9dd10..d5788ed089 100644 --- a/ansible/playbooks/roles/prometheus/defaults/main.yml +++ b/ansible/playbooks/roles/prometheus/defaults/main.yml @@ -3,11 +3,13 @@ prometheus_external_labels: environment: "{{ ansible_fqdn }}" prometheus: + version: 2.31.1 file_name: - x86_64: "prometheus-2.10.0.linux-amd64.tar.gz" - aarch64: "prometheus-2.10.0.linux-arm64.tar.gz" + x86_64: "prometheus-2.31.1.linux-amd64.tar.gz" + aarch64: "prometheus-2.31.1.linux-arm64.tar.gz" alertmanager: + version: 0.23.0 file_name: - x86_64: "alertmanager-0.17.0.linux-amd64.tar.gz" - aarch64: "alertmanager-0.17.0.linux-arm64.tar.gz" + x86_64: "alertmanager-0.23.0.linux-amd64.tar.gz" + aarch64: "alertmanager-0.23.0.linux-arm64.tar.gz" diff --git a/ansible/playbooks/roles/prometheus/defaults/upgrade.yml b/ansible/playbooks/roles/prometheus/defaults/upgrade.yml new file mode 100644 index 0000000000..62db76f835 --- /dev/null +++ b/ansible/playbooks/roles/prometheus/defaults/upgrade.yml @@ -0,0 +1,2 @@ +--- +state_file_path: /var/lib/epiphany/upgrade/state/prometheus.uncompleted diff --git a/ansible/playbooks/roles/prometheus/tasks/upgrade/main.yml b/ansible/playbooks/roles/prometheus/tasks/upgrade/main.yml new file mode 100644 index 0000000000..18b7fa6754 --- /dev/null +++ b/ansible/playbooks/roles/prometheus/tasks/upgrade/main.yml @@ -0,0 +1,137 @@ +--- +# General things needed +- name: Include new version of defaults + include_vars: + file: defaults/main.yml + name: prometheus_defaults + +- name: Include upgrade defaults + include_vars: + file: defaults/upgrade.yml + name: upgrade_defaults + +- name: Populate service facts + service_facts: + when: ansible_facts.service is undefined + +# If state file exists it means the previous run failed +- name: Check for upgrade state file + stat: + path: "{{ upgrade_defaults.state_file_path }}" + get_attributes: false + get_checksum: false + get_mime: false + register: state_file_status + +- name: Create upgrade state file + copy: + dest: "{{ upgrade_defaults.state_file_path }}" + content: Upgrade started + mode: u=rw,g=r,o= + +# Prometheus upgrade +- name: Set prometheus_will_upgrade fact + set_fact: + prometheus_will_upgrade: false + +- name: Check Prometheus binary existance + stat: + path: /usr/local/bin/prometheus + register: prometheus_stat + +- name: Start Prometheus upgrade + when: + - ansible_facts.services["prometheus.service"] is defined + - prometheus_stat.stat.exists + block: + - name: Get Prometheus version + command: prometheus --version + changed_when: false + check_mode: false + become: false + register: _prometheus_version + + - name: Set Prometheus version fact + set_fact: + prometheus_version: "{{ _prometheus_version[_stream][0].split(' ')[2] }}" + vars: + _stream: "{{ 'stdout_lines' if _prometheus_version.stdout_lines else 'stderr_lines' }}" + + - name: Print Prometheus versions + debug: + msg: + - "Installed version: {{ prometheus_version }}" + - "Target version: {{ prometheus_defaults.prometheus.version }}" + + - name: Check versions and set prometheus_will_upgrade + set_fact: + prometheus_will_upgrade: "{{ prometheus_defaults.prometheus.version is version(prometheus_version, '>') or state_file_status.stat.exists }}" + + - name: Perform upgrade + when: prometheus_will_upgrade + block: + - name: Stop services + include_tasks: upgrade/stop-services.yml + + - name: Upgrade Prometheus + include_tasks: upgrade/upgrade-prometheus.yml + + +# AlertManager upgrade +- name: Set alertmanager_will_upgrade fact + set_fact: + alertmanager_will_upgrade: false + +- name: Check binary AlertManager existance + stat: + path: /usr/local/bin/alertmanager + register: alertmanager_stat + +- name: Start AlertManager upgrade + when: + - ansible_facts.services["alertmanager.service"] is defined + - alertmanager_stat.stat.exists + block: + - name: Get AlertManager version + command: alertmanager --version + become: false + changed_when: false + check_mode: false + register: _alertmanager_version + + - name: Set AlertManager version fact + set_fact: + alertmanager_version: "{{ _alertmanager_version[_stream][0].split(' ')[2] }}" + vars: + _stream: "{{ 'stdout_lines' if _alertmanager_version.stdout_lines else 'stderr_lines' }}" + + - name: Print AlertManager versions + debug: + msg: + - "Installed version: {{ alertmanager_version }}" + - "Target version: {{ prometheus_defaults.alertmanager.version }}" + + - name: Check versions and set alertmanager_will_upgrade + set_fact: + alertmanager_will_upgrade: "{{ prometheus_defaults.alertmanager.version is version(alertmanager_version, '>') or state_file_status.stat.exists }}" + + - name: Perform upgrade + when: alertmanager_will_upgrade + block: + - name: Stop services + include_tasks: upgrade/stop-services.yml + when: not prometheus_will_upgrade + + - name: Upgrade AlertManager + include_tasks: upgrade/upgrade-alertmanager.yml + + +# Start services again if needed +- name: Start services + include_tasks: upgrade/start-services.yml + when: prometheus_will_upgrade or alertmanager_will_upgrade + +- name: "Remove upgrade state file" + file: + path: "{{ upgrade_defaults.state_file_path }}" + state: absent diff --git a/ansible/playbooks/roles/prometheus/tasks/upgrade/start-services.yml b/ansible/playbooks/roles/prometheus/tasks/upgrade/start-services.yml new file mode 100644 index 0000000000..d00cfbdccd --- /dev/null +++ b/ansible/playbooks/roles/prometheus/tasks/upgrade/start-services.yml @@ -0,0 +1,21 @@ +--- +- name: Start Prometheus service + systemd: + state: started + name: prometheus + when: + - ansible_facts.services["prometheus.service"] is defined + +- name: Start AlertManager service + systemd: + state: started + name: alertmanager + when: + - ansible_facts.services["alertmanager.service"] is defined + +- name: Start Grafana service + systemd: + state: started + name: grafana-server + when: + - ansible_facts.services["grafana-server.service"] is defined diff --git a/ansible/playbooks/roles/prometheus/tasks/upgrade/stop-services.yml b/ansible/playbooks/roles/prometheus/tasks/upgrade/stop-services.yml new file mode 100644 index 0000000000..2f2453919e --- /dev/null +++ b/ansible/playbooks/roles/prometheus/tasks/upgrade/stop-services.yml @@ -0,0 +1,21 @@ +--- +- name: Stop Prometheus service + systemd: + state: stopped + name: prometheus + when: + - ansible_facts.services["prometheus.service"] is defined + +- name: Stop AlertManager service + systemd: + state: stopped + name: alertmanager + when: + - ansible_facts.services["alertmanager.service"] is defined + +- name: Stop Grafana service + systemd: + state: stopped + name: grafana-server + when: + - ansible_facts.services["grafana-server.service"] is defined diff --git a/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-alertmanager.yml b/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-alertmanager.yml new file mode 100644 index 0000000000..a75ae4c75e --- /dev/null +++ b/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-alertmanager.yml @@ -0,0 +1,29 @@ +--- +- name: Remove old AlertManager version ({{alertmanager_version}}) + file: + path: /usr/local/bin/alertmanager + state: absent + +- name: Set AlertManager file name to install + set_fact: + binary_file_name: "{{ prometheus_defaults.alertmanager.file_name[ansible_architecture] }}" + +- name: Download new AlertManager binary ({{ prometheus_defaults.alertmanager.version }}) + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ binary_file_name }}" + +- name: Unpack and install new AlertManager binary ({{ prometheus_defaults.alertmanager.version }}) + become: true + unarchive: + src: "{{ download_directory }}/{{ binary_file_name }}" + remote_src: yes + dest: "/usr/local/bin" + creates: "/usr/local/bin/alertmanager" + extra_opts: [--strip-components=1] + mode: u=rwx,go=rx + owner: prometheus + group: prometheus + check_mode: false diff --git a/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-prometheus.yml b/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-prometheus.yml new file mode 100644 index 0000000000..e9ade6309a --- /dev/null +++ b/ansible/playbooks/roles/prometheus/tasks/upgrade/upgrade-prometheus.yml @@ -0,0 +1,29 @@ +--- +- name: Remove old Prometheus version ({{prometheus_version}}) + file: + path: /usr/local/bin/prometheus + state: absent + +- name: Set Prometheus file name to install + set_fact: + binary_file_name: "{{ prometheus_defaults.prometheus.file_name[ansible_architecture] }}" + +- name: Download new Prometheus binary ({{ prometheus_defaults.prometheus.version }}) + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ binary_file_name }}" + +- name: Unpack and install new Prometheus binary ({{ prometheus_defaults.prometheus.version }}) + become: true + unarchive: + src: "{{ download_directory }}/{{ binary_file_name }}" + remote_src: yes + dest: "/usr/local/bin" + creates: "/usr/local/bin/prometheus" + extra_opts: [--strip-components=1] + mode: u=rwx,go=rx + owner: prometheus + group: prometheus + check_mode: false diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 3bef74f7b7..e710b36cfe 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -154,8 +154,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-arm64.tar.gz # --- Misc --- https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz -https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-arm64.tar.gz -https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-arm64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_arm64.zip diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 49bd853cb1..dce8fc0789 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -156,8 +156,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-amd64.tar.gz # --- Misc --- https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz -https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz -https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index bb3f34812b..1f2c8ca114 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -152,8 +152,8 @@ https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_export https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9.0/postgres_exporter-0.9.0.linux-amd64.tar.gz # --- Misc --- https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz -https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz -https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt index 6956413288..950efeead8 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt @@ -215,8 +215,8 @@ https://github.com/prometheus-community/postgres_exporter/releases/download/v0.9 # --- Misc --- https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz -https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz -https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz +https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz +https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz https://archive.apache.org/dist/ignite/2.9.1/apache-ignite-2.9.1-bin.zip https://releases.hashicorp.com/vault/1.7.0/vault_1.7.0_linux_amd64.zip https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index f56787302d..7fc4ecf2e3 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -276,3 +276,12 @@ name: postgresql tasks_from: upgrade/main when: "'postgresql' in upgrade_components or upgrade_components|length == 0" + +- hosts: prometheus + become: true + become_method: sudo + tasks: + - include_role: + name: prometheus + tasks_from: upgrade/main + when: "'prometheus' in upgrade_components or upgrade_components|length == 0" diff --git a/cli/epicli.py b/cli/epicli.py index 035213c01a..0e5277d0fc 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -260,6 +260,7 @@ def upgrade_parser(subparsers): 'node_exporter', 'opendistro_for_elasticsearch', 'postgresql', + 'prometheus', 'rabbitmq', 'zookeeper', ]) diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index e0d100f57e..9a3d2a5c3d 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -41,6 +41,7 @@ - [#2537](https://github.com/epiphany-platform/epiphany/issues/2537) - [PostgreSQL] [upgrade] Do not remove new packages automatically in rollback - [#2180](https://github.com/epiphany-platform/epiphany/issues/2180) - [documentation] Missing clear information about supported CNI plugins - [#2755](https://github.com/epiphany-platform/epiphany/issues/2755) - Upgrade Python dependencies to the latest +- [#2700](https://github.com/epiphany-platform/epiphany/issues/2700) - Upgrade Prometheus to 2.31.1 and AlertManager to 0.23.0 ### Deprecated diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index 015283ad99..9b976ccb48 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -25,7 +25,7 @@ Note that versions are default versions and can be changed in certain cases thro | Filebeat | 7.9.2 | https://github.com/elastic/beats | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Filebeat Helm Chart | 7.9.2 | https://github.com/elastic/helm-charts | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Logstash OSS | 7.12.0 | https://github.com/elastic/logstash | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Prometheus | 2.10.0 | https://github.com/prometheus/prometheus | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Prometheus | 2.31.1 | https://github.com/prometheus/prometheus | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Grafana | 7.3.5 | https://github.com/grafana/grafana | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Node Exporter | 1.0.1 | https://github.com/prometheus/node_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Bitnami Node Exporter Helm Chart | 1.1.2 | https://github.com/bitnami/charts | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | @@ -39,7 +39,7 @@ Note that versions are default versions and can be changed in certain cases thro | PgBouncer | 1.16.0 | https://github.com/pgbouncer/pgbouncer | [ISC License](https://opensource.org/licenses/isc) | | repmgr | 5.2.1 | https://github.com/EnterpriseDB/repmgr | [GNU General Public License 3.0](https://github.com/EnterpriseDB/repmgr/blob/master/LICENSE) | | Pgpool | 4.2.4 | https://www.pgpool.net/ | [License](https://www.pgpool.net/mediawiki/index.php/pgpool-II_License) | -| Alertmanager | 0.17.0 | https://github.com/prometheus/alertmanager | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) | +| Alertmanager | 0.23.0 | https://github.com/prometheus/alertmanager | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) | | Apache Ignite | 2.9.1 | https://github.com/apache/ignite | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Apache2 | 2.4.29 | https://httpd.apache.org/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Hashicorp Vault | 1.7.0 | https://github.com/hashicorp/vault | [Mozilla Public License 2.0](https://github.com/hashicorp/vault/blob/master/LICENSE) |