From 51917d5b66e47e1daa63e18df7d8e3daab3260bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Bia=C5=82obrzecki?= Date: Tue, 26 Oct 2021 14:54:24 +0200 Subject: [PATCH] RabbitMQ monitoring automation (#1487) * Enable RabbitMQ's plugin for Prometheus metrics exposure * Add target for Prometheus to be able to scrape metrics from rabbitmq nodes * Download Grafana dashboard for displaying scraped metrics from RabbitMQ --- .../roles/grafana/tasks/dashboards.yml | 12 ++++++++++ .../playbooks/roles/grafana/tasks/main.yml | 2 +- .../prometheus/templates/prometheus.yml.j2 | 7 ++++++ .../rabbitmq/tasks/configure-rabbitmq.yml | 9 +++++++ cli/engine/ansible/AnsibleVarsGenerator.py | 5 ++-- docs/home/howto/MONITORING.md | 24 +++++++++++++++++++ .../defaults/configuration/shared-config.yml | 1 + 7 files changed, 57 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/grafana/tasks/dashboards.yml b/ansible/playbooks/roles/grafana/tasks/dashboards.yml index 3cf343323b..967ec2757e 100644 --- a/ansible/playbooks/roles/grafana/tasks/dashboards.yml +++ b/ansible/playbooks/roles/grafana/tasks/dashboards.yml @@ -23,6 +23,18 @@ loop: "{{ grafana_online_dashboards }}" when: grafana_online_dashboards != [] + - name: Set facts for RabbitMQ dashboard + set_fact: + rabbitmq_dashboard: + - dashboard_id: '10991' + datasource: 'Prometheus' + when: rabbitmq_monitoring_enabled + + - name: Extend external dashboards with RabbitMQ monitoring dashboard + set_fact: + grafana_external_dashboards: "{{ grafana_external_dashboards + rabbitmq_dashboard }}" + when: rabbitmq_monitoring_enabled + - name: Download Grafana dashboards from local repository to temporary directory include_role: name: download diff --git a/ansible/playbooks/roles/grafana/tasks/main.yml b/ansible/playbooks/roles/grafana/tasks/main.yml index 9125ec7096..2f2247e2a4 100644 --- a/ansible/playbooks/roles/grafana/tasks/main.yml +++ b/ansible/playbooks/roles/grafana/tasks/main.yml @@ -34,4 +34,4 @@ - name: Include dashboards tasks include_tasks: dashboards.yml - when: (grafana_online_dashboards != []) or (grafana_external_dashboards != []) + when: (grafana_online_dashboards != []) or (grafana_external_dashboards != []) or (rabbitmq_monitoring_enabled) diff --git a/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 b/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 index c0d2568eda..e280891595 100644 --- a/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2 @@ -41,6 +41,13 @@ scrape_configs: - files: - "{{ specification.config_directory }}/file_sd/*.yml" +{% if rabbitmq_monitoring_enabled %} + - job_name: 'rabbitmq-exporter' + static_configs: + - targets: + - "{{ groups['rabbitmq'][0] }}:15692" +{% endif %} + {% if bearer_token is defined %} - job_name: 'kubernetes-apiservers' kubernetes_sd_configs: diff --git a/ansible/playbooks/roles/rabbitmq/tasks/configure-rabbitmq.yml b/ansible/playbooks/roles/rabbitmq/tasks/configure-rabbitmq.yml index 23c17bfa3b..624a0e65e3 100644 --- a/ansible/playbooks/roles/rabbitmq/tasks/configure-rabbitmq.yml +++ b/ansible/playbooks/roles/rabbitmq/tasks/configure-rabbitmq.yml @@ -38,6 +38,15 @@ broker_state: offline register: rabbitmq_plugins +- name: Enable rabbitmq monitoring plugin + when: rabbitmq_monitoring_enabled + rabbitmq_plugin: + names: rabbitmq_prometheus,rabbitmq_management_agent,rabbitmq_web_dispatch + prefix: /usr/lib/rabbitmq + state: enabled + broker_state: offline + new_only: true + - name: Restart service when: - config_file_stat.changed or env_file_stat.changed or rabbitmq_plugins.changed diff --git a/cli/engine/ansible/AnsibleVarsGenerator.py b/cli/engine/ansible/AnsibleVarsGenerator.py index 50081319ba..6b41069fe8 100644 --- a/cli/engine/ansible/AnsibleVarsGenerator.py +++ b/cli/engine/ansible/AnsibleVarsGenerator.py @@ -190,8 +190,7 @@ def get_clean_cluster_model(self): self.clear_object(cluster_model, 'credentials') return cluster_model - def get_shared_config_from_manifest(self): - # Reuse shared config from existing manifest + def get_shared_config_from_manifest(self): # Reuse shared config from existing manifest # Shared config contains the use_ha_control_plane flag which is required during upgrades cluster_model = select_single(self.manifest_docs, lambda x: x.kind == 'epiphany-cluster') @@ -210,6 +209,8 @@ def get_shared_config_from_manifest(self): if hasattr(shared_config_doc.specification, 'supported_os'): del shared_config_doc.specification['supported_os'] + shared_config_doc.specification['rabbitmq_monitoring_enabled'] = False + # Merge the shared config doc with defaults with DefaultMerger([shared_config_doc]) as doc_merger: shared_config_doc = doc_merger.run()[0] diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index 17eb1ee5e1..f80aee313a 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -16,6 +16,10 @@ Kibana: - [How to configure Kibana](#how-to-configure-kibana) - [How to configure default user password in Kibana](#how-to-configure-default-user-password-in-kibana) +RabbitMQ: + +- [How to enable RabbitMQ monitoring](#how-to-enable-rabbitmq-monitoring) + Azure: - [How to configure Azure additional monitoring and alerting](#how-to-configure-azure-additional-monitoring-and-alerting) @@ -326,6 +330,26 @@ specification: During upgrade Epiphany takes `kibanaserver` (for Kibana) and `logstash` (for Filebeat) user passwords and re-applies them to upgraded configuration of Filebeat and Kibana. Epiphany upgrade of Open Distro, Kibana or Filebeat will fail if `kibanaserver` or `logstash` usernames were changed in configuration of Kibana, Filebeat or Open Distro for Elasticsearch. +# RabbitMQ + +## How to enable RabbitMQ monitoring + +To enable RabbitMQ monitoring set `specification/rabbitmq_monitoring_enabled` in `configuration/default-config` section to `true`. +This will: +* enable RabbitMQ's plugin for Prometheus metrics exposure +* add target for Prometheus to be able to scrape metrics from rabbitmq nodes +* download Grafana dashboard for displaying scraped metrics from RabbitMQ + +```yaml +--- +kind: configuration/shared-config +title: Shared configuration that will be visible to all roles +name: default +specification: + ... + rabbitmq_monitoring_enabled: true +``` + # Azure ## How to configure Azure additional monitoring and alerting diff --git a/schema/common/defaults/configuration/shared-config.yml b/schema/common/defaults/configuration/shared-config.yml index 219a653414..1bb62bf2cc 100644 --- a/schema/common/defaults/configuration/shared-config.yml +++ b/schema/common/defaults/configuration/shared-config.yml @@ -9,3 +9,4 @@ specification: vault_tmp_file_location: SET_BY_AUTOMATION use_ha_control_plane: False promote_to_ha: False + rabbitmq_monitoring_enabled: False