add pgbouncer rotation and adjust filebeat (#1858)

Co-authored-by: Tomasz Arendt <[email protected]>

CHANGELOG-0.9.md

@@ -4,6 +4,8 @@
 
 ### Added
 
+- [#921](https://github.com/epiphany-platform/epiphany/issues/921) - Implement log rotation for PgBouncer
+
 ### Fixed
 
 - [#1273](https://github.com/epiphany-platform/epiphany/issues/1273) - PostgreSQL replication with repmgr: hot_standby not set in config file

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml

@@ -1,6 +1,12 @@
 ---
 # This file is meant to be also used by upgrade playbook
 
+- name: Load defaults from postgresql role
+  include_vars:
+    file: roles/postgresql/defaults/main.yml
+    name: postgresql_defaults
+  when: "'postgresql' in group_names"
+
 - name: Copy configuration file (filebeat.yml)
   template:
     dest: /etc/filebeat/filebeat.yml

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2

@@ -93,6 +93,12 @@ filebeat.inputs:
   {{ k }}: {{ v }}
   {% endfor %}
   {% endif %}
+
+- type: log
+  enabled: true
+  paths:
+    - {{ postgresql_defaults.pgbouncer.logfile[ansible_os_family] }}*
+  exclude_files: [".gz$"]
 {% endif %}
 
 {% if 'kubernetes_master' in group_names or 'kubernetes_node' in group_names %}

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/defaults/main.yml

@@ -29,4 +29,18 @@ repmgr_service_name:
 
 repmgr_config_dir:
   RedHat: /etc/repmgr/10
-  Debian: /etc/postgresql/10/main
+  Debian: /etc/postgresql/10/main
+
+pgbouncer:
+  logfile:
+    RedHat: /var/log/pgbouncer/pgbouncer.log
+    Debian: /var/log/postgresql/pgbouncer.log
+  pidfile:
+    RedHat: /var/run/pgbouncer/pgbouncer.pid
+    Debian: /var/run/postgresql/pgbouncer.pid
+  user:
+    RedHat: pgbouncer
+    Debian: postgres
+  group:
+    RedHat: pgbouncer
+    Debian: postgres

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/additional-components/configure-pgbouncer.yml

@@ -19,6 +19,14 @@
     create: yes
     backup: yes
 
+- name: Create pgbouncer logrotate configuration file
+  template:
+    src: logrotate-pgbouncer.conf.j2
+    dest: /etc/logrotate.d/pgbouncer
+    owner: root
+    group: root
+    mode: u=rw,go=r
+
 - name: Enable pgbouncer service
   systemd:
     name: pgbouncer

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/configure-postgresql.yml

@@ -104,23 +104,21 @@
     name: "{{ pg_service_name[ansible_os_family] }}"
     enabled: yes
 
-- name: Configure logrotate
+- name: Configure postgresql logrotate
   block:
     - name: Remove logrotate configuration provided by 'postgresql-common' package
       file:
         path: /etc/logrotate.d/postgresql-common
         state: absent
       when: ansible_os_family == 'Debian'
 
-    - name: Create logrotate configuration file
+    - name: Create postgresql logrotate configuration file
       template:
-        src: logrotate.conf.j2
+        src: logrotate-postgresql.conf.j2
         dest: /etc/logrotate.d/postgresql
         owner: root
         group: root
         mode: 0644
-  when:
-    - specification.logrotate.config is defined
 
 - name: Set up replication
   include_tasks: replication-{{ ansible_os_family }}.yml

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/logrotate-pgbouncer.conf.j2

@@ -0,0 +1,18 @@
+# {{ ansible_managed }}
+
+{{ pgbouncer.logfile[ansible_os_family] }} {
+    missingok
+    copytruncate
+    # delaycompress is for Filebeat
+    delaycompress
+    compress
+    notifempty
+    sharedscripts
+    create 0640 {{ pgbouncer.user[ansible_os_family] }} {{ pgbouncer.group[ansible_os_family] }}
+    nodateext
+    {{ specification.logrotate.pgbouncer.period }}
+    rotate {{ specification.logrotate.pgbouncer.rotations }}
+    postrotate
+        /bin/kill -HUP `cat {{ pgbouncer.pidfile[ansible_os_family] }} 2>/dev/null` 2> /dev/null || true
+    endscript
+}

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/logrotate-postgresql.conf.j2

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+
+{{ specification.logrotate.postgresql }}

core/src/epicli/data/common/defaults/configuration/postgresql.yml

@@ -97,6 +97,9 @@ specification:
       shared_preload_libraries:
         - repmgr
   logrotate:
+    pgbouncer:
+      period: weekly
+      rotations: 5
     # Configuration partly based on /etc/logrotate.d/postgresql-common provided by 'postgresql-common' package from Ubuntu repo.
       # PostgreSQL from Ubuntu repo:
         # By default 'logging_collector' is disabled, so 'log_directory' parameter is ignored.
@@ -105,19 +108,19 @@ specification:
         # By default 'logging_collector' is enabled and there is up to 7 files named 'daily' (e.g. postgresql-Wed.log)
         # and thus they can be overwritten by built-in log facility to provide rotation.
     # To have similar configuration for both distros (with logrotate), 'log_filename' parameter is modified.
-    config: |-
+    postgresql: |-
       /var/log/postgresql/postgresql*.log {
           maxsize 10M
           daily
           rotate 6
           copytruncate
-      # delaycompress is for Filebeat
+          # delaycompress is for Filebeat
           delaycompress
           compress
           notifempty
           missingok
           su root root
           nomail
-      # to have multiple unique filenames per day when dateext option is set
+          # to have multiple unique filenames per day when dateext option is set
           dateformat -%Y%m%dH%H
       }