diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 89f608b273..95dec61855 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: - upgrade-from-stable-integration agent-versions: - "2.9.45" # renovate: latest juju 2 - - "3.1.5" # renovate: latest juju 3 + - "3.1.6" # renovate: latest juju 3 name: ${{ matrix.tox-environments }} | ${{ matrix.agent-versions }} needs: - lint diff --git a/config.yaml b/config.yaml index 8885803da9..daa72aa7fb 100644 --- a/config.yaml +++ b/config.yaml @@ -2,30 +2,127 @@ # See LICENSE file for licensing details. options: + durability_synchronous_commit: + description: | + Sets the current transactions synchronization level. This charm allows only the + “on”, “remote_apply” and “remote_write” values to avoid data loss if the primary + crashes and there are replicas. + type: string + default: "on" + instance_default_text_search_config: + description: | + Selects the text search configuration that is used by those variants of the text + search functions that do not have an explicit argument specifying it. + Allowed values start with “pg_catalog.” followed by a language name, like + “pg_catalog.english”. + type: string + default: "pg_catalog.simple" + instance_password_encryption: + description: | + Determines the algorithm to use to encrypt the password. + Allowed values are: “md5” and “scram-sha-256”. + type: string + default: "scram-sha-256" + logging_log_connections: + description: | + Logs each successful connection. + type: boolean + default: false + logging_log_disconnections: + description: | + Logs end of a session, including duration. + type: boolean + default: false + logging_log_lock_waits: + description: | + Logs long lock waits. + type: boolean + default: false + logging_log_min_duration_statement: + description: | + Sets the minimum running time (milliseconds) above which statements will be logged. + Allowed values are: from -1 to 2147483647 (-1 disables logging + statement durations). + type: int + default: -1 + memory_maintenance_work_mem: + description: | + Sets the maximum memory (KB) to be used for maintenance operations. + Allowed values are: from 1024 to 2147483647. + type: int + default: 65536 + memory_max_prepared_transactions: + description: | + Sets the maximum number of simultaneously prepared transactions. + Allowed values are: from 0 to 262143. + type: int + default: 0 + memory_shared_buffers: + description: | + Sets the number of shared memory buffers (8 kB) used by the server. This charm allows + to set this value up to 40% of the available memory from the unit, as it is unlikely + that an allocation of more than that will work better than a smaller amount. + Allowed values are: from 16 to 1073741823. + type: int + memory_temp_buffers: + description: | + Sets the maximum number of temporary buffers (8 kB) used by each session. + Allowed values are: from 100 to 1073741823. + type: int + default: 1024 + memory_work_mem: + description: | + Sets the maximum memory (KB) to be used for query workspaces. + Allowed values are: from 64 to 2147483647. + type: int + default: 4096 + optimizer_constraint_exclusion: + description: | + Enables the planner to use constraints to optimize queries. + Allowed values are: “on”, “off” and “partition”. + type: string + default: "partition" + optimizer_default_statistics_target: + description: | + Sets the default statistics target. Allowed values are: from 1 to 10000. + type: int + default: 100 + optimizer_from_collapse_limit: + description: | + Sets the FROM-list size beyond which subqueries are not collapsed. + Allowed values are: from 1 to 2147483647. + type: int + default: 8 + optimizer_join_collapse_limit: + description: | + Sets the FROM-list size beyond which JOIN constructs are not flattened. + Allowed values are: from 1 to 2147483647. + type: int + default: 8 plugin_citext_enable: default: false type: boolean - description: Enable citext extension + description: Enable citext extension. plugin_debversion_enable: default: false type: boolean - description: Enable debversion extension + description: Enable debversion extension. plugin_hstore_enable: default: false type: boolean - description: Enable hstore extension + description: Enable hstore extension. plugin_pg_trgm_enable: default: false type: boolean - description: Enable pg_trgm extension + description: Enable pg_trgm extension. plugin_plpython3u_enable: default: false type: boolean - description: Enable PL/Python extension + description: Enable PL/Python extension. plugin_unaccent_enable: default: false type: boolean - description: Enable unaccent extension + description: Enable unaccent extension. profile: description: | Profile representing the scope of deployment, and used to tune resource allocation. @@ -33,10 +130,91 @@ options: Production will tune postgresql for maximum performance while testing will tune for minimal running performance. type: string - default: production + default: "production" profile-limit-memory: type: int description: | Amount of memory in Megabytes to limit PostgreSQL and associated process to. If unset, this will be decided according to the default memory limit in the selected profile. Only comes into effect when the `production` profile is selected. + request_date_style: + description: | + Sets the display format for date and time values. Allowed formats are explained + in https://www.postgresql.org/docs/14/runtime-config-client.html#GUC-DATESTYLE. + type: string + default: "ISO, MDY" + request_standard_conforming_strings: + description: | + Causes ... strings to treat backslashes literally. + type: boolean + default: true + request_time_zone: + description: | + Sets the time zone for displaying and interpreting time stamps. + Allowed values are the ones from IANA time zone data, a time zone abbreviation + like PST and POSIX-style time zone specifications. + type: string + default: "UTC" + response_bytea_output: + description: | + Sets the output format for bytes. + Allowed values are: “escape” and “hex”. + type: string + default: "hex" + response_lc_monetary: + description: | + Sets the locale for formatting monetary amounts. + Allowed values are the locales available in the unit. + type: string + default: "C" + response_lc_numeric: + description: | + Sets the locale for formatting numbers. + Allowed values are the locales available in the unit. + type: string + default: "C" + response_lc_time: + description: | + Sets the locale for formatting date and time values. + Allowed values are the locales available in the unit. + type: string + default: "C" + vacuum_autovacuum_analyze_scale_factor: + description: | + Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when + deciding whether to trigger a VACUUM. The default, 0.1, means 10% of table size. + Allowed values are: from 0 to 100. + type: float + default: 0.1 + vacuum_autovacuum_analyze_threshold: + description: | + Sets the minimum number of inserted, updated or deleted tuples needed to trigger + an ANALYZE in any one table. Allowed values are: from 0 to 2147483647. + type: int + default: 50 + vacuum_autovacuum_freeze_max_age: + description: | + Maximum age (in transactions) before triggering autovacuum on a table to prevent + transaction ID wraparound. Allowed values are: from 100000 to 2000000000. + type: int + default: 200000000 + vacuum_autovacuum_vacuum_cost_delay: + description: | + Sets cost delay value (milliseconds) that will be used in automatic VACUUM operations. + Allowed values are: from -1 to 100 (-1 tells PostgreSQL to use the regular + vacuum_cost_delay value). + type: float + default: 2.0 + vacuum_autovacuum_vacuum_scale_factor: + description: | + Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when + deciding whether to trigger a VACUUM. The default, 0.2, means 20% of table size. + Allowed values are: from 0 to 100. + type: float + default: 0.2 + vacuum_vacuum_freeze_table_age: + description: | + Age (in transactions) at which VACUUM should scan whole table to freeze tuples. + Allowed values are: from 0 to 2000000000. + type: int + default: 150000000 diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index 3efcda0c41..debac9ae11 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -32,7 +32,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 17 +LIBPATCH = 18 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -310,6 +310,32 @@ def enable_disable_extension(self, extension: str, enable: bool, database: str = if connection is not None: connection.close() + def get_postgresql_text_search_configs(self) -> Set[str]: + """Returns the PostgreSQL available text search configs. + + Returns: + Set of PostgreSQL text search configs. + """ + with self._connect_to_database( + connect_to_current_host=True + ) as connection, connection.cursor() as cursor: + cursor.execute("SELECT CONCAT('pg_catalog.', cfgname) FROM pg_ts_config;") + text_search_configs = cursor.fetchall() + return {text_search_config[0] for text_search_config in text_search_configs} + + def get_postgresql_timezones(self) -> Set[str]: + """Returns the PostgreSQL available timezones. + + Returns: + Set of PostgreSQL timezones. + """ + with self._connect_to_database( + connect_to_current_host=True + ) as connection, connection.cursor() as cursor: + cursor.execute("SELECT name FROM pg_timezone_names;") + timezones = cursor.fetchall() + return {timezone[0] for timezone in timezones} + def get_postgresql_version(self) -> str: """Returns the PostgreSQL version. @@ -445,12 +471,12 @@ def is_restart_pending(self) -> bool: @staticmethod def build_postgresql_parameters( - profile: str, available_memory: int, limit_memory: Optional[int] = None - ) -> Optional[Dict[str, str]]: + config_options: Dict, available_memory: int, limit_memory: Optional[int] = None + ) -> Optional[Dict]: """Builds the PostgreSQL parameters. Args: - profile: the profile to use. + config_options: charm config options containing profile and PostgreSQL parameters. available_memory: available memory to use in calculation in bytes. limit_memory: (optional) limit memory to use in calculation in bytes. @@ -459,19 +485,60 @@ def build_postgresql_parameters( """ if limit_memory: available_memory = min(available_memory, limit_memory) + profile = config_options["profile"] logger.debug(f"Building PostgreSQL parameters for {profile=} and {available_memory=}") + parameters = {} + for config, value in config_options.items(): + # Filter config option not related to PostgreSQL parameters. + if not config.startswith( + ( + "durability", + "instance", + "logging", + "memory", + "optimizer", + "request", + "response", + "vacuum", + ) + ): + continue + parameter = "_".join(config.split("_")[1:]) + if parameter in ["date_style", "time_zone"]: + parameter = "".join(x.capitalize() for x in parameter.split("_")) + parameters[parameter] = value + shared_buffers_max_value = int(int(available_memory * 0.4) / 10**6) + if parameters.get("shared_buffers", 0) > shared_buffers_max_value: + raise Exception( + f"Shared buffers config option should be at most 40% of the available memory, which is {shared_buffers_max_value}MB" + ) if profile == "production": # Use 25% of the available memory for shared_buffers. # and the remaind as cache memory. shared_buffers = int(available_memory * 0.25) effective_cache_size = int(available_memory - shared_buffers) - - parameters = { - "shared_buffers": f"{int(shared_buffers/10**6)}MB", - "effective_cache_size": f"{int(effective_cache_size/10**6)}MB", - } - - return parameters + parameters.setdefault("shared_buffers", f"{int(shared_buffers/10**6)}MB") + parameters.update({"effective_cache_size": f"{int(effective_cache_size/10**6)}MB"}) else: # Return default - return {"shared_buffers": "128MB"} + parameters.setdefault("shared_buffers", "128MB") + return parameters + + def validate_date_style(self, date_style: str) -> bool: + """Validate a date style against PostgreSQL. + + Returns: + Whether the date style is valid. + """ + try: + with self._connect_to_database( + connect_to_current_host=True + ) as connection, connection.cursor() as cursor: + cursor.execute( + sql.SQL( + "SET DateStyle to {};", + ).format(sql.Identifier(date_style)) + ) + return True + except psycopg2.Error: + return False diff --git a/src/backups.py b/src/backups.py index 084567799a..cbe224c487 100644 --- a/src/backups.py +++ b/src/backups.py @@ -197,7 +197,7 @@ def _empty_data_files(self) -> None: def _change_connectivity_to_database(self, connectivity: bool) -> None: """Enable or disable the connectivity to the database.""" self.charm.unit_peer_data.update({"connectivity": "on" if connectivity else "off"}) - self.charm.update_config() + self.charm.update_config(is_creating_backup=True) def _execute_command( self, command: List[str], timeout: float = None diff --git a/src/charm.py b/src/charm.py index 6797c86c17..1a17f40824 100755 --- a/src/charm.py +++ b/src/charm.py @@ -6,7 +6,8 @@ import itertools import json import logging -from typing import Dict, List, Optional +import time +from typing import Dict, List, Optional, Tuple from charms.data_platform_libs.v0.data_models import TypedCharmBase from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider @@ -1378,8 +1379,9 @@ def update_config(self, is_creating_backup: bool = False) -> bool: limit_memory = self.config.profile_limit_memory * 10**6 else: limit_memory = None + available_cpu_cores, available_memory = self.get_available_resources() postgresql_parameters = self.postgresql.build_postgresql_parameters( - self.config.profile, self.get_available_memory(), limit_memory + self.model.config, available_memory, limit_memory ) logger.info("Updating Patroni config file") @@ -1394,6 +1396,7 @@ def update_config(self, is_creating_backup: bool = False) -> bool: restore_stanza=self.app_peer_data.get("restore-stanza"), parameters=postgresql_parameters, ) + if not self._is_workload_running: # If Patroni/PostgreSQL has not started yet and TLS relations was initialised, # then mark TLS as enabled. This commonly happens when the charm is deployed @@ -1407,10 +1410,22 @@ def update_config(self, is_creating_backup: bool = False) -> bool: logger.debug("Early exit update_config: Patroni not started yet") return False - restart_postgresql = ( - self.is_tls_enabled != self.postgresql.is_tls_enabled() - ) or self.postgresql.is_restart_pending() + if not is_creating_backup: + self._validate_config_options() + + self._patroni.update_parameter_controller_by_patroni( + "max_connections", max(4 * available_cpu_cores, 100) + ) + self._patroni.update_parameter_controller_by_patroni( + "max_prepared_transactions", self.config.memory_max_prepared_transactions + ) + + restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled() self._patroni.reload_patroni_configuration() + # Sleep the same time as Patroni's loop_wait default value, which tells how much time + # Patroni will wait before checking the configuration file again to reload it. + time.sleep(10) + restart_postgresql = restart_postgresql or self.postgresql.is_restart_pending() self.unit_peer_data.update({"tls": "enabled" if self.is_tls_enabled else ""}) # Restart PostgreSQL if TLS configuration has changed @@ -1438,6 +1453,38 @@ def update_config(self, is_creating_backup: bool = False) -> bool: return True + def _validate_config_options(self) -> None: + """Validates specific config options that need access to the database or to the TLS status.""" + if ( + self.config.instance_default_text_search_config is not None + and self.config.instance_default_text_search_config + not in self.postgresql.get_postgresql_text_search_configs() + ): + raise Exception( + "instance_default_text_search_config config option has an invalid value" + ) + + if self.config.request_date_style is not None and not self.postgresql.validate_date_style( + self.config.request_date_style + ): + raise Exception("request_date_style config option has an invalid value") + + if ( + self.config.request_time_zone is not None + and self.config.request_time_zone not in self.postgresql.get_postgresql_timezones() + ): + raise Exception("request_time_zone config option has an invalid value") + + container = self.unit.get_container("postgresql") + output, _ = container.exec(["locale", "-a"]).wait_output() + locales = list(output.splitlines()) + for parameter in ["response_lc_monetary", "response_lc_numeric", "response_lc_time"]: + value = self.model.config.get(parameter) + if value is not None and value not in locales: + raise ValueError( + f"Value for {parameter} not one of the locales available in the system" + ) + def _update_pebble_layers(self) -> None: """Update the pebble layers to keep the health check URL up-to-date.""" container = self.unit.get_container("postgresql") @@ -1496,28 +1543,36 @@ def get_resources_limits(self, container_name: str) -> Dict: return {} def get_node_allocable_memory(self) -> int: - """Return the allocable memory in bytes for a given node. - - Args: - node_name: name of the node to get the allocable memory for - """ + """Return the allocable memory in bytes for the current K8S node.""" client = Client() node = client.get(Node, name=self._get_node_name_for_pod(), namespace=self._namespace) return any_memory_to_bytes(node.status.allocatable["memory"]) - def get_available_memory(self) -> int: - """Get available memory for the container in bytes.""" + def get_node_cpu_cores(self) -> int: + """Return the number of CPU cores for the current K8S node.""" + client = Client() + node = client.get(Node, name=self._get_node_name_for_pod(), namespace=self._namespace) + return int(node.status.allocatable["cpu"]) + + def get_available_resources(self) -> Tuple[int, int]: + """Get available CPU cores and memory (in bytes) for the container.""" + cpu_cores = self.get_node_cpu_cores() allocable_memory = self.get_node_allocable_memory() container_limits = self.get_resources_limits(container_name="postgresql") + if "cpu" in container_limits: + cpu_str = container_limits["cpu"] + constrained_cpu = int(cpu_str) + if constrained_cpu < cpu_cores: + logger.debug(f"CPU constrained to {cpu_str} cores from resource limit") + cpu_cores = constrained_cpu if "memory" in container_limits: memory_str = container_limits["memory"] constrained_memory = any_memory_to_bytes(memory_str) if constrained_memory < allocable_memory: logger.debug(f"Memory constrained to {memory_str} from resource limit") - return constrained_memory + allocable_memory = constrained_memory - logger.debug("Memory constrained by node allocable memory") - return allocable_memory + return cpu_cores, allocable_memory if __name__ == "__main__": diff --git a/src/config.py b/src/config.py index c612d90702..96886ef514 100644 --- a/src/config.py +++ b/src/config.py @@ -15,6 +15,22 @@ class CharmConfig(BaseConfigModel): """Manager for the structured configuration.""" + durability_synchronous_commit: Optional[str] + instance_default_text_search_config: Optional[str] + instance_password_encryption: Optional[str] + logging_log_connections: Optional[bool] + logging_log_disconnections: Optional[bool] + logging_log_lock_waits: Optional[bool] + logging_log_min_duration_statement: Optional[int] + memory_maintenance_work_mem: Optional[int] + memory_max_prepared_transactions: Optional[int] + memory_shared_buffers: Optional[int] + memory_temp_buffers: Optional[int] + memory_work_mem: Optional[int] + optimizer_constraint_exclusion: Optional[str] + optimizer_default_statistics_target: Optional[int] + optimizer_from_collapse_limit: Optional[int] + optimizer_join_collapse_limit: Optional[int] profile: str profile_limit_memory: Optional[int] plugin_citext_enable: bool @@ -23,6 +39,19 @@ class CharmConfig(BaseConfigModel): plugin_pg_trgm_enable: bool plugin_plpython3u_enable: bool plugin_unaccent_enable: bool + request_date_style: Optional[str] + request_standard_conforming_strings: Optional[bool] + request_time_zone: Optional[str] + response_bytea_output: Optional[str] + response_lc_monetary: Optional[str] + response_lc_numeric: Optional[str] + response_lc_time: Optional[str] + vacuum_autovacuum_analyze_scale_factor: Optional[float] + vacuum_autovacuum_analyze_threshold: Optional[int] + vacuum_autovacuum_freeze_max_age: Optional[int] + vacuum_autovacuum_vacuum_cost_delay: Optional[float] + vacuum_autovacuum_vacuum_scale_factor: Optional[float] + vacuum_vacuum_freeze_table_age: Optional[int] @classmethod def keys(cls) -> list[str]: @@ -34,6 +63,106 @@ def plugin_keys(cls) -> filter: """Return plugin config names in a iterable.""" return filter(lambda x: x.startswith("plugin_"), cls.keys()) + @validator("durability_synchronous_commit") + @classmethod + def durability_synchronous_commit_values(cls, value: str) -> Optional[str]: + """Check durability_synchronous_commit config option is one of `on`, `remote_apply` or `remote_write`.""" + if value not in ["on", "remote_apply", "remote_write"]: + raise ValueError("Value not one of 'on', 'remote_apply' or 'remote_write'") + + return value + + @validator("instance_password_encryption") + @classmethod + def instance_password_encryption_values(cls, value: str) -> Optional[str]: + """Check instance_password_encryption config option is one of `md5` or `scram-sha-256`.""" + if value not in ["md5", "scram-sha-256"]: + raise ValueError("Value not one of 'md5' or 'scram-sha-256'") + + return value + + @validator("logging_log_min_duration_statement") + @classmethod + def logging_log_min_duration_statement_values(cls, value: int) -> Optional[int]: + """Check logging_log_min_duration_statement config option is between -1 and 2147483647.""" + if value < -1 or value > 2147483647: + raise ValueError("Value is not between -1 and 2147483647") + + return value + + @validator("memory_maintenance_work_mem") + @classmethod + def memory_maintenance_work_mem_values(cls, value: int) -> Optional[int]: + """Check memory_maintenance_work_mem config option is between 1024 and 2147483647.""" + if value < 1024 or value > 2147483647: + raise ValueError("Value is not between 1024 and 2147483647") + + return value + + @validator("memory_max_prepared_transactions") + @classmethod + def memory_max_prepared_transactions_values(cls, value: int) -> Optional[int]: + """Check memory_max_prepared_transactions config option is between 0 and 262143.""" + if value < 0 or value > 262143: + raise ValueError("Value is not between 0 and 262143") + + return value + + @validator("memory_shared_buffers") + @classmethod + def memory_shared_buffers_values(cls, value: int) -> Optional[int]: + """Check memory_shared_buffers config option is greater or equal than 16.""" + if value < 16 or value > 1073741823: + raise ValueError("Shared buffers config option should be at least 16") + + return value + + @validator("memory_temp_buffers") + @classmethod + def memory_temp_buffers_values(cls, value: int) -> Optional[int]: + """Check memory_temp_buffers config option is between 100 and 1073741823.""" + if value < 100 or value > 1073741823: + raise ValueError("Value is not between 100 and 1073741823") + + return value + + @validator("memory_work_mem") + @classmethod + def memory_work_mem_values(cls, value: int) -> Optional[int]: + """Check memory_work_mem config option is between 64 and 2147483647.""" + if value < 64 or value > 2147483647: + raise ValueError("Value is not between 64 and 2147483647") + + return value + + @validator("optimizer_constraint_exclusion") + @classmethod + def optimizer_constraint_exclusion_values(cls, value: str) -> Optional[str]: + """Check optimizer_constraint_exclusion config option is one of `on`, `off` or `partition`.""" + if value not in ["on", "off", "partition"]: + raise ValueError("Value not one of 'on', 'off' or 'partition'") + + return value + + @validator("optimizer_default_statistics_target") + @classmethod + def optimizer_default_statistics_target_values(cls, value: int) -> Optional[int]: + """Check optimizer_default_statistics_target config option is between 1 and 10000.""" + if value < 1 or value > 10000: + raise ValueError("Value is not between 1 and 10000") + + return value + + @validator("optimizer_from_collapse_limit", allow_reuse=True) + @validator("optimizer_join_collapse_limit", allow_reuse=True) + @classmethod + def optimizer_collapse_limit_values(cls, value: int) -> Optional[int]: + """Check optimizer collapse_limit config option is between 1 and 2147483647.""" + if value < 1 or value > 2147483647: + raise ValueError("Value is not between 1 and 2147483647") + + return value + @validator("profile") @classmethod def profile_values(cls, value: str) -> Optional[str]: @@ -53,3 +182,58 @@ def profile_limit_memory_validator(cls, value: int) -> Optional[int]: raise ValueError("`profile-limit-memory` limited to 7 digits (9999999MB)") return value + + @validator("response_bytea_output") + @classmethod + def response_bytea_output_values(cls, value: str) -> Optional[str]: + """Check response_bytea_output config option is one of `escape` or `hex`.""" + if value not in ["escape", "hex"]: + raise ValueError("Value not one of 'escape' or 'hex'") + + return value + + @validator("vacuum_autovacuum_analyze_scale_factor", allow_reuse=True) + @validator("vacuum_autovacuum_vacuum_scale_factor", allow_reuse=True) + @classmethod + def vacuum_autovacuum_vacuum_scale_factor_values(cls, value: float) -> Optional[float]: + """Check autovacuum scale_factor config option is between 0 and 100.""" + if value < 0 or value > 100: + raise ValueError("Value is not between 0 and 100") + + return value + + @validator("vacuum_autovacuum_analyze_threshold") + @classmethod + def vacuum_autovacuum_analyze_threshold_values(cls, value: int) -> Optional[int]: + """Check vacuum_autovacuum_analyze_threshold config option is between 0 and 2147483647.""" + if value < 0 or value > 2147483647: + raise ValueError("Value is not between 0 and 2147483647") + + return value + + @validator("vacuum_autovacuum_freeze_max_age") + @classmethod + def vacuum_autovacuum_freeze_max_age_values(cls, value: int) -> Optional[int]: + """Check vacuum_autovacuum_freeze_max_age config option is between 100000 and 2000000000.""" + if value < 100000 or value > 2000000000: + raise ValueError("Value is not between 100000 and 2000000000") + + return value + + @validator("vacuum_autovacuum_vacuum_cost_delay") + @classmethod + def vacuum_autovacuum_vacuum_cost_delay_values(cls, value: float) -> Optional[float]: + """Check vacuum_autovacuum_vacuum_cost_delay config option is between -1 and 100.""" + if value < -1 or value > 100: + raise ValueError("Value is not between -1 and 100") + + return value + + @validator("vacuum_vacuum_freeze_table_age") + @classmethod + def vacuum_vacuum_freeze_table_age_values(cls, value: int) -> Optional[int]: + """Check vacuum_vacuum_freeze_table_age config option is between 0 and 2000000000.""" + if value < 0 or value > 2000000000: + raise ValueError("Value is not between 0 and 2000000000") + + return value diff --git a/src/patroni.py b/src/patroni.py index 0187633b4f..01aeaec17a 100644 --- a/src/patroni.py +++ b/src/patroni.py @@ -7,7 +7,7 @@ import logging import os import pwd -from typing import List, Optional +from typing import Any, List, Optional import requests import yaml @@ -279,6 +279,18 @@ def is_database_running(self) -> bool: # Check whether the PostgreSQL process has a state equal to T (frozen). return any(process for process in postgresql_processes if process.split()[7] != "T") + @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10)) + def update_parameter_controller_by_patroni(self, parameter: str, value: Any) -> None: + """Update the value of a parameter controller by Patroni. + + For more information, check https://patroni.readthedocs.io/en/latest/patroni_configuration.html#postgresql-parameters-controlled-by-patroni. + """ + requests.patch( + f"{self._patroni_url}/config", + verify=self._verify, + json={"postgresql": {"parameters": {parameter: value}}}, + ) + @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10)) def reinitialize_postgresql(self) -> None: """Reinitialize PostgreSQL.""" diff --git a/templates/patroni.yml.j2 b/templates/patroni.yml.j2 index bdf0515b76..6d3c50802f 100644 --- a/templates/patroni.yml.j2 +++ b/templates/patroni.yml.j2 @@ -8,7 +8,6 @@ bootstrap: remove_data_directory_on_diverged_timelines: true bin_dir: /usr/lib/postgresql/{{ version }}/bin parameters: - synchronous_commit: on synchronous_standby_names: "*" {%- if enable_pgbackrest %} archive_command: 'pgbackrest --stanza={{ stanza }} archive-push %p' @@ -16,6 +15,10 @@ bootstrap: archive_command: /bin/true {%- endif %} archive_mode: on + autovacuum: true + fsync: true + full_page_writes: true + lc_messages: 'en_US.UTF8' log_autovacuum_min_duration: 60000 log_checkpoints: 'on' log_destination: 'stderr' diff --git a/tests/integration/ha_tests/test_self_healing.py b/tests/integration/ha_tests/test_self_healing.py index 3e1d20ef72..b95ca32504 100644 --- a/tests/integration/ha_tests/test_self_healing.py +++ b/tests/integration/ha_tests/test_self_healing.py @@ -75,6 +75,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(status="active", timeout=1000) +@pytest.mark.notjuju3 @pytest.mark.parametrize("process", DB_PROCESSES) async def test_kill_db_process( ops_test: OpsTest, process: str, continuous_writes, primary_start_timeout diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 0ef634241d..0cc71cdac6 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -294,21 +294,6 @@ async def deploy_and_relate_application_with_postgresql( return relation.id -async def enable_connections_logging(ops_test: OpsTest, unit_name: str) -> None: - """Turn on the log of all connections made to a PostgreSQL instance. - - Args: - ops_test: The ops test framework instance - unit_name: The name of the unit to turn on the connection logs - """ - unit_address = await get_unit_address(ops_test, unit_name) - requests.patch( - f"https://{unit_address}:8008/config", - json={"postgresql": {"parameters": {"log_connections": True}}}, - verify=False, - ) - - async def execute_query_on_unit( unit_address: str, password: str, diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index c6c8a4df4f..6fd2872ad4 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -108,10 +108,20 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): settings_names = [ "archive_command", "archive_mode", + "autovacuum", "data_directory", "cluster_name", "data_checksums", + "fsync", + "full_page_writes", + "lc_messages", "listen_addresses", + "log_autovacuum_min_duration", + "log_checkpoints", + "log_destination", + "log_temp_files", + "log_timezone", + "max_connections", "wal_level", ] cursor.execute( @@ -126,10 +136,20 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): # Validate each configuration set by Patroni on PostgreSQL. assert settings["archive_command"] == "/bin/true" assert settings["archive_mode"] == "on" + assert settings["autovacuum"] == "on" assert settings["cluster_name"] == f"patroni-{APP_NAME}" assert settings["data_directory"] == f"{STORAGE_PATH}/pgdata" assert settings["data_checksums"] == "on" + assert settings["fsync"] == "on" + assert settings["full_page_writes"] == "on" + assert settings["lc_messages"] == "en_US.UTF8" assert settings["listen_addresses"] == "0.0.0.0" + assert settings["log_autovacuum_min_duration"] == "60000" + assert settings["log_checkpoints"] == "on" + assert settings["log_destination"] == "stderr" + assert settings["log_temp_files"] == "1" + assert settings["log_timezone"] == "UTC" + assert settings["max_connections"] == "100" assert settings["wal_level"] == "logical" # Retrieve settings from Patroni REST API. @@ -137,9 +157,45 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): settings = result.json() # Validate configuration exposed by Patroni. - assert settings["postgresql"]["use_pg_rewind"] - assert settings["postgresql"]["remove_data_directory_on_rewind_failure"] - assert settings["postgresql"]["remove_data_directory_on_diverged_timelines"] + assert settings["postgresql"]["use_pg_rewind"] is True + assert settings["postgresql"]["remove_data_directory_on_rewind_failure"] is True + assert settings["postgresql"]["remove_data_directory_on_diverged_timelines"] is True + + +async def test_postgresql_parameters_change(ops_test: OpsTest) -> None: + """Test that's possible to change PostgreSQL parameters.""" + await ops_test.model.applications[APP_NAME].set_config( + { + "memory_max_prepared_transactions": "100", + "memory_shared_buffers": "128", + "response_lc_monetary": "en_GB.utf8", + } + ) + await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", idle_period=30) + password = await get_password(ops_test) + + # Connect to PostgreSQL. + for unit_id in UNIT_IDS: + host = await get_unit_address(ops_test, f"{APP_NAME}/{unit_id}") + logger.info("connecting to the database host: %s", host) + with psycopg2.connect( + f"dbname='postgres' user='operator' host='{host}' password='{password}' connect_timeout=1" + ) as connection, connection.cursor() as cursor: + settings_names = ["max_prepared_transactions", "shared_buffers", "lc_monetary"] + cursor.execute( + sql.SQL("SELECT name,setting FROM pg_settings WHERE name IN ({});").format( + sql.SQL(", ").join(sql.Placeholder() * len(settings_names)) + ), + settings_names, + ) + records = cursor.fetchall() + settings = convert_records_to_dict(records) + connection.close() + + # Validate each configuration set by Patroni on PostgreSQL. + assert settings["max_prepared_transactions"] == "100" + assert settings["shared_buffers"] == "128" + assert settings["lc_monetary"] == "en_GB.utf8" async def test_cluster_is_stable_after_leader_deletion(ops_test: OpsTest) -> None: diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index bbd56ad8ca..a09a63a085 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -89,7 +89,7 @@ async def test_finos_waltz_db(ops_test: OpsTest) -> None: await ops_test.model.remove_application(DATABASE_APP_NAME, block_until_done=True) -@pytest.mark.notjuju3 +@pytest.mark.skip(reason="Should be ported and moved to the new relation tests") async def test_indico_db_blocked(ops_test: OpsTest) -> None: """Tests if deploying and relating to Indico charm will block due to requested extensions.""" async with ops_test.fast_forward(fast_interval="30s"): diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 29919bbe79..1a3b90263d 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -17,7 +17,6 @@ check_tls_patroni_api, db_connect, deploy_and_relate_application_with_postgresql, - enable_connections_logging, get_password, get_primary, get_unit_address, @@ -95,7 +94,12 @@ async def test_mattermost_db(ops_test: OpsTest) -> None: # Enable additional logs on the PostgreSQL instance to check TLS # being used in a later step. - await enable_connections_logging(ops_test, primary) + await ops_test.model.applications[DATABASE_APP_NAME].set_config( + {"logging_log_connections": "True"} + ) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME], status="active", idle_period=30 + ) for attempt in Retrying(stop=stop_after_delay(60), wait=wait_fixed(2), reraise=True): with attempt: