[DPE-1782] Configuration support (canonical#281)

* Initial configuration support

* Add pending restart check

* Add missing parameters and make configuration support work

* Fix parameters validation

* Adjust config management

* Fix configuration support and pending restart

* Fix test

* Add validations

* Add shared_buffers min and max values

* Add validation for instance_default_text_search_config

* Add validation for request_time_zone

* Add validation for request_date_style

* Add validation for memory_max_prepared_transactions

* Fix CPU core count retrieval

* Remove check for primary endpoint being ready

* Remove logs

* Add comment

* Uncomment test code

* Remove default value

* Remove unused constant

* Minor changes in the config class

* Check new settings on integration test

* Fix validation of parameters relying on locales

* Update LIBPATCH

* Remove connection_ssl config option, fix CPU limits check and shared_buffers upper limit

* Improve checks on test

* Add config options defaults, allowed values and unit, also removing connection_ssl config option

* One more improvement in the test

* Fix tests

* Update Juju 3 version

* Skip test on Juju 3

* Skip Indico test

.github/workflows/ci.yaml

@@ -75,7 +75,7 @@ jobs:
           - upgrade-from-stable-integration
         agent-versions:
           - "2.9.45"  # renovate: latest juju 2
-          - "3.1.5"  # renovate: latest juju 3
+          - "3.1.6"  # renovate: latest juju 3
     name: ${{ matrix.tox-environments }} | ${{ matrix.agent-versions }}
     needs:
       - lint

config.yaml

@@ -2,41 +2,219 @@
 # See LICENSE file for licensing details.
 
 options:
+  durability_synchronous_commit:
+    description: |
+      Sets the current transactions synchronization level. This charm allows only the
+      “on”, “remote_apply” and “remote_write” values to avoid data loss if the primary
+      crashes and there are replicas.
+    type: string
+    default: "on"
+  instance_default_text_search_config:
+    description: |
+      Selects the text search configuration that is used by those variants of the text
+      search functions that do not have an explicit argument specifying it.
+      Allowed values start with “pg_catalog.” followed by a language name, like
+      “pg_catalog.english”.
+    type: string
+    default: "pg_catalog.simple"
+  instance_password_encryption:
+    description: |
+      Determines the algorithm to use to encrypt the password.
+      Allowed values are: “md5” and “scram-sha-256”.
+    type: string
+    default: "scram-sha-256"
+  logging_log_connections:
+    description: |
+      Logs each successful connection.
+    type: boolean
+    default: false
+  logging_log_disconnections:
+    description: |
+      Logs end of a session, including duration.
+    type: boolean
+    default: false
+  logging_log_lock_waits:
+    description: |
+      Logs long lock waits.
+    type: boolean
+    default: false
+  logging_log_min_duration_statement:
+    description: |
+      Sets the minimum running time (milliseconds) above which statements will be logged.
+      Allowed values are: from -1 to 2147483647 (-1 disables logging
+      statement durations).
+    type: int
+    default: -1
+  memory_maintenance_work_mem:
+    description: |
+      Sets the maximum memory (KB) to be used for maintenance operations.
+      Allowed values are: from 1024 to 2147483647.
+    type: int
+    default: 65536
+  memory_max_prepared_transactions:
+    description: |
+      Sets the maximum number of simultaneously prepared transactions.
+      Allowed values are: from 0 to 262143.
+    type: int
+    default: 0
+  memory_shared_buffers:
+    description: |
+      Sets the number of shared memory buffers (8 kB) used by the server. This charm allows
+      to set this value up to 40% of the available memory from the unit, as it is unlikely
+      that an allocation of more than that will work better than a smaller amount.
+      Allowed values are: from 16 to 1073741823.
+    type: int
+  memory_temp_buffers:
+    description: |
+      Sets the maximum number of temporary buffers (8 kB) used by each session.
+      Allowed values are: from 100 to 1073741823.
+    type: int
+    default: 1024
+  memory_work_mem:
+    description: |
+      Sets the maximum memory (KB) to be used for query workspaces.
+      Allowed values are: from 64 to 2147483647.
+    type: int
+    default: 4096
+  optimizer_constraint_exclusion:
+    description: |
+      Enables the planner to use constraints to optimize queries.
+      Allowed values are: “on”, “off” and “partition”.
+    type: string
+    default: "partition"
+  optimizer_default_statistics_target:
+    description: |
+      Sets the default statistics target. Allowed values are: from 1 to 10000.
+    type: int
+    default: 100
+  optimizer_from_collapse_limit:
+    description: |
+      Sets the FROM-list size beyond which subqueries are not collapsed.
+      Allowed values are: from 1 to 2147483647.
+    type: int
+    default: 8
+  optimizer_join_collapse_limit:
+    description: |
+      Sets the FROM-list size beyond which JOIN constructs are not flattened.
+      Allowed values are: from 1 to 2147483647.
+    type: int
+    default: 8
   plugin_citext_enable:
     default: false
     type: boolean
-    description: Enable citext extension
+    description: Enable citext extension.
   plugin_debversion_enable:
     default: false
     type: boolean
-    description: Enable debversion extension
+    description: Enable debversion extension.
   plugin_hstore_enable:
     default: false
     type: boolean
-    description: Enable hstore extension
+    description: Enable hstore extension.
   plugin_pg_trgm_enable:
     default: false
     type: boolean
-    description: Enable pg_trgm extension
+    description: Enable pg_trgm extension.
   plugin_plpython3u_enable:
     default: false
     type: boolean
-    description: Enable PL/Python extension
+    description: Enable PL/Python extension.
   plugin_unaccent_enable:
     default: false
     type: boolean
-    description: Enable unaccent extension
+    description: Enable unaccent extension.
   profile:
    description: |
       Profile representing the scope of deployment, and used to tune resource allocation.
       Allowed values are: “production” and “testing”.
       Production will tune postgresql for maximum performance while testing will tune for
       minimal running performance.
    type: string
-   default: production
+   default: "production"
   profile-limit-memory:
     type: int
     description: |
       Amount of memory in Megabytes to limit PostgreSQL and associated process to.
       If unset, this will be decided according to the default memory limit in the selected profile.
       Only comes into effect when the `production` profile is selected.
+  request_date_style:
+    description: |
+      Sets the display format for date and time values. Allowed formats are explained
+      in https://www.postgresql.org/docs/14/runtime-config-client.html#GUC-DATESTYLE.
+    type: string
+    default: "ISO, MDY"
+  request_standard_conforming_strings:
+    description: |
+      Causes ... strings to treat backslashes literally.
+    type: boolean
+    default: true
+  request_time_zone:
+    description: |
+      Sets the time zone for displaying and interpreting time stamps.
+      Allowed values are the ones from IANA time zone data, a time zone abbreviation
+      like PST and POSIX-style time zone specifications.
+    type: string
+    default: "UTC"
+  response_bytea_output:
+    description: |
+      Sets the output format for bytes.
+      Allowed values are: “escape” and “hex”.
+    type: string
+    default: "hex"
+  response_lc_monetary:
+    description: |
+      Sets the locale for formatting monetary amounts.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  response_lc_numeric:
+    description: |
+      Sets the locale for formatting numbers.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  response_lc_time:
+    description: |
+      Sets the locale for formatting date and time values.
+      Allowed values are the locales available in the unit.
+    type: string
+    default: "C"
+  vacuum_autovacuum_analyze_scale_factor:
+    description: |
+      Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when
+      deciding whether to trigger a VACUUM. The default, 0.1, means 10% of table size.
+      Allowed values are: from 0 to 100.
+    type: float
+    default: 0.1
+  vacuum_autovacuum_analyze_threshold:
+    description: |
+      Sets the minimum number of inserted, updated or deleted tuples needed to trigger
+      an ANALYZE in any one table. Allowed values are: from 0 to 2147483647.
+    type: int
+    default: 50
+  vacuum_autovacuum_freeze_max_age:
+    description: |
+      Maximum age (in transactions) before triggering autovacuum on a table to prevent
+      transaction ID wraparound. Allowed values are: from 100000 to 2000000000.
+    type: int
+    default: 200000000
+  vacuum_autovacuum_vacuum_cost_delay:
+    description: |
+      Sets cost delay value (milliseconds) that will be used in automatic VACUUM operations.
+      Allowed values are: from -1 to 100 (-1 tells PostgreSQL to use the regular
+      vacuum_cost_delay value).
+    type: float
+    default: 2.0
+  vacuum_autovacuum_vacuum_scale_factor:
+    description: |
+      Specifies a fraction of the table size to add to autovacuum_vacuum_threshold when
+      deciding whether to trigger a VACUUM. The default, 0.2, means 20% of table size.
+      Allowed values are: from 0 to 100.
+    type: float
+    default: 0.2
+  vacuum_vacuum_freeze_table_age:
+    description: |
+      Age (in transactions) at which VACUUM should scan whole table to freeze tuples.
+      Allowed values are: from 0 to 2000000000.
+    type: int
+    default: 150000000

lib/charms/postgresql_k8s/v0/postgresql.py

@@ -32,7 +32,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 17
+LIBPATCH = 18
 
 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
 
@@ -310,6 +310,32 @@ def enable_disable_extension(self, extension: str, enable: bool, database: str =
             if connection is not None:
                 connection.close()
 
+    def get_postgresql_text_search_configs(self) -> Set[str]:
+        """Returns the PostgreSQL available text search configs.
+
+        Returns:
+            Set of PostgreSQL text search configs.
+        """
+        with self._connect_to_database(
+            connect_to_current_host=True
+        ) as connection, connection.cursor() as cursor:
+            cursor.execute("SELECT CONCAT('pg_catalog.', cfgname) FROM pg_ts_config;")
+            text_search_configs = cursor.fetchall()
+            return {text_search_config[0] for text_search_config in text_search_configs}
+
+    def get_postgresql_timezones(self) -> Set[str]:
+        """Returns the PostgreSQL available timezones.
+
+        Returns:
+            Set of PostgreSQL timezones.
+        """
+        with self._connect_to_database(
+            connect_to_current_host=True
+        ) as connection, connection.cursor() as cursor:
+            cursor.execute("SELECT name FROM pg_timezone_names;")
+            timezones = cursor.fetchall()
+            return {timezone[0] for timezone in timezones}
+
     def get_postgresql_version(self) -> str:
         """Returns the PostgreSQL version.
 
@@ -445,12 +471,12 @@ def is_restart_pending(self) -> bool:
 
     @staticmethod
     def build_postgresql_parameters(
-        profile: str, available_memory: int, limit_memory: Optional[int] = None
-    ) -> Optional[Dict[str, str]]:
+        config_options: Dict, available_memory: int, limit_memory: Optional[int] = None
+    ) -> Optional[Dict]:
         """Builds the PostgreSQL parameters.
 
         Args:
-            profile: the profile to use.
+            config_options: charm config options containing profile and PostgreSQL parameters.
             available_memory: available memory to use in calculation in bytes.
             limit_memory: (optional) limit memory to use in calculation in bytes.
 
@@ -459,19 +485,60 @@ def build_postgresql_parameters(
         """
         if limit_memory:
             available_memory = min(available_memory, limit_memory)
+        profile = config_options["profile"]
         logger.debug(f"Building PostgreSQL parameters for {profile=} and {available_memory=}")
+        parameters = {}
+        for config, value in config_options.items():
+            # Filter config option not related to PostgreSQL parameters.
+            if not config.startswith(
+                (
+                    "durability",
+                    "instance",
+                    "logging",
+                    "memory",
+                    "optimizer",
+                    "request",
+                    "response",
+                    "vacuum",
+                )
+            ):
+                continue
+            parameter = "_".join(config.split("_")[1:])
+            if parameter in ["date_style", "time_zone"]:
+                parameter = "".join(x.capitalize() for x in parameter.split("_"))
+            parameters[parameter] = value
+        shared_buffers_max_value = int(int(available_memory * 0.4) / 10**6)
+        if parameters.get("shared_buffers", 0) > shared_buffers_max_value:
+            raise Exception(
+                f"Shared buffers config option should be at most 40% of the available memory, which is {shared_buffers_max_value}MB"
+            )
         if profile == "production":
             # Use 25% of the available memory for shared_buffers.
             # and the remaind as cache memory.
             shared_buffers = int(available_memory * 0.25)
             effective_cache_size = int(available_memory - shared_buffers)
-
-            parameters = {
-                "shared_buffers": f"{int(shared_buffers/10**6)}MB",
-                "effective_cache_size": f"{int(effective_cache_size/10**6)}MB",
-            }
-
-            return parameters
+            parameters.setdefault("shared_buffers", f"{int(shared_buffers/10**6)}MB")
+            parameters.update({"effective_cache_size": f"{int(effective_cache_size/10**6)}MB"})
         else:
             # Return default
-            return {"shared_buffers": "128MB"}
+            parameters.setdefault("shared_buffers", "128MB")
+        return parameters
+
+    def validate_date_style(self, date_style: str) -> bool:
+        """Validate a date style against PostgreSQL.
+
+        Returns:
+            Whether the date style is valid.
+        """
+        try:
+            with self._connect_to_database(
+                connect_to_current_host=True
+            ) as connection, connection.cursor() as cursor:
+                cursor.execute(
+                    sql.SQL(
+                        "SET DateStyle to {};",
+                    ).format(sql.Identifier(date_style))
+                )
+            return True
+        except psycopg2.Error:
+            return False

src/backups.py

@@ -197,7 +197,7 @@ def _empty_data_files(self) -> None:
     def _change_connectivity_to_database(self, connectivity: bool) -> None:
         """Enable or disable the connectivity to the database."""
         self.charm.unit_peer_data.update({"connectivity": "on" if connectivity else "off"})
-        self.charm.update_config()
+        self.charm.update_config(is_creating_backup=True)
 
     def _execute_command(
         self, command: List[str], timeout: float = None