Add logic for upgrading from Postgres 13 to 15

README.md

@@ -19,6 +19,7 @@ Table of Contents
     - [Prerequisites](#prerequisites)
   - [Install the EDA Server Operator](#install-the-eda-server-operator)
   - [Deploy EDA](#deploy-eda)
+  - [Upgrades](#upgrades)
   - [Advanced Configuration](#advanced-configuration)
     - [Admin user account configuration](#admin-user-account-configuration)
     - [Database Fields Encryption Configuration](#database-fields-encryption-configuration)
@@ -144,6 +145,11 @@ $ kubectl get secret eda-demo-admin-password -o jsonpath="{.data.password}" | ba
 yDL2Cx5Za94g9MvBP6B73nzVLlmfgPjR
 ```
 
+## Upgrades
+
+We recommend you take an backup by creating an EDABackup resource before upgrading, particularly if the new version includes a PostgreSQL database version change.
+
+For information on how to upgrade, please see the [upgrading.md](./docs/upgrade/upgrading.md).
 
 ## Advanced Configuration
 
@@ -217,6 +223,7 @@ spec:
 - [EDA application settings](./docs/user-guide/advanced-configuration/settings.md)
 - [Deploy a Specific Version of EDA](./docs/user-guide/advanced-configuration/deploying-a-specific-version.md)
 - [Trusting a Custom Certificate Authority](./docs/user-guide/advanced-configuration/trusting-a-custom-certificate-authority.md)
+- [Database Configuration](./docs/user-guide/database-configuration.md)
 
 ## Maintainers Docs
 

config/crd/bases/eda.ansible.com_edas.yaml

@@ -2286,10 +2286,6 @@ spec:
                       type: string
                     description: NodeSelector for the database pod.
                     type: object
-                  postgres_data_path:
-                    description: 'Registry path to the PostgreSQL container to use.
-                      Default: "/var/lib/pgsql/data/userdata"'
-                    type: string
                   postgres_extra_args:
                     description: Arguments to pass to postgres process
                     items:
@@ -2305,6 +2301,10 @@ spec:
                     description: 'Configure PostgreSQL connection sslmode option.
                       Default: "prefer"'
                     type: string
+                  postgres_keep_pvc_after_upgrade:
+                    description: Specify whether or not to keep the old PVC after PostgreSQL upgrades
+                    type: boolean
+                    default: true
                   storage_requirements:
                     description: Storage requirements for the PostgreSQL container
                     properties:
@@ -2492,6 +2492,9 @@ spec:
               dbFieldsEncryptionSecret:
                 description: Database Fields Encryption secret name of the deployed instance
                 type: string
+              upgradedPostgresVersion:
+                description: Status to indicate that the database has been upgraded to the version in the status
+                type: string
               image:
                 description: URL of the image used for the deployed instance
                 type: string

config/manifests/bases/eda-server-operator.clusterserviceversion.yaml

@@ -265,11 +265,6 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
-      - displayName: Database data path
-        path: database.postgres_data_path
-        x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
-        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Database Extra Arguments
         path: database.postgres_extra_args
         x-descriptors:
@@ -298,6 +293,11 @@ spec:
         path: database.priority_class
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Should PostgreSQL data for managed databases be kept after upgrades?
+        path: database.postgres_keep_pvc_after_upgrade
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
       - displayName: API Image
         path: image
         x-descriptors:

config/rbac/role.yaml

@@ -96,6 +96,12 @@ rules:
       - patch
       - update
       - watch
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    verbs:
+      - patch
   - apiGroups:
       - ""
     resources:

docs/upgrade/upgrading.md

@@ -0,0 +1,61 @@
+### Upgrading
+
+Before upgrading, please review the changelog for any breaking or notable changes in the releases between your current version and the one you are upgrading to. These changes can be found on the [Release page](https://github.com/ansible/eda-server-operator/releases).
+
+
+All operator versions pin to the most recent eda-server and ui image version at the time of the operator release by default. This is so that the application version only changes when you upgrade your operator (unless overriden).
+
+To find the version of eda-server that will be installed by the operator, check the version specified in the `DEFAULT_EDA_VERSION` and `DEFAULT_EDA_UI_VERSION` variable for that particular release. You can do so by running the following command
+
+```shell
+EDA_OPERATOR_VERSION=1.0.1
+docker run --entrypoint="" quay.io/ansible/eda-server-operator:$EDA_OPERATOR_VERSION bash -c "env | egrep "DEFAULT_EDA_VERSION|DEFAULT_EDA_UI_VERSION"
+```
+
+Follow the installation instructions ('make deploy', 'kustomization', etc.) using the new release version to apply the new operator yaml and upgrade the operator. This will in turn also upgrade your EDA deployment.
+
+For example, if you installed with kustomize, you could modify the version in your kustomization.yaml file from 1.0.0 to 1.0.1, then apply it.
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - config/default
+
+# Set the image tags to match the git version from above
+images:
+  - name: quay.io/ansible/eda-server-operator
+    newTag: 1.0.1
+
+# Specify a custom namespace in which to install EDA
+namespace: eda
+```
+
+Then run this to apply it:
+
+```
+kustomize build . | kubectl apply -f -
+```
+
+#### Backup
+
+The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](./roles/backup/README.md) for information on how to backup your database and secrets.
+
+In the event you need to recover the backup see the [restore role documentation](./roles/restore/README.md). *Before Restoring from a backup*, be sure to:
+* delete the old existing EDA CR
+* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
+
+**Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
+
+
+#### PostgreSQL Upgrade Considerations
+
+If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. By default, the postgres pvc from the previous version will remain unless you manually remove it, or have the `database.postgres_keep_pvc_after_upgrade` parameter set to false. You can configure it to be deleted automatically
+after a successful upgrade by setting the following variable on the EDA spec.
+
+```yaml
+  spec:
+    database:
+        postgres_keep_pvc_after_upgrade: false
+```

docs/user-guide/database-configuration.md

@@ -0,0 +1,93 @@
+### Database Configuration
+
+#### PostgreSQL Version
+
+The default PostgreSQL version for the version of EDA bundled with the latest version of the eda-server-operator is PostgreSQL 15. You can find this default for a given version by at the default value for [supported_pg_version](./roles/eda/vars/main.yml).
+
+We only have coverage for the default version of PostgreSQL. Newer versions of PostgreSQL will likely work, but should only be configured as an external database. If your database is managed by the operator (default if you don't specify a `postgres_configuration_secret`), then you should not override the default version as this may cause issues when operator tries to upgrade your postgresql pod.
+
+#### External PostgreSQL Service
+
+To configure EDA to use an external database, the Custom Resource needs to know about the connection details. To do this, create a k8s secret with those connection details and specify the name of the secret as `postgres_configuration_secret` at the CR spec level.
+
+
+The secret should be formatted as follows:
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: <resourcename>-postgres-configuration
+  namespace: <target namespace>
+stringData:
+  host: <external ip or url resolvable by the cluster>
+  port: <external port, this usually defaults to 5432>
+  database: <desired database name>
+  username: <username to connect as>
+  password: <password to connect with>
+  sslmode: prefer
+  type: unmanaged
+type: Opaque
+```
+
+> Please ensure that the value for the variable `password` should _not_ contain single or double quotes (`'`, `"`) or backslashes (`\`) to avoid any issues during deployment, [backup](./roles/backup) or [restoration](./roles/restore).
+
+> It is possible to set a specific username, password, port, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added.
+
+**Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
+
+Once the secret is created, you can specify it on your spec:
+
+```yaml
+---
+spec:
+  ...
+  postgres_configuration_secret: <name-of-your-secret>
+```
+
+#### Managed PostgreSQL Service
+
+If you don't have access to an external PostgreSQL service, the EDA operator can deploy one for you along side the EDA instance itself.
+
+The following variables are customizable for the managed PostgreSQL service
+
+| Name                                          | Description                                   | Default                                |
+| --------------------------------------------- | --------------------------------------------- | -------------------------------------- |
+| postgres_image                                | Path of the image to pull                     | quay.io/sclorg/postgresql-15-c9s       |
+| postgres_image_version                        | Image version to pull                         | c9s                                    |
+| database.resource_requirements                | PostgreSQL container resource requirements    | requests: {cpu: 50m, memory: 100Mi}    |
+| database.storage_requirements                 | PostgreSQL container storage requirements     | requests: {storage: 8Gi}               |
+| database.postgres_storage_class               | PostgreSQL PV storage class                   | Empty string                           |
+| database.priority_class                       | Priority class used for PostgreSQL pod        | Empty string                           |
+
+Example of customization could be:
+
+```yaml
+---
+spec:
+  ...
+  database:
+    resource_requirements:
+      requests:
+        cpu: 500m
+        memory: 2Gi
+      limits:
+        cpu: '1'
+        memory: 4Gi
+    storage_requirements:
+      requests:
+        storage: 8Gi
+    postgres_storage_class: fast-ssd
+    postgres_extra_args:
+      - '-c'
+      - 'max_connections=1000'
+```
+
+**Note**: If `database.postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
+
+#### Note about overriding the postgres image
+
+We recommend you use the default image sclorg image. If you override the postgres image to use a custom postgres image like `postgres:15` for example, the default data directory path may be different. These images cannot be used interchangeably.
+
+You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.

roles/backup/vars/main.yml

@@ -1,7 +1,7 @@
 ---
 deployment_type: "eda"
-_postgres_image: quay.io/sclorg/postgresql-13-c9s
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
 _postgres_image_version: latest
 backup_complete: false
 database_type: "unmanaged"
-supported_pg_version: 13
+supported_pg_version: 15

roles/eda/tasks/update_status.yml

@@ -53,3 +53,13 @@
         status:
           URL: "https://{{ route_url['resources'][0]['status']['ingress'][0]['host'] }}"
   when: ingress_type | lower == 'route'
+
+- name: Update upgradedPostgresVersion status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    status:
+      upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}"
+  when: upgraded_postgres_version is defined

roles/postgres/defaults/main.yml

@@ -4,7 +4,7 @@ deployment_type: eda
 database_name: "{{ deployment_type }}"
 database_username: "{{ deployment_type }}"
 
-_postgres_image: quay.io/sclorg/postgresql-13-c9s
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
 _postgres_image_version: latest
 
 # Add a nodeSelector for the Postgres pods.
@@ -39,9 +39,9 @@ _database:
   node_selector: ''
   priority_class: ''
   tolerations: ''
-  postgres_data_path: '/var/lib/pgsql/data/userdata'
   postgres_extra_args: ''
-  database_secret: '' # Name of k8s secret containing postgres host, username, password, database, port, and type
+  postgres_keep_pvc_after_upgrade: true  # Specify whether or not to keep the old PVC after PostgreSQL upgrades
+  database_secret: ''  # Name of k8s secret containing postgres host, username, password, database, port, and type
 
 
 image_pull_policy: IfNotPresent

roles/postgres/tasks/check_postgres_version.yml

@@ -0,0 +1,76 @@
+---
+
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
+# So we have to take that into account and preferentially set the most recent one.
+- name: Get the old postgres pod (N-1)
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    field_selectors:
+      - status.phase=Running
+  register: _running_pods
+
+- block:
+  - name: Filter pods by name
+    set_fact:
+      filtered_old_postgres_pods: "{{ _running_pods.resources |
+        selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+        rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
+        list }}"
+
+  # Sort pods by name in reverse order (most recent PG version first) and set
+  - name: Set info for previous postgres pod
+    set_fact:
+      sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
+        sort(attribute='metadata.name') |
+        reverse | list }}"
+    when: filtered_old_postgres_pods | length
+
+  - name: Set info for previous postgres pod
+    set_fact:
+      old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
+    when: filtered_old_postgres_pods | length
+  when: _running_pods.resources | length
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: this_eda
+
+# If this deployment has been upgraded before or if upgrade has already been started, set this var
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_eda['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - "'upgradedPostgresVersion' in this_eda['resources'][0]['status']"
+
+- name: Check if postgres pod is running an older version
+  block:
+    - name: Set path to PG_VERSION file for given container image
+      set_fact:
+        path_to_pg_version: '{{ _postgres_data_path }}/PG_VERSION'
+
+    - name: Get old PostgreSQL version
+      k8s_exec:
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
+        command: |
+          bash -c """
+          cat {{ path_to_pg_version }}
+          """
+      register: _old_pg_version
+
+    - debug:
+        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
+    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
+      include_tasks: upgrade_postgres.yml
+      when:
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
+  when:
+    - managed_database
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete

roles/postgres/tasks/main.yml

@@ -13,6 +13,11 @@
 - name: Set variables to be used in Postgres templates
   import_tasks: set_variables.yml
 
+# Managed Database block
+- name: Check PostgreSQL version to determine if an upgrade is needed
+  import_tasks: check_postgres_version.yml
+  when: managed_database
+
 - name: Create managed Postgres StatefulSet if no external db is defined
   import_tasks: create_managed_postgres.yml
   when: managed_database