Rework the upgrade logic to be more resiliant for multiple upgrades

config/manager/kustomization.yaml

@@ -5,13 +5,13 @@ generatorOptions:
   disableNameSuffixHash: true
 
 configMapGenerator:
-- name: awx-manager-config
-  files:
+- files:
   - controller_manager_config.yaml
+  name: awx-manager-config
 
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/ansible/awx-operator
-  newTag: latest
+  newName: quay.io/chadams/awx-operator
+  newTag: dev14

config/manager/manager.yaml

@@ -39,6 +39,7 @@ spec:
         - --leader-elect
         - --leader-election-id=awx-operator
         image: controller:latest
+        imagePullPolicy: Always
         name: awx-manager
         env:
         - name: ANSIBLE_GATHERING

roles/installer/tasks/database_configuration.yml

@@ -106,14 +106,31 @@
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
 
-- name: Get the old postgres pod information
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades. So we have to take that into account and preferentially set the most recent one.
+# Check for -13-0 pg pod
+- name: Get the old postgres pod (N-1)
   k8s_info:
     kind: Pod
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres-0"
+    name: "{{ ansible_operator_meta.name }}-postgres-13-0"
     field_selectors:
       - status.phase=Running
-  register: old_postgres_pod
+  register: _old_postgres_pods
+
+- name: Filter pods by name
+  set_fact:
+    filtered_old_postgres_pods: "{{ _old_postgres_pods.resources |
+      selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+      rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-') |
+      list }}"
+
+# Sort pods by name in reverse order (most recent PG version first) and set
+- name: Set info for previous postgres pod
+  set_fact:
+    old_postgres_pod: "{{ filtered_old_postgres_pods | 
+      sort(attribute='metadata.name') |
+      reverse | first }}"
+  when: _old_postgres_pods.resources | length
 
 - name: Look up details for this deployment
   k8s_info:
@@ -123,6 +140,16 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
   register: this_awx
 
+# If this deployment has been upgraded before or if upgrade has already been started, set this var.
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"
+
+- debug:
+    msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
 - name: Check if postgres pod is running an older version
   block:
     - name: Set path to PG_VERSION file for given container image
@@ -132,7 +159,7 @@
     - name: Get old PostgreSQL version
       k8s_exec:
         namespace: "{{ ansible_operator_meta.namespace }}"
-        pod: "{{ ansible_operator_meta.name }}-postgres-0"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
           cat {{ path_to_pg_version }}
@@ -142,12 +169,11 @@
     - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
       include_tasks: upgrade_postgres.yml
       when:
-        - _old_pg_version.stdout | default('0') | trim < supported_pg_version
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
   when:
     - managed_database
-    - "'upgradedPostgresVersion' is in this_awx['resources'][0]['status']"
-    - (this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false)) | ternary (this_awx['resources'][0]['status']['upgradedPostgresVersion'] < supported_pg_version, false)
-    - old_postgres_pod['resources'] | length  # upgrade is complete and old pg pod has been removed
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete
 
 - block:
     - name: Create Database if no database is specified

roles/installer/tasks/update_status.yml

@@ -109,5 +109,5 @@
     name: "{{ ansible_operator_meta.name }}"
     namespace: "{{ ansible_operator_meta.namespace }}"
     status:
-      upgradedPostgresVersion: "{{ upgraded_postgres_version }}"
+      upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}"
   when: upgraded_postgres_version is defined

roles/installer/tasks/upgrade_postgres.yml

@@ -62,9 +62,19 @@
   set_fact:
     postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
 
+- name: Get the name of the service for the old postgres pod
+  k8s_info:
+    kind: Service
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+    - "app.kubernetes.io/component=database"
+    - "app.kubernetes.io/instance={{ old_postgres_pod.metadata.labels['app.kubernetes.io/instance'] }}"
+    - "app.kubernetes.io/managed-by=awx-operator"
+  register: old_postgres_svc
+
 - name: Set full resolvable host name for postgres pod
   set_fact:
-    resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
+    resolvable_db_host: "{{ old_postgres_svc['resources'][0]['metadata']['name'] }}.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
 - name: Set pg_dump command
@@ -110,23 +120,32 @@
     kind: StatefulSet
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres"
+    name: "{{ item }}"
     state: absent
     wait: true
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
 
 - name: Remove old Postgres Service
   k8s:
     kind: Service
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres"
+    name: "{{ item }}"
     state: absent
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
 
 - name: Remove old persistent volume claim
   k8s:
     kind: PersistentVolumeClaim
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    name: "{{ item }}"
     state: absent
+  loop:
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-13-0"
   when: postgres_keep_pvc_after_upgrade

roles/installer/vars/main.yml

@@ -5,3 +5,4 @@ ldap_cacert_ca_crt: ''
 bundle_ca_crt: ''
 projects_existing_claim: ''
 supported_pg_version: 15
+_previous_upgraded_pg_version: 0