Merge "Normalise all pacemaker resource upgrade tasks for staged upgr…

…ades"
openstack-archive · Oct 17, 2019 · 6df0fbf · 6df0fbf
2 parents bf60423 + dfd8b73
commit 6df0fbf
Show file tree

Hide file tree

Showing 7 changed files with 280 additions and 224 deletions.
diff --git a/deployment/cinder/cinder-backup-pacemaker-puppet.yaml b/deployment/cinder/cinder-backup-pacemaker-puppet.yaml
@@ -224,65 +224,76 @@ outputs:
             container_image: {get_param: ContainerCinderBackupImage}
             container_image_latest: *cinder_backup_image_pcmklatest
       update_tasks:
-        - name: Cinder-Backup fetch and retag container image for pacemaker
+        - name: cinder_backup fetch and retag container image for pacemaker
           when: step|int == 2
           block: &cinder_backup_fetch_retag_container_tasks
-            - name: Get docker Cinder-Backup image
+            - name: Get container cinder_backup image
               set_fact:
-                docker_image: {get_param: ContainerCinderBackupImage}
-                docker_image_latest: *cinder_backup_image_pcmklatest
-            - name: Get previous Cinder-Backup image id
-              shell: "{{container_cli}} images | awk '/cinder-backup.* pcmklatest/{print $3}' | uniq"
-              register: cinder_backup_image_id
+                cinder_backup_image: {get_param: ContainerCinderBackupImage}
+                cinder_backup_image_latest: *cinder_backup_image_pcmklatest
+            - name: Pull latest cinder_backup images
+              command: "{{container_cli}} pull {{cinder_backup_image}}"
+            - name: Get previous cinder_backup image id
+              shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{cinder_backup_image_latest}}"
+              register: old_cinder_backup_image_id
+              failed_when: false
+            - name: Get new cinder_backup image id
+              shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{cinder_backup_image}}"
+              register: new_cinder_backup_image_id
+            - name: Retag pcmklatest to latest cinder_backup image
+              include_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{cinder_backup_image}}"
+                container_image_latest: "{{cinder_backup_image_latest}}"
+              when:
+                - old_cinder_backup_image_id.stdout != new_cinder_backup_image_id.stdout
             - block:
-                - name: Get a list of container using Cinder-Backup image
-                  shell: "{{container_cli}} ps -a -q -f 'ancestor={{cinder_backup_image_id.stdout}}'"
+                - name: Get a list of containers using cinder_backup image
+                  shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_cinder_backup_image_id.stdout}}'"
                   register: cinder_backup_containers_to_destroy
                 # It will be recreated with the deploy step.
-                - name: Remove any container using the same Cinder-Backup image
+                - name: Remove any containers using the same cinder_backup image
                   shell: "{{container_cli}} rm -fv {{item}}"
                   with_items: "{{ cinder_backup_containers_to_destroy.stdout_lines }}"
-                - name: Remove previous Cinder-Backup images
-                  shell: "{{container_cli}} rmi -f {{cinder_backup_image_id.stdout}}"
+                - name: Remove previous cinder_backup images
+                  shell: "{{container_cli}} rmi -f {{old_cinder_backup_image_id.stdout}}"
               when:
-                - cinder_backup_image_id.stdout != ''
-            - name: Pull latest Cinder-Backup images
-              command: "{{container_cli}} pull {{docker_image}}"
-            - name: Retag pcmklatest to latest Cinder-Backup image
-              import_role:
-                name: tripleo-container-tag
-              vars:
-                container_image: "{{docker_image}}"
-                container_image_latest: "{{docker_image_latest}}"
-            # Got to check that pacemaker_is_active is working fine with bundle.
-            # TODO: pacemaker_is_active resource doesn't support bundle.
+                - old_cinder_backup_image_id.stdout != ''
+                - old_cinder_backup_image_id.stdout != new_cinder_backup_image_id.stdout
+
       upgrade_tasks:
-        - when: step|int == 0
-          tags: common
+        - name: Prepare switch of cinder_backup image name
+          when:
+            - step|int == 0
           block:
-            - name: Get docker Cinder-Backup image
+            - name: Get cinder_backup image id currently used by pacemaker
+              shell: "pcs resource config openstack-cinder-backup | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'"
+              register: cinder_backup_image_current_res
+              failed_when: false
+            - name: cinder_backup image facts
               set_fact:
-                cinder_backup_docker_image_latest: *cinder_backup_image_pcmklatest
-            - name: Prepare the switch to new cinder_backup container image name in pacemaker
-              when: cinder_backup_containerized|bool
-              block:
-                - name: Get cinder_backup image id currently used by pacemaker
-                  shell: "{{container_cli}} images | awk '/cinder-backup.* pcmklatest/{print $3}' | uniq"
-                  register: cinder_backup_current_pcmklatest_id
-                - name: Temporarily tag the current cinder_backup image id with the upgraded image name
-                  when: cinder_backup_current_pcmklatest_id.stdout != ''
-                  import_role:
-                    name: tripleo-container-tag
-                  vars:
-                    container_image: "{{cinder_backup_current_pcmklatest_id.stdout}}"
-                    container_image_latest: "{{cinder_backup_docker_image_latest}}"
-                    pull_image: false
+                cinder_backup_image_latest: *cinder_backup_image_pcmklatest
+                cinder_backup_image_current: "{{cinder_backup_image_current_res.stdout}}"
+            - name: Temporarily tag the current cinder_backup image id with the upgraded image name
+              import_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{cinder_backup_current_pcmklatest_id.stdout}}"
+                container_image_latest: "{{cinder_backup_docker_image_latest}}"
+                pull_image: false
+              when:
+                - cinder_backup_image_current != ''
+                - cinder_backup_image_current != cinder_backup_image_latest
+            # During an OS Upgrade, the cluster may not exist so we use
+            # the shell module instead.
+            # TODO(odyssey4me):
+            # Fix the pacemaker_resource module to handle the exception
+            # for a non-existant cluster more gracefully.
             - name: Check openstack-cinder-backup cluster resource status
-              pacemaker_resource:
-                resource: openstack-cinder-backup
-                state: show
-                check_mode: false
-              ignore_errors: true
+              shell: pcs resource config openstack-cinder-backup
+              failed_when: false
+              changed_when: false
               register: cinder_backup_pcs_res_result
             - name: Set fact cinder_backup_pcs_res
               set_fact:
@@ -295,6 +306,7 @@ outputs:
             - step|int == 1
             - is_cinder_backup_bootstrap_node
             - cinder_backup_pcs_res|bool
+            - cinder_backup_image_current != cinder_backup_image_latest
           block:
             - name: Disable the cinder_backup cluster resource before container upgrade
               pacemaker_resource:
@@ -305,7 +317,7 @@ outputs:
               retries: 5
               until: output.rc == 0
             - name: Update the cinder_backup bundle to use the new container image name
-              command: "pcs resource bundle update openstack-cinder-backup container image={{cinder_backup_docker_image_latest}}"
+              command: "pcs resource bundle update openstack-cinder-backup container image={{cinder_backup_image_latest}}"
             - name: Enable the cinder_backup cluster resource
               pacemaker_resource:
                 resource: openstack-cinder-backup
@@ -318,6 +330,7 @@ outputs:
           when:
             - step|int == 3
           block: *cinder_backup_fetch_retag_container_tasks
+
       fast_forward_upgrade_tasks:
         - when:
             - step|int == 0
@@ -329,7 +342,7 @@ outputs:
                 resource: openstack-cinder-backup
                 state: show
                 check_mode: false
-              ignore_errors: true
+              failed_when: false
               register: cinder_backup_res_result
             - name: Set fact cinder_backup_res
               set_fact:

diff --git a/deployment/cinder/cinder-volume-pacemaker-puppet.yaml b/deployment/cinder/cinder-volume-pacemaker-puppet.yaml
@@ -235,11 +235,11 @@ outputs:
               when:
                 - old_cinder_volume_image_id.stdout != new_cinder_volume_image_id.stdout
             - block:
-                - name: Get a list of container using cinder_volume image
+                - name: Get a list of containers using cinder_volume image
                   shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_cinder_volume_image_id.stdout}}'"
                   register: cinder_volume_containers_to_destroy
                 # It will be recreated with the delpoy step.
-                - name: Remove any container using the same cinder_volume image
+                - name: Remove any containers using the same cinder_volume image
                   shell: "{{container_cli}} rm -fv {{item}}"
                   with_items: "{{ cinder_volume_containers_to_destroy.stdout_lines }}"
                 - name: Remove previous cinder_volume images
@@ -261,20 +261,24 @@ outputs:
               set_fact:
                 cinder_volume_image_latest: *cinder_volume_image_pcmklatest
                 cinder_volume_image_current: "{{cinder_volume_image_current_res.stdout}}"
-            - name: Prepare the switch to new cinder_volume container image name in pacemaker
-              block:
-                - name: Temporarily tag the current cinder_volume image id with the upgraded image name
-                  import_role:
-                    name: tripleo-container-tag
-                  vars:
-                    container_image: "{{cinder_volume_image_current}}"
-                    container_image_latest: "{{cinder_volume_image_latest}}"
-                    pull_image: false
-                  when:
-                    - cinder_volume_image_current != ''
-                    - cinder_volume_image_current != cinder_volume_image_latest
+            - name: Temporarily tag the current cinder_volume image id with the upgraded image name
+              import_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{cinder_volume_image_current}}"
+                container_image_latest: "{{cinder_volume_image_latest}}"
+                pull_image: false
+              when:
+                - cinder_volume_image_current != ''
+                - cinder_volume_image_current != cinder_volume_image_latest
+            # During an OS Upgrade, the cluster may not exist so we use
+            # the shell module instead.
+            # TODO(odyssey4me):
+            # Fix the pacemaker_resource module to handle the exception
+            # for a non-existant cluster more gracefully.
             - name: Check openstack-cinder-volume cluster resource status
               shell: pcs resource config openstack-cinder-volume
+              changed_when: false
               failed_when: false
               register: cinder_volume_pcs_res_result
             - name: Set fact cinder_volume_pcs_res
@@ -301,7 +305,6 @@ outputs:
             - name: pcs resource bundle update cinder_volume for new container image name
               command: "pcs resource bundle update openstack-cinder-volume container image={{cinder_volume_image_latest}}"
             - name: Enable the cinder_volume cluster resource
-              when:
               pacemaker_resource:
                 resource: openstack-cinder-volume
                 state: enable

diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml
@@ -407,21 +407,25 @@ outputs:
               set_fact:
                 galera_image_latest: *mysql_image_pcmklatest
                 galera_image_current: "{{galera_image_current_res.stdout}}"
-            - name: Prepare the switch to new galera container image name in pacemaker
-              block:
-                - name: Temporarily tag the current galera image id with the upgraded image name
-                  import_role:
-                    name: tripleo-container-tag
-                  vars:
-                    container_image: "{{galera_image_current}}"
-                    container_image_latest: "{{galera_image_latest}}"
-                    pull_image: false
-                  when:
-                    - galera_image_current != ''
-                    - galera_image_current != galera_image_latest
+            - name: Temporarily tag the current galera image id with the upgraded image name
+              import_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{galera_image_current}}"
+                container_image_latest: "{{galera_image_latest}}"
+                pull_image: false
+              when:
+                - galera_image_current != ''
+                - galera_image_current != galera_image_latest
+            # During an OS Upgrade, the cluster may not exist so we use
+            # the shell module instead.
+            # TODO(odyssey4me):
+            # Fix the pacemaker_resource module to handle the exception
+            # for a non-existant cluster more gracefully.
             - name: Check galera cluster resource status
               shell: pcs resource config galera-bundle
               failed_when: false
+              changed_when: false
               register: galera_pcs_res_result
             - name: Set fact galera_pcs_res
               set_fact:

diff --git a/deployment/database/redis-pacemaker-puppet.yaml b/deployment/database/redis-pacemaker-puppet.yaml
@@ -327,11 +327,11 @@ outputs:
               when:
                 - old_redis_image_id.stdout != new_redis_image_id.stdout
             - block:
-                - name: Get a list of container using redis image
+                - name: Get a list of containers using redis image
                   shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_redis_image_id.stdout}}'"
                   register: redis_containers_to_destroy
                 # It will be recreated with the delpoy step.
-                - name: Remove any container using the same redis image
+                - name: Remove any containers using the same redis image
                   shell: "{{container_cli}} rm -fv {{item}}"
                   with_items: "{{ redis_containers_to_destroy.stdout_lines }}"
                 - name: Remove previous redis images
@@ -353,21 +353,25 @@ outputs:
               set_fact:
                 redis_image_latest: *redis_image_pcmklatest
                 redis_image_current: "{{redis_image_current_res.stdout}}"
-            - name: Prepare the switch to new redis container image name in pacemaker
-              block:
-                - name: Temporarily tag the current redis image id with the upgraded image name
-                  import_role:
-                    name: tripleo-container-tag
-                  vars:
-                    container_image: "{{redis_image_current}}"
-                    container_image_latest: "{{redis_image_latest}}"
-                    pull_image: false
-                  when:
-                    - redis_image_current != ''
-                    - redis_image_current != redis_image_latest
+            - name: Temporarily tag the current redis image id with the upgraded image name
+              import_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{redis_image_current}}"
+                container_image_latest: "{{redis_image_latest}}"
+                pull_image: false
+              when:
+                - redis_image_current != ''
+                - redis_image_current != redis_image_latest
+            # During an OS Upgrade, the cluster may not exist so we use
+            # the shell module instead.
+            # TODO(odyssey4me):
+            # Fix the pacemaker_resource module to handle the exception
+            # for a non-existant cluster more gracefully.
             - name: Check redis cluster resource status
               shell: pcs resource config redis-bundle
               failed_when: false
+              changed_when: false
               register: redis_pcs_res_result
             - name: Set upgrade redis facts
               set_fact:

diff --git a/deployment/haproxy/haproxy-pacemaker-puppet.yaml b/deployment/haproxy/haproxy-pacemaker-puppet.yaml
@@ -399,7 +399,8 @@ outputs:
               until: output.rc == 0
               when: haproxy_cert_mounted.rc == 6
         - name: Haproxy fetch and retag container image for pacemaker
-          when: step|int == 2
+          when:
+            - step|int == 2
           block: &haproxy_fetch_retag_container_tasks
             - name: Get container haproxy image
               set_fact:
@@ -449,21 +450,25 @@ outputs:
               set_fact:
                 haproxy_image_latest: *haproxy_image_pcmklatest
                 haproxy_image_current: "{{haproxy_image_current_res.stdout}}"
-            - name: Prepare the switch to new haproxy container image name in pacemaker
-              block:
-                - name: Temporarily tag the current haproxy image id with the upgraded image name
-                  import_role:
-                    name: tripleo-container-tag
-                  vars:
-                    container_image: "{{haproxy_image_current}}"
-                    container_image_latest: "{{haproxy_image_latest}}"
-                    pull_image: false
-                  when:
-                    - haproxy_image_current != ''
-                    - haproxy_image_current != haproxy_image_latest
+            - name: Temporarily tag the current haproxy image id with the upgraded image name
+              import_role:
+                name: tripleo-container-tag
+              vars:
+                container_image: "{{haproxy_image_current}}"
+                container_image_latest: "{{haproxy_image_latest}}"
+                pull_image: false
+              when:
+                - haproxy_image_current != ''
+                - haproxy_image_current != haproxy_image_latest
+            # During an OS Upgrade, the cluster may not exist so we use
+            # the shell module instead.
+            # TODO(odyssey4me):
+            # Fix the pacemaker_resource module to handle the exception
+            # for a non-existant cluster more gracefully.
             - name: Check haproxy cluster resource status
               shell: pcs resource config haproxy-bundle
               failed_when: false
+              changed_when: false
               register: haproxy_pcs_res_result
             - name: Set upgrade haproxy facts
               set_fact: