Setup cluster definition based on the ansible limit option.

During upgrade we're re-creating the pacemaker cluster one node at a time. So we need to override the cluster definition. We do this by collecting the ansible_limit variable and mapping the controller defined as member of the cluster. Co-Authored-By: Jose Luis Franco Arza <[email protected]> Co-Authored-By: Jiri Stransky <[email protected]> Implements: blueprint upgrades-with-os Closes-Bug: #1832361 Depends-On: Ibd95f4536c033af2094bd079cc81eeba18a36352 Depends-On: Ia108af3799dfd1be24c6ea738e42f0cde19c90c3 Depends-On: I3fa0717203f3f6eae852d019bec71e97eb485aa2 Change-Id: I3cfd4272449b207874723682903505d2bbdb435b
openstack-archive · Jun 18, 2019 · cb8caaa · cb8caaa
1 parent df3b4b3
commit cb8caaa
Show file tree

Hide file tree

Showing 8 changed files with 359 additions and 19 deletions.
diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml
@@ -408,7 +408,63 @@ outputs:
                 tasks_from: restore.yml
               vars:
                 tripleo_persist_dir: /var/lib/mysql
-        - when: step|int == 0
+        - name: Create hiera data to upgrade mysql in a stepwise manner.
+          when:
+            - step|int == 1
+          block:
+            - name: set mysql upgrade node facts in a single-node environment
+              set_fact:
+                mysql_short_node_names_upgraded: "{{ mysql_short_node_names }}"
+                mysql_node_names_upgraded: "{{ mysql_node_names }}"
+                cacheable: no
+              when: groups['mysql'] | length == 1
+            - name: set mysql upgrade node facts from the limit option
+              set_fact:
+                mysql_short_node_names_upgraded: "{{ mysql_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                mysql_node_names_upgraded: "{{ mysql_node_names_upgraded|default([]) + [item] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ mysql_node_names }}"
+
+            - debug:
+                msg: "Prepare galera upgrade for {{ mysql_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade galera without staged upgrade.
+                  You need to use the limit option in order to do so.
+              when: >-
+                mysql_short_node_names_upgraded is not defined or
+                mysql_short_node_names_upgraded | length == 0 or
+                mysql_node_names_upgraded is not defined or
+                mysql_node_names_upgraded | length == 0
+
+            - name: add the mysql short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: mysql_short_node_names_override
+                tripleo_upgrade_value: "{{mysql_short_node_names_upgraded}}"
+            - name: add the mysql long name to hiera data for the upgrade
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: mysql_node_names_override
+                tripleo_upgrade_value: "{{mysql_node_names_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: "{{item}}"
+              loop:
+                - mysql_short_node_names_override
+                - mysql_node_names_override
+              when: mysql_short_node_names_upgraded | length == mysql_node_names | length
+        # Manage image update for pacemaker managed container.
+        - name: mysql baremetal to container upgrade tasks.
+          when: step|int == 0
           tags: common
           block:
             - name: Get docker Mysql image

diff --git a/deployment/database/redis-pacemaker-puppet.yaml b/deployment/database/redis-pacemaker-puppet.yaml
@@ -348,6 +348,46 @@ outputs:
             # Got to check that pacemaker_is_active is working fine with bundle.
             # TODO: pacemaker_is_active resource doesn't support bundle.
       upgrade_tasks:
+        - when:
+            - step|int == 1
+          block:
+            - name: set redis upgrade node facts in a single-node environment
+              set_fact:
+                redis_short_node_names_upgraded: "{{ redis_short_node_names }}"
+                cacheable: no
+              when: groups['redis'] | length == 1
+            - name: set redis upgrade node facts from the limit option
+              set_fact:
+                redis_short_node_names_upgraded: "{{ redis_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ redis_short_node_names }}"
+
+            - debug:
+                msg: "Prepare redis upgrade for {{ redis_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade redis without staged
+                  upgrade. You need to use the limit option in order
+                  to do so.
+              when: >-
+                redis_short_node_names_upgraded is not defined or
+                redis_short_node_names_upgraded | length == 0
+
+            - name: add the redis short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: redis_short_node_names_override
+                tripleo_upgrade_value: "{{redis_short_node_names_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: redis_short_node_names_override
+              when: redis_short_node_names_upgraded | length == redis_short_node_names | length
         - when: step|int == 0
           tags: common
           block:
@@ -409,14 +449,6 @@ outputs:
                       command: pcs resource bundle update redis-bundle storage-map add id=redis-log source-dir=/var/log/containers/redis target-dir=/var/log/redis options=rw
             - name: Update the redis bundle to use the new container image name
               command: "pcs resource bundle update redis-bundle container image={{redis_docker_image_latest}}"
-            - name: Enable the redis cluster resource
-              pacemaker_resource:
-                resource: redis-bundle
-                state: enable
-                wait_for_resource: true
-              register: output
-              retries: 5
-              until: output.rc == 0
         - name: Retag the pacemaker image if containerized
           when:
             - step|int == 3

diff --git a/deployment/haproxy/haproxy-pacemaker-puppet.yaml b/deployment/haproxy/haproxy-pacemaker-puppet.yaml
@@ -429,7 +429,50 @@ outputs:
             # Got to check that pacemaker_is_active is working fine with bundle.
             # TODO: pacemaker_is_active resource doesn't support bundle.
       upgrade_tasks:
-        - when: step|int == 0
+        - name: Create hiera data to upgrade haproxy in a stepwise manner.
+          when:
+            - step|int == 1
+          block:
+            - name: set haproxy upgrade node facts in a single-node environment
+              set_fact:
+                haproxy_short_node_names_upgraded: "{{ haproxy_short_node_names }}"
+                cacheable: no
+              when: groups['haproxy'] | length == 1
+            - name: set haproxy upgrade node facts from the limit option
+              set_fact:
+                haproxy_short_node_names_upgraded: "{{ haproxy_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ haproxy_short_node_names }}"
+
+            - debug:
+                msg: "Prepare haproxy upgrade for {{ haproxy_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade haproxy without staged
+                  upgrade. You need to use the limit option in order
+                  to do so.
+              when: >-
+                haproxy_short_node_names_upgraded is not defined or
+                haproxy_short_node_names_upgraded | length == 0
+
+            - name: add the haproxy short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: haproxy_short_node_names_override
+                tripleo_upgrade_value: "{{haproxy_short_node_names_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: haproxy_short_node_names_override
+              when: haproxy_short_node_names_upgraded | length == haproxy_short_node_names | length
+        - name: upgrade haproxy bundle-init image.
+          when:
+            - step|int == 0
           tags: common
           block:
             - name: Get docker haproxy image

diff --git a/deployment/pacemaker/pacemaker-baremetal-puppet.yaml b/deployment/pacemaker/pacemaker-baremetal-puppet.yaml
@@ -97,6 +97,10 @@ parameters:
         \[(?<pid>[^ ]*)\]
         (?<host>[^ ]*)
         (?<message>.*)$/
+  UpgradeLeappEnabled:
+    description: Use Leapp for operating system upgrade
+    type: boolean
+    default: true
 
 outputs:
   role_data:
@@ -154,18 +158,94 @@ outputs:
               pacemaker_cluster: state=offline
             - name: destroy pacemaker cluster
               command: /usr/sbin/pcs cluster destroy
-        - name: Check pacemaker cluster running before upgrade
+        - name: upgrade step 0
           when: step|int == 0
-          tags: validation
-          pacemaker_cluster: state=online check_and_fail=true
-          async: 30
-          poll: 4
-        - name: Stop pacemaker cluster
+          vars:
+            upgrade_leapp_enabled: {get_param: UpgradeLeappEnabled}
+          block:
+            - name: Check pacemaker cluster running before upgrade
+              when: step|int == 0
+              tags: validation
+              pacemaker_cluster: state=online check_and_fail=true
+              async: 30
+              poll: 4
+              when: not upgrade_leapp_enabled|bool
+        - name: upgrade step 1
+          when:
+            - step|int == 1
+          block:
+            - name: set pacemaker upgrade node facts in a single-node environment
+              set_fact:
+                pacemaker_short_node_names_upgraded: "{{ pacemaker_short_node_names }}"
+                cacheable: no
+              when: groups['pacemaker'] | length == 1
+            - name: set pacemaker upgrade node facts from the limit option
+              set_fact:
+                pacemaker_short_node_names_upgraded: "{{ pacemaker_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ pacemaker_short_node_names }}"
+
+            - debug:
+                msg: "Prepare pacemaker upgrade for {{ pacemaker_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade pacemaker without staged
+                  upgrade. You need to use the limit option in order
+                  to do so.
+              when: >-
+                pacemaker_short_node_names_upgraded is not defined or
+                pacemaker_short_node_names_upgraded | length == 0
+            - name: set pacemaker node ips fact from the names fact
+              set_fact:
+                # Generate matching IPs for the names, e.g. for these varaible values:
+                #     pacemaker_node_ips: [ "1", "2", "3" ]
+                #     pacemaker_short_node_names: [ "a", "b", "c" ]
+                #     pacemaker_short_node_names_override: [ "b" ]
+                # it will set:
+                #     pacemaker_node_ips_override: [ "2" ]
+                pacemaker_node_ips_upgraded: "{{
+                  dict(pacemaker_short_node_names|zip(pacemaker_node_ips))
+                  | dict2items
+                  | selectattr('key', 'in', pacemaker_short_node_names_upgraded)
+                  | map(attribute='value')
+                  | list }}"
+                cacheable: no
+
+            - name: add the pacemaker short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: pacemaker_short_node_names_override
+                tripleo_upgrade_value: "{{pacemaker_short_node_names_upgraded}}"
+            - name: add the pacemaker ips to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: pacemaker_node_ips_override
+                tripleo_upgrade_value: "{{pacemaker_node_ips_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: "{{item}}"
+              loop:
+                - pacemaker_short_node_names_override
+                - pacemaker_node_ips_override
+              when: pacemaker_short_node_names_upgraded | length == pacemaker_short_node_names | length
+        - name: upgrade step 2
           when: step|int == 2
-          pacemaker_cluster: state=offline
-        - name: Start pacemaker cluster
+          block:
+            - name: Stop pacemaker cluster
+              pacemaker_cluster: state=offline
+        - name: upgrade step 4
           when: step|int == 4
-          pacemaker_cluster: state=online
+          block:
+            - name: Start pacemaker cluster
+              pacemaker_cluster: state=online
       update_tasks:
         - name: Check pacemaker cluster running before the minor update
           when: step|int == 0  # TODO(marios) disabling validations?

diff --git a/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml b/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml
@@ -307,6 +307,47 @@ outputs:
             # Got to check that pacemaker_is_active is working fine with bundle.
             # TODO: pacemaker_is_active resource doesn't support bundle.
       upgrade_tasks:
+        - name: Create hiera data to upgrade oslo messaging notify in a stepwise manner.
+          when:
+            - step|int == 1
+          block:
+            - name: set oslo_messaging_notify upgrade node facts in a single-node environment
+              set_fact:
+                oslo_messaging_notify_short_node_names_upgraded: "{{ oslo_messaging_notify_short_node_names }}"
+                cacheable: no
+              when: groups['oslo_messaging_notify'] | length == 1
+            - name: set oslo_messaging_notify upgrade node facts from the limit option
+              set_fact:
+                oslo_messaging_notify_short_node_names_upgraded: "{{ oslo_messaging_notify_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ oslo_messaging_notify_short_node_names }}"
+
+            - debug:
+                msg: "Prepare oslo_messaging_notify upgrade for {{ oslo_messaging_notify_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade oslo_messaging_notify without staged
+                  upgrade. You need to use the limit option in order
+                  to do so.
+              when: >-
+                oslo_messaging_notify_short_node_names_upgraded is not defined or
+                oslo_messaging_notify_short_node_names_upgraded | length == 0
+
+            - name: add the oslo_messaging_notify short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: oslo_messaging_notify_short_node_names_override
+                tripleo_upgrade_value: "{{oslo_messaging_notify_short_node_names_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: oslo_messaging_notify_short_node_names_override
+              when: oslo_messaging_notify_short_node_names_upgraded | length == oslo_messaging_notify_short_node_names | length
         - when: step|int == 0
           tags: common
           block:
@@ -350,6 +391,7 @@ outputs:
           when:
             - step|int == 3
           block: *rabbitmq_fetch_retag_container_tasks
+
       fast_forward_upgrade_tasks:
         - when:
             - step|int == 0

diff --git a/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml b/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml
@@ -306,6 +306,47 @@ outputs:
             # Got to check that pacemaker_is_active is working fine with bundle.
             # TODO: pacemaker_is_active resource doesn't support bundle.
       upgrade_tasks:
+        - name: Create hiera data to upgrade rabbitmq in a stepwise manner.
+          when:
+            - step|int == 1
+          block:
+            - name: set rabbitmq upgrade node facts in a single-node environment
+              set_fact:
+                rabbitmq_short_node_names_upgraded: "{{ rabbitmq_short_node_names }}"
+                cacheable: no
+              when: groups['rabbitmq'] | length == 1
+            - name: set rabbitmq upgrade node facts from the limit option
+              set_fact:
+                rabbitmq_short_node_names_upgraded: "{{ rabbitmq_short_node_names_upgraded|default([]) + [item.split('.')[0]] }}"
+                cacheable: no
+              when: item.split('.')[0] in ansible_limit.split(',')
+              loop: "{{ rabbitmq_short_node_names }}"
+
+            - debug:
+                msg: "Prepare rabbitmq upgrade for {{ rabbitmq_short_node_names_upgraded }}"
+            - fail:
+                msg: >
+                  You can't upgrade rabbitmq without staged
+                  upgrade. You need to use the limit option in order
+                  to do so.
+              when: >-
+                rabbitmq_short_node_names_upgraded is not defined or
+                rabbitmq_short_node_names_upgraded | length == 0
+
+            - name: add the rabbitmq short name to hiera data for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: set.yml
+              vars:
+                tripleo_upgrade_key: rabbitmq_short_node_names_override
+                tripleo_upgrade_value: "{{rabbitmq_short_node_names_upgraded}}"
+            - name: remove the extra hiera data needed for the upgrade.
+              include_role:
+                name: tripleo-upgrade-hiera
+                tasks_from: remove.yml
+              vars:
+                tripleo_upgrade_key: rabbitmq_short_node_names_override
+              when: rabbitmq_short_node_names_upgraded | length == rabbitmq_short_node_names | length
         - name: Get docker Rabbitmq image
           set_fact:
             rabbitmq_docker_image_latest: *rabbitmq_image_pcmklatest