Add post upgrade Cb Check

aci-preupgrade-validation-script.py

@@ -39,6 +39,7 @@
 FAIL_UF = 'FAIL - UPGRADE FAILURE!!'
 ERROR = 'ERROR !!'
 MANUAL = 'MANUAL CHECK REQUIRED'
+POST = 'POST UPGRADE CHECK REQUIRED'
 NA = 'N/A'
 node_regex = r'topology/pod-(?P<pod>\d+)/node-(?P<node>\d+)'
 ver_regex = r'(?:dk9\.)?[1]?(?P<major1>\d)\.(?P<major2>\d)(?:\.|\()(?P<maint>\d+)\.?(?P<patch>(?:[a-b]|[0-9a-z]+))\)?'
@@ -2960,6 +2961,81 @@ def access_untagged_check(index, total_checks, **kwargs):
     return result
 
 
+def post_upgrade_cb_check(index, total_checks, cversion, tversion, **kwargs):
+    title = 'Post Upgrade Callback Integrity'
+    result = PASS
+    msg = ''
+    headers = ["Missed Objects", "Impact"]
+    data = []
+    recommended_action = 'Contact Cisco TAC with Output'
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#post-upgrade-callback-integrity'
+    print_title(title, index, total_checks)
+
+    new_mo_dict = {
+        "infraImplicitSetPol": {
+            "CreatedBy": "",
+            "SinceVersion": "3.2(10e)",
+            "Impact": "Infra implicit settings will not be deployed",
+        },
+        "infraRsToImplicitSetPol": {
+            "CreatedBy": "infraImplicitSetPol",
+            "SinceVersion": "3.2(10e)",
+            "Impact": "Infra implicit settings will not be deployed",
+        },
+        "fvSlaDef": {
+            "CreatedBy": "fvIPSLAMonitoringPol",
+            "SinceVersion": "4.1(1i)",
+            "Impact": "IPSLA monitor policy will not be deployed",
+        },
+        "infraRsConnectivityProfileOpt": {
+            "CreatedBy": "infraRsConnectivityProfile",
+            "SinceVersion": "5.2(4d)",
+            "Impact": "VPC for missing Mo will not be deployed to leaf",
+        },
+        "infraAssocEncapInstDef": {
+            "CreatedBy": "infraRsToEncapInstDef",
+            "SinceVersion": "5.2(4d)",
+            "Impact": "VLAN for missing Mo will not be deployed to leaf",
+        },
+        "compatSwitchHw": {
+            "CreatedBy": "",  # suppBit attribute is available from 6.0(2h)
+            "SinceVersion": "6.0(2h)",
+            "Impact": "Unexpected 64/32 bit image can deploy to switches",
+        },
+    }
+    if not tversion or (tversion and cversion.older_than(str(tversion))):
+        print_result(title, POST, 'Re-run script after APICs are upgraded and back to Fully-Fit')
+        return POST
+
+    for new_mo in new_mo_dict:
+        since_version = AciVersion(new_mo_dict[new_mo]['SinceVersion'])
+        created_by_mo = new_mo_dict[new_mo]['CreatedBy']
+
+        if since_version.newer_than(str(cversion)):
+            continue
+
+        api = "{}.json?rsp-subtree-include=count"
+        if new_mo == "compatSwitchHw":
+            # Expected to see suppBit in 32 or 64. Zero 32 means a failed postUpgradeCb.
+            api += '&query-target-filter=eq(compatSwitchHw.suppBit,"32")'
+
+        temp_new_mo_count = icurl("class", api.format(new_mo))
+        new_mo_count = int(temp_new_mo_count[0]['moCount']['attributes']['count'])
+        if created_by_mo == "":
+            if new_mo_count == 0:
+                data.append([new_mo, new_mo_dict[new_mo]["Impact"]])
+        else:
+            temp_createdby_mo_count = icurl('class', api.format(created_by_mo))
+            created_by_mo_count = int(temp_createdby_mo_count[0]['moCount']['attributes']['count'])
+            if created_by_mo_count != new_mo_count:
+                data.append([new_mo, new_mo_dict[new_mo]["Impact"]])
+
+    if data:
+        result = FAIL_O
+    print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
+    return result
+
+
 def eecdh_cipher_check(index, total_checks, cversion, **kwargs):
     title = 'EECDH SSL Cipher'
     result = FAIL_UF
@@ -3094,6 +3170,7 @@ def fabric_port_down_check(index, total_checks, **kwargs):
         features_to_disable_check,
         switch_group_guideline_check,
         mini_aci_6_0_2_check,
+        post_upgrade_cb_check,
 
         # Faults
         apic_disk_space_faults_check,
@@ -3145,7 +3222,7 @@ def fabric_port_down_check(index, total_checks, **kwargs):
         sup_a_high_memory_check,
         vmm_active_uplinks_check,
     ]
-    summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, NA: 0, 'TOTAL': len(checks)}
+    summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, POST: 0, NA: 0, 'TOTAL': len(checks)}
     for idx, check in enumerate(checks):
         try:
             r = check(idx + 1, len(checks), **inputs)
@@ -3168,7 +3245,7 @@ def fabric_port_down_check(index, total_checks, **kwargs):
         f.write(jsonString)
 
     subprocess.check_output(['tar', '-czf', BUNDLE_NAME, DIR])
-    summary_headers = [PASS, FAIL_O, FAIL_UF, MANUAL, NA, ERROR, 'TOTAL']
+    summary_headers = [PASS, FAIL_O, FAIL_UF, MANUAL, POST, NA, ERROR, 'TOTAL']
     res = max(summary_headers, key=len)
     max_header_len = len(res)
     for key in summary_headers:

docs/docs/validations.md

@@ -32,6 +32,8 @@ Items                                                        | This Script
 [Switch Upgrade Group Guidelines][g12]                       | :white_check_mark: | :grey_exclamation: 4.2(4)<br>Only RR spines (IPN connectivity not checked) | :white_check_mark:
 [Intersight Device Connector upgrade status][g13]            | :white_check_mark: | :white_check_mark: 4.2(5) | :white_check_mark:
 [Mini ACI Upgrade to 6.0(2)+][g14]                           | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
+[Post Upgrade CallBack Integrity][g15]                       | :white_check_mark: | :no_entry_sign:           | :no_entry_sign:
+
 
 [g1]: #compatibility-target-aci-version
 [g2]: #compatibility-cimc-version
@@ -47,6 +49,7 @@ Items                                                        | This Script
 [g12]: #switch-upgrade-group-guidelines
 [g13]: #intersight-device-connector-upgrade-status
 [g14]: #mini-aci-upgrade-to-602-or-later
+[g15]: #post-upgrade-callback-integrity
 
 
 ### Fault Checks
@@ -342,6 +345,55 @@ When upgrading from ACI release 6.0(1) or earlier to release 6.0(2) or later, an
     --- omit ---
     ```
 
+### Post Upgrade CallBack Integrity
+
+Post APIC cluster upgrade, APICs may invoke the post-upgrade callback against an existing object class, which will introduce an object of the new class corresponding to the existing class to extend/enhance/optimize an existing feature. This is also called data conversion.
+
+Once successfully completed, the number of objects for the existing and newly created classes should be the same.
+
+If the post-upgrade callback has failed or hasn't been able to fully complete for some reason, the data conversion will be incomplete.The impact of incomplete data conversion varies depending on the class that is missing as a result.
+
+The post-upgrade callback may fail due to a software defect, or due to an inappropriate human intervention during APIC upgrades such as decommissioning an APIC, rebooting an APIC etc.
+
+This validation checks whether the number of objects for the existing and newly created classes are the same.
+
+!!! tip
+    This validation **must** be performed after an APIC upgrade but before a switch upgrade.
+
+    Regardless of this validation, it is a best practice to run this script twice, before an APIC upgrade AND before a switch upgrade, especially when the upgrade of the entire fabric cannot be completed within the same maintenance window. If you have been doing it, please keep doing it. If not, make sure to do that because this specific validation must be run after an APIC upgrade.
+    Because of this reason, this validation warns users to run the script again if the current APIC version and the target version are different when the script is run because it implies the script was run before an APIC upgrade.
+
+
+!!! example
+    If the existing class and the new class are `infraRsToEncapInstDef` and `infraAssocEncapInstDef`, the impact of incomplete post-upgrade callback (i.e. incomplete data conversion) is that VLANs will not be deployed on leaf switches after the upgrade of the switch.
+    You can check the number of objects for these classes via moquery and query option `rsp-subtree-include=count` as shown below.
+    If the Mo count for both classes are the same, it means that `infraAssocEncapInstDef` was created successfully after the APIC upgrade by `infraRsToEncapInstDef`'s post-upgrade callback.
+    ```
+    apic1# moquery -c infraAssocEncapInstDef -x rsp-subtree-include=count
+    Total Objects shown: 1
+
+    # mo.Count
+    childAction  :
+    count        : 11
+    dn           : cnt
+    lcOwn        : local
+    modTs        : never
+    rn           : cnt
+    status       :
+
+    apic1# moquery -c infraRsToEncapInstDef -x rsp-subtree-include=count
+    Total Objects shown: 1
+
+    # mo.Count
+    childAction  :
+    count        : 11
+    dn           : cnt
+    lcOwn        : local
+    modTs        : never
+    rn           : cnt
+    status       :
+    ```
+
 ## Fault Check Details
 
 ### APIC Disk Space Usage

tests/post_upgrade_cb_check/moCount_0.json

@@ -0,0 +1,12 @@
+[
+  {
+    "moCount": {
+      "attributes": {
+        "childAction": "",
+        "count": "0",
+        "dn": "",
+        "status": ""
+      }
+    }
+  }
+]

tests/post_upgrade_cb_check/moCount_10.json

@@ -0,0 +1,12 @@
+[
+  {
+    "moCount": {
+      "attributes": {
+        "childAction": "",
+        "count": "10",
+        "dn": "",
+        "status": ""
+      }
+    }
+  }
+]

tests/post_upgrade_cb_check/moCount_8.json

@@ -0,0 +1,12 @@
+[
+  {
+    "moCount": {
+      "attributes": {
+        "childAction": "",
+        "count": "8",
+        "dn": "",
+        "status": ""
+      }
+    }
+  }
+]

tests/post_upgrade_cb_check/test_post_upgrade_cb_check.py

@@ -0,0 +1,93 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+
+# icurl queries
+mo1_new = "infraRsToImplicitSetPol.json?rsp-subtree-include=count"
+mo1_old = "infraImplicitSetPol.json?rsp-subtree-include=count"
+mo2_new = "fvSlaDef.json?rsp-subtree-include=count"
+mo2_old = "fvIPSLAMonitoringPol.json?rsp-subtree-include=count"
+mo3_new = "infraRsConnectivityProfileOpt.json?rsp-subtree-include=count"
+mo3_old = "infraRsConnectivityProfile.json?rsp-subtree-include=count"
+mo4_new = "infraAssocEncapInstDef.json?rsp-subtree-include=count"
+mo4_old = "infraRsToEncapInstDef.json?rsp-subtree-include=count"
+mo5_new = 'compatSwitchHw.json?rsp-subtree-include=count&query-target-filter=eq(compatSwitchHw.suppBit,"32")'
+
+# icurl output sets
+mo_count_pass = {
+    mo1_new: read_data(dir, "moCount_10.json"),
+    mo1_old: read_data(dir, "moCount_10.json"),
+    mo2_new: read_data(dir, "moCount_10.json"),
+    mo2_old: read_data(dir, "moCount_10.json"),
+    mo3_new: read_data(dir, "moCount_10.json"),
+    mo3_old: read_data(dir, "moCount_10.json"),
+    mo4_new: read_data(dir, "moCount_10.json"),
+    mo4_old: read_data(dir, "moCount_10.json"),
+    mo5_new: read_data(dir, "moCount_10.json"),
+}
+mo_count_fail = {
+    # Both infraImplicitSetPol and infraRsToImplicitSetPol are brand new classes.
+    # When postUpgradeCb failed, MO counts are zero as those are newly created
+    # instead of converted from the old class.
+    mo1_new: read_data(dir, "moCount_0.json"),
+    mo1_old: read_data(dir, "moCount_0.json"),
+    # Others are number mismatch
+    mo2_new: read_data(dir, "moCount_8.json"),
+    mo2_old: read_data(dir, "moCount_10.json"),
+    mo3_new: read_data(dir, "moCount_8.json"),
+    mo3_old: read_data(dir, "moCount_10.json"),
+    mo4_new: read_data(dir, "moCount_8.json"),
+    mo4_old: read_data(dir, "moCount_10.json"),
+    # suppBit is a new attribute in 6.0.2 that can be either 32 or 64.
+    # When postUpgradeCb failed, there may be no compatSwitch with suppBit being 32.
+    mo5_new: read_data(dir, "moCount_0.json"),
+}
+
+
+@pytest.mark.parametrize(
+    "icurl_outputs, cversion, tversion, expected_result",
+    [
+        # Target Version not supplied
+        (mo_count_fail, "3.2(8f)", None, script.POST),
+        # Target Version newer than current (i.e. APIC upgrade not done yet)
+        (mo_count_fail, "4.2(7v)", "5.2(8g)", script.POST),
+        # No new class
+        (mo_count_fail, "3.2(9h)", "3.2(9h)", script.PASS),
+
+        # New classes
+        # infraRsToImplicitSetPol, infraImplicitSetPol
+        (mo_count_pass, "3.2(10g)", "3.2(10g)", script.PASS),
+        (mo_count_fail, "3.2(10g)", "3.2(10g)", script.FAIL_O),
+
+        # New classes
+        # infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef
+        (mo_count_pass, "4.2(7v)", "4.2(7v)", script.PASS),
+        (mo_count_fail, "4.2(7v)", "4.2(7v)", script.FAIL_O),
+
+        # New classes
+        # infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef, infraRsConnectivityProfileOpt, infraAssocEncapInstDef
+        (mo_count_pass, "5.2(8g)", "5.2(8g)", script.PASS),
+        (mo_count_fail, "5.2(8g)", "5.2(8g)", script.FAIL_O),
+
+        # New classes
+        # infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef, infraRsConnectivityProfileOpt, infraAssocEncapInstDef, compatSwitchHw.suppBit
+        (mo_count_pass, "6.0(3e)", "6.0(3e)", script.PASS),
+        (mo_count_fail, "6.0(3e)", "6.0(3e)", script.FAIL_O),
+    ]
+)
+def test_logic(mock_icurl, cversion, tversion, expected_result):
+    result = script.post_upgrade_cb_check(
+        1,
+        1,
+        script.AciVersion(cversion),
+        script.AciVersion(tversion) if tversion else None,
+    )
+    assert result == expected_result