Skip to content

Commit

Permalink
Add post upgrade Cb Check (#110)
Browse files Browse the repository at this point in the history
* Added post_upgrade_cb_check

* Updated made as per Takuya's suggestion

* Fix doc typo

* Add pytest

* Rebase on to master

* updated post-upgrade-db function

* Added 32/64 bit image Valisation Mo

* Fix handling of infraImplicitSetPol, Add compatSwitchHw.suppBit

* Fix quotes in a query, Add POST in summary

* Fix quotes in a query for pytest

* compatSwHw Impact text modification

---------

Co-authored-by: welkin0 <[email protected]>
Co-authored-by: tkishida <[email protected]>
  • Loading branch information
3 people authored May 8, 2024
1 parent e1de09f commit 766ff1f
Show file tree
Hide file tree
Showing 6 changed files with 260 additions and 2 deletions.
81 changes: 79 additions & 2 deletions aci-preupgrade-validation-script.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
FAIL_UF = 'FAIL - UPGRADE FAILURE!!'
ERROR = 'ERROR !!'
MANUAL = 'MANUAL CHECK REQUIRED'
POST = 'POST UPGRADE CHECK REQUIRED'
NA = 'N/A'
node_regex = r'topology/pod-(?P<pod>\d+)/node-(?P<node>\d+)'
ver_regex = r'(?:dk9\.)?[1]?(?P<major1>\d)\.(?P<major2>\d)(?:\.|\()(?P<maint>\d+)\.?(?P<patch>(?:[a-b]|[0-9a-z]+))\)?'
Expand Down Expand Up @@ -2960,6 +2961,81 @@ def access_untagged_check(index, total_checks, **kwargs):
return result


def post_upgrade_cb_check(index, total_checks, cversion, tversion, **kwargs):
title = 'Post Upgrade Callback Integrity'
result = PASS
msg = ''
headers = ["Missed Objects", "Impact"]
data = []
recommended_action = 'Contact Cisco TAC with Output'
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#post-upgrade-callback-integrity'
print_title(title, index, total_checks)

new_mo_dict = {
"infraImplicitSetPol": {
"CreatedBy": "",
"SinceVersion": "3.2(10e)",
"Impact": "Infra implicit settings will not be deployed",
},
"infraRsToImplicitSetPol": {
"CreatedBy": "infraImplicitSetPol",
"SinceVersion": "3.2(10e)",
"Impact": "Infra implicit settings will not be deployed",
},
"fvSlaDef": {
"CreatedBy": "fvIPSLAMonitoringPol",
"SinceVersion": "4.1(1i)",
"Impact": "IPSLA monitor policy will not be deployed",
},
"infraRsConnectivityProfileOpt": {
"CreatedBy": "infraRsConnectivityProfile",
"SinceVersion": "5.2(4d)",
"Impact": "VPC for missing Mo will not be deployed to leaf",
},
"infraAssocEncapInstDef": {
"CreatedBy": "infraRsToEncapInstDef",
"SinceVersion": "5.2(4d)",
"Impact": "VLAN for missing Mo will not be deployed to leaf",
},
"compatSwitchHw": {
"CreatedBy": "", # suppBit attribute is available from 6.0(2h)
"SinceVersion": "6.0(2h)",
"Impact": "Unexpected 64/32 bit image can deploy to switches",
},
}
if not tversion or (tversion and cversion.older_than(str(tversion))):
print_result(title, POST, 'Re-run script after APICs are upgraded and back to Fully-Fit')
return POST

for new_mo in new_mo_dict:
since_version = AciVersion(new_mo_dict[new_mo]['SinceVersion'])
created_by_mo = new_mo_dict[new_mo]['CreatedBy']

if since_version.newer_than(str(cversion)):
continue

api = "{}.json?rsp-subtree-include=count"
if new_mo == "compatSwitchHw":
# Expected to see suppBit in 32 or 64. Zero 32 means a failed postUpgradeCb.
api += '&query-target-filter=eq(compatSwitchHw.suppBit,"32")'

temp_new_mo_count = icurl("class", api.format(new_mo))
new_mo_count = int(temp_new_mo_count[0]['moCount']['attributes']['count'])
if created_by_mo == "":
if new_mo_count == 0:
data.append([new_mo, new_mo_dict[new_mo]["Impact"]])
else:
temp_createdby_mo_count = icurl('class', api.format(created_by_mo))
created_by_mo_count = int(temp_createdby_mo_count[0]['moCount']['attributes']['count'])
if created_by_mo_count != new_mo_count:
data.append([new_mo, new_mo_dict[new_mo]["Impact"]])

if data:
result = FAIL_O
print_result(title, result, msg, headers, data, recommended_action=recommended_action, doc_url=doc_url)
return result


def eecdh_cipher_check(index, total_checks, cversion, **kwargs):
title = 'EECDH SSL Cipher'
result = FAIL_UF
Expand Down Expand Up @@ -3122,6 +3198,7 @@ def fabric_dpp_check(index, total_checks, tversion, **kwargs):
features_to_disable_check,
switch_group_guideline_check,
mini_aci_6_0_2_check,
post_upgrade_cb_check,

# Faults
apic_disk_space_faults_check,
Expand Down Expand Up @@ -3175,7 +3252,7 @@ def fabric_dpp_check(index, total_checks, tversion, **kwargs):
fabric_dpp_check,

]
summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, NA: 0, 'TOTAL': len(checks)}
summary = {PASS: 0, FAIL_O: 0, FAIL_UF: 0, ERROR: 0, MANUAL: 0, POST: 0, NA: 0, 'TOTAL': len(checks)}
for idx, check in enumerate(checks):
try:
r = check(idx + 1, len(checks), **inputs)
Expand All @@ -3198,7 +3275,7 @@ def fabric_dpp_check(index, total_checks, tversion, **kwargs):
f.write(jsonString)

subprocess.check_output(['tar', '-czf', BUNDLE_NAME, DIR])
summary_headers = [PASS, FAIL_O, FAIL_UF, MANUAL, NA, ERROR, 'TOTAL']
summary_headers = [PASS, FAIL_O, FAIL_UF, MANUAL, POST, NA, ERROR, 'TOTAL']
res = max(summary_headers, key=len)
max_header_len = len(res)
for key in summary_headers:
Expand Down
52 changes: 52 additions & 0 deletions docs/docs/validations.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ Items | This Script
[Switch Upgrade Group Guidelines][g12] | :white_check_mark: | :grey_exclamation: 4.2(4)<br>Only RR spines (IPN connectivity not checked) | :white_check_mark:
[Intersight Device Connector upgrade status][g13] | :white_check_mark: | :white_check_mark: 4.2(5) | :white_check_mark:
[Mini ACI Upgrade to 6.0(2)+][g14] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:
[Post Upgrade CallBack Integrity][g15] | :white_check_mark: | :no_entry_sign: | :no_entry_sign:


[g1]: #compatibility-target-aci-version
[g2]: #compatibility-cimc-version
Expand All @@ -47,6 +49,7 @@ Items | This Script
[g12]: #switch-upgrade-group-guidelines
[g13]: #intersight-device-connector-upgrade-status
[g14]: #mini-aci-upgrade-to-602-or-later
[g15]: #post-upgrade-callback-integrity


### Fault Checks
Expand Down Expand Up @@ -344,6 +347,55 @@ When upgrading from ACI release 6.0(1) or earlier to release 6.0(2) or later, an
--- omit ---
```

### Post Upgrade CallBack Integrity

Post APIC cluster upgrade, APICs may invoke the post-upgrade callback against an existing object class, which will introduce an object of the new class corresponding to the existing class to extend/enhance/optimize an existing feature. This is also called data conversion.

Once successfully completed, the number of objects for the existing and newly created classes should be the same.

If the post-upgrade callback has failed or hasn't been able to fully complete for some reason, the data conversion will be incomplete.The impact of incomplete data conversion varies depending on the class that is missing as a result.

The post-upgrade callback may fail due to a software defect, or due to an inappropriate human intervention during APIC upgrades such as decommissioning an APIC, rebooting an APIC etc.

This validation checks whether the number of objects for the existing and newly created classes are the same.

!!! tip
This validation **must** be performed after an APIC upgrade but before a switch upgrade.

Regardless of this validation, it is a best practice to run this script twice, before an APIC upgrade AND before a switch upgrade, especially when the upgrade of the entire fabric cannot be completed within the same maintenance window. If you have been doing it, please keep doing it. If not, make sure to do that because this specific validation must be run after an APIC upgrade.
Because of this reason, this validation warns users to run the script again if the current APIC version and the target version are different when the script is run because it implies the script was run before an APIC upgrade.


!!! example
If the existing class and the new class are `infraRsToEncapInstDef` and `infraAssocEncapInstDef`, the impact of incomplete post-upgrade callback (i.e. incomplete data conversion) is that VLANs will not be deployed on leaf switches after the upgrade of the switch.
You can check the number of objects for these classes via moquery and query option `rsp-subtree-include=count` as shown below.
If the Mo count for both classes are the same, it means that `infraAssocEncapInstDef` was created successfully after the APIC upgrade by `infraRsToEncapInstDef`'s post-upgrade callback.
```
apic1# moquery -c infraAssocEncapInstDef -x rsp-subtree-include=count
Total Objects shown: 1

# mo.Count
childAction :
count : 11
dn : cnt
lcOwn : local
modTs : never
rn : cnt
status :

apic1# moquery -c infraRsToEncapInstDef -x rsp-subtree-include=count
Total Objects shown: 1

# mo.Count
childAction :
count : 11
dn : cnt
lcOwn : local
modTs : never
rn : cnt
status :
```

## Fault Check Details

### APIC Disk Space Usage
Expand Down
12 changes: 12 additions & 0 deletions tests/post_upgrade_cb_check/moCount_0.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[
{
"moCount": {
"attributes": {
"childAction": "",
"count": "0",
"dn": "",
"status": ""
}
}
}
]
12 changes: 12 additions & 0 deletions tests/post_upgrade_cb_check/moCount_10.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[
{
"moCount": {
"attributes": {
"childAction": "",
"count": "10",
"dn": "",
"status": ""
}
}
}
]
12 changes: 12 additions & 0 deletions tests/post_upgrade_cb_check/moCount_8.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[
{
"moCount": {
"attributes": {
"childAction": "",
"count": "8",
"dn": "",
"status": ""
}
}
}
]
93 changes: 93 additions & 0 deletions tests/post_upgrade_cb_check/test_post_upgrade_cb_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import os
import pytest
import logging
import importlib
from helpers.utils import read_data

script = importlib.import_module("aci-preupgrade-validation-script")

log = logging.getLogger(__name__)
dir = os.path.dirname(os.path.abspath(__file__))


# icurl queries
mo1_new = "infraRsToImplicitSetPol.json?rsp-subtree-include=count"
mo1_old = "infraImplicitSetPol.json?rsp-subtree-include=count"
mo2_new = "fvSlaDef.json?rsp-subtree-include=count"
mo2_old = "fvIPSLAMonitoringPol.json?rsp-subtree-include=count"
mo3_new = "infraRsConnectivityProfileOpt.json?rsp-subtree-include=count"
mo3_old = "infraRsConnectivityProfile.json?rsp-subtree-include=count"
mo4_new = "infraAssocEncapInstDef.json?rsp-subtree-include=count"
mo4_old = "infraRsToEncapInstDef.json?rsp-subtree-include=count"
mo5_new = 'compatSwitchHw.json?rsp-subtree-include=count&query-target-filter=eq(compatSwitchHw.suppBit,"32")'

# icurl output sets
mo_count_pass = {
mo1_new: read_data(dir, "moCount_10.json"),
mo1_old: read_data(dir, "moCount_10.json"),
mo2_new: read_data(dir, "moCount_10.json"),
mo2_old: read_data(dir, "moCount_10.json"),
mo3_new: read_data(dir, "moCount_10.json"),
mo3_old: read_data(dir, "moCount_10.json"),
mo4_new: read_data(dir, "moCount_10.json"),
mo4_old: read_data(dir, "moCount_10.json"),
mo5_new: read_data(dir, "moCount_10.json"),
}
mo_count_fail = {
# Both infraImplicitSetPol and infraRsToImplicitSetPol are brand new classes.
# When postUpgradeCb failed, MO counts are zero as those are newly created
# instead of converted from the old class.
mo1_new: read_data(dir, "moCount_0.json"),
mo1_old: read_data(dir, "moCount_0.json"),
# Others are number mismatch
mo2_new: read_data(dir, "moCount_8.json"),
mo2_old: read_data(dir, "moCount_10.json"),
mo3_new: read_data(dir, "moCount_8.json"),
mo3_old: read_data(dir, "moCount_10.json"),
mo4_new: read_data(dir, "moCount_8.json"),
mo4_old: read_data(dir, "moCount_10.json"),
# suppBit is a new attribute in 6.0.2 that can be either 32 or 64.
# When postUpgradeCb failed, there may be no compatSwitch with suppBit being 32.
mo5_new: read_data(dir, "moCount_0.json"),
}


@pytest.mark.parametrize(
"icurl_outputs, cversion, tversion, expected_result",
[
# Target Version not supplied
(mo_count_fail, "3.2(8f)", None, script.POST),
# Target Version newer than current (i.e. APIC upgrade not done yet)
(mo_count_fail, "4.2(7v)", "5.2(8g)", script.POST),
# No new class
(mo_count_fail, "3.2(9h)", "3.2(9h)", script.PASS),
# New classes
# infraRsToImplicitSetPol, infraImplicitSetPol
(mo_count_pass, "3.2(10g)", "3.2(10g)", script.PASS),
(mo_count_fail, "3.2(10g)", "3.2(10g)", script.FAIL_O),
# New classes
# infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef
(mo_count_pass, "4.2(7v)", "4.2(7v)", script.PASS),
(mo_count_fail, "4.2(7v)", "4.2(7v)", script.FAIL_O),
# New classes
# infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef, infraRsConnectivityProfileOpt, infraAssocEncapInstDef
(mo_count_pass, "5.2(8g)", "5.2(8g)", script.PASS),
(mo_count_fail, "5.2(8g)", "5.2(8g)", script.FAIL_O),
# New classes
# infraRsToImplicitSetPol, infraImplicitSetPol, fvSlaDef, infraRsConnectivityProfileOpt, infraAssocEncapInstDef, compatSwitchHw.suppBit
(mo_count_pass, "6.0(3e)", "6.0(3e)", script.PASS),
(mo_count_fail, "6.0(3e)", "6.0(3e)", script.FAIL_O),
]
)
def test_logic(mock_icurl, cversion, tversion, expected_result):
result = script.post_upgrade_cb_check(
1,
1,
script.AciVersion(cversion),
script.AciVersion(tversion) if tversion else None,
)
assert result == expected_result

0 comments on commit 766ff1f

Please sign in to comment.