From aa7d8bfa937b2e6f4fd465a1c6f8982e007f6e99 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 13 Feb 2020 14:33:37 -0800 Subject: [PATCH] [_sonic_yang_ext.py]: Parse multilist in YANG Container. (#38) * [_sonic_yang_ext.py]: Parse multilist in YANG Container. This is needed to support VRF feature in SONiC. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com * [sonic-vlan.yang]: Change Vlan yang models to remove admin-status as mandatory attribute. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com * [_sonic_yang_ext.py]: Minor fix in _sonic_yang_ext.py Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- src/sonic-yang-mgmt/_sonic_yang_ext.py | 135 +++++++++++------- .../tests/yang-model-tests/yangTest.json | 7 + .../yang-models/sonic-loopback-interface.yang | 2 + .../yang-models/sonic-vlan.yang | 2 - 4 files changed, 92 insertions(+), 54 deletions(-) diff --git a/src/sonic-yang-mgmt/_sonic_yang_ext.py b/src/sonic-yang-mgmt/_sonic_yang_ext.py index 67d355ef5b9a..63c61f60cbea 100644 --- a/src/sonic-yang-mgmt/_sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/_sonic_yang_ext.py @@ -129,31 +129,28 @@ def cropConfigDB(self, croppedFile=None, allowExtraTables=True): """ Extract keys from table entry in Config DB and return in a dict -For Example: regex = | and tableKey = "Vlan111|2a04:5555:45:6709::1/64" -1.) first code will extract key list from regex, i.e. vlan_name and ip_prefix. -2.) then will create another regex(regexV) to extract Values from tableKey by - replacing " --> extractor i.e. (.*?)" in regex. -3.) Then will extract values from tableKey with regexV. -4.) Resulting Dict will be: +Input: +tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64" +keys: key string from YANG list, i.e. 'vlan_name ip-prefix'. +regex: A regex to extract keys from tableKeys, good to have it as accurate as possible. + +Return: KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"} """ -def extractKey(self, tableKey, regex): +def extractKey(self, tableKey, keys, regex): - # get the keys from regex of key extractor - keyList = re.findall(r'<(.*?)>', regex) - # create a regex to get values from tableKey - # and change separator to text in regexV - regexV = re.sub('<.*?>', '(.*?)', regex) - regexV = re.sub('\|', '\\|', regexV) + keyList = keys.split() + #self.logInFile("extractKey {}".format(keyList)) # get the value groups - value = re.match(r'^'+regexV+'$', tableKey) + value = re.match(regex, tableKey) # create the keyDict i = 1 keyDict = dict() for k in keyList: if value.group(i): keyDict[k] = value.group(i) + # self.logInFile("extractKey {} {}".format(k, keyDict[k])) else: raise Exception("Value not found for {} in {}".format(k, tableKey)) i = i + 1 @@ -254,13 +251,17 @@ def xlateList(self, model, yang, config, table): # TODO: define a keyExt dict as of now, but we should be able to extract # this from YANG model extentions. keyExt = { - "VLAN_INTERFACE": "|", - "ACL_RULE": "|", - "VLAN": "", - "VLAN_MEMBER": "|", - "ACL_TABLE": "", - "INTERFACE": "|", - "PORT": "" + "VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", + "VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", + "VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)", + "VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$", + "ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$", + "ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$", + "INTERFACE_LIST": "^(Ethernet[0-9]+)$", + "INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$", + "PORT_LIST": "^(Ethernet[0-9]+)$", + "LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$", + "LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$", } #create a dict to map each key under primary key with a dict yang model. #This is done to improve performance of mapping from values of TABLEs in @@ -268,23 +269,31 @@ def xlateList(self, model, yang, config, table): leafDict = self.createLeafDict(model) - self.logInFile("Xlate {}".format(table)) - # Find and extracts key from each dict in config - for pkey in config: + keyRegEx = keyExt[model['@name']] + # get keys from YANG model list itself + listKeys = model['key']['@value'] + + for pkey in config.keys(): try: vKey = None - self.logInFile("xlate Extract pkey {} {}".format(pkey,keyExt[table])) - keyDict = self.extractKey(pkey, keyExt[table]) + self.logInFile("xlateList Extract pkey {}".format(pkey)) + # Find and extracts key from each dict in config + keyDict = self.extractKey(pkey, listKeys, keyRegEx) # fill rest of the values in keyDict for vKey in config[pkey]: - self.logInFile("xlate vkey {}".format(vKey), keyExt[table]) + self.logInFile("xlateList vkey {}".format(vKey)) keyDict[vKey] = self.findYangTypedValue(vKey, \ config[pkey][vKey], leafDict) yang.append(keyDict) + # delete pkey from config, done to match one key with one list + del config[pkey] + except Exception as e: - print("Exception while Config DB --> YANG: pkey:{}, "\ + self.logInFile("xlateList Exception {}".format(e)) + self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\ "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey))) - raise e + # with multilist, we continue matching other keys. + continue return @@ -295,19 +304,36 @@ def xlateList(self, model, yang, config, table): """ def xlateContainer(self, model, yang, config, table): - # if container contains single list with containerName_LIST and - # config is not empty then xLate the list + # To Handle multiple List, Make a copy of config, because we delete keys + # from config after each match. This is done to match one pkey with one list. + configC = config.copy() + clist = model.get('list') + # If single list exists in container, if clist and isinstance(clist, dict) and \ - clist['@name'] == model['@name']+"_LIST" and bool(config): + clist['@name'] == model['@name']+"_LIST" and bool(configC): #print(clist['@name']) yang[clist['@name']] = list() - self.xlateList(model['list'], yang[clist['@name']], \ - config, table) + self.logInFile("xlateContainer listD {}".format(clist['@name'])) + self.xlateList(clist, yang[clist['@name']], \ + configC, table) + # clean empty lists + if len(yang[clist['@name']]) == 0: + del yang[clist['@name']] #print(yang[clist['@name']]) - # TODO: Handle mupltiple list and rest of the field in Container. - # We do not have any such instance in Yang model today. + # If multi-list exists in container, + elif clist and isinstance(clist, list) and bool(configC): + for modelList in clist: + yang[modelList['@name']] = list() + self.logInFile("xlateContainer listL {}".format(modelList['@name'])) + self.xlateList(modelList, yang[modelList['@name']], configC, table) + # clean empty lists + if len(yang[modelList['@name']]) == 0: + del yang[modelList['@name']] + + if len(configC): + raise(Exception("All Keys are not parsed in {}".format(table))) return @@ -325,6 +351,7 @@ def xlateConfigDBtoYang(self, jIn, yangJ): # Add new top level container for first table in this container yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key] yangJ[key][subkey] = dict() + self.logInFile("xlateConfigDBtoYang {}:{}".format(key, subkey)) self.xlateContainer(cmap['container'], yangJ[key][subkey], \ jIn[table], table) @@ -389,7 +416,6 @@ def revYangConvert(val): return vValue - """ Rev xlate from _LIST to table in config DB """ @@ -398,26 +424,31 @@ def revXlateList(self, model, yang, config, table): # TODO: define a keyExt dict as of now, but we should be able to # extract this from YANG model extentions. keyExt = { - "VLAN_INTERFACE": "|", - "ACL_RULE": "|", - "VLAN": "", - "VLAN_MEMBER": "|", - "ACL_TABLE": "", - "INTERFACE": "|", - "PORT": "" + "VLAN_INTERFACE_IPPREFIX_LIST": "|", + "VLAN_INTERFACE_LIST": "", + "VLAN_MEMBER_LIST": "|", + "VLAN_LIST": "", + "ACL_RULE_LIST": "|", + "ACL_TABLE_LIST": "", + "INTERFACE_LIST": "", + "INTERFACE_IPPREFIX_LIST": "|", + "LOOPBACK_INTERFACE_LIST": "", + "LOOPBACK_INTERFACE_IPPREFIX_LIST": "|", + "PORT_LIST": "" + } + keyRegEx = keyExt[model['@name']] # create a dict to map each key under primary key with a dict yang model. # This is done to improve performance of mapping from values of TABLEs in # config DB to leaf in YANG LIST. leafDict = self.createLeafDict(model) - # list with name
_LIST should be removed, - # right now we have only this instance of LIST - if model['@name'] == table + "_LIST": + # list with name _LIST should be removed, + if "_LIST" in model['@name']: for entry in yang: # create key of config DB table - pkey, pkeydict = self.createKey(entry, keyExt[table]) + pkey, pkeydict = self.createKey(entry, keyRegEx) config[pkey]= dict() # fill rest of the entries for key in entry: @@ -438,9 +469,8 @@ def revXlateContainer(self, model, yang, config, table): modelList = model['list'] # Pass matching list from Yang Json self.revXlateList(modelList, yang[modelList['@name']], config, table) - else: - # TODO: Container[TABLE] contains multiple lists. [Test Pending] - # No instance now. + + elif isinstance(model['list'], list): for modelList in model['list']: self.revXlateList(modelList, yang[modelList['@name']], config, table) @@ -575,10 +605,11 @@ def load_data(self, configdbJson, allowExtraTables=True): # reset xlate self.xlateJson = dict() # self.jIn will be cropped - self.cropConfigDB("cropped.json", allowExtraTables) + self.cropConfigDB(allowExtraTables=allowExtraTables) # xlated result will be in self.xlateJson self.xlateConfigDB() #print(self.xlateJson) + self.logInFile("Try to load Data in the tree") self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \ ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT) diff --git a/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json b/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json index 450d28a55a62..523eedf3c4b1 100644 --- a/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json +++ b/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json @@ -627,6 +627,8 @@ "SAMPLE_CONFIG_DB_JSON": { "VLAN_INTERFACE": { + "Vlan111": {}, + "Vlan777": {}, "Vlan111|2a04:5555:45:6709::1/64": { "scope": "global", "family": "IPv6" @@ -1178,6 +1180,10 @@ } }, "INTERFACE": { + "Ethernet112": {}, + "Ethernet14": {}, + "Ethernet16": {}, + "Ethernet18": {}, "Ethernet112|2a04:5555:40:a709::2/126": { "scope": "global", "family": "IPv6" @@ -1259,6 +1265,7 @@ } }, "LOOPBACK_INTERFACE": { + "Loopback0": {}, "Loopback0|2a04:5555:40:4::4e9/128": { "scope": "global", "family": "IPv6" diff --git a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang index 944f7d3ec88b..0ecf5f2216ff 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang @@ -44,9 +44,11 @@ module sonic-loopback-interface { /* end of LOOPBACK_INTERFACE_LIST */ list LOOPBACK_INTERFACE_IPPREFIX_LIST { + key "loopback_interface_name ip-prefix"; leaf loopback_interface_name{ + /* This node must be present in LOOPBACK_INTERFACE_LIST */ must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)" { diff --git a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang index eee721e47f71..17d922246bc2 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang @@ -140,7 +140,6 @@ module sonic-vlan { } leaf admin_status { - mandatory true; type head:admin_status; } @@ -172,7 +171,6 @@ module sonic-vlan { leaf port { /* key elements are mandatory by default */ - mandatory true; type leafref { path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name; }