CVL Changes #10: Custom Validation infra, Unit Test reorganization and new test cases addition

cvl/Makefile

@@ -55,7 +55,7 @@ schema:
 
 test-schema: | schema
 	$(MAKE) -C testdata/schema
-	cp $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
+	cp -n $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
 
 tests:
 	$(MAKE) -C tests

cvl/README.md

@@ -20,6 +20,7 @@ Below steps need to be done to enable CVL logging.
 
 2. Change the logging flags from "false" to "true" as below:
 
+```
 	{
 		"TRACE_CACHE": "true",
 		"TRACE_LIBYANG": "true",
@@ -29,15 +30,22 @@ Below steps need to be done to enable CVL logging.
 		"TRACE_DELETE": "true",
 		"TRACE_SEMANTIC": "true",
 		"TRACE_SYNTAX": "true",
+		"TRACE_ONERROR": "true",
 		"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
-		"LOGTOSTDERR": "true",
-		"__comment2__": "Display log upto INFO level",
-		"STDERRTHRESHOLD": "INFO",
-		"__comment3__": "Display log upto INFO level 8",
-		"VERBOSITY": "8",
+		"LOGTOSTDERR": "false",
+		"__comment2__": "Log messages to standard error at or above this severity level",
+		"STDERRTHRESHOLD": "ERROR",
+		"__comment3__": "Log to /tmp/cvl.log file",
+		"LOG_TO_FILE": "true",
+		"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
+		"LOG_FILE_SIZE": "10485760",
+		"__comment5__": "Set verbosity level(1 to 8) for verbose logs",
+		"VERBOSITY": "0",
 		"SKIP_VALIDATION": "false",
 		"SKIP_SEMANTIC_VALIDATION": "false"
 	}
+```
+
 3. Below environment variables need to be set at the end in /usr/bin/rest-server.sh in mgmt-framework docker. 
 
    export CVL_DEBUG=1

cvl/conf/cvl_cfg.json

@@ -7,13 +7,16 @@
 	"TRACE_DELETE": "false",
 	"TRACE_SEMANTIC": "false",
 	"TRACE_SYNTAX": "false",
-	"__comment1__": "Log trace data when error occurs",
-	"TRACE_ONERROR": "true",
-	"__comment2__": "Set LOGTOSTDER to 'true' to log on standard error",
+	"TRACE_ONERROR": "false",
+	"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
 	"LOGTOSTDERR": "false",
-	"__comment3__": "Display log upto INFO level",
+	"__comment2__": "Display log upto INFO level",
 	"STDERRTHRESHOLD": "ERROR",
-	"__comment4__": "Display log upto INFO level 8",
+	"__comment3__": "Log to /tmp/cvl.log file",
+	"LOG_TO_FILE": "false",
+	"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
+	"LOG_FILE_SIZE": "10485760",
+	"__comment5__": "Display log upto INFO level 8",
 	"VERBOSITY": "0",
 	"SKIP_VALIDATION": "false",
 	"SKIP_SEMANTIC_VALIDATION": "false"

cvl/custom_validation/common.go

@@ -0,0 +1,132 @@
+////////////////////////////////////////////////////////////////////////////////
+//                                                                            //
+//  Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+//  its subsidiaries.                                                         //
+//                                                                            //
+//  Licensed under the Apache License, Version 2.0 (the "License");           //
+//  you may not use this file except in compliance with the License.          //
+//  You may obtain a copy of the License at                                   //
+//                                                                            //
+//     http://www.apache.org/licenses/LICENSE-2.0                             //
+//                                                                            //
+//  Unless required by applicable law or agreed to in writing, software       //
+//  distributed under the License is distributed on an "AS IS" BASIS,         //
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
+//  See the License for the specific language governing permissions and       //
+//  limitations under the License.                                            //
+//                                                                            //
+////////////////////////////////////////////////////////////////////////////////
+
+package custom_validation
+
+import (
+	"reflect"
+	"github.com/antchfx/xmlquery"
+	"github.com/go-redis/redis/v7"
+	"github.com/Azure/sonic-mgmt-common/cvl/internal/util"
+	"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
+	)
+
+type CustomValidation struct {}
+
+type CVLValidateType uint
+const (
+	VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data
+	VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data
+	VALIDATE_SEMANTICS //Semantics is checked
+	VALIDATE_ALL //Syntax and Semantics are checked
+)
+
+type CVLOperation uint
+const (
+	OP_NONE   CVLOperation = 0 //Used to just validate the config without any operation
+	OP_CREATE = 1 << 0//For Create operation 
+	OP_UPDATE = 1 << 1//For Update operation
+	OP_DELETE = 1 << 2//For Delete operation
+)
+
+//CVLRetCode CVL Error codes
+type CVLRetCode int
+const (
+	CVL_SUCCESS CVLRetCode = iota
+	CVL_ERROR
+	CVL_NOT_IMPLEMENTED
+	CVL_INTERNAL_UNKNOWN
+	CVL_FAILURE
+	CVL_SYNTAX_ERROR =  CVLRetCode(yparser.YP_SYNTAX_ERROR)
+	CVL_SEMANTIC_ERROR = CVLRetCode(yparser.YP_SEMANTIC_ERROR)
+	CVL_SYNTAX_MISSING_FIELD = CVLRetCode(yparser.YP_SYNTAX_MISSING_FIELD)
+	CVL_SYNTAX_INVALID_FIELD = CVLRetCode(yparser.YP_SYNTAX_INVALID_FIELD)   /* Invalid Field  */
+	CVL_SYNTAX_INVALID_INPUT_DATA = CVLRetCode(yparser.YP_SYNTAX_INVALID_INPUT_DATA) /*Invalid Input Data */
+	CVL_SYNTAX_MULTIPLE_INSTANCE = CVLRetCode(yparser.YP_SYNTAX_MULTIPLE_INSTANCE)   /* Multiple Field Instances */
+	CVL_SYNTAX_DUPLICATE  = CVLRetCode(yparser.YP_SYNTAX_DUPLICATE)      /* Duplicate Fields  */
+	CVL_SYNTAX_ENUM_INVALID  = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID) /* Invalid enum value */
+	CVL_SYNTAX_ENUM_INVALID_NAME = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID_NAME) /* Invalid enum name  */
+	CVL_SYNTAX_ENUM_WHITESPACE = CVLRetCode(yparser.YP_SYNTAX_ENUM_WHITESPACE)     /* Enum name with leading/trailing whitespaces */
+	CVL_SYNTAX_OUT_OF_RANGE = CVLRetCode(yparser.YP_SYNTAX_OUT_OF_RANGE)    /* Value out of range/length/pattern (data) */
+	CVL_SYNTAX_MINIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MINIMUM_INVALID)       /* min-elements constraint not honored  */
+	CVL_SYNTAX_MAXIMUM_INVALID  = CVLRetCode(yparser.YP_SYNTAX_MAXIMUM_INVALID)      /* max-elements constraint not honored */
+	CVL_SEMANTIC_DEPENDENT_DATA_MISSING  = CVLRetCode(yparser.YP_SEMANTIC_DEPENDENT_DATA_MISSING)  /* Dependent Data is missing */
+	CVL_SEMANTIC_MANDATORY_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_MANDATORY_DATA_MISSING) /* Mandatory Data is missing */
+	CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */
+	CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */
+	CVL_SEMANTIC_KEY_DUPLICATE  = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */
+        CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID)
+)
+
+//CVLEditConfigData Strcture for key and data in API
+type CVLEditConfigData struct {
+	VType CVLValidateType //Validation type
+	VOp CVLOperation      //Operation type
+	Key string      //Key format : "PORT|Ethernet4"
+	Data map[string]string //Value :  {"alias": "40GE0/28", "mtu" : 9100,  "admin_status":  down}
+}
+
+//CVLErrorInfo CVL Error Structure
+type CVLErrorInfo struct {
+	TableName string      /* Table having error */
+	ErrCode  CVLRetCode   /* CVL Error return Code. */
+	CVLErrDetails string  /* CVL Error Message details. */ 
+	Keys    []string      /* Keys of the Table having error. */
+        Value    string        /* Field Value throwing error */
+	Field	 string        /* Field Name throwing error . */
+	Msg     string        /* Detailed error message. */
+	ConstraintErrMsg  string  /* Constraint error message. */
+	ErrAppTag string
+}
+
+type CustValidationCache struct {
+	Data interface{}
+}
+
+//CustValidationCtxt Custom validation context passed to custom validation function 
+type CustValidationCtxt struct {
+	ReqData []CVLEditConfigData //All request data
+	CurCfg *CVLEditConfigData //Current request data for which validation should be done
+	YNodeName string //YANG node name
+	YNodeVal string  //YANG node value, leaf-list will have "," separated value
+	YCur *xmlquery.Node //YANG data tree
+	SessCache *CustValidationCache //Session cache, can be used for storing data, persistent in session
+	RClient *redis.Client //Redis client
+}
+
+//InvokeCustomValidation Common function to invoke custom validation
+//TBD should we do this using GO plugin feature ?
+func InvokeCustomValidation(cv *CustomValidation, name string, args... interface{}) CVLErrorInfo {
+	inputs := make([]reflect.Value, len(args))
+	for i := range args {
+		inputs[i] = reflect.ValueOf(args[i])
+	}
+
+	f := reflect.ValueOf(cv).MethodByName(name)
+	if !f.IsNil() {
+		v := f.Call(inputs)
+		util.TRACE_LEVEL_LOG(util.TRACE_SEMANTIC,
+			"InvokeCustomValidation: %s(), CVLErrorInfo: %v", name, v[0])
+
+		return (v[0].Interface()).(CVLErrorInfo)
+	}
+
+	return CVLErrorInfo{ErrCode: CVL_SUCCESS}
+}
+

cvl/cvl.go

@@ -34,6 +34,8 @@ import (
 	"sync"
 	"io/ioutil"
 	"path/filepath"
+	custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
+	"unsafe"
 )
 
 //DB number 
@@ -106,6 +108,7 @@ type modelTableInfo struct {
 	whenExpr map[string][]*whenInfo
 	tablesForMustExp map[string]CVLOperation
 	refFromTables []tblFieldPair //list of table or table/field referring to this table
+	custValidation map[string]string // Map for custom validation node and function name
 	dfltLeafVal map[string]string //map of leaf names and default value
 	mandatoryNodes map[string]bool  //map of leaf names and mandatory flag
 }
@@ -140,6 +143,7 @@ type CVL struct {
 	maxTableElem map[string]int //max element count per table
 	batchLeaf []*yparser.YParserLeafValue //field name and value
 	yv *YValidator //Custom YANG validator for validating external dependencies
+	custvCache custv.CustValidationCache //Custom validation cache per session
 }
 
 // Struct for model namepsace and prefix
@@ -449,6 +453,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) {
 		tInfo.redisTableSize = lInfo.RedisTableSize
 		tInfo.keys = lInfo.Keys
 		tInfo.mapLeaf = lInfo.MapLeaf
+		tInfo.custValidation = lInfo.CustValidation
 		tInfo.mandatoryNodes = lInfo.MandatoryNodes
 
 		//store default values used in must and when exp
@@ -955,6 +960,62 @@ func (c *CVL) addCfgDataItem(configData *map[string]interface{},
 	return tblName, key
 }
 
+//Perform user defined custom validation
+func (c *CVL) doCustomValidation(node *xmlquery.Node,
+	custvCfg []custv.CVLEditConfigData,
+	curCustvCfg *custv.CVLEditConfigData, yangListName,
+	tbl, key string) CVLErrorInfo {
+
+	cvlErrObj := CVLErrorInfo{ErrCode : CVL_SUCCESS}
+
+	// yangListName provides the correct table name defined in sonic-yang
+	// For ex. VLAN_INTERFACE_LIST and VLAN_INTERFACE_IPADDR_LIST are in same container
+	for nodeName, custFunc := range modelInfo.tableInfo[yangListName].custValidation {
+		//find the node value
+		//node value is empty for custom validation function at list level
+		nodeVal := ""
+		if !strings.HasSuffix(nodeName, "_LIST") {
+			for nodeLeaf := node.FirstChild; nodeLeaf != nil;
+			nodeLeaf = nodeLeaf.NextSibling {
+				if (nodeName != nodeLeaf.Data) {
+					continue
+				}
+
+				if (len(nodeLeaf.Attr) > 0) &&
+				(nodeLeaf.Attr[0].Name.Local == "leaf-list") {
+					nodeVal = curCustvCfg.Data[nodeName]
+				} else {
+					nodeVal = nodeLeaf.FirstChild.Data
+				}
+			}
+
+		}
+
+		//Call custom validation functions
+		CVL_LOG(INFO_TRACE, "Calling custom validation function %s", custFunc)
+		pCustv := &custv.CustValidationCtxt{
+			ReqData: custvCfg,
+			CurCfg: curCustvCfg,
+			YNodeName: nodeName,
+			YNodeVal: nodeVal,
+			YCur: node,
+			SessCache: &(c.custvCache),
+			RClient: redisClient}
+
+		errObj := custv.InvokeCustomValidation(&custv.CustomValidation{},
+		custFunc, pCustv)
+
+		cvlErrObj = *(*CVLErrorInfo)(unsafe.Pointer(&errObj))
+
+		if (cvlErrObj.ErrCode != CVL_SUCCESS) {
+			CVL_LOG(WARNING, "Custom validation failed, Error = %v", cvlErrObj)
+			return cvlErrObj
+		}
+	}
+
+	return cvlErrObj
+}
+
 // getLeafRefInfo This function returns leafrefInfo structure based on table name,
 // target table name and leaf node name where leafRef is present
 func getLeafRefInfo(tblName, fldName, targetTblName string) *leafRefInfo {
@@ -978,4 +1039,4 @@ func isMandatoryTrueNode(tblName, field string) bool {
 	}
 
 	return false
-}
+}

cvl/cvl_api.go

@@ -21,6 +21,7 @@ package cvl
 
 import (
 	"fmt"
+	"reflect"
 	"encoding/json"
 	"github.com/go-redis/redis/v7"
 	toposort "github.com/philopon/go-toposort"
@@ -29,7 +30,9 @@ import (
 	. "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
 	"strings"
 	"github.com/antchfx/xmlquery"
+	"unsafe"
 	"runtime"
+	custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
 	"time"
 	"sync"
 )
@@ -321,14 +324,16 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
 		caller = f.Name()
 	}
 
-        CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)
+  CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)
 
 	if SkipValidation() {
 		CVL_LOG(INFO_TRACE, "Skipping CVL validation.")
 		return cvlErrObj, CVL_SUCCESS
 	}
 
 	//Type cast to custom validation cfg data
+	sliceHeader := *(*reflect.SliceHeader)(unsafe.Pointer(&cfgData))
+	custvCfg := *(*[]custv.CVLEditConfigData)(unsafe.Pointer(&sliceHeader))
 
 	c.clearTmpDbCache()
 	//c.yv.root.FirstChild = nil
@@ -544,6 +549,13 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
 			continue
 		}
 
+		//Step 3.2 : Run all custom validations
+		cvlErrObj= c.doCustomValidation(node, custvCfg, &custvCfg[i], yangListName,
+		tbl, key)
+		if cvlErrObj.ErrCode != CVL_SUCCESS {
+			return cvlErrObj,cvlErrObj.ErrCode
+		}
+
 		//Step 3.3 : Perform semantic validation
 		if cvlErrObj = c.validateSemantics(node, yangListName, key, &cfgData[i]);
 		cvlErrObj.ErrCode != CVL_SUCCESS {
@@ -1087,4 +1099,4 @@ func (c *CVL) GetAllReferringTables(tableName string) (map[string][]string) {
 	}
 
 	return refTbls
-}
+}