go format

analyzer.go

@@ -83,15 +83,15 @@ type Analyzer struct {
 //pattern to be used as the starting block for
 //all conversions
 type AnalyzerResult struct {
-	Service		    models.Service
+	Service         models.Service
 	PatternId       string
 	Pattern         string
 	TagPositions    string
 	ExampleCount    int
 	Examples        []LogRecord
 	DateCreated     time.Time
 	DateLastMatched time.Time
-	ComplexityScore	float64
+	ComplexityScore float64
 }
 
 type analyzerNode struct {
@@ -166,15 +166,15 @@ func AddExampleToAnalyzerResult(this *AnalyzerResult, lr LogRecord) {
 //This ensures that the same pattern will always have the same id, returns a sha1 hash of the pattern + service name.
 func GenerateIDFromString(pattern string, service string) string {
 	h := sha1.New()
-	h.Write([]byte(pattern+service))
-	sha := h.Sum(nil)              // "sha" is uint8 type, encoded in base16
+	h.Write([]byte(pattern + service))
+	sha := h.Sum(nil)                 // "sha" is uint8 type, encoded in base16
 	shaStr := hex.EncodeToString(sha) // String representation
 	return shaStr
 }
 
 //This calculates the complexity of the pattern, looking at the ratio if non-string tokens to string tokens.
 //This value helps the reviewer of the pattern  filter out the patterns that may be over tokenized.
-func CalculatePatternComplexity(seq Sequence, lgt int) float64{
+func CalculatePatternComplexity(seq Sequence, lgt int) float64 {
 
 	var (
 		strct float64
@@ -185,28 +185,28 @@ func CalculatePatternComplexity(seq Sequence, lgt int) float64{
 	//length of the sequence
 	lt := float64(len(seq))
 
-	for _, tok := range seq{
-		if tok.Type == TokenString{
+	for _, tok := range seq {
+		if tok.Type == TokenString {
 			strct += 1
-		} else if tok.Type == TokenLiteral{
-			if len(tok.Value) > 1{
+		} else if tok.Type == TokenLiteral {
+			if len(tok.Value) > 1 {
 				litct += 1
-			}else if tok.Value != "="{
+			} else if tok.Value != "=" {
 				lt -= 1
 			}
-		} else if tok.Type == TokenInteger{
+		} else if tok.Type == TokenInteger {
 			intct += 1
-		} else{
+		} else {
 			othct += 1
 		}
 	}
 
 	//Complexity score
 	//all literal tokens
-	if lt == litct{
+	if lt == litct {
 		return 0.0
-	}else if strct > 0{
-		return float64(strct/lt)
+	} else if strct > 0 {
+		return float64(strct / lt)
 	}
 
 	return 0.05

cmd/sequence_db/sequence_db_main.go

@@ -117,7 +117,7 @@ func scan(cmd *cobra.Command, args []string) {
 		_, lrMap, _ = sequence.ReadLogRecordAsMap(iscan, informat, lrMap, batchsize)
 		for _, lrc := range lrMap {
 			for _, l := range lrc.Records {
-				seq, _, _:= sequence.ScanMessage(scanner, l.Message, format)
+				seq, _, _ := sequence.ScanMessage(scanner, l.Message, format)
 				fmt.Fprintf(ofile, "%s\n\n", seq.PrintTokens())
 			}
 		}
@@ -157,9 +157,9 @@ func analyzebyservice(cmd *cobra.Command, args []string) {
 	start("analyzebyservice")
 	scanner := sequence.NewScanner()
 	var (
-		err      error
-		aseq     sequence.Sequence
-		mtype    string
+		err   error
+		aseq  sequence.Sequence
+		mtype string
 	)
 	iscan, ifile, err := sequence.OpenInputFile(infile)
 	if err != nil {

config.go

@@ -98,7 +98,6 @@ func ReadConfig(file string) error {
 	config.connectionInfo = configInfo.ConnectionInfo
 	config.databaseType = configInfo.DatabaseType
 
-
 	timesettings.formats = make(map[int][]string, len(configInfo.Timesettings.Formats))
 	for i, f := range configInfo.Timesettings.Formats {
 		x, err := strconv.Atoi(i)

databasehandler.go

@@ -27,7 +27,7 @@ func CreateDatabase(cinfo string, driver string, path string, dbname string) {
 	if err != nil {
 		logger.HandleFatal(err.Error())
 	}
-    if driver == "sqlite3" {
+	if driver == "sqlite3" {
 		s, file, err := OpenInputFile("database_scripts/sqlite3.txt")
 		defer file.Close()
 		for s.Scan() {
@@ -242,7 +242,7 @@ func addPattern(ctx context.Context, tx *sql.Tx, result AnalyzerResult, tr int)
 	}
 	tp := null.String{String: result.TagPositions, Valid: true}
 	p := models.Pattern{ID: result.PatternId, ServiceID: result.Service.ID, SequencePattern: result.Pattern, DateCreated: time.Now(),
-		CumulativeMatchCount: int64(result.ExampleCount), OriginalMatchCount: int64(result.ExampleCount), DateLastMatched: time.Now(), IgnorePattern: false, TagPositions: tp, ComplexityScore:result.ComplexityScore}
+		CumulativeMatchCount: int64(result.ExampleCount), OriginalMatchCount: int64(result.ExampleCount), DateLastMatched: time.Now(), IgnorePattern: false, TagPositions: tp, ComplexityScore: result.ComplexityScore}
 	err := p.Insert(ctx, tx, boil.Whitelist("id", "service_id", "sequence_pattern", "date_created", "date_last_matched", "original_match_count", "cumulative_match_count", "ignore_pattern", "tag_positions", "complexity_score"))
 	if err != nil {
 		logger.DatabaseInsertFailed("pattern", result.PatternId, err.Error())
@@ -254,10 +254,10 @@ func addPattern(ctx context.Context, tx *sql.Tx, result AnalyzerResult, tr int)
 	return true
 }
 
-func SaveIgnoredPatterns(pattids []string){
+func SaveIgnoredPatterns(pattids []string) {
 	db, ctx := OpenDbandSetContext()
 	defer db.Close()
-	for _, p := range pattids{
+	for _, p := range pattids {
 		ignorePattern(ctx, db, p)
 	}
 }

logrecord.go

@@ -70,7 +70,6 @@ func ReadLogRecord(fname string, format string, lr []LogRecord, batchLimit int)
 	return lr
 }
 
-
 //This method expects records in the format {"service": "service-name", message: "log message"}
 //eg {"service":"remctld","message":"error receiving initial token: unexpected end of file"} if json or for text
 //service [space] message, eg: remctld error receiving initial token: unexpected end of file.

logstash_grok/logstash_grok.go

@@ -9,8 +9,8 @@ package logstash_grok
 import (
 	"fmt"
 	"github.com/BurntSushi/toml"
-	"index/suffixarray"
 	"gitlab.in2p3.fr/cc-in2p3-system/sequence"
+	"index/suffixarray"
 	"sort"
 	"strconv"
 	"strings"
@@ -25,7 +25,6 @@ var (
 	logger *sequence.StandardLogger
 )
 
-
 func SetLogger(log *sequence.StandardLogger) {
 	logger = log
 }
@@ -57,10 +56,10 @@ func readConfig(file string) error {
 //Transformed patterns need to be reviewed before use in production.
 func OutputToFiles(outfile string, config string, complexitylevel float64, cmap map[string]sequence.AnalyzerResult, thresholdType string, thresholdValue string) (int, string, error) {
 	var (
-		err   error
-		count int
-		top5     string
-		patmap   map[string]sequence.AnalyzerResult
+		err    error
+		count  int
+		top5   string
+		patmap map[string]sequence.AnalyzerResult
 	)
 
 	if config == "" {
@@ -74,7 +73,7 @@ func OutputToFiles(outfile string, config string, complexitylevel float64, cmap
 		db, ctx := sequence.OpenDbandSetContext()
 		defer db.Close()
 		//get from the config instead
-		if thresholdType == ""{
+		if thresholdType == "" {
 			thresholdType = sequence.GetThresholdType()
 			thresholdValue = sequence.GetThresholdValue()
 		}

message.go

@@ -276,10 +276,10 @@ func (this *Message) scanToken(data string, nt int) (int, Token, error) {
 			} else if hexLen > 0 && this.state.hexColons > 1 {
 				if this.state.hexColons == 5 && this.state.hexMaxSuccColons == 1 {
 					//if the end of the message there won't be an extra space
-					if i == l-1{
+					if i == l-1 {
 						return hexLen, Token{Type: TokenMac, Tag: tagType}, nil
 					} else {
-						return hexLen-1, Token{Type: TokenMac, Tag: tagType}, nil
+						return hexLen - 1, Token{Type: TokenMac, Tag: tagType}, nil
 					}
 
 				} else if this.state.hexSuccColonsSeries == 1 ||
@@ -316,7 +316,7 @@ func (this *Message) scanToken(data string, nt int) (int, Token, error) {
 					}
 				//if the sentence finishes with a . it is likely to be punctuation remove it so it makes its own token.
 				case TokenLiteral:
-					if r == '.' && i == l-1 && tokenLen > 1{
+					if r == '.' && i == l-1 && tokenLen > 1 {
 						tokenLen--
 					}
 				}
@@ -535,7 +535,7 @@ func (this *Message) tokenStep(i int, r rune, nr string) bool {
 
 	//glog.Debugf("2. i=%d, r=%c, tokenStop=%t, tokenType=%s", i, r, this.state.tokenStop, this.state.tokenType)
 	//this sets the backslash for escaping values in the next round
-	this.state.backslash =  r == '\\'
+	this.state.backslash = r == '\\'
 	return this.state.tokenStop
 }
 

scanner.go

@@ -127,8 +127,8 @@ func (this *Scanner) Scan(s string, isParse bool, pos []int) (Sequence, bool, er
 	this.seq = this.seq[:0]
 
 	var (
-		err error
-		tok Token
+		err    error
+		tok    Token
 		isJson = false
 	)
 
@@ -262,13 +262,13 @@ func (this *Scanner) ScanJson(s string) (Sequence, bool, error) {
 				// start quote, ignore, move on
 				//state = jsonObjectStart
 				if kquote = !kquote; !kquote {
-					return nil,isJson, fmt.Errorf("Invalid message. Expecting start quote for key, got end quote.")
+					return nil, isJson, fmt.Errorf("Invalid message. Expecting start quote for key, got end quote.")
 				}
 
 			case "}":
 				// got something like {}, ignore this key
 				if len(keys)-1 < 0 {
-					return nil,isJson, fmt.Errorf("Invalid message. Too many } characters.")
+					return nil, isJson, fmt.Errorf("Invalid message. Too many } characters.")
 				}
 
 				keys = keys[:len(keys)-1]
@@ -279,7 +279,7 @@ func (this *Scanner) ScanJson(s string) (Sequence, bool, error) {
 					//glog.Debugf("depth=%d, keys=%v", len(keys), keys)
 					switch len(keys) {
 					case 0:
-						return nil,isJson, fmt.Errorf("Invalid message. Expecting inside object, not so.")
+						return nil, isJson, fmt.Errorf("Invalid message. Expecting inside object, not so.")
 
 					case 1:
 						keys[0] = tok.Value
@@ -504,10 +504,10 @@ func (this *Scanner) ScanJson(s string) (Sequence, bool, error) {
 	}
 
 	if err != nil && err != io.EOF {
-		return nil, isJson,  err
+		return nil, isJson, err
 	}
 
-	return this.seq, isJson,  nil
+	return this.seq, isJson, nil
 }
 
 //This is essentially the same function as Scan Json about but it preserves the structure of the message for text matching.
@@ -650,7 +650,7 @@ func (this *Scanner) ScanJson_Preserve(s string) (Sequence, bool, error) {
 			default:
 				state = jsonObjectValue
 				tok.isValue = true
-				if tok.Type == TokenLiteral{
+				if tok.Type == TokenLiteral {
 					tok.Type = TokenString
 				}
 			}
@@ -745,7 +745,7 @@ func (this *Scanner) ScanJson_Preserve(s string) (Sequence, bool, error) {
 			default:
 				state = jsonArrayValue
 				tok.isValue = true
-				if tok.Type == TokenLiteral{
+				if tok.Type == TokenLiteral {
 					tok.Type = TokenString
 				}
 				this.insertToken(tok)

syslog_ng_pattern_db/output_pattern_db_yaml.go

@@ -2,10 +2,10 @@ package syslog_ng_pattern_db
 
 import (
 	"fmt"
+	"gitlab.in2p3.fr/cc-in2p3-system/sequence"
 	"gopkg.in/yaml.v3"
 	"math"
 	"os"
-	"gitlab.in2p3.fr/cc-in2p3-system/sequence"
 	"time"
 )
 
@@ -25,10 +25,10 @@ type yRule struct {
 }
 
 type yRuleValues struct {
-	Complexity      float64`yaml:"seq-complexity"`
-	Seqmatches      int    `yaml:"seq-matches"`
-	DateCreated     string `yaml:"seq-created"`
-	DateLastMatched string `yaml:"seq-last-match"`
+	Complexity      float64 `yaml:"seq-complexity"`
+	Seqmatches      int     `yaml:"seq-matches"`
+	DateCreated     string  `yaml:"seq-created"`
+	DateLastMatched string  `yaml:"seq-last-match"`
 }
 
 //This represents a ruleset section in the sys-log ng yaml file
@@ -91,7 +91,7 @@ func buildRule(result sequence.AnalyzerResult, rsName string) yRule {
 	}
 	rule.Values.DateCreated = result.DateCreated.Format("2006-01-02")
 	rule.Values.DateLastMatched = result.DateLastMatched.Format("2006-01-02")
-	rule.Values.Complexity = math.Round(result.ComplexityScore*100)/100
+	rule.Values.Complexity = math.Round(result.ComplexityScore*100) / 100
 	//create a new UUID
 	rule.ID = result.PatternId
 	return rule

syslog_ng_pattern_db/pattern_db.go

@@ -56,7 +56,7 @@ func readConfig(file string) error {
 //first we replace the easy ones that are surrounded by spaces
 //then we deal with the compound ones
 func replaceTags(pattern string) string {
-	if len(pattern) < 1{
+	if len(pattern) < 1 {
 		return pattern
 	}
 	//make sure @ are escaped @@ before we start
@@ -235,7 +235,6 @@ func getTimeRegex(p string) (string, string) {
 	return p, rg
 }
 
-
 //This is the function that drives the output to file.
 //The user can pass the pattern map if no database is used or
 //pass the map created during the analysis
@@ -264,7 +263,7 @@ func OutputToFiles(outformat string, outfile string, config string, complexityle
 		db, ctx := sequence.OpenDbandSetContext()
 		defer db.Close()
 		//get from the config instead
-		if thresholdType == ""{
+		if thresholdType == "" {
 			thresholdType = sequence.GetThresholdType()
 			thresholdValue = sequence.GetThresholdValue()
 		}

utils.go

@@ -13,10 +13,10 @@ import (
 //Scans the message using the appropriate format
 func ScanMessage(scanner *Scanner, data string, format string) (Sequence, bool, error) {
 	var (
-		seq Sequence
+		seq    Sequence
 		isJson bool
-		err error
-		pos []int
+		err    error
+		pos    []int
 	)
 
 	if testJson(data) {
@@ -172,4 +172,4 @@ func SaveLogMessages(lr LogRecordCollection, fname string) {
 	for _, r := range lr.Records {
 		fmt.Fprintf(ofile, "%s  %s\n", r.Service, r.Message)
 	}
-}
+}