add bench

.github/workflows/bench-go.yml

@@ -48,7 +48,12 @@ jobs:
         go version
         sudo go version
         
-    - name: Bench
+    - name: Bench dnsutils
       run: |
         cd dnsutils/
-        go test -run=^$ -bench=.
+        go test -benchmem -run=^$ -bench=.
+    
+    - name: Bench transformers
+      run: |
+        cd ../transformers/
+        go test -benchmem -run=^$ -bench=.

docs/performance.md

@@ -17,13 +17,14 @@ The conversion of DNS logs to JSON, text, or PCAP can incur CPU costs. Here's a
 goos: linux
 goarch: amd64
 pkg: github.com/dmachard/go-dnscollector/dnsutils
-cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz
-BenchmarkDnsMessage_ToTextFormat-4               2600718             460.7 ns/op
-BenchmarkDnsMessage_ToPacketLayer-4              1171467             969.5 ns/op
-BenchmarkDnsMessage_ToDNSTap-4                    993242              1130 ns/op
-BenchmarkDnsMessage_ToExtendedDNSTap-4            618400              1951 ns/op
-BenchmarkDnsMessage_ToJSON-4                      190939              6584 ns/op
-BenchmarkDnsMessage_ToFlatJSON-4                   19868             55533 ns/op
+cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz 
+BenchmarkDnsMessage_ToTextFormat-4               2555529             450.2 ns/op              80 B/op          4 allocs/op
+BenchmarkDnsMessage_ToPacketLayer-4              1138892             952.0 ns/op            1144 B/op         12 allocs/op
+BenchmarkDnsMessage_ToDNSTap-4                   1036468              1136 ns/op             592 B/op         18 allocs/op
+BenchmarkDnsMessage_ToExtendedDNSTap-4            612438              1970 ns/op            1056 B/op         25 allocs/op
+BenchmarkDnsMessage_ToJSON-4                      188379              6724 ns/op            3632 B/op          3 allocs/op
+BenchmarkDnsMessage_ToFlatten-4                   121525             10151 ns/op            8215 B/op         29 allocs/op
+BenchmarkDnsMessage_ToFlatJSON-4                   20704             58365 ns/op           22104 B/op        220 allocs/op
 ```
 
 ## Memory usage

go.mod

@@ -26,7 +26,6 @@ require (
 	github.com/klauspost/compress v1.17.6
 	github.com/miekg/dns v1.1.58
 	github.com/natefinch/lumberjack v2.0.0+incompatible
-	github.com/nqd/flat v0.2.0
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/rs/tzsp v0.0.0-20161230003637-8ce729c826b9
@@ -78,7 +77,6 @@ require (
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/memberlist v0.5.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
-	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect

go.sum

@@ -720,9 +720,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
-github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk=
 github.com/influxdata/influxdb-client-go v1.4.0/go.mod h1:S+oZsPivqbcP1S9ur+T+QqXvrYS3NCZeMQtBoH4D1dw=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
@@ -821,8 +818,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nqd/flat v0.2.0 h1:g6lXtMxsxrz6PZOO+rNnAJUn/GGRrK4FgVEhy/v+cHI=
-github.com/nqd/flat v0.2.0/go.mod h1:FOuslZmNY082wVfVUUb7qAGWKl8z8Nor9FMg+Xj2Nss=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI=

transformers/latency_test.go

@@ -143,6 +143,7 @@ func Test_HashQueries_Expire(t *testing.T) {
 	}
 }
 
+// Bench
 func Benchmark_HashQueries_Set(b *testing.B) {
 	mapexpire := NewHashQueries(10 * time.Second)
 

transformers/subprocessors_test.go

@@ -17,30 +17,29 @@ const (
 )
 
 // Bench to init DNS message
-func BenchmarkTransforms_Init(b *testing.B) {
+func BenchmarkTransforms_InitAndProcess(b *testing.B) {
 	config := pkgconfig.GetFakeConfigTransformers()
 	config.Suspicious.Enable = true
 	config.GeoIP.Enable = true
-	config.GeoIP.DBCountryFile = "../testsdata/GeoLite2-Country.mmdb"
-	config.GeoIP.Enable = true
-	config.GeoIP.DBASNFile = "../testsdata/GeoLite2-ASN.mmdb"
+	config.GeoIP.DBCountryFile = ".././testsdata/GeoLite2-Country.mmdb"
+	config.GeoIP.DBASNFile = ".././testsdata/GeoLite2-ASN.mmdb"
 	config.UserPrivacy.Enable = true
 	config.UserPrivacy.MinimazeQname = true
-	config.UserPrivacy.Enable = true
 	config.UserPrivacy.AnonymizeIP = true
 	config.Normalize.Enable = true
 	config.Normalize.QnameLowerCase = true
 	config.Filtering.Enable = true
-	config.Filtering.KeepDomainFile = "../testsdata/filtering_keep_domains.txt"
+	config.Filtering.KeepDomainFile = ".././testsdata/filtering_keep_domains.txt"
 
 	channels := []chan dnsutils.DNSMessage{}
-	subprocessors := NewTransforms(config, logger.New(false), "test", channels, 0)
+	transformers := NewTransforms(config, logger.New(false), "test", channels, 0)
 
 	dm := dnsutils.GetFakeDNSMessage()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		subprocessors.InitDNSMessageFormat(&dm)
+		transformers.InitDNSMessageFormat(&dm)
+		transformers.ProcessMessage(&dm)
 	}
 }
 
@@ -173,10 +172,10 @@ func TestTransformsReduceQname(t *testing.T) {
 	}
 
 	// test 3: local.home
-	dm.DNS.Qname = "localhost.domain.local.home"
+	dm.DNS.Qname = "localhost.domain.localtest.home"
 	returnCode = subprocessors.ProcessMessage(&dm)
 
-	if dm.DNS.Qname != "local.home" {
+	if dm.DNS.Qname != "localtest.home" {
 		t.Errorf("Qname minimization failed, got %s", dm.DNS.Qname)
 	}
 	if returnCode != ReturnSuccess {

transformers/userprivacy.go

@@ -130,7 +130,7 @@ func (s *UserPrivacyProcessor) HashIP(ip string) string {
 		hash := sha256.New()
 		hash.Write([]byte(ip))
 		return fmt.Sprintf("%x", hash.Sum(nil))
-	case "sha512":
+	case "sha512": // nolint
 		hash := sha512.New()
 		hash.Write([]byte(ip))
 		return fmt.Sprintf("%x", hash.Sum(nil))

transformers/userprivacy_test.go

@@ -13,6 +13,74 @@ var (
 	TestIP6 = "fe80::6111:626:c1b2:2353"
 )
 
+// bench
+func BenchmarkUserPrivacy_ReduceQname(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.MinimazeQname = true
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+	qname := "localhost.domain.local.home"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.MinimazeQname(qname)
+	}
+}
+
+func BenchmarkUserPrivacy_HashIP(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.HashIP = true
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.HashIP(TestIP4)
+	}
+}
+
+func BenchmarkUserPrivacy_HashIPSha512(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.HashIP = true
+	config.UserPrivacy.HashIPAlgo = "sha512"
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.HashIP(TestIP4)
+	}
+}
+
+func BenchmarkUserPrivacy_AnonymizeIP(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.UserPrivacy.Enable = true
+	config.UserPrivacy.AnonymizeIP = true
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.AnonymizeIP(TestIP4)
+	}
+}
+
+// other tests
 func TestUserPrivacy_ReduceQname(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()