From d0eef4633349e4ad0483d689ee8277ac145d99e7 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 6 Mar 2024 08:46:57 +0100 Subject: [PATCH] add bench --- .github/workflows/bench-go.yml | 9 +++- docs/performance.md | 15 ++++--- go.mod | 2 - go.sum | 5 --- transformers/latency_test.go | 1 + transformers/subprocessors_test.go | 19 ++++----- transformers/userprivacy.go | 2 +- transformers/userprivacy_test.go | 68 ++++++++++++++++++++++++++++++ 8 files changed, 94 insertions(+), 27 deletions(-) diff --git a/.github/workflows/bench-go.yml b/.github/workflows/bench-go.yml index 61aef575..a4ee650f 100644 --- a/.github/workflows/bench-go.yml +++ b/.github/workflows/bench-go.yml @@ -48,7 +48,12 @@ jobs: go version sudo go version - - name: Bench + - name: Bench dnsutils run: | cd dnsutils/ - go test -run=^$ -bench=. \ No newline at end of file + go test -benchmem -run=^$ -bench=. + + - name: Bench transformers + run: | + cd ../transformers/ + go test -benchmem -run=^$ -bench=. \ No newline at end of file diff --git a/docs/performance.md b/docs/performance.md index 19a3fb1b..1833a98c 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -17,13 +17,14 @@ The conversion of DNS logs to JSON, text, or PCAP can incur CPU costs. Here's a goos: linux goarch: amd64 pkg: github.com/dmachard/go-dnscollector/dnsutils -cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz -BenchmarkDnsMessage_ToTextFormat-4 2600718 460.7 ns/op -BenchmarkDnsMessage_ToPacketLayer-4 1171467 969.5 ns/op -BenchmarkDnsMessage_ToDNSTap-4 993242 1130 ns/op -BenchmarkDnsMessage_ToExtendedDNSTap-4 618400 1951 ns/op -BenchmarkDnsMessage_ToJSON-4 190939 6584 ns/op -BenchmarkDnsMessage_ToFlatJSON-4 19868 55533 ns/op +cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz +BenchmarkDnsMessage_ToTextFormat-4 2555529 450.2 ns/op 80 B/op 4 allocs/op +BenchmarkDnsMessage_ToPacketLayer-4 1138892 952.0 ns/op 1144 B/op 12 allocs/op +BenchmarkDnsMessage_ToDNSTap-4 1036468 1136 ns/op 592 B/op 18 allocs/op +BenchmarkDnsMessage_ToExtendedDNSTap-4 612438 1970 ns/op 1056 B/op 25 allocs/op +BenchmarkDnsMessage_ToJSON-4 188379 6724 ns/op 3632 B/op 3 allocs/op +BenchmarkDnsMessage_ToFlatten-4 121525 10151 ns/op 8215 B/op 29 allocs/op +BenchmarkDnsMessage_ToFlatJSON-4 20704 58365 ns/op 22104 B/op 220 allocs/op ``` ## Memory usage diff --git a/go.mod b/go.mod index 482eb0bc..9082e55b 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/klauspost/compress v1.17.6 github.com/miekg/dns v1.1.58 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/nqd/flat v0.2.0 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.18.0 github.com/rs/tzsp v0.0.0-20161230003637-8ce729c826b9 @@ -78,7 +77,6 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/memberlist v0.5.0 // indirect github.com/hashicorp/serf v0.10.1 // indirect - github.com/imdario/mergo v0.3.15 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect diff --git a/go.sum b/go.sum index 5b0bf7e6..eda396e2 100644 --- a/go.sum +++ b/go.sum @@ -720,9 +720,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk= github.com/influxdata/influxdb-client-go v1.4.0/go.mod h1:S+oZsPivqbcP1S9ur+T+QqXvrYS3NCZeMQtBoH4D1dw= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= @@ -821,8 +818,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nqd/flat v0.2.0 h1:g6lXtMxsxrz6PZOO+rNnAJUn/GGRrK4FgVEhy/v+cHI= -github.com/nqd/flat v0.2.0/go.mod h1:FOuslZmNY082wVfVUUb7qAGWKl8z8Nor9FMg+Xj2Nss= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= diff --git a/transformers/latency_test.go b/transformers/latency_test.go index 70e19540..4213d532 100644 --- a/transformers/latency_test.go +++ b/transformers/latency_test.go @@ -143,6 +143,7 @@ func Test_HashQueries_Expire(t *testing.T) { } } +// Bench func Benchmark_HashQueries_Set(b *testing.B) { mapexpire := NewHashQueries(10 * time.Second) diff --git a/transformers/subprocessors_test.go b/transformers/subprocessors_test.go index 7c1f97c1..9d608548 100644 --- a/transformers/subprocessors_test.go +++ b/transformers/subprocessors_test.go @@ -17,30 +17,29 @@ const ( ) // Bench to init DNS message -func BenchmarkTransforms_Init(b *testing.B) { +func BenchmarkTransforms_InitAndProcess(b *testing.B) { config := pkgconfig.GetFakeConfigTransformers() config.Suspicious.Enable = true config.GeoIP.Enable = true - config.GeoIP.DBCountryFile = "../testsdata/GeoLite2-Country.mmdb" - config.GeoIP.Enable = true - config.GeoIP.DBASNFile = "../testsdata/GeoLite2-ASN.mmdb" + config.GeoIP.DBCountryFile = ".././testsdata/GeoLite2-Country.mmdb" + config.GeoIP.DBASNFile = ".././testsdata/GeoLite2-ASN.mmdb" config.UserPrivacy.Enable = true config.UserPrivacy.MinimazeQname = true - config.UserPrivacy.Enable = true config.UserPrivacy.AnonymizeIP = true config.Normalize.Enable = true config.Normalize.QnameLowerCase = true config.Filtering.Enable = true - config.Filtering.KeepDomainFile = "../testsdata/filtering_keep_domains.txt" + config.Filtering.KeepDomainFile = ".././testsdata/filtering_keep_domains.txt" channels := []chan dnsutils.DNSMessage{} - subprocessors := NewTransforms(config, logger.New(false), "test", channels, 0) + transformers := NewTransforms(config, logger.New(false), "test", channels, 0) dm := dnsutils.GetFakeDNSMessage() b.ResetTimer() for i := 0; i < b.N; i++ { - subprocessors.InitDNSMessageFormat(&dm) + transformers.InitDNSMessageFormat(&dm) + transformers.ProcessMessage(&dm) } } @@ -173,10 +172,10 @@ func TestTransformsReduceQname(t *testing.T) { } // test 3: local.home - dm.DNS.Qname = "localhost.domain.local.home" + dm.DNS.Qname = "localhost.domain.localtest.home" returnCode = subprocessors.ProcessMessage(&dm) - if dm.DNS.Qname != "local.home" { + if dm.DNS.Qname != "localtest.home" { t.Errorf("Qname minimization failed, got %s", dm.DNS.Qname) } if returnCode != ReturnSuccess { diff --git a/transformers/userprivacy.go b/transformers/userprivacy.go index d3eb3f61..568eef21 100644 --- a/transformers/userprivacy.go +++ b/transformers/userprivacy.go @@ -130,7 +130,7 @@ func (s *UserPrivacyProcessor) HashIP(ip string) string { hash := sha256.New() hash.Write([]byte(ip)) return fmt.Sprintf("%x", hash.Sum(nil)) - case "sha512": + case "sha512": // nolint hash := sha512.New() hash.Write([]byte(ip)) return fmt.Sprintf("%x", hash.Sum(nil)) diff --git a/transformers/userprivacy_test.go b/transformers/userprivacy_test.go index 5970d204..b2572b2d 100644 --- a/transformers/userprivacy_test.go +++ b/transformers/userprivacy_test.go @@ -13,6 +13,74 @@ var ( TestIP6 = "fe80::6111:626:c1b2:2353" ) +// bench +func BenchmarkUserPrivacy_ReduceQname(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.MinimazeQname = true + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + qname := "localhost.domain.local.home" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.MinimazeQname(qname) + } +} + +func BenchmarkUserPrivacy_HashIP(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.HashIP = true + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.HashIP(TestIP4) + } +} + +func BenchmarkUserPrivacy_HashIPSha512(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.HashIP = true + config.UserPrivacy.HashIPAlgo = "sha512" + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.HashIP(TestIP4) + } +} + +func BenchmarkUserPrivacy_AnonymizeIP(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.AnonymizeIP = true + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.AnonymizeIP(TestIP4) + } +} + +// other tests func TestUserPrivacy_ReduceQname(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers()