From 774e96fdc54ef7a486198e800c7ef755a5dd7135 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 8 Mar 2024 14:14:01 +0100 Subject: [PATCH 1/2] add bench tests for normalize trasnform add icann managed flag in dnsmessage --- .github/workflows/bench-go.yml | 2 +- README.md | 2 +- config.yml | 1 + dnsutils/message.go | 8 ++ dnsutils/message_test.go | 4 +- docs/transformers/transform_normalize.md | 2 + transformers/normalize.go | 4 +- transformers/normalize_test.go | 167 ++++++++++++++++------- 8 files changed, 138 insertions(+), 52 deletions(-) diff --git a/.github/workflows/bench-go.yml b/.github/workflows/bench-go.yml index 661198cd..3df2d30b 100644 --- a/.github/workflows/bench-go.yml +++ b/.github/workflows/bench-go.yml @@ -56,4 +56,4 @@ jobs: - name: Bench transformers run: | cd transformers/ - go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*$ \ No newline at end of file + go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*\|BenchmarkNormalize.*$ \ No newline at end of file diff --git a/README.md b/README.md index 71313958..057c572d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Go Report Go version Go tests -Go bench +Go bench Go lines

diff --git a/config.yml b/config.yml index 93ee7a79..63d60be4 100644 --- a/config.yml +++ b/config.yml @@ -826,6 +826,7 @@ multiplexer: # # additionnals directive for text format # # - publicsuffix-tld: tld # # - publicsuffix-etld+1: effective tld plus one +# # - publicsuffix-managed-icann: flag for managed icann domains # normalize: # # Wwww.GooGlE.com will be equal to www.google.com # qname-lowercase: true diff --git a/dnsutils/message.go b/dnsutils/message.go index c5ecd722..571825a7 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -186,6 +186,7 @@ type TransformSuspicious struct { type TransformPublicSuffix struct { QnamePublicSuffix string `json:"tld"` QnameEffectiveTLDPlusOne string `json:"etld+1"` + ManagedByICANN bool `json:"managed-icann"` } type TransformExtracted struct { @@ -456,6 +457,12 @@ func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings. s.WriteString(dm.PublicSuffix.QnamePublicSuffix) case directive == "publixsuffix-etld+1": s.WriteString(dm.PublicSuffix.QnameEffectiveTLDPlusOne) + case directive == "publixsuffix-managed-icann": + if dm.PublicSuffix.ManagedByICANN { + s.WriteString("managed") + } else { + s.WriteString("private") + } default: return errors.New(ErrorUnexpectedDirective + directive) } @@ -1169,6 +1176,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { if dm.PublicSuffix != nil { dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne + dnsFields["publicsuffix.managed-icann"] = dm.PublicSuffix.ManagedByICANN } // Add TransformExtracted fields diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 6c947e13..a50be42d 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -363,12 +363,14 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) { PublicSuffix: &TransformPublicSuffix{ QnamePublicSuffix: "com", QnameEffectiveTLDPlusOne: "hello.com", + ManagedByICANN: true, }, }, jsonRef: `{ "publicsuffix": { "tld": "com", - "etld+1": "hello.com" + "etld+1": "hello.com", + "managed-icann: true } }`, }, diff --git a/docs/transformers/transform_normalize.md b/docs/transformers/transform_normalize.md index 13edc053..72db641f 100644 --- a/docs/transformers/transform_normalize.md +++ b/docs/transformers/transform_normalize.md @@ -51,6 +51,7 @@ Example: "publicsuffix": { "etld+1": "eu.org", "tld": "org", + "managed-icann": true } ``` @@ -58,3 +59,4 @@ Specific directives added for text format: - `publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME - `publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME +- `publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains diff --git a/transformers/normalize.go b/transformers/normalize.go index 5226b242..dee77562 100644 --- a/transformers/normalize.go +++ b/transformers/normalize.go @@ -138,6 +138,7 @@ func (p *NormalizeProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) { dm.PublicSuffix = &dnsutils.TransformPublicSuffix{ QnamePublicSuffix: "-", QnameEffectiveTLDPlusOne: "-", + ManagedByICANN: false, } } } @@ -174,8 +175,9 @@ func (p *NormalizeProcessor) GetEffectiveTld(dm *dnsutils.DNSMessage) int { etld, icann := publicsuffixlist.PublicSuffix(qname) if icann { dm.PublicSuffix.QnamePublicSuffix = etld + dm.PublicSuffix.ManagedByICANN = true } else { - p.logError("suffix unmanaged by icann: %s", qname) + dm.PublicSuffix.ManagedByICANN = false } return ReturnSuccess } diff --git a/transformers/normalize_test.go b/transformers/normalize_test.go index f8fa5195..9e8805c7 100644 --- a/transformers/normalize_test.go +++ b/transformers/normalize_test.go @@ -1,8 +1,6 @@ package transformers import ( - "encoding/json" - "reflect" "strings" "testing" @@ -11,52 +9,6 @@ import ( "github.com/dmachard/go-logger" ) -func TestNormalize_Json(t *testing.T) { - // enable feature - config := pkgconfig.GetFakeConfigTransformers() - - log := logger.New(false) - outChans := []chan dnsutils.DNSMessage{} - - // get fake - dm := dnsutils.GetFakeDNSMessage() - dm.Init() - - // init subproccesor - qnameNorm := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - qnameNorm.InitDNSMessage(&dm) - - // expected json - refJSON := ` - { - "publicsuffix": { - "tld":"-", - "etld+1":"-" - } - } - ` - - var dmMap map[string]interface{} - err := json.Unmarshal([]byte(dm.ToJSON()), &dmMap) - if err != nil { - t.Fatalf("could not unmarshal dm json: %s\n", err) - } - - var refMap map[string]interface{} - err = json.Unmarshal([]byte(refJSON), &refMap) - if err != nil { - t.Fatalf("could not unmarshal ref json: %s\n", err) - } - - if _, ok := dmMap["publicsuffix"]; !ok { - t.Fatalf("transformer key is missing") - } - - if !reflect.DeepEqual(dmMap["publicsuffix"], refMap["publicsuffix"]) { - t.Errorf("json format different from reference") - } -} - func TestNormalize_LowercaseQname(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers() @@ -199,3 +151,122 @@ func TestNormalize_AddTldPlusOne(t *testing.T) { }) } } + +func TestNormalize_SuffixUnmanaged(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + log := logger.New(true) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error) + + dm := dnsutils.GetFakeDNSMessage() + // https://publicsuffix.org/list/effective_tld_names.dat + // // ===BEGIN ICANN DOMAINS=== + // .... + // // ===END ICANN DOMAINS=== + // ===BEGIN PRIVATE DOMAINS=== + // .. + dm.DNS.Qname = "play.googleapis.com" + // // ===END PRIVATE DOMAINS=== + + psl.InitDNSMessage(&dm) + psl.GetEffectiveTld(&dm) + if dm.PublicSuffix.ManagedByICANN { + t.Errorf("Qname %s should be private domains", dm.DNS.Qname) + } +} + +func TestNormalize_SuffixICANNManaged(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + log := logger.New(true) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error) + + dm := dnsutils.GetFakeDNSMessage() + // https://publicsuffix.org/list/effective_tld_names.dat + // // ===BEGIN ICANN DOMAINS=== + dm.DNS.Qname = "en.wikipedia.org" + // // ===END ICANN DOMAINS=== + // ===BEGIN PRIVATE DOMAINS=== + // .. + // // ===END PRIVATE DOMAINS=== + + psl.InitDNSMessage(&dm) + psl.GetEffectiveTld(&dm) + if !dm.PublicSuffix.ManagedByICANN { + t.Errorf("Qname %s should be ICANN managed", dm.DNS.Qname) + } +} + +// bench tests + +func BenchmarkNormalize_GetEffectiveTld(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + dm := dnsutils.GetFakeDNSMessage() + dm.DNS.Qname = "en.wikipedia.org" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.InitDNSMessage(&dm) + subprocessor.GetEffectiveTld(&dm) + } +} + +func BenchmarkNormalize_GetEffectiveTldPlusOne(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + dm := dnsutils.GetFakeDNSMessage() + dm.DNS.Qname = "en.wikipedia.org" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.InitDNSMessage(&dm) + subprocessor.GetEffectiveTld(&dm) + } +} + +func BenchmarkNormalize_QnameLowercase(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + dm := dnsutils.GetFakeDNSMessage() + dm.DNS.Qname = "EN.Wikipedia.Org" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.LowercaseQname(&dm) + } +} + +func BenchmarkNormalize_QuietText(b *testing.B) { + config := pkgconfig.GetFakeConfigTransformers() + + log := logger.New(false) + channels := []chan dnsutils.DNSMessage{} + + subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error) + dm := dnsutils.GetFakeDNSMessage() + dm.DNS.Qname = "EN.Wikipedia.Org" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + subprocessor.QuietText(&dm) + } +} From 23382211da74b3d39fd8e3fd2f31eb1275709de0 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 8 Mar 2024 20:06:06 +0100 Subject: [PATCH 2/2] fix test --- dnsutils/message_test.go | 2 +- transformers/normalize_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index a50be42d..fe7a08d0 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -370,7 +370,7 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) { "publicsuffix": { "tld": "com", "etld+1": "hello.com", - "managed-icann: true + "managed-icann": true } }`, }, diff --git a/transformers/normalize_test.go b/transformers/normalize_test.go index 9e8805c7..6026f1a6 100644 --- a/transformers/normalize_test.go +++ b/transformers/normalize_test.go @@ -190,7 +190,7 @@ func TestNormalize_SuffixICANNManaged(t *testing.T) { dm := dnsutils.GetFakeDNSMessage() // https://publicsuffix.org/list/effective_tld_names.dat // // ===BEGIN ICANN DOMAINS=== - dm.DNS.Qname = "en.wikipedia.org" + dm.DNS.Qname = "fr.wikipedia.org" // // ===END ICANN DOMAINS=== // ===BEGIN PRIVATE DOMAINS=== // ..