diff --git a/.github/workflows/bench-go.yml b/.github/workflows/bench-go.yml
index 661198cd..3df2d30b 100644
--- a/.github/workflows/bench-go.yml
+++ b/.github/workflows/bench-go.yml
@@ -56,4 +56,4 @@ jobs:
- name: Bench transformers
run: |
cd transformers/
- go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*$
\ No newline at end of file
+ go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*\|BenchmarkNormalize.*$
\ No newline at end of file
diff --git a/README.md b/README.md
index 71313958..057c572d 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
-
+
diff --git a/config.yml b/config.yml
index 93ee7a79..63d60be4 100644
--- a/config.yml
+++ b/config.yml
@@ -826,6 +826,7 @@ multiplexer:
# # additionnals directive for text format
# # - publicsuffix-tld: tld
# # - publicsuffix-etld+1: effective tld plus one
+# # - publicsuffix-managed-icann: flag for managed icann domains
# normalize:
# # Wwww.GooGlE.com will be equal to www.google.com
# qname-lowercase: true
diff --git a/dnsutils/message.go b/dnsutils/message.go
index c5ecd722..571825a7 100644
--- a/dnsutils/message.go
+++ b/dnsutils/message.go
@@ -186,6 +186,7 @@ type TransformSuspicious struct {
type TransformPublicSuffix struct {
QnamePublicSuffix string `json:"tld"`
QnameEffectiveTLDPlusOne string `json:"etld+1"`
+ ManagedByICANN bool `json:"managed-icann"`
}
type TransformExtracted struct {
@@ -456,6 +457,12 @@ func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings.
s.WriteString(dm.PublicSuffix.QnamePublicSuffix)
case directive == "publixsuffix-etld+1":
s.WriteString(dm.PublicSuffix.QnameEffectiveTLDPlusOne)
+ case directive == "publixsuffix-managed-icann":
+ if dm.PublicSuffix.ManagedByICANN {
+ s.WriteString("managed")
+ } else {
+ s.WriteString("private")
+ }
default:
return errors.New(ErrorUnexpectedDirective + directive)
}
@@ -1169,6 +1176,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
if dm.PublicSuffix != nil {
dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix
dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne
+ dnsFields["publicsuffix.managed-icann"] = dm.PublicSuffix.ManagedByICANN
}
// Add TransformExtracted fields
diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go
index 6c947e13..fe7a08d0 100644
--- a/dnsutils/message_test.go
+++ b/dnsutils/message_test.go
@@ -363,12 +363,14 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) {
PublicSuffix: &TransformPublicSuffix{
QnamePublicSuffix: "com",
QnameEffectiveTLDPlusOne: "hello.com",
+ ManagedByICANN: true,
},
},
jsonRef: `{
"publicsuffix": {
"tld": "com",
- "etld+1": "hello.com"
+ "etld+1": "hello.com",
+ "managed-icann": true
}
}`,
},
diff --git a/docs/transformers/transform_normalize.md b/docs/transformers/transform_normalize.md
index 13edc053..72db641f 100644
--- a/docs/transformers/transform_normalize.md
+++ b/docs/transformers/transform_normalize.md
@@ -51,6 +51,7 @@ Example:
"publicsuffix": {
"etld+1": "eu.org",
"tld": "org",
+ "managed-icann": true
}
```
@@ -58,3 +59,4 @@ Specific directives added for text format:
- `publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME
- `publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME
+- `publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains
diff --git a/transformers/normalize.go b/transformers/normalize.go
index 5226b242..dee77562 100644
--- a/transformers/normalize.go
+++ b/transformers/normalize.go
@@ -138,6 +138,7 @@ func (p *NormalizeProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
dm.PublicSuffix = &dnsutils.TransformPublicSuffix{
QnamePublicSuffix: "-",
QnameEffectiveTLDPlusOne: "-",
+ ManagedByICANN: false,
}
}
}
@@ -174,8 +175,9 @@ func (p *NormalizeProcessor) GetEffectiveTld(dm *dnsutils.DNSMessage) int {
etld, icann := publicsuffixlist.PublicSuffix(qname)
if icann {
dm.PublicSuffix.QnamePublicSuffix = etld
+ dm.PublicSuffix.ManagedByICANN = true
} else {
- p.logError("suffix unmanaged by icann: %s", qname)
+ dm.PublicSuffix.ManagedByICANN = false
}
return ReturnSuccess
}
diff --git a/transformers/normalize_test.go b/transformers/normalize_test.go
index f8fa5195..6026f1a6 100644
--- a/transformers/normalize_test.go
+++ b/transformers/normalize_test.go
@@ -1,8 +1,6 @@
package transformers
import (
- "encoding/json"
- "reflect"
"strings"
"testing"
@@ -11,52 +9,6 @@ import (
"github.com/dmachard/go-logger"
)
-func TestNormalize_Json(t *testing.T) {
- // enable feature
- config := pkgconfig.GetFakeConfigTransformers()
-
- log := logger.New(false)
- outChans := []chan dnsutils.DNSMessage{}
-
- // get fake
- dm := dnsutils.GetFakeDNSMessage()
- dm.Init()
-
- // init subproccesor
- qnameNorm := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
- qnameNorm.InitDNSMessage(&dm)
-
- // expected json
- refJSON := `
- {
- "publicsuffix": {
- "tld":"-",
- "etld+1":"-"
- }
- }
- `
-
- var dmMap map[string]interface{}
- err := json.Unmarshal([]byte(dm.ToJSON()), &dmMap)
- if err != nil {
- t.Fatalf("could not unmarshal dm json: %s\n", err)
- }
-
- var refMap map[string]interface{}
- err = json.Unmarshal([]byte(refJSON), &refMap)
- if err != nil {
- t.Fatalf("could not unmarshal ref json: %s\n", err)
- }
-
- if _, ok := dmMap["publicsuffix"]; !ok {
- t.Fatalf("transformer key is missing")
- }
-
- if !reflect.DeepEqual(dmMap["publicsuffix"], refMap["publicsuffix"]) {
- t.Errorf("json format different from reference")
- }
-}
-
func TestNormalize_LowercaseQname(t *testing.T) {
// enable feature
config := pkgconfig.GetFakeConfigTransformers()
@@ -199,3 +151,122 @@ func TestNormalize_AddTldPlusOne(t *testing.T) {
})
}
}
+
+func TestNormalize_SuffixUnmanaged(t *testing.T) {
+ // enable feature
+ config := pkgconfig.GetFakeConfigTransformers()
+ log := logger.New(true)
+ outChans := []chan dnsutils.DNSMessage{}
+
+ // init the processor
+ psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error)
+
+ dm := dnsutils.GetFakeDNSMessage()
+ // https://publicsuffix.org/list/effective_tld_names.dat
+ // // ===BEGIN ICANN DOMAINS===
+ // ....
+ // // ===END ICANN DOMAINS===
+ // ===BEGIN PRIVATE DOMAINS===
+ // ..
+ dm.DNS.Qname = "play.googleapis.com"
+ // // ===END PRIVATE DOMAINS===
+
+ psl.InitDNSMessage(&dm)
+ psl.GetEffectiveTld(&dm)
+ if dm.PublicSuffix.ManagedByICANN {
+ t.Errorf("Qname %s should be private domains", dm.DNS.Qname)
+ }
+}
+
+func TestNormalize_SuffixICANNManaged(t *testing.T) {
+ // enable feature
+ config := pkgconfig.GetFakeConfigTransformers()
+ log := logger.New(true)
+ outChans := []chan dnsutils.DNSMessage{}
+
+ // init the processor
+ psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error)
+
+ dm := dnsutils.GetFakeDNSMessage()
+ // https://publicsuffix.org/list/effective_tld_names.dat
+ // // ===BEGIN ICANN DOMAINS===
+ dm.DNS.Qname = "fr.wikipedia.org"
+ // // ===END ICANN DOMAINS===
+ // ===BEGIN PRIVATE DOMAINS===
+ // ..
+ // // ===END PRIVATE DOMAINS===
+
+ psl.InitDNSMessage(&dm)
+ psl.GetEffectiveTld(&dm)
+ if !dm.PublicSuffix.ManagedByICANN {
+ t.Errorf("Qname %s should be ICANN managed", dm.DNS.Qname)
+ }
+}
+
+// bench tests
+
+func BenchmarkNormalize_GetEffectiveTld(b *testing.B) {
+ config := pkgconfig.GetFakeConfigTransformers()
+
+ log := logger.New(false)
+ channels := []chan dnsutils.DNSMessage{}
+
+ subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+ dm := dnsutils.GetFakeDNSMessage()
+ dm.DNS.Qname = "en.wikipedia.org"
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ subprocessor.InitDNSMessage(&dm)
+ subprocessor.GetEffectiveTld(&dm)
+ }
+}
+
+func BenchmarkNormalize_GetEffectiveTldPlusOne(b *testing.B) {
+ config := pkgconfig.GetFakeConfigTransformers()
+
+ log := logger.New(false)
+ channels := []chan dnsutils.DNSMessage{}
+
+ subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+ dm := dnsutils.GetFakeDNSMessage()
+ dm.DNS.Qname = "en.wikipedia.org"
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ subprocessor.InitDNSMessage(&dm)
+ subprocessor.GetEffectiveTld(&dm)
+ }
+}
+
+func BenchmarkNormalize_QnameLowercase(b *testing.B) {
+ config := pkgconfig.GetFakeConfigTransformers()
+
+ log := logger.New(false)
+ channels := []chan dnsutils.DNSMessage{}
+
+ subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+ dm := dnsutils.GetFakeDNSMessage()
+ dm.DNS.Qname = "EN.Wikipedia.Org"
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ subprocessor.LowercaseQname(&dm)
+ }
+}
+
+func BenchmarkNormalize_QuietText(b *testing.B) {
+ config := pkgconfig.GetFakeConfigTransformers()
+
+ log := logger.New(false)
+ channels := []chan dnsutils.DNSMessage{}
+
+ subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+ dm := dnsutils.GetFakeDNSMessage()
+ dm.DNS.Qname = "EN.Wikipedia.Org"
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ subprocessor.QuietText(&dm)
+ }
+}