From 07a1c5a1c1dba599b97046c9a37bd650641576fc Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:48:36 +0100 Subject: [PATCH 1/3] add flexible config #473 --- README.md | 2 +- config.yml | 8 +- docs/transformers/transform_userprivacy.md | 8 +- pkgconfig/transformers.go | 14 +++- transformers/userprivacy.go | 84 +++++++++++++++++--- transformers/userprivacy_test.go | 89 +++++++++++++++++++++- 6 files changed, 185 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1f6c78b4..b43fed16 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Multiplexer - *Send to remote host with generic transport protocol* - [`TCP`](docs/loggers/logger_tcp.md) - [`Syslog`](docs/loggers/logger_syslog.md) with TLS support - - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages + - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages with TLS support - *Send to various sinks* - [`Fluentd`](docs/loggers/logger_fluentd.md) - [`InfluxDB`](docs/loggers/logger_influxdb.md) diff --git a/config.yml b/config.yml index 7a8e6ed9..cdcbffa7 100644 --- a/config.yml +++ b/config.yml @@ -705,10 +705,16 @@ multiplexer: # user-privacy: # # IP-Addresses are anonymities by zeroing the host-part of an address. # anonymize-ip: false +# # summarize IPv4 down to the /integer level, default is /16 +# anonymize-v4bits: "/8" +# # summarize IPv6 down to the /integer level, default is /64 +# anonymize-v6bits: "::/64" # # Reduce Qname to second level only, for exemple mail.google.com be replaced by google.com # minimaze-qname: false -# # Hash query and response IP +# # Hashes the query and response IP with the specified algorithm. # hash-ip: false +# # Algorithm to use for IP hashing, currently supported `sha1` (default), `sha256`, `sha512` +# hash-ip-algo: sha1 # # Use this option to add top level domain and tld+1, based on public suffix list https://publicsuffix.org/ # # or convert all domain to lowercase diff --git a/docs/transformers/transform_userprivacy.md b/docs/transformers/transform_userprivacy.md index 43b316a4..84b041d0 100644 --- a/docs/transformers/transform_userprivacy.md +++ b/docs/transformers/transform_userprivacy.md @@ -9,13 +9,19 @@ For example: Options: - `anonymize-ip`: (boolean) enable or disable anomymiser ip -- `hash-ip`: (boolean) hash query and response IP with sha1 +- `anonymize-v4bits`: (string) summarize IPv4 down to the /integer level, default is `/16` +- `anonymize-v6bits`: (string) summarize IPv6 down to the /integer level, default is `::/64` +- `hash-ip`: (boolean) hashes the query and response IP with the specified algorithm. +- `hash-ip-algo`: (string) algorithm to use for IP hashing, currently supported `sha1` (default), `sha256`, `sha512` - `minimaze-qname`: (boolean) keep only the second level domain ```yaml transforms: user-privacy: anonymize-ip: false + anonymize-v4bits: "/16" + anonymize-v6bits: "::/64" hash-ip: false + hash-ip-algo: "sha1" minimaze-qname: false ``` diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index ba325232..d52b9aa0 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -2,10 +2,13 @@ package pkgconfig type ConfigTransformers struct { UserPrivacy struct { - Enable bool `yaml:"enable"` - AnonymizeIP bool `yaml:"anonymize-ip"` - MinimazeQname bool `yaml:"minimaze-qname"` - HashIP bool `yaml:"hash-ip"` + Enable bool `yaml:"enable"` + AnonymizeIP bool `yaml:"anonymize-ip"` + AnonymizeIPV4Bits string `yaml:"anonymize-v4bits"` + AnonymizeIPV6Bits string `yaml:"anonymize-v6bits"` + MinimazeQname bool `yaml:"minimaze-qname"` + HashIP bool `yaml:"hash-ip"` + HashIPAlgo string `yaml:"hash-ip-algo"` } `yaml:"user-privacy"` Normalize struct { Enable bool `yaml:"enable"` @@ -79,8 +82,11 @@ func (c *ConfigTransformers) SetDefault() { c.UserPrivacy.Enable = false c.UserPrivacy.AnonymizeIP = false + c.UserPrivacy.AnonymizeIPV4Bits = "0.0.0.0/16" + c.UserPrivacy.AnonymizeIPV6Bits = "::/64" c.UserPrivacy.MinimazeQname = false c.UserPrivacy.HashIP = false + c.UserPrivacy.HashIPAlgo = "sha1" c.Normalize.Enable = false c.Normalize.QnameLowerCase = false diff --git a/transformers/userprivacy.go b/transformers/userprivacy.go index 6711b6c8..8c5d8110 100644 --- a/transformers/userprivacy.go +++ b/transformers/userprivacy.go @@ -2,8 +2,11 @@ package transformers import ( "crypto/sha1" + "crypto/sha256" + "crypto/sha512" "fmt" "net" + "strconv" "strings" "github.com/dmachard/go-dnscollector/dnsutils" @@ -12,10 +15,25 @@ import ( "golang.org/x/net/publicsuffix" ) -var ( - defaultIPv4Mask = net.IPv4Mask(255, 255, 0, 0) // /24 - defaultIPv6Mask = net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0} // /64 -) +func parseCIDRMask(mask string) (net.IPMask, error) { + parts := strings.Split(mask, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid mask format, expected /integer: %s", mask) + } + + ones, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid /%s cidr", mask) + } + + if strings.Contains(parts[0], ":") { + ipv6Mask := net.CIDRMask(ones, 128) + return ipv6Mask, nil + } + + ipv4Mask := net.CIDRMask(ones, 32) + return ipv4Mask, nil +} type UserPrivacyProcessor struct { config *pkgconfig.ConfigTransformers @@ -33,21 +51,46 @@ func NewUserPrivacySubprocessor(config *pkgconfig.ConfigTransformers, logger *lo ) UserPrivacyProcessor { s := UserPrivacyProcessor{ config: config, - v4Mask: defaultIPv4Mask, - v6Mask: defaultIPv6Mask, instance: instance, outChannels: outChannels, logInfo: logInfo, logError: logError, } - + s.ReadConfig() return s } +func (s *UserPrivacyProcessor) ReadConfig() { + + var err error + s.v4Mask, err = parseCIDRMask(s.config.UserPrivacy.AnonymizeIPV4Bits) + if err != nil { + s.LogError("unable to init v4 mask: %v", err) + } + + if !strings.Contains(s.config.UserPrivacy.AnonymizeIPV6Bits, ":") { + s.LogError("invalid v6 mask, expect format ::/integer") + } + s.v6Mask, err = parseCIDRMask(s.config.UserPrivacy.AnonymizeIPV6Bits) + if err != nil { + s.LogError("unable to init v6 mask: %v", err) + } +} + func (s *UserPrivacyProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) { s.config = config } +func (p *UserPrivacyProcessor) LogInfo(msg string, v ...interface{}) { + log := fmt.Sprintf("transformer=userprivacy#%d - ", p.instance) + p.logInfo(log+msg, v...) +} + +func (p *UserPrivacyProcessor) LogError(msg string, v ...interface{}) { + log := fmt.Sprintf("transformer=userprivacy#%d - ", p.instance) + p.logError(log+msg, v...) +} + func (s *UserPrivacyProcessor) MinimazeQname(qname string) string { if etpo, err := publicsuffix.EffectiveTLDPlusOne(qname); err == nil { return etpo @@ -57,6 +100,14 @@ func (s *UserPrivacyProcessor) MinimazeQname(qname string) string { } func (s *UserPrivacyProcessor) AnonymizeIP(ip string) string { + // if mask is nil, something is wrong + if s.v4Mask == nil { + return ip + } + if s.v6Mask == nil { + return ip + } + ipaddr := net.ParseIP(ip) isipv4 := strings.LastIndex(ip, ".") @@ -70,7 +121,20 @@ func (s *UserPrivacyProcessor) AnonymizeIP(ip string) string { } func (s *UserPrivacyProcessor) HashIP(ip string) string { - hash := sha1.New() - hash.Write([]byte(ip)) - return fmt.Sprintf("%x", hash.Sum(nil)) + switch s.config.UserPrivacy.HashIPAlgo { + case "sha1": + hash := sha1.New() + hash.Write([]byte(ip)) + return fmt.Sprintf("%x", hash.Sum(nil)) + case "sha256": + hash := sha256.New() + hash.Write([]byte(ip)) + return fmt.Sprintf("%x", hash.Sum(nil)) + case "sha512": + hash := sha512.New() + hash.Write([]byte(ip)) + return fmt.Sprintf("%x", hash.Sum(nil)) + default: + return ip + } } diff --git a/transformers/userprivacy_test.go b/transformers/userprivacy_test.go index d51ee986..11de03a4 100644 --- a/transformers/userprivacy_test.go +++ b/transformers/userprivacy_test.go @@ -8,7 +8,7 @@ import ( "github.com/dmachard/go-logger" ) -func TestReduceQname(t *testing.T) { +func TestUserPrivacy_ReduceQname(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers() config.UserPrivacy.Enable = true @@ -39,7 +39,48 @@ func TestReduceQname(t *testing.T) { } } -func TestAnonymizeIPv4(t *testing.T) { +func TestUserPrivacy_HashIPDefault(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.HashIP = true + + log := logger.New(false) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) + + ip := "192.168.1.2" + + ret := userPrivacy.HashIP(ip) + if ret != "c0ca1efec6aaf505e943397662c28f89ac8f3bc2" { + t.Errorf("IP hashing failed, got %s", ret) + } +} + +func TestUserPrivacy_HashIPSha512(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.HashIP = true + config.UserPrivacy.HashIPAlgo = "sha512" + + log := logger.New(false) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) + + ip := "192.168.1.2" + + ret := userPrivacy.HashIP(ip) + if ret != "800e8f97a29404b7031dfb8d7185b2d30a3cd326b535cda3dcec20a0f4749b1099f98e49245d67eb188091adfba9a45dc0c15e612b554ae7181d8f8a479b67a0" { + t.Errorf("IP hashing failed, got %s", ret) + } +} + +func TestUserPrivacy_AnonymizeIPv4DefaultMask(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers() config.UserPrivacy.Enable = true @@ -59,7 +100,7 @@ func TestAnonymizeIPv4(t *testing.T) { } } -func TestAnonymizeIPv6(t *testing.T) { +func TestUserPrivacy_AnonymizeIPv6DefaultMask(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers() config.UserPrivacy.Enable = true @@ -78,3 +119,45 @@ func TestAnonymizeIPv6(t *testing.T) { t.Errorf("Ipv6 anonymization failed, got %s", ret) } } + +func TestUserPrivacy_AnonymizeIPv4RemoveIP(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.AnonymizeIP = true + config.UserPrivacy.AnonymizeIPV4Bits = "/0" + + log := logger.New(false) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) + + ip := "192.168.1.2" + + ret := userPrivacy.AnonymizeIP(ip) + if ret != "0.0.0.0" { + t.Errorf("Ipv4 anonymization failed with mask %s, got %s", config.UserPrivacy.AnonymizeIPV4Bits, ret) + } +} + +func TestUserPrivacy_AnonymizeIPv6RemoveIP(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.UserPrivacy.Enable = true + config.UserPrivacy.AnonymizeIP = true + config.UserPrivacy.AnonymizeIPV6Bits = "::/0" + + log := logger.New(false) + outChans := []chan dnsutils.DNSMessage{} + + // init the processor + userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) + + ip := "fe80::6111:626:c1b2:2353" + + ret := userPrivacy.AnonymizeIP(ip) + if ret != "::" { + t.Errorf("Ipv6 anonymization failed, got %s", ret) + } +} From 9239ff5caab19928808ca5992a015c513c1acdd8 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:54:00 +0100 Subject: [PATCH 2/3] fix linter --- transformers/userprivacy.go | 12 ++++++------ transformers/userprivacy_test.go | 29 +++++++++++------------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/transformers/userprivacy.go b/transformers/userprivacy.go index 8c5d8110..0eebf9ec 100644 --- a/transformers/userprivacy.go +++ b/transformers/userprivacy.go @@ -81,14 +81,14 @@ func (s *UserPrivacyProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers s.config = config } -func (p *UserPrivacyProcessor) LogInfo(msg string, v ...interface{}) { - log := fmt.Sprintf("transformer=userprivacy#%d - ", p.instance) - p.logInfo(log+msg, v...) +func (s *UserPrivacyProcessor) LogInfo(msg string, v ...interface{}) { + log := fmt.Sprintf("transformer=userprivacy#%d - ", s.instance) + s.logInfo(log+msg, v...) } -func (p *UserPrivacyProcessor) LogError(msg string, v ...interface{}) { - log := fmt.Sprintf("transformer=userprivacy#%d - ", p.instance) - p.logError(log+msg, v...) +func (s *UserPrivacyProcessor) LogError(msg string, v ...interface{}) { + log := fmt.Sprintf("transformer=userprivacy#%d - ", s.instance) + s.logError(log+msg, v...) } func (s *UserPrivacyProcessor) MinimazeQname(qname string) string { diff --git a/transformers/userprivacy_test.go b/transformers/userprivacy_test.go index 11de03a4..5970d204 100644 --- a/transformers/userprivacy_test.go +++ b/transformers/userprivacy_test.go @@ -8,6 +8,11 @@ import ( "github.com/dmachard/go-logger" ) +var ( + TestIP4 = "192.168.1.2" + TestIP6 = "fe80::6111:626:c1b2:2353" +) + func TestUserPrivacy_ReduceQname(t *testing.T) { // enable feature config := pkgconfig.GetFakeConfigTransformers() @@ -51,9 +56,7 @@ func TestUserPrivacy_HashIPDefault(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "192.168.1.2" - - ret := userPrivacy.HashIP(ip) + ret := userPrivacy.HashIP(TestIP4) if ret != "c0ca1efec6aaf505e943397662c28f89ac8f3bc2" { t.Errorf("IP hashing failed, got %s", ret) } @@ -72,9 +75,7 @@ func TestUserPrivacy_HashIPSha512(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "192.168.1.2" - - ret := userPrivacy.HashIP(ip) + ret := userPrivacy.HashIP(TestIP4) if ret != "800e8f97a29404b7031dfb8d7185b2d30a3cd326b535cda3dcec20a0f4749b1099f98e49245d67eb188091adfba9a45dc0c15e612b554ae7181d8f8a479b67a0" { t.Errorf("IP hashing failed, got %s", ret) } @@ -92,9 +93,7 @@ func TestUserPrivacy_AnonymizeIPv4DefaultMask(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "192.168.1.2" - - ret := userPrivacy.AnonymizeIP(ip) + ret := userPrivacy.AnonymizeIP(TestIP4) if ret != "192.168.0.0" { t.Errorf("Ipv4 anonymization failed, got %s", ret) } @@ -112,9 +111,7 @@ func TestUserPrivacy_AnonymizeIPv6DefaultMask(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "fe80::6111:626:c1b2:2353" - - ret := userPrivacy.AnonymizeIP(ip) + ret := userPrivacy.AnonymizeIP(TestIP6) if ret != "fe80::" { t.Errorf("Ipv6 anonymization failed, got %s", ret) } @@ -133,9 +130,7 @@ func TestUserPrivacy_AnonymizeIPv4RemoveIP(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "192.168.1.2" - - ret := userPrivacy.AnonymizeIP(ip) + ret := userPrivacy.AnonymizeIP(TestIP4) if ret != "0.0.0.0" { t.Errorf("Ipv4 anonymization failed with mask %s, got %s", config.UserPrivacy.AnonymizeIPV4Bits, ret) } @@ -154,9 +149,7 @@ func TestUserPrivacy_AnonymizeIPv6RemoveIP(t *testing.T) { // init the processor userPrivacy := NewUserPrivacySubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error) - ip := "fe80::6111:626:c1b2:2353" - - ret := userPrivacy.AnonymizeIP(ip) + ret := userPrivacy.AnonymizeIP(TestIP6) if ret != "::" { t.Errorf("Ipv6 anonymization failed, got %s", ret) } From a9cc661b6989aa9ea7926bca03aded91827838a0 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:34:56 +0100 Subject: [PATCH 3/3] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b43fed16..cdb3390b 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/dmachard/go-dns-collector)](https://goreportcard.com/report/dmachard/go-dns-collector) ![Go version](https://img.shields.io/badge/go%20version-min%201.20-blue) -![Go tests](https://img.shields.io/badge/go%20tests-366-green) -![Go lines](https://img.shields.io/badge/go%20lines-32797-red) +![Go tests](https://img.shields.io/badge/go%20tests-370-green) +![Go lines](https://img.shields.io/badge/go%20lines-32932-red) ![Go Tests](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-go.yml/badge.svg) ![Github Actions](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-dnstap.yml/badge.svg) ![Github Actions PDNS](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-powerdns.yml/badge.svg)