From cd4b97197cfafea739d81d860903d79d85cb73af Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 17 May 2024 20:20:29 +0200 Subject: [PATCH 01/12] set defaults for global --- go.mod | 1 + go.sum | 2 ++ pkgconfig/global.go | 32 ++++++++++++-------------------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 4060d31e..457b9d7d 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.12.4 // indirect github.com/dennwc/varint v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3c7fd43c..c6e2757a 100644 --- a/go.sum +++ b/go.sum @@ -457,6 +457,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/pkgconfig/global.go b/pkgconfig/global.go index 1ea6650c..e45351ed 100644 --- a/pkgconfig/global.go +++ b/pkgconfig/global.go @@ -2,34 +2,26 @@ package pkgconfig import ( "reflect" + + "github.com/creasty/defaults" ) type ConfigGlobal struct { - TextFormat string `yaml:"text-format"` - TextFormatDelimiter string `yaml:"text-format-delimiter"` - TextFormatBoundary string `yaml:"text-format-boundary"` + TextFormat string `yaml:"text-format" default:"timestamp identity operation rcode queryip queryport family protocol length-unit qname qtype latency"` + TextFormatDelimiter string `yaml:"text-format-delimiter" default:" "` + TextFormatBoundary string `yaml:"text-format-boundary" default:"\""` Trace struct { - Verbose bool `yaml:"verbose"` - LogMalformed bool `yaml:"log-malformed"` - Filename string `yaml:"filename"` - MaxSize int `yaml:"max-size"` - MaxBackups int `yaml:"max-backups"` + Verbose bool `yaml:"verbose" default:"false"` + LogMalformed bool `yaml:"log-malformed" default:"false"` + Filename string `yaml:"filename" default:""` + MaxSize int `yaml:"max-size" default:"10"` + MaxBackups int `yaml:"max-backups" default:"10"` } `yaml:"trace"` - ServerIdentity string `yaml:"server-identity"` + ServerIdentity string `yaml:"server-identity" default:""` } func (c *ConfigGlobal) SetDefault() { - // global config - c.TextFormat = "timestamp identity operation rcode queryip queryport family protocol length-unit qname qtype latency" - c.TextFormatDelimiter = " " - c.TextFormatBoundary = "\"" - - c.Trace.Verbose = false - c.Trace.LogMalformed = false - c.Trace.Filename = "" - c.Trace.MaxSize = 10 - c.Trace.MaxBackups = 10 - c.ServerIdentity = "" + defaults.Set(c) } func (c *ConfigGlobal) Check(userCfg map[string]interface{}) error { From ccdf120d9ef30d2bdd97c13e383665da0299b56c Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 17 May 2024 20:38:41 +0200 Subject: [PATCH 02/12] update default for collectors --- pkgconfig/collectors.go | 187 +++++++++++++--------------------------- 1 file changed, 62 insertions(+), 125 deletions(-) diff --git a/pkgconfig/collectors.go b/pkgconfig/collectors.go index 9df3b344..23e94f91 100644 --- a/pkgconfig/collectors.go +++ b/pkgconfig/collectors.go @@ -2,159 +2,96 @@ package pkgconfig import ( "reflect" + + "github.com/creasty/defaults" ) type ConfigCollectors struct { DNSMessage struct { - Enable bool `yaml:"enable"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` Matching struct { Include map[string]interface{} `yaml:"include"` Exclude map[string]interface{} `yaml:"exclude"` } `yaml:"matching"` } `yaml:"dnsmessage"` Tail struct { - Enable bool `yaml:"enable"` - TimeLayout string `yaml:"time-layout"` - PatternQuery string `yaml:"pattern-query"` - PatternReply string `yaml:"pattern-reply"` - FilePath string `yaml:"file-path"` + Enable bool `yaml:"enable" default:"false"` + TimeLayout string `yaml:"time-layout" default:""` + PatternQuery string `yaml:"pattern-query" default:""` + PatternReply string `yaml:"pattern-reply" default:""` + FilePath string `yaml:"file-path" default:""` } `yaml:"tail"` Dnstap struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - SockPath string `yaml:"sock-path"` - TLSSupport bool `yaml:"tls-support"` - TLSMinVersion string `yaml:"tls-min-version"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - RcvBufSize int `yaml:"sock-rcvbuf"` - ResetConn bool `yaml:"reset-conn"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - DisableDNSParser bool `yaml:"disable-dnsparser"` - ExtendedSupport bool `yaml:"extended-support"` - Compression string `yaml:"compression"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"0.0.0.0"` + ListenPort int `yaml:"listen-port" default:"6000"` + SockPath string `yaml:"sock-path" default:""` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + RcvBufSize int `yaml:"sock-rcvbuf" default:"0"` + ResetConn bool `yaml:"reset-conn" default:"true"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + DisableDNSParser bool `yaml:"disable-dnsparser" default:"false"` + ExtendedSupport bool `yaml:"extended-support" default:"false"` + Compression string `yaml:"compression" default:"none"` } `yaml:"dnstap"` DnstapProxifier struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - SockPath string `yaml:"sock-path"` - TLSSupport bool `yaml:"tls-support"` - TLSMinVersion string `yaml:"tls-min-version"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"0.0.0.0"` + ListenPort int `yaml:"listen-port" default:"6000"` + SockPath string `yaml:"sock-path" default:""` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` } `yaml:"dnstap-relay"` AfpacketLiveCapture struct { - Enable bool `yaml:"enable"` - Port int `yaml:"port"` - Device string `yaml:"device"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - FragmentSupport bool `yaml:"enable-defrag-ip"` + Enable bool `yaml:"enable" default:"false"` + Port int `yaml:"port" default:"53"` + Device string `yaml:"device" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + FragmentSupport bool `yaml:"enable-defrag-ip" default:"true"` } `yaml:"afpacket-sniffer"` XdpLiveCapture struct { - Enable bool `yaml:"enable"` - Port int `yaml:"port"` - Device string `yaml:"device"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Port int `yaml:"port" default:"53"` + Device string `yaml:"device" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"xdp-sniffer"` PowerDNS struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - TLSSupport bool `yaml:"tls-support"` - TLSMinVersion string `yaml:"tls-min-version"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - AddDNSPayload bool `yaml:"add-dns-payload"` - RcvBufSize int `yaml:"sock-rcvbuf"` - ResetConn bool `yaml:"reset-conn"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"0.0.0.0"` + ListenPort int `yaml:"listen-port" default:"6001"` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2 "` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + AddDNSPayload bool `yaml:"add-dns-payload" default:"false"` + RcvBufSize int `yaml:"sock-rcvbuf" default:"0"` + ResetConn bool `yaml:"reset-conn" default:"true"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"powerdns"` FileIngestor struct { - Enable bool `yaml:"enable"` - WatchDir string `yaml:"watch-dir"` - WatchMode string `yaml:"watch-mode"` - PcapDNSPort int `yaml:"pcap-dns-port"` - DeleteAfter bool `yaml:"delete-after"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + WatchDir string `yaml:"watch-dir" default:""` + WatchMode string `yaml:"watch-mode" default:"pcap"` + PcapDNSPort int `yaml:"pcap-dns-port" default:"53"` + DeleteAfter bool `yaml:"delete-after" default:"false"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"file-ingestor"` Tzsp struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"0.0.0.0"` + ListenPort int `yaml:"listen-port" default:"10000"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"tzsp"` } func (c *ConfigCollectors) SetDefault() { - c.DNSMessage.Enable = false - c.DNSMessage.ChannelBufferSize = 65535 - - c.Tail.Enable = false - c.Tail.TimeLayout = "" - c.Tail.PatternQuery = "" - c.Tail.PatternReply = "" - c.Tail.FilePath = "" - - c.Dnstap.Enable = false - c.Dnstap.ListenIP = AnyIP - c.Dnstap.ListenPort = 6000 - c.Dnstap.SockPath = "" - c.Dnstap.TLSSupport = false - c.Dnstap.TLSMinVersion = TLSV12 - c.Dnstap.CertFile = "" - c.Dnstap.KeyFile = "" - c.Dnstap.RcvBufSize = 0 - c.Dnstap.ResetConn = true - c.Dnstap.ChannelBufferSize = 65535 - c.Dnstap.DisableDNSParser = false - c.Dnstap.ExtendedSupport = false - c.Dnstap.Compression = CompressNone - - c.DnstapProxifier.Enable = false - c.DnstapProxifier.ListenIP = AnyIP - c.DnstapProxifier.ListenPort = 6000 - c.DnstapProxifier.SockPath = "" - c.DnstapProxifier.TLSSupport = false - c.DnstapProxifier.TLSMinVersion = TLSV12 - c.DnstapProxifier.CertFile = "" - c.DnstapProxifier.KeyFile = "" - - c.XdpLiveCapture.Enable = false - c.XdpLiveCapture.Device = "" - c.XdpLiveCapture.ChannelBufferSize = 65535 - - c.AfpacketLiveCapture.Enable = false - c.AfpacketLiveCapture.Port = 53 - c.AfpacketLiveCapture.Device = "" - c.AfpacketLiveCapture.ChannelBufferSize = 65535 - c.AfpacketLiveCapture.FragmentSupport = true - - c.PowerDNS.Enable = false - c.PowerDNS.ListenIP = AnyIP - c.PowerDNS.ListenPort = 6001 - c.PowerDNS.TLSSupport = false - c.PowerDNS.TLSMinVersion = TLSV12 - c.PowerDNS.CertFile = "" - c.PowerDNS.KeyFile = "" - c.PowerDNS.AddDNSPayload = false - c.PowerDNS.RcvBufSize = 0 - c.PowerDNS.ResetConn = true - c.PowerDNS.ChannelBufferSize = 65535 - - c.FileIngestor.Enable = false - c.FileIngestor.WatchDir = "" - c.FileIngestor.PcapDNSPort = 53 - c.FileIngestor.WatchMode = ModePCAP - c.FileIngestor.DeleteAfter = false - c.FileIngestor.ChannelBufferSize = 65535 - - c.Tzsp.Enable = false - c.Tzsp.ListenIP = AnyIP - c.Tzsp.ListenPort = 10000 - c.Tzsp.ChannelBufferSize = 65535 + defaults.Set(c) } func (c *ConfigCollectors) IsValid(userCfg map[string]interface{}) error { From c75565619ed1803771314c6fc0014b472b822b42 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 17 May 2024 20:43:28 +0200 Subject: [PATCH 03/12] fix default tls version --- pkgconfig/collectors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgconfig/collectors.go b/pkgconfig/collectors.go index 23e94f91..64197618 100644 --- a/pkgconfig/collectors.go +++ b/pkgconfig/collectors.go @@ -66,7 +66,7 @@ type ConfigCollectors struct { ListenIP string `yaml:"listen-ip" default:"0.0.0.0"` ListenPort int `yaml:"listen-port" default:"6001"` TLSSupport bool `yaml:"tls-support" default:"false"` - TLSMinVersion string `yaml:"tls-min-version" default:"1.2 "` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` CertFile string `yaml:"cert-file" default:""` KeyFile string `yaml:"key-file" default:""` AddDNSPayload bool `yaml:"add-dns-payload" default:"false"` From 364fcce95e5db43b2affe63c1e1c012477dd182c Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 17 May 2024 20:48:59 +0200 Subject: [PATCH 04/12] remove configchecker --- pkgconfig/config.go | 76 +++++++++++ pkgconfig/config_test.go | 226 +++++++++++++++++++++++++++++++ pkgconfig/configchecker.go | 83 ------------ pkgconfig/configchecker_test.go | 232 -------------------------------- 4 files changed, 302 insertions(+), 315 deletions(-) delete mode 100644 pkgconfig/configchecker.go delete mode 100644 pkgconfig/configchecker_test.go diff --git a/pkgconfig/config.go b/pkgconfig/config.go index c22b2437..9e77cdea 100644 --- a/pkgconfig/config.go +++ b/pkgconfig/config.go @@ -1,10 +1,12 @@ package pkgconfig import ( + "io" "os" "reflect" "github.com/pkg/errors" + "gopkg.in/yaml.v3" ) func IsValidMode(mode string) bool { @@ -133,3 +135,77 @@ func CheckConfigWithTags(v reflect.Value, userCfg map[string]interface{}) error } return nil } + +func ReloadConfig(configPath string, config *Config) error { + // Open config file + configFile, err := os.Open(configPath) + if err != nil { + return nil + } + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configFile); err != nil { + return err + } + + // Init new YAML decode + configFile.Seek(0, 0) + d := yaml.NewDecoder(configFile) + + // Start YAML decoding from file + if err := d.Decode(&config); err != nil { + return err + } + return nil +} + +func LoadConfig(configPath string) (*Config, error) { + // Open config file + configFile, err := os.Open(configPath) + if err != nil { + return nil, err + } + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configFile); err != nil { + return nil, err + } + + // Init new YAML decode + configFile.Seek(0, 0) + d := yaml.NewDecoder(configFile) + + // Start YAML decoding to go + config := &Config{} + config.SetDefault() + + if err := d.Decode(&config); err != nil { + return nil, err + } + + return config, nil +} + +func CheckConfig(configFile *os.File) error { + // Read config file bytes + configBytes, err := io.ReadAll(configFile) + if err != nil { + return errors.Wrap(err, "Error reading configuration file") + } + + // Unmarshal YAML to map + userCfg := make(map[string]interface{}) + err = yaml.Unmarshal(configBytes, &userCfg) + if err != nil { + return errors.Wrap(err, "error parsing YAML file") + } + + // check the user config with the default one + config := &Config{} + config.SetDefault() + + // check if the provided config is valid + return config.IsValid(userCfg) +} diff --git a/pkgconfig/config_test.go b/pkgconfig/config_test.go index 9a684fc6..6bf7eb20 100644 --- a/pkgconfig/config_test.go +++ b/pkgconfig/config_test.go @@ -32,3 +32,229 @@ func TestConfig_GetServerIdentity_Hostname(t *testing.T) { t.Errorf("Expected %s, but got %s", expected2, result2) } } + +func createTempConfigFile(content string) (string, error) { + tempFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + return "", err + } + defer tempFile.Close() + + if _, err := tempFile.WriteString(content); err != nil { + return "", err + } + + return tempFile.Name(), nil +} + +func TestConfig_CheckConfig(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + }{ + { + name: "Valid multiplexer configuration", + content: ` +global: + trace: + verbose: true + server-identity: "dns-collector" +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + transforms: + normalize: + qname-lowercase: false + loggers: + - name: console + stdout: + mode: text + routes: + - from: [ tap ] + to: [ console ] +`, + wantErr: false, + }, + { + name: "Valid pipeline configuration", + content: ` +global: + trace: + verbose: true + server-identity: "dns-collector" +pipelines: + - name: dnsdist-main + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + routing-policy: + default: [ console ] + + - name: console + stdout: + mode: text +`, + wantErr: false, + }, + { + name: "Invalid key", + content: ` +global: + logger: bad-position +`, + wantErr: true, + }, + { + name: "Invalid multiplexer config format", + content: ` +multiplexer: + - name: block + dnstap: + listen-ip: 0.0.0.0 + transforms: + normalize: + qname-lowercase: true +`, + wantErr: true, + }, + { + name: "Invalid multiplexer logger", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + loggers: + - name: tapOut + dnstap: + listen-ip: 0.0.0.0 + routes: + - from: [ tapIn ] + to: [ tapOut ] +`, + wantErr: true, + }, + { + name: "Invalid pipeline transform", + content: ` +pipelines: + - name: dnsdist-main + dnstap: + listen-ip: 0.0.0.0 + transforms: + normalize: + qname-lowercase: true + routing-policy: + default: [ console ] +`, + wantErr: true, + }, + { + name: "Invalid multiplexer route", + content: ` +multiplexer: + routes: + - from: [test-route] + unknown-key: invalid +`, + wantErr: true, + }, + { + name: "pipeline dynamic keys", + content: ` +pipelines: + - name: match + dnsmessage: + matching: + include: + atags.tags.*: test + atags.tags.2: test + dns.resources-records.*: test +`, + wantErr: false, + }, + { + name: "freeform loki #643", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + loggers: + - name: loki + lokiclient: + server-url: "https://grafana-loki.example.com/loki/api/v1/push" + job-name: "dnscollector" + mode: "flat-json" + tls-insecure: true + tenant-id: fake + relabel-configs: + - source_labels: ["__dns_qtype"] + target_label: "qtype" + replacement: "test" + action: "update" + separator: "," + regex: "test" + routes: + - from: [ tap ] + to: [ loki ] +`, + wantErr: false, + }, + { + name: "freeform scalyr #676", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + loggers: + - name: scalyr + scalyrclient: + apikey: XXXXX + attrs: + service: dnstap + type: queries + flush-interval: 10 + mode: flat-json + sessioninfo: + cloud_provider: Azure + cloud_region: westeurope + routes: + - from: [ tap ] + to: [ scalyr ] +`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempFile, err := createTempConfigFile(tt.content) + if err != nil { + t.Fatalf("Error creating temporary file: %v", err) + } + defer os.Remove(tempFile) + configFile, err := os.Open(tempFile) + if err != nil { + t.Fatalf("Read temporary file: %v", err) + } + defer configFile.Close() + + err = CheckConfig(configFile) + if (err != nil) != tt.wantErr { + t.Errorf("CheckConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/pkgconfig/configchecker.go b/pkgconfig/configchecker.go deleted file mode 100644 index a24d5e66..00000000 --- a/pkgconfig/configchecker.go +++ /dev/null @@ -1,83 +0,0 @@ -package pkgconfig - -import ( - "io" - "os" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -func ReloadConfig(configPath string, config *Config) error { - // Open config file - configFile, err := os.Open(configPath) - if err != nil { - return nil - } - defer configFile.Close() - - // Check config to detect unknown keywords - if err := CheckConfig(configFile); err != nil { - return err - } - - // Init new YAML decode - configFile.Seek(0, 0) - d := yaml.NewDecoder(configFile) - - // Start YAML decoding from file - if err := d.Decode(&config); err != nil { - return err - } - return nil -} - -func LoadConfig(configPath string) (*Config, error) { - // Open config file - configFile, err := os.Open(configPath) - if err != nil { - return nil, err - } - defer configFile.Close() - - // Check config to detect unknown keywords - if err := CheckConfig(configFile); err != nil { - return nil, err - } - - // Init new YAML decode - configFile.Seek(0, 0) - d := yaml.NewDecoder(configFile) - - // Start YAML decoding to go - config := &Config{} - config.SetDefault() - - if err := d.Decode(&config); err != nil { - return nil, err - } - - return config, nil -} - -func CheckConfig(configFile *os.File) error { - // Read config file bytes - configBytes, err := io.ReadAll(configFile) - if err != nil { - return errors.Wrap(err, "Error reading configuration file") - } - - // Unmarshal YAML to map - userCfg := make(map[string]interface{}) - err = yaml.Unmarshal(configBytes, &userCfg) - if err != nil { - return errors.Wrap(err, "error parsing YAML file") - } - - // check the user config with the default one - config := &Config{} - config.SetDefault() - - // check if the provided config is valid - return config.IsValid(userCfg) -} diff --git a/pkgconfig/configchecker_test.go b/pkgconfig/configchecker_test.go deleted file mode 100644 index e5bf683f..00000000 --- a/pkgconfig/configchecker_test.go +++ /dev/null @@ -1,232 +0,0 @@ -package pkgconfig - -import ( - "os" - "testing" -) - -func createTempConfigFile(content string) (string, error) { - tempFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - return "", err - } - defer tempFile.Close() - - if _, err := tempFile.WriteString(content); err != nil { - return "", err - } - - return tempFile.Name(), nil -} - -func TestConfig_CheckConfig(t *testing.T) { - tests := []struct { - name string - content string - wantErr bool - }{ - { - name: "Valid multiplexer configuration", - content: ` -global: - trace: - verbose: true - server-identity: "dns-collector" -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - transforms: - normalize: - qname-lowercase: false - loggers: - - name: console - stdout: - mode: text - routes: - - from: [ tap ] - to: [ console ] -`, - wantErr: false, - }, - { - name: "Valid pipeline configuration", - content: ` -global: - trace: - verbose: true - server-identity: "dns-collector" -pipelines: - - name: dnsdist-main - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - routing-policy: - default: [ console ] - - - name: console - stdout: - mode: text -`, - wantErr: false, - }, - { - name: "Invalid key", - content: ` -global: - logger: bad-position -`, - wantErr: true, - }, - { - name: "Invalid multiplexer config format", - content: ` -multiplexer: - - name: block - dnstap: - listen-ip: 0.0.0.0 - transforms: - normalize: - qname-lowercase: true -`, - wantErr: true, - }, - { - name: "Invalid multiplexer logger", - content: ` -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - loggers: - - name: tapOut - dnstap: - listen-ip: 0.0.0.0 - routes: - - from: [ tapIn ] - to: [ tapOut ] -`, - wantErr: true, - }, - { - name: "Invalid pipeline transform", - content: ` -pipelines: - - name: dnsdist-main - dnstap: - listen-ip: 0.0.0.0 - transforms: - normalize: - qname-lowercase: true - routing-policy: - default: [ console ] -`, - wantErr: true, - }, - { - name: "Invalid multiplexer route", - content: ` -multiplexer: - routes: - - from: [test-route] - unknown-key: invalid -`, - wantErr: true, - }, - { - name: "pipeline dynamic keys", - content: ` -pipelines: - - name: match - dnsmessage: - matching: - include: - atags.tags.*: test - atags.tags.2: test - dns.resources-records.*: test -`, - wantErr: false, - }, - { - name: "freeform loki #643", - content: ` -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - loggers: - - name: loki - lokiclient: - server-url: "https://grafana-loki.example.com/loki/api/v1/push" - job-name: "dnscollector" - mode: "flat-json" - tls-insecure: true - tenant-id: fake - relabel-configs: - - source_labels: ["__dns_qtype"] - target_label: "qtype" - replacement: "test" - action: "update" - separator: "," - regex: "test" - routes: - - from: [ tap ] - to: [ loki ] -`, - wantErr: false, - }, - { - name: "freeform scalyr #676", - content: ` -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - loggers: - - name: scalyr - scalyrclient: - apikey: XXXXX - attrs: - service: dnstap - type: queries - flush-interval: 10 - mode: flat-json - sessioninfo: - cloud_provider: Azure - cloud_region: westeurope - routes: - - from: [ tap ] - to: [ scalyr ] -`, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tempFile, err := createTempConfigFile(tt.content) - if err != nil { - t.Fatalf("Error creating temporary file: %v", err) - } - defer os.Remove(tempFile) - configFile, err := os.Open(tempFile) - if err != nil { - t.Fatalf("Read temporary file: %v", err) - } - defer configFile.Close() - - err = CheckConfig(configFile) - if (err != nil) != tt.wantErr { - t.Errorf("CheckConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} From c59028a1c35da41224ce28cef1821e81ca0086cf Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 09:00:12 +0200 Subject: [PATCH 05/12] remove defaults for transformers --- go.mod | 2 +- pkgconfig/transformers.go | 174 +++++++++++++------------------------- 2 files changed, 59 insertions(+), 117 deletions(-) diff --git a/go.mod b/go.mod index 457b9d7d..ded5e985 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/IBM/fluent-forward-go v0.2.2 github.com/IBM/sarama v1.43.2 github.com/cilium/ebpf v0.15.0 + github.com/creasty/defaults v1.7.0 github.com/dmachard/go-clientsyslog v0.4.0 github.com/dmachard/go-dnstap-protobuf v1.0.1 github.com/dmachard/go-framestream v0.10.0 @@ -48,7 +49,6 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.12.4 // indirect github.com/dennwc/varint v1.0.0 // indirect diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index e3c8c699..7cf31824 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -1,6 +1,10 @@ package pkgconfig -import "reflect" +import ( + "reflect" + + "github.com/creasty/defaults" +) type RelabelingConfig struct { Regex string `yaml:"regex"` @@ -9,147 +13,85 @@ type RelabelingConfig struct { type ConfigTransformers struct { UserPrivacy struct { - Enable bool `yaml:"enable"` - AnonymizeIP bool `yaml:"anonymize-ip"` - AnonymizeIPV4Bits string `yaml:"anonymize-v4bits"` - AnonymizeIPV6Bits string `yaml:"anonymize-v6bits"` - MinimazeQname bool `yaml:"minimaze-qname"` - HashIP bool `yaml:"hash-ip"` - HashIPAlgo string `yaml:"hash-ip-algo"` + Enable bool `yaml:"enable" default:"false"` + AnonymizeIP bool `yaml:"anonymize-ip" default:"false"` + AnonymizeIPV4Bits string `yaml:"anonymize-v4bits" default:"0.0.0.0/16"` + AnonymizeIPV6Bits string `yaml:"anonymize-v6bits" default:"::/64"` + MinimazeQname bool `yaml:"minimaze-qname" default:"false"` + HashIP bool `yaml:"hash-ip" default:"false"` + HashIPAlgo string `yaml:"hash-ip-algo" default:"sha1"` } `yaml:"user-privacy"` Normalize struct { - Enable bool `yaml:"enable"` - QnameLowerCase bool `yaml:"qname-lowercase"` - RRLowerCase bool `yaml:"rr-lowercase"` - QuietText bool `yaml:"quiet-text"` - AddTld bool `yaml:"add-tld"` - AddTldPlusOne bool `yaml:"add-tld-plus-one"` + Enable bool `yaml:"enable" default:"false"` + QnameLowerCase bool `yaml:"qname-lowercase" default:"false"` + RRLowerCase bool `yaml:"rr-lowercase" default:"false"` + QuietText bool `yaml:"quiet-text" default:"false"` + AddTld bool `yaml:"add-tld" default:"false"` + AddTldPlusOne bool `yaml:"add-tld-plus-one" default:"false"` } `yaml:"normalize"` Latency struct { - Enable bool `yaml:"enable"` - MeasureLatency bool `yaml:"measure-latency"` - UnansweredQueries bool `yaml:"unanswered-queries"` - QueriesTimeout int `yaml:"queries-timeout"` + Enable bool `yaml:"enable" default:"false"` + MeasureLatency bool `yaml:"measure-latency" default:"false"` + UnansweredQueries bool `yaml:"unanswered-queries" default:"false"` + QueriesTimeout int `yaml:"queries-timeout" default:"2"` } `yaml:"latency"` Reducer struct { - Enable bool `yaml:"enable"` - RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector"` - QnamePlusOne bool `yaml:"qname-plus-one"` - WatchInterval int `yaml:"watch-interval"` + Enable bool `yaml:"enable" default:"false"` + RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"` + QnamePlusOne bool `yaml:"qname-plus-one" default:"false"` + WatchInterval int `yaml:"watch-interval" default:"5"` } `yaml:"reducer"` Filtering struct { - Enable bool `yaml:"enable"` - DropFqdnFile string `yaml:"drop-fqdn-file"` - DropDomainFile string `yaml:"drop-domain-file"` - KeepFqdnFile string `yaml:"keep-fqdn-file"` - KeepDomainFile string `yaml:"keep-domain-file"` - DropQueryIPFile string `yaml:"drop-queryip-file"` - KeepQueryIPFile string `yaml:"keep-queryip-file"` - KeepRdataFile string `yaml:"keep-rdata-file"` - DropRcodes []string `yaml:"drop-rcodes,flow"` - LogQueries bool `yaml:"log-queries"` - LogReplies bool `yaml:"log-replies"` - Downsample int `yaml:"downsample"` + Enable bool `yaml:"enable" default:"false"` + DropFqdnFile string `yaml:"drop-fqdn-file" default:""` + DropDomainFile string `yaml:"drop-domain-file" default:""` + KeepFqdnFile string `yaml:"keep-fqdn-file" default:""` + KeepDomainFile string `yaml:"keep-domain-file" default:""` + DropQueryIPFile string `yaml:"drop-queryip-file" default:""` + KeepQueryIPFile string `yaml:"keep-queryip-file" default:""` + KeepRdataFile string `yaml:"keep-rdata-file" default:""` + DropRcodes []string `yaml:"drop-rcodes,flow" default:"[]"` + LogQueries bool `yaml:"log-queries" default:"true"` + LogReplies bool `yaml:"log-replies" default:"true"` + Downsample int `yaml:"downsample" default:"0"` } `yaml:"filtering"` GeoIP struct { - Enable bool `yaml:"enable"` - DBCountryFile string `yaml:"mmdb-country-file"` - DBCityFile string `yaml:"mmdb-city-file"` - DBASNFile string `yaml:"mmdb-asn-file"` + Enable bool `yaml:"enable" default:"false"` + DBCountryFile string `yaml:"mmdb-country-file" default:""` + DBCityFile string `yaml:"mmdb-city-file" default:""` + DBASNFile string `yaml:"mmdb-asn-file" default:""` } `yaml:"geoip"` Suspicious struct { - Enable bool `yaml:"enable"` - ThresholdQnameLen int `yaml:"threshold-qname-len"` - ThresholdPacketLen int `yaml:"threshold-packet-len"` - ThresholdSlow float64 `yaml:"threshold-slow"` - CommonQtypes []string `yaml:"common-qtypes,flow"` - UnallowedChars []string `yaml:"unallowed-chars,flow"` - ThresholdMaxLabels int `yaml:"threshold-max-labels"` - WhitelistDomains []string `yaml:"whitelist-domains,flow"` + Enable bool `yaml:"enable" default:"false"` + ThresholdQnameLen int `yaml:"threshold-qname-len" default:"100"` + ThresholdPacketLen int `yaml:"threshold-packet-len" default:"1000"` + ThresholdSlow float64 `yaml:"threshold-slow" default:"1.0"` + CommonQtypes []string `yaml:"common-qtypes,flow" default:"[\"A\", \"AAAA\", \"TXT\", \"CNAME\", \"PTR\", \"NAPTR\", \"DNSKEY\", \"SRV\", \"SOA\", \"NS\", \"MX\", \"DS\", \"HTTPS\"]"` + UnallowedChars []string `yaml:"unallowed-chars,flow" default:"[\"\"\", \"==\", \"/\", \":\"]"` + ThresholdMaxLabels int `yaml:"threshold-max-labels" default:"10"` + WhitelistDomains []string `yaml:"whitelist-domains,flow" default:"[\"\\.ip6\\.arpa\"]"` } `yaml:"suspicious"` Extract struct { - Enable bool `yaml:"enable"` - AddPayload bool `yaml:"add-payload"` + Enable bool `yaml:"enable" default:"false"` + AddPayload bool `yaml:"add-payload" default:"false"` } `yaml:"extract"` MachineLearning struct { - Enable bool `yaml:"enable"` - AddFeatures bool `yaml:"add-features"` + Enable bool `yaml:"enable" default:"false"` + AddFeatures bool `yaml:"add-features" default:"false"` } `yaml:"machine-learning"` ATags struct { - Enable bool `yaml:"enable"` - Tags []string `yaml:"tags,flow"` + Enable bool `yaml:"enable" default:"false"` + Tags []string `yaml:"tags,flow" default:"[]"` } `yaml:"atags"` Relabeling struct { - Enable bool `yaml:"enable"` + Enable bool `yaml:"enable" default:"false"` Rename []RelabelingConfig `yaml:"rename,flow"` Remove []RelabelingConfig `yaml:"remove,flow"` } `yaml:"relabeling"` } func (c *ConfigTransformers) SetDefault() { - c.Suspicious.Enable = false - c.Suspicious.ThresholdQnameLen = 100 - c.Suspicious.ThresholdPacketLen = 1000 - c.Suspicious.ThresholdSlow = 1.0 - c.Suspicious.CommonQtypes = []string{"A", "AAAA", "TXT", "CNAME", "PTR", - "NAPTR", "DNSKEY", "SRV", "SOA", "NS", "MX", "DS", "HTTPS"} - c.Suspicious.UnallowedChars = []string{"\"", "==", "/", ":"} - c.Suspicious.ThresholdMaxLabels = 10 - c.Suspicious.WhitelistDomains = []string{"\\.ip6\\.arpa"} - - c.UserPrivacy.Enable = false - c.UserPrivacy.AnonymizeIP = false - c.UserPrivacy.AnonymizeIPV4Bits = "0.0.0.0/16" - c.UserPrivacy.AnonymizeIPV6Bits = "::/64" - c.UserPrivacy.MinimazeQname = false - c.UserPrivacy.HashIP = false - c.UserPrivacy.HashIPAlgo = "sha1" - - c.Normalize.Enable = false - c.Normalize.QnameLowerCase = false - c.Normalize.RRLowerCase = false - c.Normalize.QuietText = false - c.Normalize.AddTld = false - c.Normalize.AddTldPlusOne = false - - c.Latency.Enable = false - c.Latency.MeasureLatency = false - c.Latency.UnansweredQueries = false - c.Latency.QueriesTimeout = 2 - - c.Reducer.Enable = false - c.Reducer.RepetitiveTrafficDetector = false - c.Reducer.QnamePlusOne = false - c.Reducer.WatchInterval = 5 - - c.Filtering.Enable = false - c.Filtering.DropFqdnFile = "" - c.Filtering.DropDomainFile = "" - c.Filtering.KeepFqdnFile = "" - c.Filtering.KeepDomainFile = "" - c.Filtering.DropQueryIPFile = "" - c.Filtering.DropRcodes = []string{} - c.Filtering.LogQueries = true - c.Filtering.LogReplies = true - c.Filtering.Downsample = 0 - - c.GeoIP.Enable = false - c.GeoIP.DBCountryFile = "" - c.GeoIP.DBCityFile = "" - c.GeoIP.DBASNFile = "" - - c.Extract.Enable = false - c.Extract.AddPayload = false - - c.MachineLearning.Enable = false - c.MachineLearning.AddFeatures = false - - c.ATags.Enable = false - c.ATags.Tags = []string{} - - c.Relabeling.Enable = false - c.Relabeling.Remove = []RelabelingConfig{} - c.Relabeling.Rename = []RelabelingConfig{} + defaults.Set(c) } func (c *ConfigTransformers) IsValid(userCfg map[string]interface{}) error { From 38175ff2b90bc15aa0663c9457a20ed505ca8179 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 09:27:32 +0200 Subject: [PATCH 06/12] fix regression in test --- go.mod | 2 +- go.sum | 2 ++ pkgconfig/transformers.go | 4 ++-- transformers/suspicious_test.go | 14 +++++++------- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index ded5e985..9501ba44 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/dmachard/go-dnstap-protobuf v1.0.1 github.com/dmachard/go-framestream v0.10.0 github.com/dmachard/go-logger v0.4.0 - github.com/dmachard/go-netutils v0.1.0 + github.com/dmachard/go-netutils v0.2.0 github.com/dmachard/go-powerdns-protobuf v1.1.1 github.com/dmachard/go-topmap v1.0.0 github.com/farsightsec/golang-framestream v0.3.0 diff --git a/go.sum b/go.sum index c6e2757a..8112e428 100644 --- a/go.sum +++ b/go.sum @@ -479,6 +479,8 @@ github.com/dmachard/go-logger v0.4.0 h1:JJJW8C5Ri6OaWIECAE6dUNqLs4ym1+WX3xD6h5Mx github.com/dmachard/go-logger v0.4.0/go.mod h1:Gf6Au3CX5l3rZ+Tb3yX31u6h4lwVeZQSBklUI3h8gCA= github.com/dmachard/go-netutils v0.1.0 h1:VdlUeUvpv7t4iPbNfHIKRC0itE/OwEZRl/GIdsc0mqc= github.com/dmachard/go-netutils v0.1.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY= +github.com/dmachard/go-netutils v0.2.0 h1:jizdNnEftIPOf+f3bUg2yWorX8wDQjBfeXYPAS6zGp8= +github.com/dmachard/go-netutils v0.2.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY= github.com/dmachard/go-powerdns-protobuf v1.1.1 h1:HhgkjPGJN9QCLVFWxiTIwP3E0He8ET9uJZaT7/+6HXw= github.com/dmachard/go-powerdns-protobuf v1.1.1/go.mod h1:3sewpdCN4u5KpXBxrLpidHAC18v24y+f4OZ4GKfLaME= github.com/dmachard/go-topmap v1.0.0 h1:FzCnB80WJMSPhpEfWt/79y97XotTQjhlrsXKR6435ow= diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index 7cf31824..edea3c93 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -67,9 +67,9 @@ type ConfigTransformers struct { ThresholdPacketLen int `yaml:"threshold-packet-len" default:"1000"` ThresholdSlow float64 `yaml:"threshold-slow" default:"1.0"` CommonQtypes []string `yaml:"common-qtypes,flow" default:"[\"A\", \"AAAA\", \"TXT\", \"CNAME\", \"PTR\", \"NAPTR\", \"DNSKEY\", \"SRV\", \"SOA\", \"NS\", \"MX\", \"DS\", \"HTTPS\"]"` - UnallowedChars []string `yaml:"unallowed-chars,flow" default:"[\"\"\", \"==\", \"/\", \":\"]"` + UnallowedChars []string `yaml:"unallowed-chars,flow" default:"[\"\\\"\", \"==\", \"/\", \":\"]"` ThresholdMaxLabels int `yaml:"threshold-max-labels" default:"10"` - WhitelistDomains []string `yaml:"whitelist-domains,flow" default:"[\"\\.ip6\\.arpa\"]"` + WhitelistDomains []string `yaml:"whitelist-domains,flow" default:"[\"\\\\.ip6\\\\.arpa\"]"` } `yaml:"suspicious"` Extract struct { Enable bool `yaml:"enable" default:"false"` diff --git a/transformers/suspicious_test.go b/transformers/suspicious_test.go index e7608662..443e209d 100644 --- a/transformers/suspicious_test.go +++ b/transformers/suspicious_test.go @@ -81,7 +81,7 @@ func TestSuspicious_MalformedPacket(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.MalformedPacket != true { @@ -110,7 +110,7 @@ func TestSuspicious_LongDomain(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.LongDomain != true { @@ -139,7 +139,7 @@ func TestSuspicious_SlowDomain(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.SlowDomain != true { @@ -168,7 +168,7 @@ func TestSuspicious_LargePacket(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.LargePacket != true { @@ -196,7 +196,7 @@ func TestSuspicious_UncommonQtype(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.UncommonQtypes != true { @@ -225,7 +225,7 @@ func TestSuspicious_ExceedMaxLabels(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.ExcessiveNumberLabels != true { @@ -253,7 +253,7 @@ func TestSuspicious_UnallowedChars(t *testing.T) { suspicious.CheckIfSuspicious(&dm) if dm.Suspicious.Score != 1.0 { - t.Errorf("suspicious score should be equal to 1.0") + t.Errorf("suspicious score should be equal to 0.0, got: %d", int(dm.Suspicious.Score)) } if dm.Suspicious.UnallowedChars != true { From b380f2376d2a5517335b6fcab587241bc05e0fd7 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 14:11:46 +0200 Subject: [PATCH 07/12] remove default for loggers --- pkgconfig/loggers.go | 812 ++++++++++++++----------------------------- 1 file changed, 268 insertions(+), 544 deletions(-) diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go index 20324043..bff4e238 100644 --- a/pkgconfig/loggers.go +++ b/pkgconfig/loggers.go @@ -3,593 +3,317 @@ package pkgconfig import ( "reflect" - "github.com/dmachard/go-netutils" + "github.com/creasty/defaults" "github.com/prometheus/prometheus/model/relabel" ) type ConfigLoggers struct { DevNull struct { - Enable bool `yaml:"enable"` - Mode string `yaml:"mode"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Mode string `yaml:"mode" default:"text"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"devnull"` Stdout struct { - Enable bool `yaml:"enable"` - Mode string `yaml:"mode"` - TextFormat string `yaml:"text-format"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Mode string `yaml:"mode" default:"text"` + TextFormat string `yaml:"text-format" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"stdout"` Prometheus struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - TLSSupport bool `yaml:"tls-support"` - TLSMutual bool `yaml:"tls-mutual"` - TLSMinVersion string `yaml:"tls-min-version"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - PromPrefix string `yaml:"prometheus-prefix"` - LabelsList []string `yaml:"prometheus-labels"` - TopN int `yaml:"top-n"` - BasicAuthLogin string `yaml:"basic-auth-login"` - BasicAuthPwd string `yaml:"basic-auth-pwd"` - BasicAuthEnabled bool `yaml:"basic-auth-enable"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - RequestersMetricsEnabled bool `yaml:"requesters-metrics-enabled"` - DomainsMetricsEnabled bool `yaml:"domains-metrics-enabled"` - NoErrorMetricsEnabled bool `yaml:"noerror-metrics-enabled"` - ServfailMetricsEnabled bool `yaml:"servfail-metrics-enabled"` - NonExistentMetricsEnabled bool `yaml:"nonexistent-metrics-enabled"` - TimeoutMetricsEnabled bool `yaml:"timeout-metrics-enabled"` - HistogramMetricsEnabled bool `yaml:"histogram-metrics-enabled"` - RequestersCacheTTL int `yaml:"requesters-cache-ttl"` - RequestersCacheSize int `yaml:"requesters-cache-size"` - DomainsCacheTTL int `yaml:"domains-cache-ttl"` - DomainsCacheSize int `yaml:"domains-cache-size"` - NoErrorDomainsCacheTTL int `yaml:"noerror-domains-cache-ttl"` - NoErrorDomainsCacheSize int `yaml:"noerror-domains-cache-size"` - ServfailDomainsCacheTTL int `yaml:"servfail-domains-cache-ttl"` - ServfailDomainsCacheSize int `yaml:"servfail-domains-cache-size"` - NXDomainsCacheTTL int `yaml:"nonexistent-domains-cache-ttl"` - NXDomainsCacheSize int `yaml:"nonexistent-domains-cache-size"` - DefaultDomainsCacheTTL int `yaml:"default-domains-cache-ttl"` - DefaultDomainsCacheSize int `yaml:"default-domains-cache-size"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"127.0.0.1"` + ListenPort int `yaml:"listen-port" default:"8081"` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSMutual bool `yaml:"tls-mutual" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + PromPrefix string `yaml:"prometheus-prefix" default:"dnscollector"` + LabelsList []string `yaml:"prometheus-labels" default:"[]"` + TopN int `yaml:"top-n" default:"10"` + BasicAuthLogin string `yaml:"basic-auth-login" default:"admin"` + BasicAuthPwd string `yaml:"basic-auth-pwd" default:"changeme"` + BasicAuthEnabled bool `yaml:"basic-auth-enable" default:"true"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + RequestersMetricsEnabled bool `yaml:"requesters-metrics-enabled" default:"true"` + DomainsMetricsEnabled bool `yaml:"domains-metrics-enabled" default:"true"` + NoErrorMetricsEnabled bool `yaml:"noerror-metrics-enabled" default:"true"` + ServfailMetricsEnabled bool `yaml:"servfail-metrics-enabled" default:"true"` + NonExistentMetricsEnabled bool `yaml:"nonexistent-metrics-enabled" default:"true"` + TimeoutMetricsEnabled bool `yaml:"timeout-metrics-enabled" default:"false"` + HistogramMetricsEnabled bool `yaml:"histogram-metrics-enabled" default:"false"` + RequestersCacheTTL int `yaml:"requesters-cache-ttl" default:"250000"` + RequestersCacheSize int `yaml:"requesters-cache-size" default:"3600"` + DomainsCacheTTL int `yaml:"domains-cache-ttl" default:"500000"` + DomainsCacheSize int `yaml:"domains-cache-size" default:"3600"` + NoErrorDomainsCacheTTL int `yaml:"noerror-domains-cache-ttl" default:"100000"` + NoErrorDomainsCacheSize int `yaml:"noerror-domains-cache-size" default:"3600"` + ServfailDomainsCacheTTL int `yaml:"servfail-domains-cache-ttl" default:"10000"` + ServfailDomainsCacheSize int `yaml:"servfail-domains-cache-size" default:"3600"` + NXDomainsCacheTTL int `yaml:"nonexistent-domains-cache-ttl" default:"10000"` + NXDomainsCacheSize int `yaml:"nonexistent-domains-cache-size" default:"3600"` + DefaultDomainsCacheTTL int `yaml:"default-domains-cache-ttl" default:"1000"` + DefaultDomainsCacheSize int `yaml:"default-domains-cache-size" default:"3600"` } `yaml:"prometheus"` RestAPI struct { - Enable bool `yaml:"enable"` - ListenIP string `yaml:"listen-ip"` - ListenPort int `yaml:"listen-port"` - BasicAuthLogin string `yaml:"basic-auth-login"` - BasicAuthPwd string `yaml:"basic-auth-pwd"` - TLSSupport bool `yaml:"tls-support"` - TLSMinVersion string `yaml:"tls-min-version"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - TopN int `yaml:"top-n"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ListenIP string `yaml:"listen-ip" default:"127.0.0.1"` + ListenPort int `yaml:"listen-port" default:"8080"` + BasicAuthLogin string `yaml:"basic-auth-login" default:"admin"` + BasicAuthPwd string `yaml:"basic-auth-pwd" default:"changeme"` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + TopN int `yaml:"top-n" default:"100"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"restapi"` LogFile struct { - Enable bool `yaml:"enable"` - FilePath string `yaml:"file-path"` - MaxSize int `yaml:"max-size"` - MaxFiles int `yaml:"max-files"` - FlushInterval int `yaml:"flush-interval"` - Compress bool `yaml:"compress"` - CompressInterval int `yaml:"compress-interval"` - CompressPostCommand string `yaml:"compress-postcommand"` - Mode string `yaml:"mode"` - PostRotateCommand string `yaml:"postrotate-command"` - PostRotateDelete bool `yaml:"postrotate-delete-success"` - TextFormat string `yaml:"text-format"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - ExtendedSupport bool `yaml:"extended-support"` + Enable bool `yaml:"enable" default:"false"` + FilePath string `yaml:"file-path" default:""` + MaxSize int `yaml:"max-size" default:"100"` + MaxFiles int `yaml:"max-files" default:"10"` + FlushInterval int `yaml:"flush-interval" default:"10"` + Compress bool `yaml:"compress" default:"false"` + CompressInterval int `yaml:"compress-interval" default:"60"` + CompressPostCommand string `yaml:"compress-postcommand" default:""` + Mode string `yaml:"mode" default:"text"` + PostRotateCommand string `yaml:"postrotate-command" default:""` + PostRotateDelete bool `yaml:"postrotate-delete-success" default:"false"` + TextFormat string `yaml:"text-format" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + ExtendedSupport bool `yaml:"extended-support" default:"false"` } `yaml:"logfile"` DNSTap struct { - Enable bool `yaml:"enable"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - Transport string `yaml:"transport"` - SockPath string `yaml:"sock-path"` - ConnectTimeout int `yaml:"connect-timeout"` - RetryInterval int `yaml:"retry-interval"` - FlushInterval int `yaml:"flush-interval"` - TLSSupport bool `yaml:"tls-support"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - ServerID string `yaml:"server-id"` - OverwriteIdentity bool `yaml:"overwrite-identity"` - BufferSize int `yaml:"buffer-size"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - ExtendedSupport bool `yaml:"extended-support"` - Compression string `yaml:"compression"` + Enable bool `yaml:"enable" default:"false"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"6000"` + Transport string `yaml:"transport" default:"tcp"` + SockPath string `yaml:"sock-path" default:""` + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + RetryInterval int `yaml:"retry-interval" default:"10"` + FlushInterval int `yaml:"flush-interval" default:"30"` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + ServerID string `yaml:"server-id" default:""` + OverwriteIdentity bool `yaml:"overwrite-identity" default:"false"` + BufferSize int `yaml:"buffer-size" default:"100"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + ExtendedSupport bool `yaml:"extended-support" default:"false"` + Compression string `yaml:"compression" default:"none"` } `yaml:"dnstapclient"` TCPClient struct { - Enable bool `yaml:"enable"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - SockPath string `yaml:"sock-path"` // deprecated - RetryInterval int `yaml:"retry-interval"` - Transport string `yaml:"transport"` - TLSSupport bool `yaml:"tls-support"` // deprecated - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - Mode string `yaml:"mode"` - TextFormat string `yaml:"text-format"` - PayloadDelimiter string `yaml:"delimiter"` - BufferSize int `yaml:"buffer-size"` - FlushInterval int `yaml:"flush-interval"` - ConnectTimeout int `yaml:"connect-timeout"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"9999"` + SockPath string `yaml:"sock-path" default:""` // deprecated + RetryInterval int `yaml:"retry-interval" default:"10"` + Transport string `yaml:"transport" default:"tcp"` + TLSSupport bool `yaml:"tls-support" default:"false"` // deprecated + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + Mode string `yaml:"mode" default:"flat-json"` + TextFormat string `yaml:"text-format" default:""` + PayloadDelimiter string `yaml:"delimiter" default:"\n"` + BufferSize int `yaml:"buffer-size" default:"100"` + FlushInterval int `yaml:"flush-interval" default:"30"` + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"tcpclient"` Syslog struct { - Enable bool `yaml:"enable"` - Severity string `yaml:"severity"` - Facility string `yaml:"facility"` - Transport string `yaml:"transport"` - RemoteAddress string `yaml:"remote-address"` - RetryInterval int `yaml:"retry-interval"` - TextFormat string `yaml:"text-format"` - Mode string `yaml:"mode"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - Formatter string `yaml:"formatter"` - Framer string `yaml:"framer"` - Hostname string `yaml:"hostname"` - AppName string `yaml:"app-name"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - Tag string `yaml:"tag"` - ReplaceNullChar string `yaml:"replace-null-char"` - FlushInterval int `yaml:"flush-interval"` - BufferSize int `yaml:"buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Severity string `yaml:"severity" default:"INFO"` + Facility string `yaml:"facility" default:"DAEMON"` + Transport string `yaml:"transport" default:"local"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1:514"` + RetryInterval int `yaml:"retry-interval" default:"10"` + TextFormat string `yaml:"text-format" default:""` + Mode string `yaml:"mode" default:"text"` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + Formatter string `yaml:"formatter" default:"rfc5424"` + Framer string `yaml:"framer" default:""` + Hostname string `yaml:"hostname" default:""` + AppName string `yaml:"app-name" default:"DNScollector"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + Tag string `yaml:"tag" default:""` + ReplaceNullChar string `yaml:"replace-null-char" default:"�"` + FlushInterval int `yaml:"flush-interval" default:"30"` + BufferSize int `yaml:"buffer-size" default:"100"` } `yaml:"syslog"` Fluentd struct { - Enable bool `yaml:"enable"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - SockPath string `yaml:"sock-path"` // deprecated - ConnectTimeout int `yaml:"connect-timeout"` - RetryInterval int `yaml:"retry-interval"` - FlushInterval int `yaml:"flush-interval"` - Transport string `yaml:"transport"` - TLSSupport bool `yaml:"tls-support"` // deprecated - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - Tag string `yaml:"tag"` - BufferSize int `yaml:"buffer-size"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"24224"` + SockPath string `yaml:"sock-path" default:""` // deprecated + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + RetryInterval int `yaml:"retry-interval" default:"10"` + FlushInterval int `yaml:"flush-interval" default:"30"` + Transport string `yaml:"transport" default:"tcp"` + TLSSupport bool `yaml:"tls-support" default:"false"` // deprecated + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + Tag string `yaml:"tag" default:"dnscollector"` + BufferSize int `yaml:"buffer-size" default:"100"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"4096"` } `yaml:"fluentd"` InfluxDB struct { - Enable bool `yaml:"enable"` - ServerURL string `yaml:"server-url"` - AuthToken string `yaml:"auth-token"` - TLSSupport bool `yaml:"tls-support"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - Bucket string `yaml:"bucket"` - Organization string `yaml:"organization"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ServerURL string `yaml:"server-url" default:"http://localhost:8086"` + AuthToken string `yaml:"auth-token" default:""` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + Bucket string `yaml:"bucket" default:""` + Organization string `yaml:"organization" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"influxdb"` LokiClient struct { - Enable bool `yaml:"enable"` - ServerURL string `yaml:"server-url"` - JobName string `yaml:"job-name"` - Mode string `yaml:"mode"` - FlushInterval int `yaml:"flush-interval"` - BatchSize int `yaml:"batch-size"` - RetryInterval int `yaml:"retry-interval"` - TextFormat string `yaml:"text-format"` - ProxyURL string `yaml:"proxy-url"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - BasicAuthLogin string `yaml:"basic-auth-login"` - BasicAuthPwd string `yaml:"basic-auth-pwd"` - BasicAuthPwdFile string `yaml:"basic-auth-pwd-file"` - TenantID string `yaml:"tenant-id"` - RelabelConfigs []*relabel.Config `yaml:"relabel-configs"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + ServerURL string `yaml:"server-url" default:"http://localhost:3100/loki/api/v1/push"` + JobName string `yaml:"job-name" default:"dnscollector"` + Mode string `yaml:"mode" default:"text"` + FlushInterval int `yaml:"flush-interval" default:"5"` + BatchSize int `yaml:"batch-size" default:"1048576"` + RetryInterval int `yaml:"retry-interval" default:"10"` + TextFormat string `yaml:"text-format" default:""` + ProxyURL string `yaml:"proxy-url" default:""` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + BasicAuthLogin string `yaml:"basic-auth-login" default:""` + BasicAuthPwd string `yaml:"basic-auth-pwd" default:""` + BasicAuthPwdFile string `yaml:"basic-auth-pwd-file" default:""` + TenantID string `yaml:"tenant-id" default:""` + RelabelConfigs []*relabel.Config `yaml:"relabel-configs" default:"[]"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"lokiclient"` Statsd struct { - Enable bool `yaml:"enable"` - Prefix string `yaml:"prefix"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - ConnectTimeout int `yaml:"connect-timeout"` - Transport string `yaml:"transport"` - FlushInterval int `yaml:"flush-interval"` - TLSSupport bool `yaml:"tls-support"` // deprecated - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Prefix string `yaml:"prefix" default:"dnscollector"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"8125"` + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + Transport string `yaml:"transport" default:"udp"` + FlushInterval int `yaml:"flush-interval" default:"10"` + CertFile string `yaml:"cert-file" default:""` + TLSSupport bool `yaml:"tls-support" default:"false"` // deprecated + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + KeyFile string `yaml:"key-file" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"statsd"` ElasticSearchClient struct { - Enable bool `yaml:"enable"` - Index string `yaml:"index"` - Server string `yaml:"server"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - BulkSize int `yaml:"bulk-size"` - BulkChannelSize int `yaml:"bulk-channel-size"` - FlushInterval int `yaml:"flush-interval"` - Compression string `yaml:"compression"` + Enable bool `yaml:"enable" default:"false"` + Index string `yaml:"index" default:"dnscollector"` + Server string `yaml:"server" default:"http://127.0.0.1:9200/"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + BulkSize int `yaml:"bulk-size" default:"5242880"` + BulkChannelSize int `yaml:"bulk-channel-size" default:"10"` + FlushInterval int `yaml:"flush-interval" default:"10"` + Compression string `yaml:"compression" default:"none"` } `yaml:"elasticsearch"` ScalyrClient struct { - Enable bool `yaml:"enable"` - Mode string `yaml:"mode"` - TextFormat string `yaml:"text-format"` - SessionInfo map[string]string `yaml:"sessioninfo"` - Attrs map[string]interface{} `yaml:"attrs"` - ServerURL string `yaml:"server-url"` - APIKey string `yaml:"apikey"` - Parser string `yaml:"parser"` - FlushInterval int `yaml:"flush-interval"` - ProxyURL string `yaml:"proxy-url"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + Mode string `yaml:"mode" default:"text"` + TextFormat string `yaml:"text-format" default:""` + SessionInfo map[string]string `yaml:"sessioninfo" default:"{}"` + Attrs map[string]interface{} `yaml:"attrs" default:"{}"` + ServerURL string `yaml:"server-url" default:"app.scalyr.com"` + APIKey string `yaml:"apikey" default:""` + Parser string `yaml:"parser" default:""` + FlushInterval int `yaml:"flush-interval" default:"10"` + ProxyURL string `yaml:"proxy-url" default:""` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"scalyrclient"` RedisPub struct { - Enable bool `yaml:"enable"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - SockPath string `yaml:"sock-path"` // deprecated - RetryInterval int `yaml:"retry-interval"` - Transport string `yaml:"transport"` - TLSSupport bool `yaml:"tls-support"` // deprecated - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - Mode string `yaml:"mode"` - TextFormat string `yaml:"text-format"` - PayloadDelimiter string `yaml:"delimiter"` - BufferSize int `yaml:"buffer-size"` - FlushInterval int `yaml:"flush-interval"` - ConnectTimeout int `yaml:"connect-timeout"` - RedisChannel string `yaml:"redis-channel"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"6379"` + SockPath string `yaml:"sock-path" default:""` // deprecated + RetryInterval int `yaml:"retry-interval" default:"10"` + Transport string `yaml:"transport" default:"tcp"` + TLSSupport bool `yaml:"tls-support" default:"false"` // deprecated + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + Mode string `yaml:"mode" default:"flat-json"` + TextFormat string `yaml:"text-format" default:""` + PayloadDelimiter string `yaml:"delimiter" default:"\n"` + BufferSize int `yaml:"buffer-size" default:"100"` + FlushInterval int `yaml:"flush-interval" default:"30"` + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + RedisChannel string `yaml:"redis-channel" default:"dns_collector"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"redispub"` KafkaProducer struct { - Enable bool `yaml:"enable"` - RemoteAddress string `yaml:"remote-address"` - RemotePort int `yaml:"remote-port"` - RetryInterval int `yaml:"retry-interval"` - TLSSupport bool `yaml:"tls-support"` - TLSInsecure bool `yaml:"tls-insecure"` - TLSMinVersion string `yaml:"tls-min-version"` - CAFile string `yaml:"ca-file"` - CertFile string `yaml:"cert-file"` - KeyFile string `yaml:"key-file"` - SaslSupport bool `yaml:"sasl-support"` - SaslUsername string `yaml:"sasl-username"` - SaslPassword string `yaml:"sasl-password"` - SaslMechanism string `yaml:"sasl-mechanism"` - Mode string `yaml:"mode"` - TextFormat string `yaml:"text-format"` - BufferSize int `yaml:"buffer-size"` - FlushInterval int `yaml:"flush-interval"` - ConnectTimeout int `yaml:"connect-timeout"` - Topic string `yaml:"topic"` - Partition int `yaml:"partition"` - ChannelBufferSize int `yaml:"chan-buffer-size"` - Compression string `yaml:"compression"` + Enable bool `yaml:"enable" default:"false"` + RemoteAddress string `yaml:"remote-address" default:"127.0.0.1"` + RemotePort int `yaml:"remote-port" default:"9092"` + RetryInterval int `yaml:"retry-interval" default:"10"` + TLSSupport bool `yaml:"tls-support" default:"false"` + TLSInsecure bool `yaml:"tls-insecure" default:"false"` + TLSMinVersion string `yaml:"tls-min-version" default:"1.2"` + CAFile string `yaml:"ca-file" default:""` + CertFile string `yaml:"cert-file" default:""` + KeyFile string `yaml:"key-file" default:""` + SaslSupport bool `yaml:"sasl-support" default:"false"` + SaslUsername string `yaml:"sasl-username" default:""` + SaslPassword string `yaml:"sasl-password" default:""` + SaslMechanism string `yaml:"sasl-mechanism" default:"PLAIN"` + Mode string `yaml:"mode" default:"flat-json"` + TextFormat string `yaml:"text-format" default:""` + BufferSize int `yaml:"buffer-size" default:"100"` + FlushInterval int `yaml:"flush-interval" default:"10"` + ConnectTimeout int `yaml:"connect-timeout" default:"5"` + Topic string `yaml:"topic" default:"dnscollector"` + Partition int `yaml:"partition" default:"0"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` + Compression string `yaml:"compression" default:"none"` } `yaml:"kafkaproducer"` FalcoClient struct { - Enable bool `yaml:"enable"` - URL string `yaml:"url"` - ChannelBufferSize int `yaml:"chan-buffer-size"` + Enable bool `yaml:"enable" default:"false"` + URL string `yaml:"url" default:"http://127.0.0.1:9200"` + ChannelBufferSize int `yaml:"chan-buffer-size" default:"65535"` } `yaml:"falco"` ClickhouseClient struct { - Enable bool `yaml:"enable"` - URL string `yaml:"url"` - User string `yaml:"user"` - Password string `yaml:"password"` - Database string `yaml:"database"` - Table string `yaml:"table"` + Enable bool `yaml:"enable" default:"false"` + URL string `yaml:"url" default:"http://localhost:8123"` + User string `yaml:"user" default:"default"` + Password string `yaml:"password" default:"password"` + Database string `yaml:"database" default:"dnscollector"` + Table string `yaml:"table" default:"records"` } `yaml:"clickhouse"` } func (c *ConfigLoggers) SetDefault() { - c.Stdout.Enable = false - c.Stdout.Mode = ModeText - c.Stdout.TextFormat = "" - c.Stdout.ChannelBufferSize = 65535 - - c.DNSTap.Enable = false - c.DNSTap.RemoteAddress = LocalhostIP - c.DNSTap.RemotePort = 6000 - c.DNSTap.Transport = netutils.SocketTCP - c.DNSTap.ConnectTimeout = 5 - c.DNSTap.RetryInterval = 10 - c.DNSTap.FlushInterval = 30 - c.DNSTap.SockPath = "" - c.DNSTap.TLSSupport = false - c.DNSTap.TLSInsecure = false - c.DNSTap.TLSMinVersion = TLSV12 - c.DNSTap.CAFile = "" - c.DNSTap.CertFile = "" - c.DNSTap.KeyFile = "" - c.DNSTap.ServerID = "" - c.DNSTap.OverwriteIdentity = false - c.DNSTap.BufferSize = 100 - c.DNSTap.ChannelBufferSize = 65535 - c.DNSTap.Compression = CompressNone - c.DNSTap.ExtendedSupport = false - - c.LogFile.Enable = false - c.LogFile.FilePath = "" - c.LogFile.FlushInterval = 10 - c.LogFile.MaxSize = 100 - c.LogFile.MaxFiles = 10 - c.LogFile.Compress = false - c.LogFile.CompressInterval = 60 - c.LogFile.CompressPostCommand = "" - c.LogFile.Mode = ModeText - c.LogFile.PostRotateCommand = "" - c.LogFile.PostRotateDelete = false - c.LogFile.TextFormat = "" - c.LogFile.ChannelBufferSize = 65535 - c.LogFile.ExtendedSupport = false - - c.Prometheus.Enable = false - c.Prometheus.ListenIP = LocalhostIP - c.Prometheus.ListenPort = 8081 - c.Prometheus.TLSSupport = false - c.Prometheus.TLSMutual = false - c.Prometheus.TLSMinVersion = TLSV12 - c.Prometheus.CertFile = "" - c.Prometheus.KeyFile = "" - c.Prometheus.PromPrefix = ProgName - c.Prometheus.TopN = 10 - c.Prometheus.BasicAuthLogin = "admin" - c.Prometheus.BasicAuthPwd = "changeme" - c.Prometheus.BasicAuthEnabled = true - c.Prometheus.ChannelBufferSize = 65535 - c.Prometheus.HistogramMetricsEnabled = false - c.Prometheus.RequestersMetricsEnabled = true - c.Prometheus.DomainsMetricsEnabled = true - c.Prometheus.NoErrorMetricsEnabled = true - c.Prometheus.ServfailMetricsEnabled = true - c.Prometheus.NonExistentMetricsEnabled = true - c.Prometheus.RequestersCacheTTL = 3600 - c.Prometheus.RequestersCacheSize = 250000 - c.Prometheus.DomainsCacheTTL = 3600 - c.Prometheus.DomainsCacheSize = 500000 - c.Prometheus.DomainsCacheTTL = 3600 - c.Prometheus.NoErrorDomainsCacheSize = 100000 - c.Prometheus.NoErrorDomainsCacheTTL = 3600 - c.Prometheus.ServfailDomainsCacheSize = 10000 - c.Prometheus.ServfailDomainsCacheTTL = 3600 - c.Prometheus.NXDomainsCacheSize = 10000 - c.Prometheus.NXDomainsCacheTTL = 3600 - c.Prometheus.DefaultDomainsCacheSize = 1000 - c.Prometheus.DefaultDomainsCacheTTL = 3600 - - c.RestAPI.Enable = false - c.RestAPI.ListenIP = LocalhostIP - c.RestAPI.ListenPort = 8080 - c.RestAPI.BasicAuthLogin = "admin" - c.RestAPI.BasicAuthPwd = "changeme" - c.RestAPI.TLSSupport = false - c.RestAPI.TLSMinVersion = TLSV12 - c.RestAPI.CertFile = "" - c.RestAPI.KeyFile = "" - c.RestAPI.TopN = 100 - c.RestAPI.ChannelBufferSize = 65535 - - c.TCPClient.Enable = false - c.TCPClient.RemoteAddress = LocalhostIP - c.TCPClient.RemotePort = 9999 - c.TCPClient.SockPath = "" - c.TCPClient.RetryInterval = 10 - c.TCPClient.Transport = netutils.SocketTCP - c.TCPClient.TLSSupport = false - c.TCPClient.TLSInsecure = false - c.TCPClient.TLSMinVersion = TLSV12 - c.TCPClient.CAFile = "" - c.TCPClient.CertFile = "" - c.TCPClient.KeyFile = "" - c.TCPClient.Mode = ModeFlatJSON - c.TCPClient.TextFormat = "" - c.TCPClient.PayloadDelimiter = "\n" - c.TCPClient.BufferSize = 100 - c.TCPClient.ConnectTimeout = 5 - c.TCPClient.FlushInterval = 30 - c.TCPClient.ChannelBufferSize = 65535 - - c.Syslog.Enable = false - c.Syslog.Severity = "INFO" - c.Syslog.Facility = "DAEMON" - c.Syslog.Transport = "local" - c.Syslog.RemoteAddress = "127.0.0.1:514" - c.Syslog.TextFormat = "" - c.Syslog.Mode = ModeText - c.Syslog.RetryInterval = 10 - c.Syslog.TLSInsecure = false - c.Syslog.TLSMinVersion = TLSV12 - c.Syslog.CAFile = "" - c.Syslog.CertFile = "" - c.Syslog.KeyFile = "" - c.Syslog.ChannelBufferSize = 65535 - c.Syslog.Tag = "" - c.Syslog.Framer = "" - c.Syslog.Formatter = "rfc5424" - c.Syslog.Hostname = "" - c.Syslog.AppName = "DNScollector" - c.Syslog.ReplaceNullChar = "�" - c.Syslog.FlushInterval = 30 - c.Syslog.BufferSize = 100 - - c.Fluentd.Enable = false - c.Fluentd.RemoteAddress = LocalhostIP - c.Fluentd.RemotePort = 24224 - c.Fluentd.SockPath = "" // deprecated - c.Fluentd.RetryInterval = 10 - c.Fluentd.ConnectTimeout = 5 - c.Fluentd.FlushInterval = 30 - c.Fluentd.Transport = netutils.SocketTCP - c.Fluentd.TLSSupport = false // deprecated - c.Fluentd.TLSInsecure = false - c.Fluentd.TLSMinVersion = TLSV12 - c.Fluentd.CAFile = "" - c.Fluentd.CertFile = "" - c.Fluentd.KeyFile = "" - c.Fluentd.Tag = ProgQname - c.Fluentd.BufferSize = 100 - c.Fluentd.ChannelBufferSize = 4096 - - c.InfluxDB.Enable = false - c.InfluxDB.ServerURL = "http://localhost:8086" - c.InfluxDB.AuthToken = "" - c.InfluxDB.TLSSupport = false - c.InfluxDB.TLSInsecure = false - c.InfluxDB.TLSMinVersion = TLSV12 - c.InfluxDB.CAFile = "" - c.InfluxDB.CertFile = "" - c.InfluxDB.KeyFile = "" - c.InfluxDB.Bucket = "" - c.InfluxDB.Organization = "" - c.InfluxDB.ChannelBufferSize = 65535 - - c.LokiClient.Enable = false - c.LokiClient.ServerURL = "http://localhost:3100/loki/api/v1/push" - c.LokiClient.JobName = ProgName - c.LokiClient.Mode = ModeText - c.LokiClient.FlushInterval = 5 - c.LokiClient.BatchSize = 1024 * 1024 - c.LokiClient.RetryInterval = 10 - c.LokiClient.TextFormat = "" - c.LokiClient.ProxyURL = "" - c.LokiClient.TLSInsecure = false - c.LokiClient.TLSMinVersion = TLSV12 - c.LokiClient.CAFile = "" - c.LokiClient.CertFile = "" - c.LokiClient.KeyFile = "" - c.LokiClient.BasicAuthLogin = "" - c.LokiClient.BasicAuthPwd = "" - c.LokiClient.BasicAuthPwdFile = "" - c.LokiClient.TenantID = "" - c.LokiClient.ChannelBufferSize = 65535 - - c.Statsd.Enable = false - c.Statsd.Prefix = ProgName - c.Statsd.RemoteAddress = LocalhostIP - c.Statsd.RemotePort = 8125 - c.Statsd.Transport = netutils.SocketUDP - c.Statsd.ConnectTimeout = 5 - c.Statsd.FlushInterval = 10 - c.Statsd.TLSSupport = false // deprecated - c.Statsd.TLSInsecure = false - c.Statsd.TLSMinVersion = TLSV12 - c.Statsd.CAFile = "" - c.Statsd.CertFile = "" - c.Statsd.KeyFile = "" - c.Statsd.ChannelBufferSize = 65535 - - c.ElasticSearchClient.Enable = false - c.ElasticSearchClient.Server = "http://127.0.0.1:9200/" - c.ElasticSearchClient.Index = ProgName - c.ElasticSearchClient.ChannelBufferSize = 4096 - c.ElasticSearchClient.BulkSize = 5242880 - c.ElasticSearchClient.FlushInterval = 10 - c.ElasticSearchClient.BulkChannelSize = 10 - c.ElasticSearchClient.Compression = CompressNone - - c.ScalyrClient.Enable = false - c.ScalyrClient.Mode = ModeText - c.ScalyrClient.TextFormat = "" - c.ScalyrClient.SessionInfo = map[string]string{} - c.ScalyrClient.Attrs = map[string]interface{}{} - c.ScalyrClient.ServerURL = "app.scalyr.com" - c.ScalyrClient.APIKey = "" - c.ScalyrClient.Parser = "" - c.ScalyrClient.FlushInterval = 10 - c.ScalyrClient.ProxyURL = "" - c.ScalyrClient.TLSInsecure = false - c.ScalyrClient.TLSMinVersion = TLSV12 - c.ScalyrClient.CAFile = "" - c.ScalyrClient.CertFile = "" - c.ScalyrClient.KeyFile = "" - c.ScalyrClient.ChannelBufferSize = 65535 - - c.RedisPub.Enable = false - c.RedisPub.RemoteAddress = LocalhostIP - c.RedisPub.RemotePort = 6379 - c.RedisPub.SockPath = "" - c.RedisPub.RetryInterval = 10 - c.RedisPub.Transport = netutils.SocketTCP - c.RedisPub.TLSSupport = false - c.RedisPub.TLSInsecure = false - c.RedisPub.TLSMinVersion = TLSV12 - c.RedisPub.CAFile = "" - c.RedisPub.CertFile = "" - c.RedisPub.KeyFile = "" - c.RedisPub.Mode = ModeFlatJSON - c.RedisPub.TextFormat = "" - c.RedisPub.PayloadDelimiter = "\n" - c.RedisPub.BufferSize = 100 - c.RedisPub.ConnectTimeout = 5 - c.RedisPub.FlushInterval = 30 - c.RedisPub.RedisChannel = "dns_collector" - c.RedisPub.ChannelBufferSize = 65535 - - c.KafkaProducer.Enable = false - c.KafkaProducer.RemoteAddress = LocalhostIP - c.KafkaProducer.RemotePort = 9092 - c.KafkaProducer.RetryInterval = 10 - c.KafkaProducer.TLSSupport = false - c.KafkaProducer.TLSInsecure = false - c.KafkaProducer.TLSMinVersion = TLSV12 - c.KafkaProducer.CAFile = "" - c.KafkaProducer.CertFile = "" - c.KafkaProducer.KeyFile = "" - c.KafkaProducer.SaslSupport = false - c.KafkaProducer.SaslUsername = "" - c.KafkaProducer.SaslPassword = "" - c.KafkaProducer.SaslMechanism = SASLMechanismPlain - c.KafkaProducer.Mode = ModeFlatJSON - c.KafkaProducer.TextFormat = "" - c.KafkaProducer.BufferSize = 100 - c.KafkaProducer.ConnectTimeout = 5 - c.KafkaProducer.FlushInterval = 10 - c.KafkaProducer.Topic = ProgName - c.KafkaProducer.Partition = 0 - c.KafkaProducer.ChannelBufferSize = 4096 - c.KafkaProducer.Compression = CompressNone - - c.FalcoClient.Enable = false - c.FalcoClient.URL = "http://127.0.0.1:9200" - c.FalcoClient.ChannelBufferSize = 65535 - - c.ClickhouseClient.Enable = false - c.ClickhouseClient.URL = "http://localhost:8123" - c.ClickhouseClient.User = "default" - c.ClickhouseClient.Password = "password" - c.ClickhouseClient.Database = ProgName - c.ClickhouseClient.Table = "records" + defaults.Set(c) } func (c *ConfigLoggers) IsValid(userCfg map[string]interface{}) error { From aa30b712021c3c25cbbe0999b6bc0b4044c9e1c2 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 14:53:54 +0200 Subject: [PATCH 08/12] fix tag in fluentd --- pkgconfig/loggers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go index bff4e238..9ca13b6a 100644 --- a/pkgconfig/loggers.go +++ b/pkgconfig/loggers.go @@ -166,7 +166,7 @@ type ConfigLoggers struct { CAFile string `yaml:"ca-file" default:""` CertFile string `yaml:"cert-file" default:""` KeyFile string `yaml:"key-file" default:""` - Tag string `yaml:"tag" default:"dnscollector"` + Tag string `yaml:"tag" default:"dns.collector"` BufferSize int `yaml:"buffer-size" default:"100"` ChannelBufferSize int `yaml:"chan-buffer-size" default:"4096"` } `yaml:"fluentd"` From f7358fe4753b9d6b707c9d849469bbf8995327d6 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 15:24:44 +0200 Subject: [PATCH 09/12] remove tls client --- README.md | 2 +- pkgconfig/constants.go | 27 +------------ pkgconfig/tls_client.go | 73 ------------------------------------ pkgconfig/tls_client_test.go | 26 ------------- workers/dnstap_relay.go | 4 +- workers/dnstapclient.go | 13 +++---- workers/dnstapserver.go | 4 +- workers/fluentd.go | 4 +- workers/influxdb.go | 5 ++- workers/kafkaproducer.go | 5 ++- workers/lokiclient.go | 5 ++- workers/powerdns.go | 4 +- workers/prometheus.go | 4 +- workers/redispub.go | 4 +- workers/restapi.go | 4 +- workers/scalyr.go | 5 ++- workers/statsd.go | 6 +-- workers/syslog.go | 6 +-- workers/tcpclient.go | 4 +- 19 files changed, 42 insertions(+), 163 deletions(-) delete mode 100644 pkgconfig/tls_client.go delete mode 100644 pkgconfig/tls_client_test.go diff --git a/README.md b/README.md index 1319b7b0..a921075f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Go version Go tests Go bench -Go lines +Go lines

diff --git a/pkgconfig/constants.go b/pkgconfig/constants.go index 4fb98b64..9c976027 100644 --- a/pkgconfig/constants.go +++ b/pkgconfig/constants.go @@ -1,9 +1,5 @@ package pkgconfig -import ( - "crypto/tls" -) - const ( StrUnknown = "UNKNOWN" @@ -19,13 +15,8 @@ const ( ValidDomain = "dnscollector.dev." BadDomainLabel = "ultramegaverytoolonglabel-ultramegaverytoolonglabel-ultramegaverytoolonglabel.dnscollector.dev." badLongLabel = "ultramegaverytoolonglabel-ultramegaverytoolonglabel-" - BadVeryLongDomain = "ultramegaverytoolonglabel.dnscollector" + - badLongLabel + - badLongLabel + - badLongLabel + - badLongLabel + - badLongLabel + - ".dev." + BadVeryLongDomain = "ultramegaverytoolonglabel.dnscollector" + badLongLabel + badLongLabel + + badLongLabel + badLongLabel + badLongLabel + ".dev." ModeText = "text" ModeJSON = "json" @@ -36,11 +27,6 @@ const ( SASLMechanismPlain = "PLAIN" SASLMechanismScram = "SCRAM-SHA-512" - TLSV10 = "1.0" - TLSV11 = "1.1" - TLSV12 = "1.2" - TLSV13 = "1.3" - CompressGzip = "gzip" CompressSnappy = "snappy" CompressLz4 = "lz4" @@ -48,15 +34,6 @@ const ( CompressNone = "none" ) -var ( - TLSVersion = map[string]uint16{ - TLSV10: tls.VersionTLS10, - TLSV11: tls.VersionTLS11, - TLSV12: tls.VersionTLS12, - TLSV13: tls.VersionTLS13, - } -) - var ( PrefixLogWorker = "worker - " PrefixLogTransformer = "transformer - " diff --git a/pkgconfig/tls_client.go b/pkgconfig/tls_client.go deleted file mode 100644 index eeb000ef..00000000 --- a/pkgconfig/tls_client.go +++ /dev/null @@ -1,73 +0,0 @@ -package pkgconfig - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "os" -) - -var clientCipherSuites = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -} - -func IsValidTLS(mode string) bool { - switch mode { - case - TLSV10, - TLSV11, - TLSV12, - TLSV13: - return true - } - return false -} - -type TLSOptions struct { - CAFile string - CertFile string - KeyFile string - InsecureSkipVerify bool - MinVersion string -} - -func TLSClientConfig(options TLSOptions) (*tls.Config, error) { - - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - InsecureSkipVerify: false, - CipherSuites: clientCipherSuites, - } - tlsConfig.InsecureSkipVerify = options.InsecureSkipVerify - - if len(options.CAFile) > 0 { - CAs := x509.NewCertPool() - pemData, err := os.ReadFile(options.CAFile) - if err != nil { - return nil, fmt.Errorf("could not read CA certificate %q: %w", options.CAFile, err) - } - if !CAs.AppendCertsFromPEM(pemData) { - return nil, fmt.Errorf("failed to append certificates from PEM file: %q", options.CAFile) - } - tlsConfig.RootCAs = CAs - } - - if len(options.CertFile) > 0 && len(options.KeyFile) > 0 { - cer, err := tls.LoadX509KeyPair(options.CertFile, options.KeyFile) - if err != nil { - return nil, fmt.Errorf("loading certificate failed: %w", err) - } - tlsConfig.Certificates = []tls.Certificate{cer} - } - - if tlsVersion, ok := TLSVersion[options.MinVersion]; ok { - tlsConfig.MinVersion = tlsVersion - } else { - return nil, fmt.Errorf("invalid minimum TLS version: %x", options.MinVersion) - } - - return tlsConfig, nil -} diff --git a/pkgconfig/tls_client_test.go b/pkgconfig/tls_client_test.go deleted file mode 100644 index db3889c4..00000000 --- a/pkgconfig/tls_client_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package pkgconfig - -import ( - "crypto/tls" - "reflect" - "testing" -) - -func TestConfigClientTLSNoVerify(t *testing.T) { - tlsConfig, err := TLSClientConfig(TLSOptions{InsecureSkipVerify: true, MinVersion: TLSV12}) - - if err != nil || tlsConfig == nil { - t.Fatal("Unable to configure client TLS", err) - } - - if !reflect.DeepEqual(tlsConfig.CipherSuites, clientCipherSuites) { - t.Fatal("Unexpected client cipher suites") - } - if tlsConfig.MinVersion != tls.VersionTLS12 { - t.Fatal("Unexpected client TLS version") - } - - if tlsConfig.Certificates != nil { - t.Fatal("Somehow client certificates were set") - } -} diff --git a/workers/dnstap_relay.go b/workers/dnstap_relay.go index 9ad7f215..bd4c5061 100644 --- a/workers/dnstap_relay.go +++ b/workers/dnstap_relay.go @@ -27,7 +27,7 @@ func NewDnstapProxifier(next []Worker, config *pkgconfig.Config, logger *logger. } func (w *DnstapProxifier) CheckConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Collectors.DnstapProxifier.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Collectors.DnstapProxifier.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "] dnstaprelay - invalid tls min version") } } @@ -124,7 +124,7 @@ func (w *DnstapProxifier) StartCollect() { listener, err := netutils.StartToListen( w.GetConfig().Collectors.DnstapProxifier.ListenIP, w.GetConfig().Collectors.DnstapProxifier.ListenPort, w.GetConfig().Collectors.DnstapProxifier.SockPath, - w.GetConfig().Collectors.DnstapProxifier.TLSSupport, pkgconfig.TLSVersion[w.GetConfig().Collectors.DnstapProxifier.TLSMinVersion], + w.GetConfig().Collectors.DnstapProxifier.TLSSupport, netutils.TLSVersion[w.GetConfig().Collectors.DnstapProxifier.TLSMinVersion], w.GetConfig().Collectors.DnstapProxifier.CertFile, w.GetConfig().Collectors.DnstapProxifier.KeyFile) if err != nil { w.LogFatal("collector dnstaprelay listening failed: ", err) diff --git a/workers/dnstapclient.go b/workers/dnstapclient.go index 5f743fe8..55dc2d9d 100644 --- a/workers/dnstapclient.go +++ b/workers/dnstapclient.go @@ -50,7 +50,7 @@ func (w *DnstapSender) ReadConfig() { w.GetConfig().Loggers.DNSTap.ServerID = w.GetConfig().GetServerIdentity() } - if !pkgconfig.IsValidTLS(w.GetConfig().Loggers.DNSTap.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Loggers.DNSTap.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "invalid tls min version") } } @@ -103,15 +103,12 @@ func (w *DnstapSender) ConnectToRemote() { var tlsConfig *tls.Config - tlsOptions := pkgconfig.TLSOptions{ - InsecureSkipVerify: w.GetConfig().Loggers.DNSTap.TLSInsecure, - MinVersion: w.GetConfig().Loggers.DNSTap.TLSMinVersion, - CAFile: w.GetConfig().Loggers.DNSTap.CAFile, - CertFile: w.GetConfig().Loggers.DNSTap.CertFile, - KeyFile: w.GetConfig().Loggers.DNSTap.KeyFile, + tlsOptions := netutils.TLSOptions{ + InsecureSkipVerify: w.GetConfig().Loggers.DNSTap.TLSInsecure, MinVersion: w.GetConfig().Loggers.DNSTap.TLSMinVersion, + CAFile: w.GetConfig().Loggers.DNSTap.CAFile, CertFile: w.GetConfig().Loggers.DNSTap.CertFile, KeyFile: w.GetConfig().Loggers.DNSTap.KeyFile, } - tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err = netutils.TLSClientConfig(tlsOptions) if err == nil { dialer := &net.Dialer{Timeout: connTimeout} conn, err = tls.DialWithDialer(dialer, netutils.SocketTCP, address, tlsConfig) diff --git a/workers/dnstapserver.go b/workers/dnstapserver.go index 229a96ad..883d1658 100644 --- a/workers/dnstapserver.go +++ b/workers/dnstapserver.go @@ -36,7 +36,7 @@ func NewDnstapServer(next []Worker, config *pkgconfig.Config, logger *logger.Log } func (w *DnstapServer) CheckConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Collectors.Dnstap.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Collectors.Dnstap.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "] dnstap - invalid tls min version") } } @@ -192,7 +192,7 @@ func (w *DnstapServer) StartCollect() { // start to listen listener, err := netutils.StartToListen( cfg.ListenIP, cfg.ListenPort, cfg.SockPath, - cfg.TLSSupport, pkgconfig.TLSVersion[cfg.TLSMinVersion], + cfg.TLSSupport, netutils.TLSVersion[cfg.TLSMinVersion], cfg.CertFile, cfg.KeyFile) if err != nil { w.LogFatal(pkgconfig.PrefixLogWorker+"["+w.GetName()+"] listen error: ", err) diff --git a/workers/fluentd.go b/workers/fluentd.go index 887684ef..1dbae418 100644 --- a/workers/fluentd.go +++ b/workers/fluentd.go @@ -93,14 +93,14 @@ func (w *FluentdClient) ConnectToRemote() { var tlsConfig *tls.Config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.Fluentd.TLSInsecure, MinVersion: w.GetConfig().Loggers.Fluentd.TLSMinVersion, CAFile: w.GetConfig().Loggers.Fluentd.CAFile, CertFile: w.GetConfig().Loggers.Fluentd.CertFile, KeyFile: w.GetConfig().Loggers.Fluentd.KeyFile, } - tlsConfig, _ = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, _ = netutils.TLSClientConfig(tlsOptions) c = client.New(client.ConnectionOptions{ Factory: &client.ConnFactory{ diff --git a/workers/influxdb.go b/workers/influxdb.go index 5c5723d7..7837e08a 100644 --- a/workers/influxdb.go +++ b/workers/influxdb.go @@ -6,6 +6,7 @@ import ( "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" + "github.com/dmachard/go-netutils" influxdb2 "github.com/influxdata/influxdb-client-go" "github.com/influxdata/influxdb-client-go/api" @@ -81,7 +82,7 @@ func (w *InfluxDBClient) StartLogging() { opts := influxdb2.DefaultOptions() opts.SetUseGZip(true) if w.GetConfig().Loggers.InfluxDB.TLSSupport { - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.InfluxDB.TLSInsecure, MinVersion: w.GetConfig().Loggers.InfluxDB.TLSMinVersion, CAFile: w.GetConfig().Loggers.InfluxDB.CAFile, @@ -89,7 +90,7 @@ func (w *InfluxDBClient) StartLogging() { KeyFile: w.GetConfig().Loggers.InfluxDB.KeyFile, } - tlsConfig, err := pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err := netutils.TLSClientConfig(tlsOptions) if err != nil { w.LogFatal("logger=influxdb - tls config failed:", err) } diff --git a/workers/kafkaproducer.go b/workers/kafkaproducer.go index cec6b9d8..442b0a75 100644 --- a/workers/kafkaproducer.go +++ b/workers/kafkaproducer.go @@ -12,6 +12,7 @@ import ( "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" + "github.com/dmachard/go-netutils" "github.com/segmentio/kafka-go" "github.com/segmentio/kafka-go/compress" "github.com/segmentio/kafka-go/sasl/plain" @@ -90,7 +91,7 @@ func (w *KafkaProducer) ConnectToKafka(ctx context.Context, readyTimer *time.Tim // enable TLS if w.GetConfig().Loggers.KafkaProducer.TLSSupport { - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.KafkaProducer.TLSInsecure, MinVersion: w.GetConfig().Loggers.KafkaProducer.TLSMinVersion, CAFile: w.GetConfig().Loggers.KafkaProducer.CAFile, @@ -98,7 +99,7 @@ func (w *KafkaProducer) ConnectToKafka(ctx context.Context, readyTimer *time.Tim KeyFile: w.GetConfig().Loggers.KafkaProducer.KeyFile, } - tlsConfig, err := pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err := netutils.TLSClientConfig(tlsOptions) if err != nil { w.LogFatal("logger=kafka - tls config failed:", err) } diff --git a/workers/lokiclient.go b/workers/lokiclient.go index cc3e3f4c..28b43e1d 100644 --- a/workers/lokiclient.go +++ b/workers/lokiclient.go @@ -16,6 +16,7 @@ import ( "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" + "github.com/dmachard/go-netutils" "github.com/gogo/protobuf/proto" "github.com/grafana/dskit/backoff" "github.com/klauspost/compress/snappy" @@ -95,7 +96,7 @@ func (w *LokiClient) ReadConfig() { } // tls client config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.LokiClient.TLSInsecure, MinVersion: w.GetConfig().Loggers.LokiClient.TLSMinVersion, CAFile: w.GetConfig().Loggers.LokiClient.CAFile, @@ -103,7 +104,7 @@ func (w *LokiClient) ReadConfig() { KeyFile: w.GetConfig().Loggers.LokiClient.KeyFile, } - tlsConfig, err := pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err := netutils.TLSClientConfig(tlsOptions) if err != nil { w.LogFatal(pkgconfig.PrefixLogWorker+"["+w.GetName()+"] loki - tls config failed:", err) } diff --git a/workers/powerdns.go b/workers/powerdns.go index 99df90e1..a2379696 100644 --- a/workers/powerdns.go +++ b/workers/powerdns.go @@ -35,7 +35,7 @@ func NewPdnsServer(next []Worker, config *pkgconfig.Config, logger *logger.Logge } func (w *PdnsServer) CheckConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Collectors.PowerDNS.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Collectors.PowerDNS.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "] invalid tls min version") } } @@ -132,7 +132,7 @@ func (w *PdnsServer) StartCollect() { // start to listen listener, err := netutils.StartToListen( cfg.ListenIP, cfg.ListenPort, "", - cfg.TLSSupport, pkgconfig.TLSVersion[cfg.TLSMinVersion], + cfg.TLSSupport, netutils.TLSVersion[cfg.TLSMinVersion], cfg.CertFile, cfg.KeyFile) if err != nil { w.LogFatal(pkgconfig.PrefixLogWorker+"["+w.GetName()+"] listening failed: ", err) diff --git a/workers/prometheus.go b/workers/prometheus.go index a98bd336..d80e57ef 100644 --- a/workers/prometheus.go +++ b/workers/prometheus.go @@ -1054,7 +1054,7 @@ func (w *Prometheus) InitProm() { } func (w *Prometheus) ReadConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Loggers.Prometheus.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Loggers.Prometheus.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "] prometheus - invalid tls min version") } } @@ -1106,7 +1106,7 @@ func (w *Prometheus) ListenAndServe() { } // update tls min version according to the user config - tlsConfig.MinVersion = pkgconfig.TLSVersion[w.GetConfig().Loggers.Prometheus.TLSMinVersion] + tlsConfig.MinVersion = netutils.TLSVersion[w.GetConfig().Loggers.Prometheus.TLSMinVersion] if w.GetConfig().Loggers.Prometheus.TLSMutual { diff --git a/workers/redispub.go b/workers/redispub.go index b8dbc2d1..be062dc3 100644 --- a/workers/redispub.go +++ b/workers/redispub.go @@ -122,7 +122,7 @@ func (w *RedisPub) ConnectToRemote() { var tlsConfig *tls.Config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.RedisPub.TLSInsecure, MinVersion: w.GetConfig().Loggers.RedisPub.TLSMinVersion, CAFile: w.GetConfig().Loggers.RedisPub.CAFile, @@ -130,7 +130,7 @@ func (w *RedisPub) ConnectToRemote() { KeyFile: w.GetConfig().Loggers.RedisPub.KeyFile, } - tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err = netutils.TLSClientConfig(tlsOptions) if err == nil { dialer := &net.Dialer{Timeout: connTimeout} conn, err = tls.DialWithDialer(dialer, netutils.SocketTCP, address, tlsConfig) diff --git a/workers/restapi.go b/workers/restapi.go index ff3b6348..2b67cbb4 100644 --- a/workers/restapi.go +++ b/workers/restapi.go @@ -87,7 +87,7 @@ func NewRestAPI(config *pkgconfig.Config, logger *logger.Logger, name string) *R } func (w *RestAPI) ReadConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Loggers.RestAPI.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Loggers.RestAPI.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "]restapi - invalid tls min version") } } @@ -612,7 +612,7 @@ func (w *RestAPI) ListenAndServe() { } // update tls min version according to the user config - tlsConfig.MinVersion = pkgconfig.TLSVersion[w.GetConfig().Loggers.RestAPI.TLSMinVersion] + tlsConfig.MinVersion = netutils.TLSVersion[w.GetConfig().Loggers.RestAPI.TLSMinVersion] listener, err = tls.Listen(netutils.SocketTCP, addrlisten, tlsConfig) diff --git a/workers/scalyr.go b/workers/scalyr.go index 59f718da..b61efc98 100644 --- a/workers/scalyr.go +++ b/workers/scalyr.go @@ -19,6 +19,7 @@ import ( "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/transformers" "github.com/dmachard/go-logger" + "github.com/dmachard/go-netutils" ) // ScalyrClient is a client for Scalyr(https://www.dataset.com/) @@ -87,7 +88,7 @@ func (w *ScalyrClient) ReadConfig() { } // tls client config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.ScalyrClient.TLSInsecure, MinVersion: w.GetConfig().Loggers.ScalyrClient.TLSMinVersion, CAFile: w.GetConfig().Loggers.ScalyrClient.CAFile, @@ -95,7 +96,7 @@ func (w *ScalyrClient) ReadConfig() { KeyFile: w.GetConfig().Loggers.ScalyrClient.KeyFile, } - tlsConfig, err := pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err := netutils.TLSClientConfig(tlsOptions) if err != nil { w.LogFatal("unable to parse tls confgi: ", err) } diff --git a/workers/statsd.go b/workers/statsd.go index ff38378c..ddded423 100644 --- a/workers/statsd.go +++ b/workers/statsd.go @@ -57,7 +57,7 @@ func NewStatsdClient(config *pkgconfig.Config, logger *logger.Logger, name strin } func (w *StatsdClient) ReadConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Loggers.Statsd.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Loggers.Statsd.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "[" + w.GetName() + "]statd - invalid tls min version") } } @@ -266,7 +266,7 @@ func (w *StatsdClient) StartLogging() { var tlsConfig *tls.Config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.Statsd.TLSInsecure, MinVersion: w.GetConfig().Loggers.Statsd.TLSMinVersion, CAFile: w.GetConfig().Loggers.Statsd.CAFile, @@ -274,7 +274,7 @@ func (w *StatsdClient) StartLogging() { KeyFile: w.GetConfig().Loggers.Statsd.KeyFile, } - tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err = netutils.TLSClientConfig(tlsOptions) if err == nil { dialer := &net.Dialer{Timeout: connTimeout} conn, err = tls.DialWithDialer(dialer, netutils.SocketTCP, address, tlsConfig) diff --git a/workers/syslog.go b/workers/syslog.go index 264d838a..2a493f98 100644 --- a/workers/syslog.go +++ b/workers/syslog.go @@ -35,7 +35,7 @@ func NewSyslog(config *pkgconfig.Config, console *logger.Logger, name string) *S } func (w *Syslog) ReadConfig() { - if !pkgconfig.IsValidTLS(w.GetConfig().Loggers.Syslog.TLSMinVersion) { + if !netutils.IsValidTLS(w.GetConfig().Loggers.Syslog.TLSMinVersion) { w.LogFatal(pkgconfig.PrefixLogWorker + "invalid tls min version") } @@ -95,7 +95,7 @@ func (w *Syslog) ConnectToRemote() { w.GetConfig().Loggers.Syslog.Transport, w.GetConfig().Loggers.Syslog.RemoteAddress) - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.Syslog.TLSInsecure, MinVersion: w.GetConfig().Loggers.Syslog.TLSMinVersion, CAFile: w.GetConfig().Loggers.Syslog.CAFile, @@ -103,7 +103,7 @@ func (w *Syslog) ConnectToRemote() { KeyFile: w.GetConfig().Loggers.Syslog.KeyFile, } - tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err = netutils.TLSClientConfig(tlsOptions) if err == nil { logWriter, err = syslog.DialWithTLSConfig(w.GetConfig().Loggers.Syslog.Transport, w.GetConfig().Loggers.Syslog.RemoteAddress, w.facility|w.severity, diff --git a/workers/tcpclient.go b/workers/tcpclient.go index 081d270a..e0cf2c85 100644 --- a/workers/tcpclient.go +++ b/workers/tcpclient.go @@ -121,7 +121,7 @@ func (w *TCPClient) ConnectToRemote() { var tlsConfig *tls.Config - tlsOptions := pkgconfig.TLSOptions{ + tlsOptions := netutils.TLSOptions{ InsecureSkipVerify: w.GetConfig().Loggers.TCPClient.TLSInsecure, MinVersion: w.GetConfig().Loggers.TCPClient.TLSMinVersion, CAFile: w.GetConfig().Loggers.TCPClient.CAFile, @@ -129,7 +129,7 @@ func (w *TCPClient) ConnectToRemote() { KeyFile: w.GetConfig().Loggers.TCPClient.KeyFile, } - tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions) + tlsConfig, err = netutils.TLSClientConfig(tlsOptions) if err == nil { dialer := &net.Dialer{Timeout: connTimeout} conn, err = tls.DialWithDialer(dialer, netutils.SocketTCP, address, tlsConfig) From b57e81720731a17be281033a017b0eb606b38cbb Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 15:42:21 +0200 Subject: [PATCH 10/12] Update README and fix Makefile --- Makefile | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bc70c5ca..036ab208 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ GO_DNSTAP_PROTOBUF := 1.0.1 GO_FRAMESTREAM := 0.10.0 GO_CLIENTSYSLOG := 0.4.0 GO_TOPMAP := 1.0.0 -GO_NETUTILS := 0.1.0 +GO_NETUTILS := 0.2.0 BUILD_TIME := $(shell LANG=en_US date +"%F_%T_%z") COMMIT := $(shell git rev-parse --short HEAD) diff --git a/README.md b/README.md index a921075f..30348ee0 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@

Go Report Go version -Go tests +Go tests Go bench -Go lines +Go lines

From 0367d0238d9fef0569adbf9e7f36d025121ba76f Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 15:52:53 +0200 Subject: [PATCH 11/12] code cleanup --- dnsutils/dns_parser.go | 145 ++++++++-------------------------------- dnsutils/edns_parser.go | 8 +-- 2 files changed, 29 insertions(+), 124 deletions(-) diff --git a/dnsutils/dns_parser.go b/dnsutils/dns_parser.go index 78132ef7..1eda2f19 100644 --- a/dnsutils/dns_parser.go +++ b/dnsutils/dns_parser.go @@ -14,111 +14,34 @@ const DNSLen = 12 const UNKNOWN = "UNKNOWN" var ( - Class = map[int]string{ - 1: "IN", - 3: "CH", - 4: "HS", - 254: "NONE", - 255: "ANY", - } + Class = map[int]string{1: "IN", 3: "CH", 4: "HS", 254: "NONE", 255: "ANY"} Rdatatypes = map[int]string{ - 0: "NONE", - 1: "A", - 2: "NS", - 3: "MD", - 4: "MF", - 5: "CNAME", - 6: "SOA", - 7: "MB", - 8: "MG", - 9: "MR", - 10: "NULL", - 11: "WKS", - 12: "PTR", - 13: "HINFO", - 14: "MINFO", - 15: "MX", - 16: "TXT", - 17: "RP", - 18: "AFSDB", - 19: "X25", - 20: "ISDN", - 21: "RT", - 22: "NSAP", - 23: "NSAP_PTR", - 24: "SIG", - 25: "KEY", - 26: "PX", - 27: "GPOS", - 28: "AAAA", - 29: "LOC", - 30: "NXT", - 33: "SRV", - 35: "NAPTR", - 36: "KX", - 37: "CERT", - 38: "A6", - 39: "DNAME", - 41: "OPT", - 42: "APL", - 43: "DS", - 44: "SSHFP", - 45: "IPSECKEY", - 46: "RRSIG", - 47: "NSEC", - 48: "DNSKEY", - 49: "DHCID", - 50: "NSEC3", - 51: "NSEC3PARAM", - 52: "TSLA", - 53: "SMIMEA", - 55: "HIP", - 56: "NINFO", - 59: "CDS", - 60: "CDNSKEY", - 61: "OPENPGPKEY", - 62: "CSYNC", - 64: "SVCB", - 65: "HTTPS", - 99: "SPF", - 103: "UNSPEC", - 108: "EUI48", - 109: "EUI64", - 249: "TKEY", - 250: "TSIG", - 251: "IXFR", - 252: "AXFR", - 253: "MAILB", - 254: "MAILA", - 255: "ANY", - 256: "URI", - 257: "CAA", - 258: "AVC", - 259: "AMTRELAY", - 32768: "TA", - 32769: "DLV", + 0: "NONE", 1: "A", 2: "NS", 3: "MD", + 4: "MF", 5: "CNAME", 6: "SOA", 7: "MB", + 8: "MG", 9: "MR", 10: "NULL", 11: "WKS", + 12: "PTR", 13: "HINFO", 14: "MINFO", 15: "MX", + 16: "TXT", 17: "RP", 18: "AFSDB", 19: "X25", + 20: "ISDN", 21: "RT", 22: "NSAP", 23: "NSAP_PTR", + 24: "SIG", 25: "KEY", 26: "PX", 27: "GPOS", + 28: "AAAA", 29: "LOC", 30: "NXT", 33: "SRV", + 35: "NAPTR", 36: "KX", 37: "CERT", 38: "A6", + 39: "DNAME", 41: "OPT", 42: "APL", 43: "DS", + 44: "SSHFP", 45: "IPSECKEY", 46: "RRSIG", 47: "NSEC", + 48: "DNSKEY", 49: "DHCID", 50: "NSEC3", 51: "NSEC3PARAM", + 52: "TSLA", 53: "SMIMEA", 55: "HIP", 56: "NINFO", + 59: "CDS", 60: "CDNSKEY", 61: "OPENPGPKEY", 62: "CSYNC", + 64: "SVCB", 65: "HTTPS", 99: "SPF", 103: "UNSPEC", + 108: "EUI48", 109: "EUI64", 249: "TKEY", 250: "TSIG", + 251: "IXFR", 252: "AXFR", 253: "MAILB", 254: "MAILA", + 255: "ANY", 256: "URI", 257: "CAA", 258: "AVC", + 259: "AMTRELAY", 32768: "TA", 32769: "DLV", } Rcodes = map[int]string{ - 0: "NOERROR", - 1: "FORMERR", - 2: "SERVFAIL", - 3: "NXDOMAIN", - 4: "NOIMP", - 5: "REFUSED", - 6: "YXDOMAIN", - 7: "YXRRSET", - 8: "NXRRSET", - 9: "NOTAUTH", - 10: "NOTZONE", - 11: "DSOTYPENI", - 16: "BADSIG", - 17: "BADKEY", - 18: "BADTIME", - 19: "BADMODE", - 20: "BADNAME", - 21: "BADALG", - 22: "BADTRUNC", - 23: "BADCOOKIE", + 0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOIMP", + 5: "REFUSED", 6: "YXDOMAIN", 7: "YXRRSET", 8: "NXRRSET", 9: "NOTAUTH", + 10: "NOTZONE", 11: "DSOTYPENI", 16: "BADSIG", 17: "BADKEY", + 18: "BADTIME", 19: "BADMODE", 20: "BADNAME", 21: "BADALG", + 22: "BADTRUNC", 23: "BADCOOKIE", } ) @@ -169,21 +92,9 @@ func (e *decodingError) Unwrap() error { } type DNSHeader struct { - ID int - Qr int - Opcode int - Aa int - Tc int - Rd int - Ra int - Z int - Ad int - Cd int - Rcode int - Qdcount int - Ancount int - Nscount int - Arcount int + ID, Qr, Opcode, Rcode int + Aa, Tc, Rd, Ra, Z, Ad, Cd int + Qdcount, Ancount, Nscount, Arcount int } /* diff --git a/dnsutils/edns_parser.go b/dnsutils/edns_parser.go index 945037b5..b7952f05 100644 --- a/dnsutils/edns_parser.go +++ b/dnsutils/edns_parser.go @@ -17,13 +17,7 @@ var ErrDecodeEdnsTooManyOpts = errors.New("edns, packet contained too many OPT R var ( OptCodes = map[int]string{ - 3: "NSID", - 8: "CSUBNET", - 9: "EXPIRE", - 10: "COOKIE", - 11: "KEEPALIVE", - 12: "PADDING", - 15: "ERRORS", + 3: "NSID", 8: "CSUBNET", 9: "EXPIRE", 10: "COOKIE", 11: "KEEPALIVE", 12: "PADDING", 15: "ERRORS", } ErrorCodeToString = map[int]string{ 0: "Other", From a172b4f8d0d1082b1cb45ac487c300e0d9c9b976 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 18 May 2024 15:58:30 +0200 Subject: [PATCH 12/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30348ee0..92efaa7b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Go version Go tests Go bench -Go lines +Go lines