-
+
-
+
diff --git a/config.yml b/config.yml index de758f94..f721ef5c 100644 --- a/config.yml +++ b/config.yml @@ -14,8 +14,7 @@ global: ################################################ # Pipelining configuration # more details: https://github.com/dmachard/go-dnscollector/blob/main/docs/running_mode.md#pipelining -# collectors: https://github.com/dmachard/go-dnscollector/blob/main/docs/collectors.md -# loggers: https://github.com/dmachard/go-dnscollector/blob/main/docs/loggers.md +# workers: https://github.com/dmachard/go-dnscollector/blob/main/docs/workers.md # transformers: https://github.com/dmachard/go-dnscollector/blob/main/docs/transformers.md ################################################ pipelines: @@ -37,8 +36,7 @@ pipelines: ################################################ # DEPRECATED - multiplexer configuration # more details: https://github.com/dmachard/go-dnscollector/blob/main/docs/running_mode.md#multiplexer -# collectors: https://github.com/dmachard/go-dnscollector/blob/main/docs/collectors.md -# loggers: https://github.com/dmachard/go-dnscollector/blob/main/docs/loggers.md +# workers: https://github.com/dmachard/go-dnscollector/blob/main/docs/workers.md # transformers: https://github.com/dmachard/go-dnscollector/blob/main/docs/transformers.md ################################################ # multiplexer: diff --git a/docs/loggers/logger_file.md b/docs/loggers/logger_file.md index 12c74038..5eba2546 100644 --- a/docs/loggers/logger_file.md +++ b/docs/loggers/logger_file.md @@ -35,7 +35,7 @@ Options: * `compress-interval` (integer) > checking every X seconds if new log files must be compressed -* `compress-command` (string) +* `compress-postcommand` (string) > run external script after file compress step * `mode` (string) @@ -64,7 +64,7 @@ logfile: flush-interval: 10 compress: false compress-interval: 5 - compress-command: null + compress-postcommand: null mode: text text-format: "" postrotate-command: null diff --git a/docs/workers.md b/docs/workers.md index 3b25c183..4e1af7fe 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -2,29 +2,29 @@ A worker can act as a collector or a logger. -| Worker | Descriptions | -| :-----------------------------------------------------|:--------------------------------------------------------| -| [DNStap](collectors/collector_dnstap.md) | DNStap receiver and proxifier | -| [PowerDNS](collectors/collector_powerdns.md) | Protobuf PowerDNS receiver | -| [Tail](collectors/collector_tail.md) | Tail on plain text file | -| [XDP Sniffer](collectors/collector_xdp.md) | Live capture on network interface with XDP | -| [AF_PACKET Sniffer](collectors/collector_afpacket.md) | Live capture on network interface with AF_PACKET socket | -| [File Ingestor](collectors/collector_fileingestor.md) | File ingestor like pcap | -| [DNS Message](collectors/collector_dnsmessage.md) | Matching specific DNS message | -| [Console](loggers/logger_stdout.md) | Print logs to stdout in text, json or binary formats. | -| [File](loggers/logger_file.md) | Save logs to file in plain text or binary formats | -| [DNStap](loggers/logger_dnstap.md) | Send logs as DNStap format to a remote collector | -| [Prometheus](loggers/logger_prometheus.md) | Expose metrics | -| [Statsd](loggers/logger_statsd.md) | Expose metrics | -| [Rest API](loggers/logger_restapi.md) | Search domains, clients in logs | -| [TCP](loggers/logger_tcp.md) | Tcp stream client logger | -| [Syslog](loggers/logger_syslog.md) | Syslog logger to local syslog system or remote one. | -| [Fluentd](loggers/logger_fluentd.md) | Send logs to Fluentd server | -| [InfluxDB](loggers/logger_influxdb.md) | Send logs to InfluxDB server | -| [Loki](loggers/logger_loki.md) | Send logs to Loki server | -| [ElasticSearch](loggers/logger_elasticsearch.md) | Send logs to Elastic instance | -| [Scalyr](loggers/logger_scalyr.md) | Client for the Scalyr/DataSet addEvents API endpoint. | -| [Redis](loggers/logger_redis.md) | Redis pub logger | -| [Kafka](loggers/logger_kafka.md) | Kafka DNS producer | -| [Falco](loggers/logger_falco.md) | Falco plugin logger | -| [ClickHouse](loggers/logger_clickhouse.md) | ClickHouse logger | +| Worker | Type | Descriptions | +| :-----------------------------------------------------|:----------|:--------------------------------------------------------| +| [DNStap Server](collectors/collector_dnstap.md) | Collector | DNStap receiver and proxifier | +| [PowerDNS](collectors/collector_powerdns.md) | Collector | Protobuf PowerDNS receiver | +| [Tail](collectors/collector_tail.md) | Collector | Tail on plain text file | +| [XDP Sniffer](collectors/collector_xdp.md) | Collector | Live capture on network interface with XDP | +| [AF_PACKET Sniffer](collectors/collector_afpacket.md) | Collector | Live capture on network interface with AF_PACKET socket | +| [File Ingestor](collectors/collector_fileingestor.md) | Collector | File ingestor like pcap | +| [DNS Message](collectors/collector_dnsmessage.md) | Collector | Matching specific DNS message | +| [Console](loggers/logger_stdout.md) | Logger | Print logs to stdout in text, json or binary formats. | +| [File](loggers/logger_file.md) | Logger | Save logs to file in plain text or binary formats | +| [DNStap Client](loggers/logger_dnstap.md) | Logger | Send logs as DNStap format to a remote collector | +| [Prometheus](loggers/logger_prometheus.md) | Logger | Expose metrics | +| [Statsd](loggers/logger_statsd.md) | Logger | Expose metrics | +| [Rest API](loggers/logger_restapi.md) | Logger | Search domains, clients in logs | +| [TCP](loggers/logger_tcp.md) | Logger | Tcp stream client logger | +| [Syslog](loggers/logger_syslog.md) | Logger | Syslog logger to local syslog system or remote one. | +| [Fluentd](loggers/logger_fluentd.md) | Logger | Send logs to Fluentd server | +| [InfluxDB](loggers/logger_influxdb.md) | Logger | Send logs to InfluxDB server | +| [Loki Client](loggers/logger_loki.md) | Logger | Send logs to Loki server | +| [ElasticSearch](loggers/logger_elasticsearch.md) | Logger | Send logs to Elastic instance | +| [Scalyr](loggers/logger_scalyr.md) | Logger | Client for the Scalyr/DataSet addEvents API endpoint. | +| [Redis publisher](loggers/logger_redis.md) | Logger | Redis pub logger | +| [Kafka Producer](loggers/logger_kafka.md) | Logger | Kafka DNS producer | +| [Falco](loggers/logger_falco.md) | Logger | Falco plugin logger | +| [ClickHouse](loggers/logger_clickhouse.md) | Logger | ClickHouse logger | diff --git a/pkgconfig/config_test.go b/pkgconfig/config_test.go index f5a9cedd..83975557 100644 --- a/pkgconfig/config_test.go +++ b/pkgconfig/config_test.go @@ -2,6 +2,7 @@ package pkgconfig import ( "os" + "strings" "testing" ) @@ -47,6 +48,38 @@ func createTempConfigFile(content string) (string, error) { return tempFile.Name(), nil } +func TestConfig_CheckConfig_InvalidKey_Logger(t *testing.T) { + content := ` +pipelines: +- name: filednstap + logfile: + invalid: null +` + + tempFile, err := createTempConfigFile(content) + if err != nil { + t.Fatalf("Error creating temporary file: %v", err) + } + defer os.Remove(tempFile) + configFile, err := os.Open(tempFile) + if err != nil { + t.Fatalf("Read temporary file: %v", err) + } + defer configFile.Close() + + err = CheckConfig(configFile) + if err == nil { + t.Errorf("CheckConfig() is nil, want error") + return + } + + if !strings.Contains(err.Error(), "unknown key") { + t.Errorf("invalid error: %s", err) + return + } + +} + func TestConfig_CheckConfig(t *testing.T) { tests := []struct { name string diff --git a/pkgconfig/pipelines.go b/pkgconfig/pipelines.go index 52b8e7b0..d8941f57 100644 --- a/pkgconfig/pipelines.go +++ b/pkgconfig/pipelines.go @@ -35,13 +35,26 @@ func (c *ConfigPipelines) IsValid(userCfg map[string]interface{}) error { delete(userCfg, "routing-policy") } - a := ConfigCollectors{} - errA := a.IsValid(userCfg) - b := ConfigLoggers{} - errB := b.IsValid(userCfg) + wc := ConfigCollectors{} + wl := ConfigLoggers{} - if errA != nil && errB != nil { - return errors.Errorf("invalid stranza - %s", errA) + for workerName := range userCfg { + collectorExist := wc.IsExists(workerName) + loggerExist := wl.IsExists(workerName) + if !collectorExist && !loggerExist { + return errors.Errorf("invalid worker type - %s", workerName) + } + + if collectorExist { + if err := wc.IsValid(userCfg); err != nil { + return errors.Errorf("%s", err) + } + } + if loggerExist { + if err := wl.IsValid(userCfg); err != nil { + return errors.Errorf("%s", err) + } + } } return nil