diff --git a/README.md b/README.md index e5faa035..7620aadd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

Go Report Go version -Go tests +Go tests Go bench Go lines

@@ -56,24 +56,24 @@ - [`Statsd`](docs/loggers/logger_statsd.md) support - [`REST API`](docs/loggers/logger_restapi.md) with [swagger](https://generator.swagger.io/?url=https://raw.githubusercontent.com/dmachard/go-dnscollector/main/docs/swagger.yml) to search DNS domains - *Send to remote host with generic transport protocol* - - [`TCP`](docs/loggers/logger_tcp.md) + - Raw [`TCP`](docs/loggers/logger_tcp.md) client - [`Syslog`](docs/loggers/logger_syslog.md) with TLS support - - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages with TLS support + - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf client - *Send to various sinks* - [`Fluentd`](docs/loggers/logger_fluentd.md) - [`InfluxDB`](docs/loggers/logger_influxdb.md) - - [`Loki`](docs/loggers/logger_loki.md) + - [`Loki`](docs/loggers/logger_loki.md) client - [`ElasticSearch`](docs/loggers/logger_elasticsearch.md) - [`Scalyr`](docs/loggers/logger_scalyr.md) - [`Redis`](docs/loggers/logger_redis.md) publisher - [`Kafka`](docs/loggers/logger_kafka.md) producer - - [`Clickhouse`](docs/loggers/logger_clickhouse.md) *(not yet production ready)* + - [`ClickHouse`](docs/loggers/logger_clickhouse.md) client *(not yet production ready)* - *Send to security tools* - [`Falco`](docs/loggers/logger_falco.md) - **[Transformers](./docs/transformers.md)** - - Custom `JSON` [Relabeling](docs/transformers/transform_relabeling.md) + - Custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON structure - Add additionnal [Tags](docs/transformers/transform_atags.md) - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md) - Latency [Computing](docs/transformers/transform_latency.md) diff --git a/dnsutils/message.go b/dnsutils/message.go index 18249f65..1634116b 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -37,6 +37,7 @@ var ( ReducerDirectives = regexp.MustCompile(`^reducer-*`) MachineLearningDirectives = regexp.MustCompile(`^ml-*`) FilteringDirectives = regexp.MustCompile(`^filtering-*`) + ATagsDirectives = regexp.MustCompile(`^atags*`) ) func GetIPPort(dm *DNSMessage) (string, int, string, int) { @@ -446,6 +447,49 @@ func (dm *DNSMessage) handlePdnsDirectives(directive string, s *strings.Builder) return nil } +func (dm *DNSMessage) handleATagsDirectives(directive string, s *strings.Builder) error { + if dm.ATags == nil { + s.WriteString("-") + } else { + var directives []string + if i := strings.IndexByte(directive, ':'); i == -1 { + directives = append(directives, directive) + } else { + directives = []string{directive[:i], directive[i+1:]} + } + + switch directive := directives[0]; { + case directive == "atags": + if len(dm.ATags.Tags) > 0 { + if len(directives) == 2 { + tagIndex, err := strconv.Atoi(directives[1]) + if err != nil { + log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1]) + } + if tagIndex >= len(dm.ATags.Tags) { + s.WriteString("-") + } else { + s.WriteString(dm.ATags.Tags[tagIndex]) + } + } else { + for i, tag := range dm.ATags.Tags { + s.WriteString(tag) + // add separator + if i+1 < len(dm.ATags.Tags) { + s.WriteString(",") + } + } + } + } else { + s.WriteString("-") + } + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + func (dm *DNSMessage) handleSuspiciousDirectives(directive string, s *strings.Builder) error { if dm.Suspicious == nil { s.WriteString("-") @@ -793,6 +837,11 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo if err != nil { return nil, err } + case ATagsDirectives.MatchString(directive): + err := dm.handleATagsDirectives(directive, &s) + if err != nil { + return nil, err + } // handle invalid directive default: diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 36003dea..21abff7a 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -396,6 +396,15 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) { } }`, }, + { + transform: "atags", + dmRef: DNSMessage{ATags: &TransformATags{Tags: []string{"test0", "test1"}}}, + jsonRef: `{ + "atags": { + "tags": [ "test0", "test1" ] + } + }`, + }, } for _, tc := range testcases { @@ -1200,6 +1209,61 @@ func TestDnsMessage_TextFormat_Directives_Pdns(t *testing.T) { } } +func TestDnsMessage_TextFormat_Directives_ATags(t *testing.T) { + config := pkgconfig.GetFakeConfig() + + testcases := []struct { + name string + format string + dm DNSMessage + expected string + }{ + { + name: "undefined", + format: "atags", + dm: DNSMessage{}, + expected: "-", + }, + { + name: "empty_attributes", + format: "atags", + dm: DNSMessage{ATags: &TransformATags{}}, + expected: "-", + }, + { + name: "tags_all", + format: "atags", + dm: DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}}, + expected: "tag1,tag2", + }, + { + name: "tags_index", + format: "atags:1", + dm: DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}}, + expected: "tag2", + }, + { + name: "tags_invalid_index", + format: "atags:3", + dm: DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}}, + expected: "-", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + line := tc.dm.String( + strings.Fields(tc.format), + config.Global.TextFormatDelimiter, + config.Global.TextFormatBoundary, + ) + if line != tc.expected { + t.Errorf("Want: %s, got: %s", tc.expected, line) + } + }) + } +} + func TestDnsMessage_TextFormat_Directives_Suspicious(t *testing.T) { config := pkgconfig.GetFakeConfig() diff --git a/docs/_integration/elasticsearch/README.md b/docs/_integration/elasticsearch/README.md index f879cd49..ad1ab444 100644 --- a/docs/_integration/elasticsearch/README.md +++ b/docs/_integration/elasticsearch/README.md @@ -1,6 +1,8 @@ # DNS-collector with Elastic and Kibana +- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/elasticsearch/docker-compose.yml)` file + - Create the `data` folder. - Start the docker stack: diff --git a/docs/_integration/fluentd/README.md b/docs/_integration/fluentd/README.md index 22c5f042..f1073cc7 100644 --- a/docs/_integration/fluentd/README.md +++ b/docs/_integration/fluentd/README.md @@ -1,6 +1,8 @@ # DNS-collector with Fluentd +- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/fluentd/docker-compose.yml)` file + - Create the `data` folder. - Start the docker stack: diff --git a/docs/_integration/kafka/README.md b/docs/_integration/kafka/README.md index ea89da61..77ea3473 100644 --- a/docs/_integration/kafka/README.md +++ b/docs/_integration/kafka/README.md @@ -1,6 +1,8 @@ # DNS-collector with Kafka +- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/kafka/docker-compose.yml)` file + - Create the `data` folder. - Start the docker stack: diff --git a/docs/collectors/collector_dnstap.md b/docs/collectors/collector_dnstap.md index e0da5bcc..5c31a2c8 100644 --- a/docs/collectors/collector_dnstap.md +++ b/docs/collectors/collector_dnstap.md @@ -7,34 +7,34 @@ The traffic can be a tcp or unix DNStap stream. TLS is also supported. Options: -- `listen-ip` (str) local address to bind to. +- `listen-ip` (str) > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0). -- `listen-port` (int) local port to bind to. +- `listen-port` (int) > Set the local port that the server will listen on. If not provided, use the default port. -- `sock-path` (str) Unix socket path. +- `sock-path` (str) > Specify the path for the Unix socket to be created. -- `tls-support` (bool) set to true to enable TLS. +- `tls-support` (bool) > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication. -- `tls-min-version` (str) Minimun TLS version to use. +- `tls-min-version` (str) > Specifies the minimum TLS version that the server will support. -- `cert-file` (str) path to a certificate server file to use. +- `cert-file` (str) > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled. -- `key-file`(str) path to a key server file to use. +- `key-file`(str) > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled. -- `sock-rcvbuf` (int) sets the socket receive buffer in bytes SO_RCVBUF. - > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down. +- `sock-rcvbuf` (int) + > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down. Sets the socket receive buffer in bytes SO_RCVBUF. > Set to zero to use the default system value. -- `reset-conn` (bool) reset TCP connection on exit. +- `reset-conn` (bool) > Set whether to send a TCP Reset to force the cleanup of the connection on the remote side when the server exits. -- `chan-buffer-size` (int) incoming channel size, number of packet before to drop it. +- `chan-buffer-size` (int) > Specifies the maximum number of packets that can be buffered before dropping additional packets. -- `disable-dnsparser"` (bool) disable the minimalist DNS parser. - > Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ... -- `extended-support` (bool) decode the extended extra field sent by DNScollector. - > If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. +- `disable-dnsparser"` (bool) + > Disable the minimalist DNS parser. Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ... +- `extended-support` (bool) + > Decode the extended extra field sent by DNScollector. If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others. -- `compression` (string) Compression for DNStap messages: `none`, `gzip`, `lz4`, `snappy`, `zstd`. - > Specifies the compression algorithm to use. +- `compression` (string) + > Specifies the compression algorithm to use. Compression for DNStap messages: `none`, `gzip`, `lz4`, `snappy`, `zstd`. Defaults: @@ -67,19 +67,19 @@ For config examples, take a look to the following [one](../_examples/use-case-12 Options: -- `listen-ip` (str) local address to bind to. +- `listen-ip` (str) > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0). -- `listen-port` (int) local port to bind to. +- `listen-port` (int) > Set the local port that the server will listen on. If not provided, use the default port. -- `sock-path` (str) Unix socket path. +- `sock-path` (str) > Specify the path for the Unix socket to be created. -- `tls-support` (bool) set to true to enable TLS. +- `tls-support` (bool) > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication. -- `tls-min-version` (str) Minimun TLS version to use. +- `tls-min-version` (str) > Specifies the minimum TLS version that the server will support. -- `cert-file` (str) path to a certificate server file to use. +- `cert-file` (str) > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled. -- `key-file`(str) path to a key server file to use. +- `key-file`(str) > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled. Defaults diff --git a/docs/dnsjson.md b/docs/dnsjson.md index bf33f9ad..e68c36a7 100644 --- a/docs/dnsjson.md +++ b/docs/dnsjson.md @@ -162,6 +162,7 @@ This JSON message can be extended by collector(s): This JSON message can be also extended by transformer(s): +- [Atags](transformers/transformer_atags.md) - [GeoIP](transformers/transformer_geoip.md) - [Suspicious traffic detector](transformers/transform_suspiciousdetector.md) - [Public suffix](transformers/transform_normalize.md) diff --git a/docs/loggers.md b/docs/loggers.md index 4eda86b4..82a638d5 100644 --- a/docs/loggers.md +++ b/docs/loggers.md @@ -18,4 +18,4 @@ | [Redis](loggers/logger_redis.md) | Redis pub logger | | [Kafka](loggers/logger_kafka.md) | Kafka DNS producer | | [Falco](loggers/logger_falco.md) | Falco plugin logger | -| [Clickhouse](loggers/logger_clickhouse.md) | ClickHouse logger | +| [ClickHouse](loggers/logger_clickhouse.md) | ClickHouse logger | diff --git a/docs/loggers/logger_clickhouse.md b/docs/loggers/logger_clickhouse.md index d230d406..53112cf1 100644 --- a/docs/loggers/logger_clickhouse.md +++ b/docs/loggers/logger_clickhouse.md @@ -1,7 +1,7 @@ -# Logger: Clickhouse client +# Logger: ClickHouse client -Clickhouse client to remote Clickhouse server +Clickhouse client to remote ClickHouse server Options: diff --git a/docs/transformers/transform_atags.md b/docs/transformers/transform_atags.md index d78bfd76..2c059bc3 100644 --- a/docs/transformers/transform_atags.md +++ b/docs/transformers/transform_atags.md @@ -1,19 +1,63 @@ # Transformer: ATags -Use this transformer to add additional flag in your DNS logs. +Use this transformer to add additional tags in your DNS logs. + +This transformation can be valuable in the `pipeline` mode, where it's possible to match specific traffic. +In such cases, you can include a tag for identification." + +A list of string is expected in the `tag` field. Configuration example: ```yaml -- name: filter +transforms: + atags: + tags: [ "TXT:google", "MX:apple" ] +``` + +When the feature is enabled, the following json field are populated in your DNS message: + +Flat JSON: + +```json +{ + "atags.tags.0": "TXT:google", + "atags.tags.1": "MX:apple" +} + +Default JSON structure: + +```json +{ + "atags": { + "tags": [ "test0", "test1" ] + } +} +``` + +Complete example with the `dnsmessage` collector + +```yaml +pipelines: + - name: filter dnsmessage: - matching: + matching: include: - dns.qname: "^.*\\.google\\.com$" + dns.qname: "^.*\\.google\\.com$" transforms: - atags: + atags: tags: [ "google"] - routing-policy: - dropped: [ outputfile ] - default: [ central ] +``` + +Custom text format: + +If you logs your DNS traffic in basic text format, you can use the specific directives: + +- `atags[:INDEX]`: get all tags separated by comma, or the tag according to the provided INDEX + +```yaml +- name: console + stdout: + mode: text + text-format: "timestamp-rfc3339ns identity qr qname qtype atags:0" ```