add more docs for the atags transformer (#656)

* add more docs
* add tests
* update docs
* add text directive

README.md

@@ -1,7 +1,7 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.20-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-435-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20tests-440-green" alt="Go tests"/>
 <img src="https://img.shields.io/badge/go%20bench-19-green" alt="Go bench"/>
 <img src="https://img.shields.io/badge/go%20lines-39765-green" alt="Go lines"/>
 </p>
@@ -56,24 +56,24 @@
     - [`Statsd`](docs/loggers/logger_statsd.md) support
     - [`REST API`](docs/loggers/logger_restapi.md) with [swagger](https://generator.swagger.io/?url=https://raw.githubusercontent.com/dmachard/go-dnscollector/main/docs/swagger.yml) to search DNS domains
   - *Send to remote host with generic transport protocol*
-    - [`TCP`](docs/loggers/logger_tcp.md)
+    - Raw [`TCP`](docs/loggers/logger_tcp.md) client
     - [`Syslog`](docs/loggers/logger_syslog.md) with TLS support
-    - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf messages with TLS support
+    - [`DNSTap`](docs/loggers/logger_dnstap.md) protobuf client
   - *Send to various sinks*
     - [`Fluentd`](docs/loggers/logger_fluentd.md)
     - [`InfluxDB`](docs/loggers/logger_influxdb.md)
-    - [`Loki`](docs/loggers/logger_loki.md)
+    - [`Loki`](docs/loggers/logger_loki.md) client
     - [`ElasticSearch`](docs/loggers/logger_elasticsearch.md)
     - [`Scalyr`](docs/loggers/logger_scalyr.md)
     - [`Redis`](docs/loggers/logger_redis.md) publisher
     - [`Kafka`](docs/loggers/logger_kafka.md) producer
-    - [`Clickhouse`](docs/loggers/logger_clickhouse.md) *(not yet production ready)*
+    - [`ClickHouse`](docs/loggers/logger_clickhouse.md) client *(not yet production ready)*
   - *Send to security tools*
     - [`Falco`](docs/loggers/logger_falco.md)
 
 - **[Transformers](./docs/transformers.md)**
 
-  - Custom `JSON` [Relabeling](docs/transformers/transform_relabeling.md)
+  - Custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON structure
   - Add additionnal [Tags](docs/transformers/transform_atags.md)
   - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md)
   - Latency [Computing](docs/transformers/transform_latency.md)

dnsutils/message.go

@@ -37,6 +37,7 @@ var (
 	ReducerDirectives         = regexp.MustCompile(`^reducer-*`)
 	MachineLearningDirectives = regexp.MustCompile(`^ml-*`)
 	FilteringDirectives       = regexp.MustCompile(`^filtering-*`)
+	ATagsDirectives           = regexp.MustCompile(`^atags*`)
 )
 
 func GetIPPort(dm *DNSMessage) (string, int, string, int) {
@@ -446,6 +447,49 @@ func (dm *DNSMessage) handlePdnsDirectives(directive string, s *strings.Builder)
 	return nil
 }
 
+func (dm *DNSMessage) handleATagsDirectives(directive string, s *strings.Builder) error {
+	if dm.ATags == nil {
+		s.WriteString("-")
+	} else {
+		var directives []string
+		if i := strings.IndexByte(directive, ':'); i == -1 {
+			directives = append(directives, directive)
+		} else {
+			directives = []string{directive[:i], directive[i+1:]}
+		}
+
+		switch directive := directives[0]; {
+		case directive == "atags":
+			if len(dm.ATags.Tags) > 0 {
+				if len(directives) == 2 {
+					tagIndex, err := strconv.Atoi(directives[1])
+					if err != nil {
+						log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1])
+					}
+					if tagIndex >= len(dm.ATags.Tags) {
+						s.WriteString("-")
+					} else {
+						s.WriteString(dm.ATags.Tags[tagIndex])
+					}
+				} else {
+					for i, tag := range dm.ATags.Tags {
+						s.WriteString(tag)
+						// add separator
+						if i+1 < len(dm.ATags.Tags) {
+							s.WriteString(",")
+						}
+					}
+				}
+			} else {
+				s.WriteString("-")
+			}
+		default:
+			return errors.New(ErrorUnexpectedDirective + directive)
+		}
+	}
+	return nil
+}
+
 func (dm *DNSMessage) handleSuspiciousDirectives(directive string, s *strings.Builder) error {
 	if dm.Suspicious == nil {
 		s.WriteString("-")
@@ -793,6 +837,11 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
 			if err != nil {
 				return nil, err
 			}
+		case ATagsDirectives.MatchString(directive):
+			err := dm.handleATagsDirectives(directive, &s)
+			if err != nil {
+				return nil, err
+			}
 
 		// handle invalid directive
 		default:

dnsutils/message_test.go

@@ -396,6 +396,15 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) {
 						}
 					}`,
 		},
+		{
+			transform: "atags",
+			dmRef:     DNSMessage{ATags: &TransformATags{Tags: []string{"test0", "test1"}}},
+			jsonRef: `{
+						"atags": {
+							"tags": [ "test0", "test1" ]
+						}
+					}`,
+		},
 	}
 
 	for _, tc := range testcases {
@@ -1200,6 +1209,61 @@ func TestDnsMessage_TextFormat_Directives_Pdns(t *testing.T) {
 	}
 }
 
+func TestDnsMessage_TextFormat_Directives_ATags(t *testing.T) {
+	config := pkgconfig.GetFakeConfig()
+
+	testcases := []struct {
+		name     string
+		format   string
+		dm       DNSMessage
+		expected string
+	}{
+		{
+			name:     "undefined",
+			format:   "atags",
+			dm:       DNSMessage{},
+			expected: "-",
+		},
+		{
+			name:     "empty_attributes",
+			format:   "atags",
+			dm:       DNSMessage{ATags: &TransformATags{}},
+			expected: "-",
+		},
+		{
+			name:     "tags_all",
+			format:   "atags",
+			dm:       DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}},
+			expected: "tag1,tag2",
+		},
+		{
+			name:     "tags_index",
+			format:   "atags:1",
+			dm:       DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}},
+			expected: "tag2",
+		},
+		{
+			name:     "tags_invalid_index",
+			format:   "atags:3",
+			dm:       DNSMessage{ATags: &TransformATags{Tags: []string{"tag1", "tag2"}}},
+			expected: "-",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			line := tc.dm.String(
+				strings.Fields(tc.format),
+				config.Global.TextFormatDelimiter,
+				config.Global.TextFormatBoundary,
+			)
+			if line != tc.expected {
+				t.Errorf("Want: %s, got: %s", tc.expected, line)
+			}
+		})
+	}
+}
+
 func TestDnsMessage_TextFormat_Directives_Suspicious(t *testing.T) {
 	config := pkgconfig.GetFakeConfig()
 

docs/_integration/elasticsearch/README.md

@@ -1,6 +1,8 @@
 
 # DNS-collector with Elastic and Kibana
 
+- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/elasticsearch/docker-compose.yml)` file
+
 - Create the `data` folder.
 
 - Start the docker stack:

docs/_integration/fluentd/README.md

@@ -1,6 +1,8 @@
 
 # DNS-collector with Fluentd
 
+- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/fluentd/docker-compose.yml)` file
+
 - Create the `data` folder.
 
 - Start the docker stack:

docs/_integration/kafka/README.md

@@ -1,6 +1,8 @@
 
 # DNS-collector with Kafka
 
+- Download the `[docker-compose](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/kafka/docker-compose.yml)` file
+
 - Create the `data` folder.
 
 - Start the docker stack:

docs/collectors/collector_dnstap.md

@@ -7,34 +7,34 @@ The traffic can be a tcp or unix DNStap stream. TLS is also supported.
 
 Options:
 
-- `listen-ip` (str) local address to bind to.
+- `listen-ip` (str)
   > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0).
-- `listen-port` (int) local port to bind to.
+- `listen-port` (int)
   > Set the local port that the server will listen on. If not provided, use the default port.
-- `sock-path` (str) Unix socket path.
+- `sock-path` (str)
   > Specify the path for the Unix socket to be created.
-- `tls-support` (bool) set to true to enable TLS.
+- `tls-support` (bool)
   > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication.
-- `tls-min-version` (str) Minimun TLS version to use.
+- `tls-min-version` (str)
   > Specifies the minimum TLS version that the server will support.
-- `cert-file` (str) path to a certificate server file to use.
+- `cert-file` (str)
   > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled.
-- `key-file`(str) path to a key server file to use.
+- `key-file`(str)
   > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled.
-- `sock-rcvbuf` (int) sets the socket receive buffer in bytes SO_RCVBUF.
-  > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down.
+- `sock-rcvbuf` (int)
+  > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down. Sets the socket receive buffer in bytes SO_RCVBUF.
   > Set to zero to use the default system value.
-- `reset-conn` (bool) reset TCP connection on exit.
+- `reset-conn` (bool)
   > Set whether to send a TCP Reset to force the cleanup of the connection on the remote side when the server exits.
-- `chan-buffer-size` (int) incoming channel size, number of packet before to drop it.
+- `chan-buffer-size` (int)
   > Specifies the maximum number of packets that can be buffered before dropping additional packets.
-- `disable-dnsparser"` (bool) disable the minimalist DNS parser.
-  > Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ...
-- `extended-support` (bool) decode the extended extra field sent by DNScollector. 
-  > If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector.
+- `disable-dnsparser"` (bool)
+  > Disable the minimalist DNS parser. Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ...
+- `extended-support` (bool)
+  > Decode the extended extra field sent by DNScollector. If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector.
   > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others.
-- `compression` (string) Compression for DNStap messages: `none`, `gzip`, `lz4`, `snappy`, `zstd`.
-  > Specifies the compression algorithm to use.
+- `compression` (string)
+  > Specifies the compression algorithm to use.  Compression for DNStap messages: `none`, `gzip`, `lz4`, `snappy`, `zstd`.
 
 Defaults:
 
@@ -67,19 +67,19 @@ For config examples, take a look to the following [one](../_examples/use-case-12
 
 Options:
 
-- `listen-ip` (str) local address to bind to.
+- `listen-ip` (str)
   > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0).
-- `listen-port` (int) local port to bind to.
+- `listen-port` (int)
   > Set the local port that the server will listen on. If not provided, use the default port.
-- `sock-path` (str) Unix socket path.
+- `sock-path` (str)
   > Specify the path for the Unix socket to be created.
-- `tls-support` (bool) set to true to enable TLS.
+- `tls-support` (bool)
   > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication.
-- `tls-min-version` (str) Minimun TLS version to use.
+- `tls-min-version` (str)
   > Specifies the minimum TLS version that the server will support.
-- `cert-file` (str) path to a certificate server file to use.
+- `cert-file` (str)
   > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled.
-- `key-file`(str) path to a key server file to use.
+- `key-file`(str)
   > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled.
 
 Defaults

docs/dnsjson.md

@@ -162,6 +162,7 @@ This JSON message can be extended by collector(s):
 
 This JSON message can be also extended by transformer(s):
 
+- [Atags](transformers/transformer_atags.md)
 - [GeoIP](transformers/transformer_geoip.md)
 - [Suspicious traffic detector](transformers/transform_suspiciousdetector.md)
 - [Public suffix](transformers/transform_normalize.md)

docs/loggers.md

@@ -18,4 +18,4 @@
 | [Redis](loggers/logger_redis.md)                | Redis pub logger                                      |
 | [Kafka](loggers/logger_kafka.md)                | Kafka DNS producer                                    |
 | [Falco](loggers/logger_falco.md)                | Falco plugin logger                                   |
-| [Clickhouse](loggers/logger_clickhouse.md)      | ClickHouse logger                                   |
+| [ClickHouse](loggers/logger_clickhouse.md)      | ClickHouse logger                                   |

docs/loggers/logger_clickhouse.md

@@ -1,7 +1,7 @@
 
-# Logger: Clickhouse client
+# Logger: ClickHouse client
 
-Clickhouse client to remote Clickhouse server
+Clickhouse client to remote ClickHouse server
 
 Options:
 

docs/transformers/transform_atags.md

@@ -1,19 +1,63 @@
 # Transformer: ATags
 
-Use this transformer to add additional flag in your DNS logs.
+Use this transformer to add additional tags in your DNS logs.
+
+This transformation can be valuable in the `pipeline` mode, where it's possible to match specific traffic. 
+In such cases, you can include a tag for identification."
+
+A list of string is expected in the `tag` field.
 
 Configuration example:
 
 ```yaml
-- name: filter
+transforms:
+  atags:
+    tags: [ "TXT:google", "MX:apple" ]
+```
+
+When the feature is enabled, the following json field are populated in your DNS message:
+
+Flat JSON:
+
+```json
+{
+  "atags.tags.0": "TXT:google",
+  "atags.tags.1": "MX:apple"
+}
+
+Default JSON structure:
+
+```json
+{
+  "atags": {
+    "tags": [ "test0", "test1" ]
+  }
+}
+```
+
+Complete example with the `dnsmessage` collector
+
+```yaml
+pipelines:
+  - name: filter
     dnsmessage:
-    matching:
+      matching:
         include:
-        dns.qname: "^.*\\.google\\.com$"
+          dns.qname: "^.*\\.google\\.com$"
     transforms:
-    atags:
+      atags:
         tags: [ "google"]
-    routing-policy:
-    dropped: [ outputfile ]
-    default: [ central ]
+```
+
+Custom text format:
+
+If you logs your DNS traffic in basic text format, you can use the specific directives:
+
+- `atags[:INDEX]`: get all tags separated by comma, or the tag according to the provided INDEX
+
+```yaml
+- name: console
+  stdout:
+    mode: text
+    text-format: "timestamp-rfc3339ns identity qr qname qtype atags:0"
 ```