Skip to content

Commit

Permalink
add rewrite transform
Browse files Browse the repository at this point in the history
escape dnstap version directive
  • Loading branch information
dmachard committed Jun 26, 2024
1 parent 6b2808b commit 0d4056c
Show file tree
Hide file tree
Showing 10 changed files with 173 additions and 24 deletions.
5 changes: 4 additions & 1 deletion config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ global:
verbose: true
server-identity: "dns-collector"
pid-file: ""
text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length-unit qname qtype latency"
text-format: "timestamp-rfc3339ns identity version operation rcode queryip queryport family protocol length-unit qname qtype latency"
text-format-delimiter: " "
text-format-boundary: "\""
text-jinja: ""
Expand Down Expand Up @@ -42,6 +42,9 @@ pipelines:
normalize:
qname-lowercase: true
qname-replace-nonprintable: true
rewrite:
identifiers:
dnstap.identity: "foo"
routing-policy:
forward: [ console ]
dropped: [ ]
Expand Down
34 changes: 20 additions & 14 deletions dnsutils/message.go
Original file line number Diff line number Diff line change
Expand Up @@ -673,26 +673,14 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
if len(qname) == 0 {
s.WriteString(".")
} else {
if len(fieldDelimiter) > 0 {
if strings.Contains(qname, fieldDelimiter) {
qnameEscaped := qname
if strings.Contains(qname, fieldBoundary) {
qnameEscaped = strings.ReplaceAll(qnameEscaped, fieldBoundary, "\\"+fieldBoundary)
}
s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, qnameEscaped))
} else {
s.WriteString(qname)
}
} else {
s.WriteString(qname)
}
escapeStringAndWrite(&s, qname, fieldDelimiter, fieldBoundary)
}
case directive == "identity":
s.WriteString(dm.DNSTap.Identity)
case directive == "peer-name":
s.WriteString(dm.DNSTap.PeerName)
case directive == "version":
s.WriteString(dm.DNSTap.Version)
escapeStringAndWrite(&s, dm.DNSTap.Version, fieldDelimiter, fieldBoundary)
case directive == "extra":
s.WriteString(dm.DNSTap.Extra)
case directive == "policy-rule":
Expand Down Expand Up @@ -1171,6 +1159,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
"dns.qtype": dm.DNS.Qtype,
"dns.qclass": dm.DNS.Qclass,
"dns.rcode": dm.DNS.Rcode,
"dns.questions-count": dm.DNS.QuestionsCount,
"dnstap.identity": dm.DNSTap.Identity,
"dnstap.latency": dm.DNSTap.LatencySec,
"dnstap.operation": dm.DNSTap.Operation,
Expand Down Expand Up @@ -1858,6 +1847,7 @@ func GetFakeDNSMessage() DNSMessage {
dm := DNSMessage{}
dm.Init()
dm.DNSTap.Identity = "collector"
dm.DNSTap.Version = "dnscollector 1.0.0"
dm.DNSTap.Operation = "CLIENT_QUERY"
dm.DNS.Type = DNSQuery
dm.DNS.Qname = pkgconfig.ProgQname
Expand Down Expand Up @@ -1913,3 +1903,19 @@ func convertToString(value interface{}) string {
return fmt.Sprintf("%v", v)
}
}

func escapeStringAndWrite(s *strings.Builder, fieldString, fieldDelimiter, fieldBoundary string) {
if len(fieldDelimiter) > 0 {
if strings.Contains(fieldString, fieldDelimiter) {
fieldEscaped := fieldString
if strings.Contains(fieldString, fieldBoundary) {
fieldEscaped = strings.ReplaceAll(fieldEscaped, fieldBoundary, "\\"+fieldBoundary)
}
s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, fieldEscaped))
} else {
s.WriteString(fieldString)
}
} else {
s.WriteString(fieldString)
}
}
25 changes: 16 additions & 9 deletions dnsutils/message_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -815,62 +815,69 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
boundary: config.Global.TextFormatBoundary,
format: config.Global.TextFormat,
qname: "dnscollector.fr",
expected: "- - - - - - - - 0b dnscollector.fr - -",
expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b dnscollector.fr A -",
},
{
name: "custom_delimiter",
delimiter: ";",
boundary: config.Global.TextFormatBoundary,
format: config.Global.TextFormat,
qname: "dnscollector.fr",
expected: "-;-;-;-;-;-;-;-;0b;dnscollector.fr;-;-",
expected: "-;collector;CLIENT_QUERY;NOERROR;1.2.3.4;1234;-;-;0b;dnscollector.fr;A;-",
},
{
name: "empty_delimiter",
delimiter: "",
boundary: config.Global.TextFormatBoundary,
format: config.Global.TextFormat,
qname: "dnscollector.fr",
expected: "--------0bdnscollector.fr--",
expected: "-collectorCLIENT_QUERYNOERROR1.2.3.41234--0bdnscollector.frA-",
},
{
name: "qname_quote",
delimiter: config.Global.TextFormatDelimiter,
boundary: config.Global.TextFormatBoundary,
format: config.Global.TextFormat,
qname: "dns collector.fr",
expected: "- - - - - - - - 0b \"dns collector.fr\" - -",
expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns collector.fr\" A -",
},
{
name: "default_boundary",
delimiter: config.Global.TextFormatDelimiter,
boundary: config.Global.TextFormatBoundary,
format: config.Global.TextFormat,
qname: "dns\"coll tor\".fr",
expected: "- - - - - - - - 0b \"dns\\\"coll tor\\\".fr\" - -",
expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns\\\"coll tor\\\".fr\" A -",
},
{
name: "custom_boundary",
delimiter: config.Global.TextFormatDelimiter,
boundary: "!",
format: config.Global.TextFormat,
qname: "dnscoll tor.fr",
expected: "- - - - - - - - 0b !dnscoll tor.fr! - -",
expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b !dnscoll tor.fr! A -",
},
{
name: "custom_text",
delimiter: config.Global.TextFormatDelimiter,
boundary: config.Global.TextFormatBoundary,
format: "qname {IN} qtype",
qname: "dnscollector.fr",
expected: "dnscollector.fr IN -",
expected: "dnscollector.fr IN A",
},
{
name: "quote_dnstap_version",
delimiter: config.Global.TextFormatDelimiter,
boundary: config.Global.TextFormatBoundary,
format: "identity version qname",
qname: "dnscollector.fr",
expected: "collector \"dnscollector 1.0.0\" dnscollector.fr",
},
}

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
dm := DNSMessage{}
dm.Init()
dm := GetFakeDNSMessage()

dm.DNS.Qname = tc.qname

Expand Down
2 changes: 2 additions & 0 deletions docs/dnsjson.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Example:
"qtype": "A",
"id": 23455,
"qclass": "IN",
"questions-count": 0,
"flags": {
"qr": true,
"tc": false,
Expand Down Expand Up @@ -116,6 +117,7 @@ Here's a flat JSON output formatted using `jq`:
"dns.qtype": "A",
"dns.rcode": "NOERROR",
"dns.qclass": "IN",
"dns.questions-count": 0,
"dns.resource-records.an.0.name": "google.nl",
"dns.resource-records.an.0.rdata": "142.251.39.99",
"dns.resource-records.an.0.rdatatype": "A",
Expand Down
1 change: 1 addition & 0 deletions docs/transformers.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,4 @@ Transformers processing is currently in this order :
| [Traffic Prediction](transformers/transform_trafficprediction.md) | Features to train machine learning models |
| [Additionnal Tags](transformers/transform_atags.md) | Add additionnal tags |
| [JSON relabeling](transformers/transform_relabeling.md) | JSON relabeling to rename or remove keys |
| [DNS message rewrite](transformers/transform_rewrite.md) | Rewrite value for DNS messages structure |
3 changes: 3 additions & 0 deletions docs/transformers/transform_rewrite.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Transformer: Rewrite

Use this transformer to rewrite internal DNS messages structure.
4 changes: 4 additions & 0 deletions pkgconfig/transformers.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ type ConfigTransformers struct {
Rename []RelabelingConfig `yaml:"rename,flow"`
Remove []RelabelingConfig `yaml:"remove,flow"`
} `yaml:"relabeling"`
Rewrite struct {
Enable bool `yaml:"enable" default:"false"`
Identifiers map[string]interface{} `yaml:"identifiers,flow"`
} `yaml:"rewrite"`
}

func (c *ConfigTransformers) SetDefault() {
Expand Down
91 changes: 91 additions & 0 deletions transformers/rewrite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package transformers

import (
"errors"
"reflect"
"strings"

"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
"github.com/dmachard/go-logger"
)

type RewriteTransform struct {
GenericTransformer
}

func NewRewriteTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *RewriteTransform {
t := &RewriteTransform{GenericTransformer: NewTransformer(config, logger, "rewrite", name, instance, nextWorkers)}
return t
}

func (t *RewriteTransform) GetTransforms() ([]Subtransform, error) {
subtransforms := []Subtransform{}
if len(t.config.Rewrite.Identifiers) > 0 {
subtransforms = append(subtransforms, Subtransform{name: "rewrite", processFunc: t.UpdateValues})
}
return subtransforms, nil
}

func (t *RewriteTransform) UpdateValues(dm *dnsutils.DNSMessage) (int, error) {
dmValue := reflect.ValueOf(dm)
if dmValue.Kind() == reflect.Ptr {
dmValue = dmValue.Elem()
}

for nestedKeys, value := range t.config.Rewrite.Identifiers {
realValue, found := getFieldByTag(dmValue, nestedKeys)
switch {
case !found:
return 0, errors.New("field not found: " + nestedKeys)
case !realValue.CanSet():
return 0, errors.New("field cannot be set: " + nestedKeys)
default:
newValue := reflect.ValueOf(value)
if realValue.Kind() == newValue.Kind() {
realValue.Set(newValue)
} else {
return 0, errors.New("type mismatch: unable to set the value for " + nestedKeys)
}
}
}

return ReturnKeep, nil
}

func getFieldByTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) {
listKeys := strings.SplitN(nestedKeys, ".", 2)

for j, jsonKey := range listKeys {
// Iterate over the fields of the structure
for i := 0; i < value.NumField(); i++ {
field := value.Type().Field(i)

// Get JSON tag
tag := field.Tag.Get("json")
tagClean := strings.TrimSuffix(tag, ",omitempty")

// Check if the JSON tag matches
if tagClean == jsonKey {
switch field.Type.Kind() {
// ptr
case reflect.Ptr:
if fieldValue, found := getFieldByTag(value.Field(i).Elem(), listKeys[j+1]); found {
return fieldValue, true
}

// struct
case reflect.Struct:
if fieldValue, found := getFieldByTag(value.Field(i), listKeys[j+1]); found {
return fieldValue, true
}
// int, string
default:
return value.Field(i), true
}
}
}
}

return reflect.Value{}, false
}
31 changes: 31 additions & 0 deletions transformers/rewrite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package transformers

import (
"testing"

"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
"github.com/dmachard/go-logger"
)

func TestRewrite_CompileRegex(t *testing.T) {
// enable feature
config := pkgconfig.GetFakeConfigTransformers()
config.Relabeling.Enable = true
config.Relabeling.Rename = append(config.Relabeling.Rename, pkgconfig.RelabelingConfig{
Regex: "^dns.qname$",
Replacement: "qname_test",
})
config.Relabeling.Remove = append(config.Relabeling.Remove, pkgconfig.RelabelingConfig{
Regex: "^dns.qtype$",
})

// init the processor
outChans := []chan dnsutils.DNSMessage{}
relabeling := NewRelabelTransform(config, logger.New(false), "test", 0, outChans)
relabeling.GetTransforms()

if len(relabeling.RelabelingRules) != 2 {
t.Errorf("invalid number of rules")
}
}
1 change: 1 addition & 0 deletions transformers/transformers.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ func NewTransforms(config *pkgconfig.ConfigTransformers, logger *logger.Logger,
d.availableTransforms = append(d.availableTransforms, TransformEntry{NewMachineLearningTransform(config, logger, name, instance, nextWorkers)})
d.availableTransforms = append(d.availableTransforms, TransformEntry{NewLatencyTransform(config, logger, name, instance, nextWorkers)})
d.availableTransforms = append(d.availableTransforms, TransformEntry{NewDNSGeoIPTransform(config, logger, name, instance, nextWorkers)})
d.availableTransforms = append(d.availableTransforms, TransformEntry{NewRewriteTransform(config, logger, name, instance, nextWorkers)})

d.Prepare()
return d
Expand Down

0 comments on commit 0d4056c

Please sign in to comment.