From 1673d550387af5fdc191d7b567120551829cc68f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 11 Aug 2020 16:10:19 +0200 Subject: [PATCH] DNS over TLS support for DNS processor (#19321) (#20090) * DNS over TLS DoT support https://github.com/elastic/beats/issues/16663 * added changelog https://github.com/elastic/beats/pull/19321 * Update dns.asciidoc * added testsuite for https://github.com/elastic/beats/pull/19321 * Fix CHANGELOG entries and lint fixes * Apply suggestions from code review Co-authored-by: Marc Guasch Co-authored-by: Andrew Kroh (cherry picked from commit 89bfb6c7c034b901df84d4830fa6f15cbd847250) Co-authored-by: Andrew Kroh --- CHANGELOG.next.asciidoc | 1 + libbeat/processors/dns/config.go | 12 ++- libbeat/processors/dns/dns.go | 2 +- libbeat/processors/dns/docs/dns.asciidoc | 4 + libbeat/processors/dns/resolver.go | 21 +++- libbeat/processors/dns/resolver_test.go | 122 ++++++++++++++++++++++- 6 files changed, 156 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 01cd1a71d4e3..7ec049c1e229 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -436,6 +436,7 @@ field. You can revert this change by configuring tags for the module and omittin - Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464] - Add support to trim captured values in the dissect processor. {pull}19464[19464] - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] +- Add support for DNS over TLS for the `dns` processor. {pull}19321[19321] - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] - Add leader election for Kubernetes autodiscover. {pull}20281[20281] - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767] diff --git a/libbeat/processors/dns/config.go b/libbeat/processors/dns/config.go index ae447a20c721..9a669f187ff6 100644 --- a/libbeat/processors/dns/config.go +++ b/libbeat/processors/dns/config.go @@ -36,6 +36,7 @@ type Config struct { Action FieldAction `config:"action"` // Append or replace (defaults to append) when target exists. TagOnFailure []string `config:"tag_on_failure"` // Tags to append when a failure occurs. Fields common.MapStr `config:"fields"` // Mapping of source fields to target fields. + Transport string `config:"transport"` // Can be tls or udp. reverseFlat map[string]string } @@ -117,6 +118,14 @@ func (c *Config) Validate() error { c.reverseFlat[k] = target } + c.Transport = strings.ToLower(c.Transport) + switch c.Transport { + case "tls": + case "udp": + default: + return errors.Errorf("invalid transport method type '%v' specified in "+ + "config (valid value is: tls or udp)", c.Transport) + } return nil } @@ -155,5 +164,6 @@ var defaultConfig = Config{ MaxCapacity: 10000, }, }, - Timeout: 500 * time.Millisecond, + Transport: "udp", + Timeout: 500 * time.Millisecond, } diff --git a/libbeat/processors/dns/dns.go b/libbeat/processors/dns/dns.go index 49b4946733ed..2ac913f3016b 100644 --- a/libbeat/processors/dns/dns.go +++ b/libbeat/processors/dns/dns.go @@ -65,7 +65,7 @@ func New(cfg *common.Config) (processors.Processor, error) { ) log.Debugf("DNS processor config: %+v", c) - resolver, err := NewMiekgResolver(metrics, c.Timeout, c.Nameservers...) + resolver, err := NewMiekgResolver(metrics, c.Timeout, c.Transport, c.Nameservers...) if err != nil { return nil, err } diff --git a/libbeat/processors/dns/docs/dns.asciidoc b/libbeat/processors/dns/docs/dns.asciidoc index b75fb8bf87a4..6e6347f2dd2e 100644 --- a/libbeat/processors/dns/docs/dns.asciidoc +++ b/libbeat/processors/dns/docs/dns.asciidoc @@ -45,6 +45,7 @@ processors: - dns: type: reverse action: append + transport: tls fields: server.ip: server.hostname client.ip: client.hostname @@ -104,3 +105,6 @@ for each DNS request so if you have 2 nameservers then the total timeout will be `tag_on_failure`:: A list of tags to add to the event when any lookup fails. The tags are only added once even if multiple lookups fail. By default no tags are added upon failure. + +`transport`:: The type of transport connection that should be used can either be +`tls` (DNS over TLS) or `udp`. Defaults to `udp`. diff --git a/libbeat/processors/dns/resolver.go b/libbeat/processors/dns/resolver.go index 701ee8e49ace..366f00e165f9 100644 --- a/libbeat/processors/dns/resolver.go +++ b/libbeat/processors/dns/resolver.go @@ -64,7 +64,7 @@ type nameserverStats struct { // NewMiekgResolver returns a new MiekgResolver. It returns an error if no // nameserver are given and none can be read from /etc/resolv.conf. -func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers ...string) (*MiekgResolver, error) { +func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport string, servers ...string) (*MiekgResolver, error) { // Use /etc/resolv.conf if no nameservers are given. (Won't work for Windows). if len(servers) == 0 { config, err := dns.ClientConfigFromFile(etcResolvConf) @@ -77,7 +77,14 @@ func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers . // Add port if one was not specified. for i, s := range servers { if _, _, err := net.SplitHostPort(s); err != nil { - withPort := s + ":53" + var withPort string + switch transport { + case "tls": + withPort = s + ":853" + default: + withPort = s + ":53" + } + if _, _, retryErr := net.SplitHostPort(withPort); retryErr == nil { servers[i] = withPort continue @@ -90,9 +97,17 @@ func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers . timeout = defaultConfig.Timeout } + var clientTransferType string + switch transport { + case "tls": + clientTransferType = "tcp-tls" + default: + clientTransferType = "udp" + } + return &MiekgResolver{ client: &dns.Client{ - Net: "udp", + Net: clientTransferType, Timeout: timeout, }, servers: servers, diff --git a/libbeat/processors/dns/resolver_test.go b/libbeat/processors/dns/resolver_test.go index 0340da316d7e..904979514db8 100644 --- a/libbeat/processors/dns/resolver_test.go +++ b/libbeat/processors/dns/resolver_test.go @@ -18,6 +18,7 @@ package dns import ( + "crypto/tls" "net" "strings" "testing" @@ -38,7 +39,7 @@ func TestMiekgResolverLookupPTR(t *testing.T) { defer stop() reg := monitoring.NewRegistry() - res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, addr) + res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, "udp", addr) if err != nil { t.Fatal(err) } @@ -68,8 +69,61 @@ func TestMiekgResolverLookupPTR(t *testing.T) { assert.Equal(t, 12, metricCount) } +func TestMiekgResolverLookupPTRTLS(t *testing.T) { + //Build Cert + cert, err := tls.X509KeyPair(CertPEMBlock, KeyPEMBlock) + if err != nil { + t.Fatalf("unable to build certificate: %v", err) + } + config := tls.Config{ + Certificates: []tls.Certificate{cert}, + } + // serve TLS with cert + stop, addr, err := ServeDNSTLS(FakeDNSHandler, &config) + if err != nil { + t.Fatal(err) + } + defer stop() + + reg := monitoring.NewRegistry() + + res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, "tls", addr) + if err != nil { + t.Fatal(err) + } + // we use a self signed certificate for localhost + // we have to pass InsecureSSL to the DNS resolver + res.client.TLSConfig = &tls.Config{ + InsecureSkipVerify: true, + } + // Success + ptr, err := res.LookupPTR("8.8.8.8") + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Host) + assert.EqualValues(t, 19273, ptr.TTL) + + // NXDOMAIN + _, err = res.LookupPTR("1.1.1.1") + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "NXDOMAIN") + } + + // Validate that our metrics exist. + var metricCount int + reg.Do(monitoring.Full, func(name string, v interface{}) { + if strings.Contains(name, "processor.dns") { + metricCount++ + } + t.Logf("%v: %+v", name, v) + }) + assert.Equal(t, 12, metricCount) +} + func ServeDNS(h dns.HandlerFunc) (cancel func() error, addr string, err error) { // Setup listener on ephemeral port. + a, err := net.ResolveUDPAddr("udp4", "localhost:0") if err != nil { return nil, "", err @@ -86,6 +140,20 @@ func ServeDNS(h dns.HandlerFunc) (cancel func() error, addr string, err error) { return s.Shutdown, s.PacketConn.LocalAddr().String(), err } +func ServeDNSTLS(h dns.HandlerFunc, config *tls.Config) (cancel func() error, addr string, err error) { + // Setup listener on ephemeral port. + l, err := tls.Listen("tcp", "localhost:0", config) + if err != nil { + return nil, "", err + } + + var s dns.Server + s.Handler = h + s.Listener = l + go s.ActivateAndServe() + return s.Shutdown, l.Addr().String(), err +} + func FakeDNSHandler(w dns.ResponseWriter, msg *dns.Msg) { m := new(dns.Msg) m.SetReply(msg) @@ -98,3 +166,55 @@ func FakeDNSHandler(w dns.ResponseWriter, msg *dns.Msg) { } w.WriteMsg(m) } + +var ( + KeyPEMBlock = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2g2zpEtWaIUx5o6MEnWnGsf0Ba1SDc3AwgOmxeNIPBJYVCrk +sWe8Qt/5nymReVFcum76995ncr/zT+e4e8l+hXuGzTKZJpOj27Igb0/wa3j2hIcu +rnbzfwkJ+KMag2UUKdSo31ChMU+64bwziEXunF347Ot7dBLtw3PJKbabNCP+/oil +iUv2TzxxYosN+AEg4gNKLa3DMpbUnD+9Igb9KmaVp1FVhZted/AP4vn7h6Urb4ER +xMuvv3xqZvIKQ9/G1XAISYXk2feZ5yP+k1HF4ds7HJDwrP+Bv+EVyv38EKdmu9N3 +Oej8wKf3Acjln/ucbg1S3Dmkyg0x2388S4c35wIDAQABAoIBAB8MnGvknmU7siNW +YPOv9R+HIWQ9jdWRWsVFp9W9y2diZVl20iHA17neErlrPd+8iiux6eKptKlOU+Mo +58gYpP9023kUn2Iy275I2v1+sIldLB0q8qa9IWcRbm4NK5VSK1DZi0JhRNK0u7Ox +DNV2v8dcSjnSPj4FA/402owqCGegBQuheYE0LDEMiNAm6hZmQ5Npf0mTfJA/OuM4 +ONSR7lNncrR0pOZ3f3WWH+021eoZCgu2A64yfX5FFI7y5jvRn8KigXEDfXcdyFKO +725Slq4V2E2NmrMyRKNBLUSUC2hcy0tQsfo3+yANxA6PBNQ0EVqkF4uGn1IzNWOz +gDSyfSECgYEA2jgTpv9v0SrURdY3lOOjYZNCoJ9ZhUTxOsQQZLUJ+1/bQQ4Y0ONK +cnC/Ve76C/k+otbILAaRnOxGw5Apq25yPNoxjFFzP7tbN85IB+4db637qZNK2gfX +oEJd6wat4Urs8NbUKCE+XkbdENOIdXUiQxp9U6jXxprd5Ii4jICwRvsCgYEA/85J +1to++Td64gKfWDv4FUo5ZqVn70JdM/Knf5Pd37z/sjNowxhDz7AhismRditX02lt +T2g/raIW9Z/SpxI44VHCRJGPOvBvaMgCNGOH0FBHatFsfKwKzpMwapTfobqj3ZYa +DDDc8r9WQM8IDcLM6B7aOV46LWMEhMRSfDa9bwUCgYEAokbRVn7eSE3xTX3gF3ix +Jv67rXbSu6hpO6pSBpIaujSud9Jj4fMkibYOk3kDuaPAUJgog5Te9DNA7G1oj3Oy +wE4CSrbHXb2WOAnOxxbsDQD1BUXjhAAQ+bxg20Y8SC3Pxcn8O1t9Zd6MxtaHw9E3 +iW9Jg80rqSXBnRGPK+0HKcECgYBsRYk1WjzLSTNG1CtTslZH1JnFG3+JYoKGiU9i +DVkc6Sck6uONqAiTsI4R600ZQjEzN21f7dT+Dhw/rH0B4BGZNPzP/vgrzzaol/du +6y3B+yivSqLrhfoxA1W71vVsw8217WFrBYePa3L7jWVwRaJrIRvmqj5flYiFFX+A +Ob8mbQKBgAHhlnVzoKCq4mZ7Glpc0K6L57btVZNn0TEGyVli1ECvgC3zRm1rEofG +LatVl7h6ud25ZJYnP7DelGxHsZnDXNirLFlSB0CL4F6I5xNoBvCoH0Q8ckDSh4C7 +tlAyD5m9gwvgdkNFWq6/lcUPxGksTtTk8dGnhJz8pGlZvp6+dZCM +-----END RSA PRIVATE KEY-----`) + + CertPEMBlock = []byte(`-----BEGIN CERTIFICATE----- +MIIDaTCCAlGgAwIBAgIQGqg47wLgbjwwrZASuakmwjANBgkqhkiG9w0BAQsFADAy +MRQwEgYDVQQKEwtMb2cgQ291cmllcjEaMBgGA1UEAxMRYmVhdHMuZWxhc3RpYy5j +b20wHhcNMjAwNjIzMDY0NDEwWhcNMjEwNjIzMDY0NDEwWjAyMRQwEgYDVQQKEwtM +b2cgQ291cmllcjEaMBgGA1UEAxMRYmVhdHMuZWxhc3RpYy5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDbOkS1ZohTHmjowSdacax/QFrVINzcDC +A6bF40g8ElhUKuSxZ7xC3/mfKZF5UVy6bvr33mdyv/NP57h7yX6Fe4bNMpkmk6Pb +siBvT/BrePaEhy6udvN/CQn4oxqDZRQp1KjfUKExT7rhvDOIRe6cXfjs63t0Eu3D +c8kptps0I/7+iKWJS/ZPPHFiiw34ASDiA0otrcMyltScP70iBv0qZpWnUVWFm153 +8A/i+fuHpStvgRHEy6+/fGpm8gpD38bVcAhJheTZ95nnI/6TUcXh2zsckPCs/4G/ +4RXK/fwQp2a703c56PzAp/cByOWf+5xuDVLcOaTKDTHbfzxLhzfnAgMBAAGjezB5 +MA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8E +BTADAQH/MEEGA1UdEQQ6MDiCATqCCWxvY2FsaG9zdIcQAAAAAAAAAAAAAAAAAAAA +AIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAL6px +cjflhqqewqa9cvhFNT6E7UDnA7Mf34GIQPQrORXyOnyE11mDp5sEMGaz8bDajHHc +0JL8Q/5rDyRsSfe1pIyViAOxn+V/7qXfgowI3tkJbSaqHX7SlHF0dEiuGQ1coBMx +RgW17XhPtV+fk/DiXtUEkgtB7/q0Kc9C9C2GJIbOtupZ/mnkdk/5YT4tfXywNnWC +lLjT6T5+wZgRkcnr7lYNiTdS+GtN0YspPT+YD3ZTJCYD9KPcbA6k9XXXwmU8Ij6H +waodyGzG03YJbY3l2zSt3lG3jv9Tj+Ic0kRyEzzxk8exyi6nWXue/6a884kgAjiL +bXmdL6wkIJz1U+XtuQ== +-----END CERTIFICATE-----`) +)