diff --git a/config.yml b/config.yml index 156e547c..55b86939 100644 --- a/config.yml +++ b/config.yml @@ -130,8 +130,10 @@ subprocessors: drop-fqdn-file: "" # path file of the domain drop list, domains list can be a partial domain name with regexp expression drop-domain-file: "" - # path file of the query IP drop list + # path file of the query IP drop list, one IP address or subnet per line drop-queryip-file: "" + # path file of the query IP keep list, one IP address or subnet per line + keep-queryip-file: "" # drop specific responses according to the return code (NOERROR, ...). This list is empty by default # Example to ignore NOERROR dns packets # drop-rcodes: diff --git a/dnsutils/config.go b/dnsutils/config.go index aed0c224..257b373f 100644 --- a/dnsutils/config.go +++ b/dnsutils/config.go @@ -78,6 +78,7 @@ type Config struct { DropFqdnFile string `yaml:"drop-fqdn-file"` DropDomainFile string `yaml:"drop-domain-file"` DropQueryIpFile string `yaml:"drop-queryip-file"` + KeepQueryIpFile string `yaml:"keep-queryip-file"` DropRcodes []string `yaml:"drop-rcodes,flow"` LogQueries bool `yaml:"log-queries"` LogReplies bool `yaml:"log-replies"!` diff --git a/doc/configuration.md b/doc/configuration.md index 09ddd2f5..f3590915 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -297,7 +297,8 @@ This feature can be useful to increase logging performance.. Options: - `drop-fqdn-file`: (string) path file to a fqdn drop list, domains list must be a full qualified domain name - `drop-domain-file`: (string) path file to domain drop list, domains list can be a partial domain name with regexp expression -- `drop-queryip-file`: (string) path file to the query ip drop list +- `drop-queryip-file`: (string) path file to the query ip or ip prefix drop list +- `keep-queryip-file`: (string) path file to the query ip or ip prefix keep list, addresses in both drop and keep are always kept - `drop-rcodes`: (list of string) rcode list, empty by default - `log-queries`: (boolean) forward received queries to configured loggers - `log-replies`: (boolean) forward received replies to configured loggers @@ -308,6 +309,7 @@ subprocessors: drop-fqdn-file: "" drop-domain-file: "" drop-queryip-file: "" + keep-queryip-file: "" drop-rcodes: [] log-queries: true log-replies: true diff --git a/go.mod b/go.mod index 08c8b7cb..005fae92 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,11 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) +require ( + go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -56,4 +61,5 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 ) diff --git a/go.sum b/go.sum index 7af5cb09..5edc0a45 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -1075,6 +1076,10 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= +go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1289,6 +1294,7 @@ golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1389,6 +1395,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1557,6 +1564,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw= +inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8= k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58= k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU= diff --git a/subprocessors/filtering.go b/subprocessors/filtering.go index 38f50f3d..be88011f 100644 --- a/subprocessors/filtering.go +++ b/subprocessors/filtering.go @@ -10,6 +10,7 @@ import ( "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-logger" "gopkg.in/fsnotify.v1" + "inet.af/netaddr" ) type FilteringProcessor struct { @@ -17,7 +18,8 @@ type FilteringProcessor struct { logger *logger.Logger dropDomains bool mapRcodes map[string]bool - mapQueryIp map[string]bool + ipsetDrop *netaddr.IPSet + ipsetKeep *netaddr.IPSet listFqdns map[string]bool listDomainsRegex map[string]*regexp.Regexp fileWatcher *fsnotify.Watcher @@ -35,7 +37,8 @@ func NewFilteringProcessor(config *dnsutils.Config, logger *logger.Logger) Filte config: config, logger: logger, mapRcodes: make(map[string]bool), - mapQueryIp: make(map[string]bool), + ipsetDrop: &netaddr.IPSet{}, + ipsetKeep: &netaddr.IPSet{}, listFqdns: make(map[string]bool), listDomainsRegex: make(map[string]*regexp.Regexp), fileWatcher: watcher, @@ -55,26 +58,59 @@ func (p *FilteringProcessor) LoadRcodes() { } } +func (p *FilteringProcessor) loadQueryIpList(fname string, drop bool) (uint64, error) { + file, err := os.Open(fname) + if err != nil { + return 0, err + } + + // register the file to watch + /*if err := p.fileWatcher.Add(fname); err != nil { + p.LogError("unable to watch ip file: ", err) + }*/ + + scanner := bufio.NewScanner(file) + var read uint64 + var ipsetbuilder netaddr.IPSetBuilder + for scanner.Scan() { + read++ + ipOrPrefix := strings.ToLower(scanner.Text()) + prefix, err := netaddr.ParseIPPrefix(ipOrPrefix) + if err != nil { + ip, err := netaddr.ParseIP(ipOrPrefix) + if err != nil { + p.LogError("%s in in %s is neither an IP address nor a prefix", ipOrPrefix, fname) + continue + } + ipsetbuilder.Add(ip) + continue + } + ipsetbuilder.AddPrefix(prefix) + } + if drop { + p.ipsetDrop, err = ipsetbuilder.IPSet() + } else { + p.ipsetKeep, err = ipsetbuilder.IPSet() + } + + return read, err +} + func (p *FilteringProcessor) LoadQueryIpList() { if len(p.config.Subprocessors.Filtering.DropQueryIpFile) > 0 { - file, err := os.Open(p.config.Subprocessors.Filtering.DropQueryIpFile) + read, err := p.loadQueryIpList(p.config.Subprocessors.Filtering.DropQueryIpFile, true) if err != nil { p.LogError("unable to open query ip file: ", err) - } else { - - // register the file to watch - /*if err := p.fileWatcher.Add(p.config.Subprocessors.Filtering.DropQueryIpFile); err != nil { - p.LogError("unable to watch ip file: ", err) - }*/ - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - queryip := strings.ToLower(scanner.Text()) - p.mapQueryIp[queryip] = true - } - p.LogInfo("loaded with %d query ip to the drop list", len(p.mapQueryIp)) } + p.LogInfo("loaded with %d query ip to the drop list", read) + } + if len(p.config.Subprocessors.Filtering.KeepQueryIpFile) > 0 { + read, err := p.loadQueryIpList(p.config.Subprocessors.Filtering.KeepQueryIpFile, false) + if err != nil { + p.LogError("unable to open query ip file: ", err) + } + p.LogInfo("loaded with %d query ip to the keep list", read) } } @@ -164,8 +200,12 @@ func (p *FilteringProcessor) CheckIfDrop(dm *dnsutils.DnsMessage) bool { return true } - // drop according to the query ip ? - if _, ok := p.mapQueryIp[dm.NetworkInfo.QueryIp]; ok { + // drop or keep according to the query ip ? + ip, _ := netaddr.ParseIP(dm.NetworkInfo.QueryIp) + if p.ipsetKeep.Contains(ip) { + return false + } + if p.ipsetDrop.Contains(ip) { return true } diff --git a/subprocessors/filtering_test.go b/subprocessors/filtering_test.go index c5640508..97031c3e 100644 --- a/subprocessors/filtering_test.go +++ b/subprocessors/filtering_test.go @@ -61,6 +61,7 @@ func TestFilteringByQueryIp(t *testing.T) { // config config := dnsutils.GetFakeConfig() config.Subprocessors.Filtering.DropQueryIpFile = "../testsdata/filtering_queryip.txt" + config.Subprocessors.Filtering.KeepQueryIpFile = "../testsdata/filtering_queryip_keep.txt" // init subproccesor filtering := NewFilteringProcessor(config, logger.New(false)) @@ -75,6 +76,22 @@ func TestFilteringByQueryIp(t *testing.T) { if filtering.CheckIfDrop(&dm) == false { t.Errorf("dns query should be dropped!") } + + dm.NetworkInfo.QueryIp = "192.168.1.10" // Both in drop and keep, so keep + if filtering.CheckIfDrop(&dm) == true { + t.Errorf("dns query should not be dropped!") + } + + dm.NetworkInfo.QueryIp = "192.0.2.3" // dropped by subnet + if filtering.CheckIfDrop(&dm) == false { + t.Errorf("dns query should be dropped!") + } + + dm.NetworkInfo.QueryIp = "192.0.2.1" // dropped by subnet, but explicitly in keep + if filtering.CheckIfDrop(&dm) == true { + t.Errorf("dns query should not be dropped!") + } + } func TestFilteringByFqdn(t *testing.T) { diff --git a/testsdata/filtering_queryip.txt b/testsdata/filtering_queryip.txt index 2c563f20..ffed6cc9 100644 --- a/testsdata/filtering_queryip.txt +++ b/testsdata/filtering_queryip.txt @@ -1,3 +1,4 @@ 192.168.1.15 192.168.1.12 -192.168.1.10 \ No newline at end of file +192.168.1.10 +192.0.2.0/21 \ No newline at end of file diff --git a/testsdata/filtering_queryip_keep.txt b/testsdata/filtering_queryip_keep.txt new file mode 100644 index 00000000..5883080f --- /dev/null +++ b/testsdata/filtering_queryip_keep.txt @@ -0,0 +1,2 @@ +192.168.1.10 +192.0.2.1 \ No newline at end of file