IP filtering: Use subnets and explicit keep

This allows users to drop certain traffic (like 0.0.0.0/0), but keep
specific traffic (like 192.0.2.0/25).

Signed-off-by: Pieter Lexis <[email protected]>

config.yml

@@ -130,8 +130,10 @@ subprocessors:
     drop-fqdn-file: ""
     # path file of the domain drop list, domains list can be a partial domain name with regexp expression
     drop-domain-file: ""
-    # path file of the query IP drop list
+    # path file of the query IP drop list, one IP address or subnet per line
     drop-queryip-file: ""
+    # path file of the query IP keep list, one IP address or subnet per line
+    keep-queryip-file: ""
     # drop specific responses according to the return code (NOERROR, ...). This list is empty by default
     # Example to ignore NOERROR dns packets
     # drop-rcodes:

dnsutils/config.go

@@ -78,6 +78,7 @@ type Config struct {
 			DropFqdnFile    string   `yaml:"drop-fqdn-file"`
 			DropDomainFile  string   `yaml:"drop-domain-file"`
 			DropQueryIpFile string   `yaml:"drop-queryip-file"`
+			KeepQueryIpFile string   `yaml:"keep-queryip-file"`
 			DropRcodes      []string `yaml:"drop-rcodes,flow"`
 			LogQueries      bool     `yaml:"log-queries"`
 			LogReplies      bool     `yaml:"log-replies"!`

doc/configuration.md

@@ -297,7 +297,8 @@ This feature can be useful to increase logging performance..
 Options:
 - `drop-fqdn-file`: (string) path file to a fqdn drop list, domains list must be a full qualified domain name
 - `drop-domain-file`: (string) path file to domain drop list, domains list can be a partial domain name with regexp expression
-- `drop-queryip-file`: (string) path file to the query ip drop list
+- `drop-queryip-file`: (string) path file to the query ip or ip prefix drop list
+- `keep-queryip-file`: (string) path file to the query ip or ip prefix keep list, addresses in both drop and keep are always kept
 - `drop-rcodes`: (list of string) rcode list, empty by default
 - `log-queries`: (boolean) forward received queries to configured loggers
 - `log-replies`: (boolean)  forward received replies to configured loggers
@@ -308,6 +309,7 @@ subprocessors:
     drop-fqdn-file: ""
     drop-domain-file: ""
     drop-queryip-file: ""
+    keep-queryip-file: ""
     drop-rcodes: []
     log-queries: true
     log-replies: true

go.mod

@@ -26,6 +26,11 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
+require (
+	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
+)
+
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
@@ -56,4 +61,5 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
 )

go.sum

@@ -250,6 +250,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
@@ -1075,6 +1076,10 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
+go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE=
+go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
 golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1289,6 +1294,7 @@ golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1389,6 +1395,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1557,6 +1564,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
+inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw=
+inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8=
 k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A=
 k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58=
 k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU=

subprocessors/filtering.go

@@ -10,14 +10,16 @@ import (
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-logger"
 	"gopkg.in/fsnotify.v1"
+	"inet.af/netaddr"
 )
 
 type FilteringProcessor struct {
 	config           *dnsutils.Config
 	logger           *logger.Logger
 	dropDomains      bool
 	mapRcodes        map[string]bool
-	mapQueryIp       map[string]bool
+	ipsetDrop        *netaddr.IPSet
+	ipsetKeep        *netaddr.IPSet
 	listFqdns        map[string]bool
 	listDomainsRegex map[string]*regexp.Regexp
 	fileWatcher      *fsnotify.Watcher
@@ -35,7 +37,8 @@ func NewFilteringProcessor(config *dnsutils.Config, logger *logger.Logger) Filte
 		config:           config,
 		logger:           logger,
 		mapRcodes:        make(map[string]bool),
-		mapQueryIp:       make(map[string]bool),
+		ipsetDrop:        &netaddr.IPSet{},
+		ipsetKeep:        &netaddr.IPSet{},
 		listFqdns:        make(map[string]bool),
 		listDomainsRegex: make(map[string]*regexp.Regexp),
 		fileWatcher:      watcher,
@@ -55,26 +58,59 @@ func (p *FilteringProcessor) LoadRcodes() {
 	}
 }
 
+func (p *FilteringProcessor) loadQueryIpList(fname string, drop bool) (uint64, error) {
+	file, err := os.Open(fname)
+	if err != nil {
+		return 0, err
+	}
+
+	// register the file to watch
+	/*if err := p.fileWatcher.Add(fname); err != nil {
+		p.LogError("unable to watch ip file: ", err)
+	}*/
+
+	scanner := bufio.NewScanner(file)
+	var read uint64
+	var ipsetbuilder netaddr.IPSetBuilder
+	for scanner.Scan() {
+		read++
+		ipOrPrefix := strings.ToLower(scanner.Text())
+		prefix, err := netaddr.ParseIPPrefix(ipOrPrefix)
+		if err != nil {
+			ip, err := netaddr.ParseIP(ipOrPrefix)
+			if err != nil {
+				p.LogError("%s in in %s is neither an IP address nor a prefix", ipOrPrefix, fname)
+				continue
+			}
+			ipsetbuilder.Add(ip)
+			continue
+		}
+		ipsetbuilder.AddPrefix(prefix)
+	}
+	if drop {
+		p.ipsetDrop, err = ipsetbuilder.IPSet()
+	} else {
+		p.ipsetKeep, err = ipsetbuilder.IPSet()
+	}
+
+	return read, err
+}
+
 func (p *FilteringProcessor) LoadQueryIpList() {
 	if len(p.config.Subprocessors.Filtering.DropQueryIpFile) > 0 {
-		file, err := os.Open(p.config.Subprocessors.Filtering.DropQueryIpFile)
+		read, err := p.loadQueryIpList(p.config.Subprocessors.Filtering.DropQueryIpFile, true)
 		if err != nil {
 			p.LogError("unable to open query ip file: ", err)
-		} else {
-
-			// register the file to watch
-			/*if err := p.fileWatcher.Add(p.config.Subprocessors.Filtering.DropQueryIpFile); err != nil {
-				p.LogError("unable to watch ip file: ", err)
-			}*/
-
-			scanner := bufio.NewScanner(file)
-			for scanner.Scan() {
-				queryip := strings.ToLower(scanner.Text())
-				p.mapQueryIp[queryip] = true
-			}
-			p.LogInfo("loaded with %d query ip to the drop list", len(p.mapQueryIp))
 		}
+		p.LogInfo("loaded with %d query ip to the drop list", read)
+	}
 
+	if len(p.config.Subprocessors.Filtering.KeepQueryIpFile) > 0 {
+		read, err := p.loadQueryIpList(p.config.Subprocessors.Filtering.KeepQueryIpFile, false)
+		if err != nil {
+			p.LogError("unable to open query ip file: ", err)
+		}
+		p.LogInfo("loaded with %d query ip to the keep list", read)
 	}
 }
 
@@ -164,8 +200,12 @@ func (p *FilteringProcessor) CheckIfDrop(dm *dnsutils.DnsMessage) bool {
 		return true
 	}
 
-	// drop according to the query ip ?
-	if _, ok := p.mapQueryIp[dm.NetworkInfo.QueryIp]; ok {
+	// drop or keep according to the query ip ?
+	ip, _ := netaddr.ParseIP(dm.NetworkInfo.QueryIp)
+	if p.ipsetKeep.Contains(ip) {
+		return false
+	}
+	if p.ipsetDrop.Contains(ip) {
 		return true
 	}
 

subprocessors/filtering_test.go

@@ -61,6 +61,7 @@ func TestFilteringByQueryIp(t *testing.T) {
 	// config
 	config := dnsutils.GetFakeConfig()
 	config.Subprocessors.Filtering.DropQueryIpFile = "../testsdata/filtering_queryip.txt"
+	config.Subprocessors.Filtering.KeepQueryIpFile = "../testsdata/filtering_queryip_keep.txt"
 
 	// init subproccesor
 	filtering := NewFilteringProcessor(config, logger.New(false))
@@ -75,6 +76,22 @@ func TestFilteringByQueryIp(t *testing.T) {
 	if filtering.CheckIfDrop(&dm) == false {
 		t.Errorf("dns query should be dropped!")
 	}
+
+	dm.NetworkInfo.QueryIp = "192.168.1.10" // Both in drop and keep, so keep
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
+
+	dm.NetworkInfo.QueryIp = "192.0.2.3" // dropped by subnet
+	if filtering.CheckIfDrop(&dm) == false {
+		t.Errorf("dns query should be dropped!")
+	}
+
+	dm.NetworkInfo.QueryIp = "192.0.2.1" // dropped by subnet, but explicitly in keep
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
+
 }
 
 func TestFilteringByFqdn(t *testing.T) {

testsdata/filtering_queryip.txt

@@ -1,3 +1,4 @@
 192.168.1.15
 192.168.1.12
-192.168.1.10
+192.168.1.10
+192.0.2.0/21

testsdata/filtering_queryip_keep.txt

@@ -0,0 +1,2 @@
+192.168.1.10
+192.0.2.1