From 00d93ce1132ecadcf0facd607bce21eb562ede5f Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 23 Jul 2024 12:59:15 -0400 Subject: [PATCH 01/59] handled ipv6 nses in the ResolverConfig --- src/internal/util/util.go | 7 ++- src/zdns/lookup.go | 20 ++++-- src/zdns/resolver.go | 126 +++++++++++++++++++++++--------------- src/zdns/util.go | 3 +- 4 files changed, 99 insertions(+), 57 deletions(-) diff --git a/src/internal/util/util.go b/src/internal/util/util.go index 0176877f..1f387cfe 100644 --- a/src/internal/util/util.go +++ b/src/internal/util/util.go @@ -103,9 +103,10 @@ func BindFlags(cmd *cobra.Command, v *viper.Viper, envPrefix string) { }) } -// getDefaultResolvers returns a slice of default DNS resolvers to be used when no system resolvers could be discovered. -func GetDefaultResolvers() []string { - return []string{"8.8.8.8:53", "8.8.4.4:53", "1.1.1.1:53", "1.0.0.1:53"} +// GetDefaultResolvers returns a slice of default DNS resolvers to be used when no system resolvers could be discovered. +// Returns IPv4 and IPv6 resolvers. +func GetDefaultResolvers() ([]string, []string) { + return []string{"8.8.8.8:53", "8.8.4.4:53", "1.1.1.1:53", "1.0.0.1:53"}, []string{"2001:4860:4860::8888:53", "2001:4860:4860::8844:53", "2606:4700:4700::1111:53", "2606:4700:4700::1001:53"} } // IsStringValidDomainName checks if the given string is a valid domain name using regex diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index c7ee9e06..97ec1bed 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -27,11 +27,11 @@ import ( "github.com/zmap/zdns/src/internal/util" ) -// GetDNSServers returns a list of DNS servers from a file, or an error if one occurs -func GetDNSServers(path string) ([]string, error) { +// GetDNSServers returns a list of IPv4, IPv6 DNS servers from a file, or an error if one occurs +func GetDNSServers(path string) (ipv4, ipv6 []string, err error) { c, err := dns.ClientConfigFromFile(path) if err != nil { - return []string{}, fmt.Errorf("error reading DNS config file: %w", err) + return []string{}, []string{}, fmt.Errorf("error reading DNS config file (%s): %w", path, err) } servers := make([]string, 0, len(c.Servers)) for _, s := range c.Servers { @@ -41,7 +41,19 @@ func GetDNSServers(path string) ([]string, error) { full := strings.Join([]string{s, c.Port}, ":") servers = append(servers, full) } - return servers, nil + ipv4 = make([]string, 0, len(servers)) + ipv6 = make([]string, 0, len(servers)) + for _, s := range servers { + ip := net.ParseIP(s) + if ip.To4() != nil { + ipv4 = append(ipv4, s) + } else if ip.To16() != nil { + ipv6 = append(ipv6, s) + } else { + return []string{}, []string{}, fmt.Errorf("could not parse IP address (%s) from file: %s", s, path) + } + } + return ipv4, ipv6, nil } // Lookup client interface for help in mocking diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 9d413fb7..4469dd2f 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -60,7 +60,8 @@ type ResolverConfig struct { Blacklist *blacklist.SafeBlacklist - LocalAddrs []net.IP // local addresses to use for connections, one will be selected at random for the resolver + LocalAddrsV4 []net.IP // ipv4 local addresses to use for connections, one will be selected at random for the resolver + LocalAddrsV6 []net.IP // ipv6 local addresses to use for connections, one will be selected at random for the resolver Retries int LogLevel log.Level @@ -69,12 +70,13 @@ type ResolverConfig struct { IPVersionMode IPVersionMode ShouldRecycleSockets bool - IterativeTimeout time.Duration // applicable to iterative queries only, timeout for a single iteration step - Timeout time.Duration // timeout for the resolution of a single name - MaxDepth int - ExternalNameServers []string // name servers used for external lookups - LookupAllNameServers bool // perform the lookup via all the nameservers for the domain - DNSConfigFilePath string // path to the DNS config file, ex: /etc/resolv.conf + IterativeTimeout time.Duration // applicable to iterative queries only, timeout for a single iteration step + Timeout time.Duration // timeout for the resolution of a single name + MaxDepth int + ExternalNameServersV4 []string // v4 name servers used for external lookups + ExternalNameServersV6 []string // v6 name servers used for external lookups + LookupAllNameServers bool // perform the lookup via all the nameservers for the domain + DNSConfigFilePath string // path to the DNS config file, ex: /etc/resolv.conf DNSSecEnabled bool EdnsOptions []dns.EDNS0 @@ -126,40 +128,9 @@ func (rc *ResolverConfig) PopulateAndValidate() error { } func (rc *ResolverConfig) populateResolverConfig() error { - // Nameservers - if len(rc.ExternalNameServers) == 0 { - // if nameservers aren't set, use OS' default - ns, err := GetDNSServers(rc.DNSConfigFilePath) - if err != nil { - ns = util.GetDefaultResolvers() - log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(ns, ", ")) - } - rc.ExternalNameServers = ns - } else { - portValidatedNSs := make([]string, 0, len(rc.ExternalNameServers)) - // check that the nameservers have a port and append one if necessary - for _, ns := range rc.ExternalNameServers { - portNS, err := util.AddDefaultPortToDNSServerName(ns) - if err != nil { - return fmt.Errorf("could not parse name server: %s", ns) - } - portValidatedNSs = append(portValidatedNSs, portNS) - } - rc.ExternalNameServers = portValidatedNSs + if err := rc.populateNameServers(); err != nil { + return errors.Wrap(err, "could not populate name servers") } - // TODO - Remove when we add IPv6 support - ipv4NameServers := make([]string, 0, len(rc.ExternalNameServers)) - for _, ns := range rc.ExternalNameServers { - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return fmt.Errorf("could not split host and port for nameserver: %s", ns) - } - if ip.To4() != nil { - ipv4NameServers = append(ipv4NameServers, ns) - } - } - rc.ExternalNameServers = ipv4NameServers - // Local Addresses if len(rc.LocalAddrs) == 0 { // localAddr not set, so we need to find the default IP address @@ -183,20 +154,75 @@ func (rc *ResolverConfig) populateResolverConfig() error { log.Info("ignoring non-IPv4 local address: ", addr) } } + // TODO handle link-local IPv6 + // TODO pull this into it's own function + //if ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { + // log.Debug("ignoring link-local IPv6 nameserver: ", portNS) + // continue + //} rc.LocalAddrs = ipv4LocalAddrs return nil } +func (rc *ResolverConfig) populateNameServers() error { + if len(rc.ExternalNameServersV4) == 0 && len(rc.ExternalNameServersV6) == 0 { + // if nameservers aren't set, use OS' default + nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) + if err != nil { + nsv4, nsv6 = util.GetDefaultResolvers() + log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(append(nsv4, nsv6...), ", ")) + } + rc.ExternalNameServersV4 = nsv4 + rc.ExternalNameServersV6 = nsv6 + } + + // ensure port is set for all nameservers + portValidatedNSsV4 := make([]string, 0, len(rc.ExternalNameServersV4)) + portValidatedNSsV6 := make([]string, 0, len(rc.ExternalNameServersV6)) + // check that the nameservers have a port and append one if necessary + for _, ns := range rc.ExternalNameServersV4 { + portNS, err := util.AddDefaultPortToDNSServerName(ns) + if err != nil { + return fmt.Errorf("could not parse name server: %s", ns) + } + portValidatedNSsV4 = append(portValidatedNSsV4, portNS) + } + for _, ns := range rc.ExternalNameServersV6 { + portNS, err := util.AddDefaultPortToDNSServerName(ns) + if err != nil { + return fmt.Errorf("could not parse name server: %s", ns) + } + portValidatedNSsV6 = append(portValidatedNSsV6, portNS) + } + // remove link-local IPv6 nameservers + nonLinkLocalIPv6NSs := make([]string, 0, len(portValidatedNSsV6)) + for _, ns := range portValidatedNSsV6 { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) + } + if ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { + log.Debug("ignoring link-local IPv6 nameserver: ", ns) + continue + } + nonLinkLocalIPv6NSs = append(nonLinkLocalIPv6NSs, ns) + } + rc.ExternalNameServersV4 = portValidatedNSsV4 + rc.ExternalNameServersV6 = nonLinkLocalIPv6NSs + + return nil +} + // validateLoopbackConsistency checks that the following is true // - if using a loopback nameserver, all nameservers are loopback and vice-versa // - if using a loopback local address, all local addresses are loopback and vice-versa // - either all nameservers AND all local addresses are loopback, or none are func (rc *ResolverConfig) validateLoopbackConsistency() error { - + allNameServers := append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...) // check if all nameservers are loopback or non-loopback allNameserversLoopback := true noneNameserversLoopback := true - for _, ns := range rc.ExternalNameServers { + for _, ns := range allNameServers { ip, _, err := util.SplitHostPort(ns) if err != nil { return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) @@ -208,29 +234,31 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } } loopbackNameserverMismatch := allNameserversLoopback == noneNameserversLoopback - if len(rc.ExternalNameServers) > 0 && loopbackNameserverMismatch { - return fmt.Errorf("cannot mix loopback and non-loopback nameservers: %v", rc.ExternalNameServers) + if len(append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...)) > 0 && loopbackNameserverMismatch { + return fmt.Errorf("cannot mix loopback and non-loopback nameservers: %v", allNameServers) } allLocalAddrsLoopback := true noneLocalAddrsLoopback := true + allLocalAddrs := append(rc.LocalAddrsV4, rc.LocalAddrsV6...) // check if all local addresses are loopback or non-loopback - for _, addr := range rc.LocalAddrs { + for _, addr := range allLocalAddrs { if addr.IsLoopback() { noneLocalAddrsLoopback = false } else { allLocalAddrsLoopback = false } } - if len(rc.LocalAddrs) > 0 && allLocalAddrsLoopback == noneLocalAddrsLoopback { - return fmt.Errorf("cannot mix loopback and non-loopback local addresses: %v", rc.LocalAddrs) + if len(allLocalAddrs) > 0 && allLocalAddrsLoopback == noneLocalAddrsLoopback { + return fmt.Errorf("cannot mix loopback and non-loopback local addresses: %v", allLocalAddrs) } // Both nameservers and local addresses are completely loopback or non-loopback // if using loopback nameservers, override local addresses to be loopback and warn user if allNameserversLoopback && noneLocalAddrsLoopback { log.Warn("nameservers are loopback, setting local address to loopback to match") - rc.LocalAddrs = []net.IP{net.ParseIP(LoopbackAddrString)} + rc.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} + // we ignore link-local local addresses, so nothing to be done for IPv6 } else if noneNameserversLoopback && allLocalAddrsLoopback { return errors.New("using loopback local addresses with non-loopback nameservers is not supported. " + "Consider setting nameservers to loopback addresses assuming you have a local DNS server") @@ -240,7 +268,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { func (rc *ResolverConfig) PrintInfo() { log.Infof("using local addresses: %v", rc.LocalAddrs) - log.Infof("for non-iterative lookups, using nameservers: %s", strings.Join(rc.ExternalNameServers, ", ")) + log.Infof("for non-iterative lookups, using nameservers: %s", strings.Join(append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...), ", ")) } // NewResolverConfig creates a new ResolverConfig with default values. diff --git a/src/zdns/util.go b/src/zdns/util.go index 512eee68..67d6d4a4 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -15,11 +15,12 @@ package zdns import ( - "errors" "fmt" "net" "strings" + "github.com/pkg/errors" + "github.com/zmap/dns" ) From cba93727a92cf0f33f04c5bc12190fb8a55c3510 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 23 Jul 2024 15:17:37 -0400 Subject: [PATCH 02/59] added v6 local addrs to config --- src/zdns/resolver.go | 74 +++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 4469dd2f..d3c0a749 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -32,8 +32,9 @@ import ( const ( // TODO - we'll need to update this when we add IPv6 support - LoopbackAddrString = "127.0.0.1" - googleDNSResolverAddr = "8.8.8.8:53" + LoopbackAddrString = "127.0.0.1" + googleDNSResolverAddr = "8.8.8.8:53" + googleDNSResolverAddrV6 = "2001:4860:4860::8888:53" defaultTimeout = 15 * time.Second // timeout for resolving a single name defaultIterativeTimeout = 4 * time.Second // timeout for single iteration in an iterative query @@ -92,7 +93,8 @@ func (rc *ResolverConfig) PopulateAndValidate() error { // Potentially, a name-server could be listed multiple times by either the user or in the OS's respective /etc/resolv.conf // De-dupe - rc.ExternalNameServers = util.RemoveDuplicates(rc.ExternalNameServers) + rc.ExternalNameServersV4 = util.RemoveDuplicates(rc.ExternalNameServersV4) + rc.ExternalNameServersV6 = util.RemoveDuplicates(rc.ExternalNameServersV6) if isValid, reason := rc.TransportMode.isValid(); !isValid { return fmt.Errorf("invalid transport mode: %s", reason) @@ -105,12 +107,12 @@ func (rc *ResolverConfig) PopulateAndValidate() error { } // Check that all nameservers/local addresses are valid - for _, ns := range rc.ExternalNameServers { + for _, ns := range append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...) { if _, _, err := net.SplitHostPort(ns); err != nil { return fmt.Errorf("invalid name server: %s", ns) } } - for _, addr := range rc.LocalAddrs { + for _, addr := range append(rc.LocalAddrsV4, rc.LocalAddrsV6...) { if addr == nil { return errors.New("local address cannot be nil") } @@ -131,36 +133,57 @@ func (rc *ResolverConfig) populateResolverConfig() error { if err := rc.populateNameServers(); err != nil { return errors.Wrap(err, "could not populate name servers") } - // Local Addresses - if len(rc.LocalAddrs) == 0 { + if err := rc.populateLocalAddrs(); err != nil { + return errors.Wrap(err, "could not populate local addresses") + } + // if there is no IPv6 local addresses, we should not use IPv6 + if len(rc.LocalAddrsV6) == 0 { + log.Warn("no IPv6 local addresses found, only using IPv4") + rc.IPVersionMode = IPv4Only + } + + return rc.populateLocalAddrs() +} + +// populateLocalAddrs populates/validates the local addresses for the resolver. +// If no local addresses are set, it will find a IP address and IPv6 address, if applicable. +func (rc *ResolverConfig) populateLocalAddrs() error { + if len(rc.LocalAddrsV4) == 0 { // localAddr not set, so we need to find the default IP address conn, err := net.Dial("udp", googleDNSResolverAddr) if err != nil { return fmt.Errorf("unable to find default IP address to open socket: %w", err) } - rc.LocalAddrs = append(rc.LocalAddrs, conn.LocalAddr().(*net.UDPAddr).IP) + rc.LocalAddrsV4 = append(rc.LocalAddrsV4, conn.LocalAddr().(*net.UDPAddr).IP) // cleanup socket if err = conn.Close(); err != nil { log.Error("unable to close test connection to Google public DNS: ", err) } } - // TODO - Remove when we add IPv6 support - ipv4LocalAddrs := make([]net.IP, 0, len(rc.LocalAddrs)) - for _, addr := range rc.LocalAddrs { - if addr.To4() != nil { - ipv4LocalAddrs = append(ipv4LocalAddrs, addr) - } else { - log.Info("ignoring non-IPv4 local address: ", addr) + lookupIPv6 := rc.IPVersionMode != IPv4Only + if len(rc.LocalAddrsV6) == 0 && lookupIPv6 { + // localAddr not set, so we need to find the default IPv6 address + conn, err := net.Dial("udp", googleDNSResolverAddrV6) + if err != nil { + return fmt.Errorf("unable to find default IPv6 address to open socket: %w", err) + } + rc.LocalAddrsV6 = append(rc.LocalAddrsV6, conn.LocalAddr().(*net.UDPAddr).IP) + // cleanup socket + if err = conn.Close(); err != nil { + log.Error("unable to close test connection to Google IPv6 public DNS: ", err) + } + + nonLinkLocalIPv6 := make([]net.IP, 0, len(rc.LocalAddrsV6)) + for _, ip := range rc.LocalAddrsV6 { + if ip != nil && ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { + log.Debug("ignoring link-local IPv6 nameserver: ", ip) + continue + } + nonLinkLocalIPv6 = append(nonLinkLocalIPv6, ip) } + rc.LocalAddrsV6 = nonLinkLocalIPv6 } - // TODO handle link-local IPv6 - // TODO pull this into it's own function - //if ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { - // log.Debug("ignoring link-local IPv6 nameserver: ", portNS) - // continue - //} - rc.LocalAddrs = ipv4LocalAddrs return nil } @@ -267,7 +290,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } func (rc *ResolverConfig) PrintInfo() { - log.Infof("using local addresses: %v", rc.LocalAddrs) + log.Infof("using local addresses: %v", append(rc.LocalAddrsV4, rc.LocalAddrsV6...)) log.Infof("for non-iterative lookups, using nameservers: %s", strings.Join(append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...), ", ")) } @@ -279,8 +302,9 @@ func NewResolverConfig() *ResolverConfig { LookupClient: LookupClient{}, Cache: c, - Blacklist: blacklist.New(), - LocalAddrs: nil, + Blacklist: blacklist.New(), + LocalAddrsV4: nil, + LocalAddrsV6: nil, TransportMode: defaultTransportMode, IPVersionMode: defaultIPVersionMode, From 02e09d7ac1191798e8a076d5f080c019fe73c459 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 11:42:57 -0400 Subject: [PATCH 03/59] added most basic IPv6 NS test --- testing/integration_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 553ab9e1..962f27ef 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -546,6 +546,13 @@ def test_a(self): self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + def test_a_ipv6(self): + c = "A --name-servers=2001:4860:4860::8888:53" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + def test_cname(self): c = "CNAME" name = "www.zdns-testing.com" From 639670749a630104edf0537d99197aad408be979 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 11:51:56 -0400 Subject: [PATCH 04/59] added comments explaining we'll mix IPv4 and IPv6 addresses in the CLI side and distinguish later --- src/cli/config_validation.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 622a979d..befab603 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -39,6 +39,7 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.Wrap(err, "client subnet did not pass validation") } + // local address - the user can enter both IPv4 and IPv6 addresses. We'll differentiate them later in the Resolver if GC.LocalAddrString != "" { for _, la := range strings.Split(GC.LocalAddrString, ",") { ip := net.ParseIP(la) @@ -51,6 +52,7 @@ func populateNetworkingConfig(gc *CLIConf) error { gc.LocalAddrSpecified = true } + // local interface - same as local addresses, an interface could have both IPv4 and IPv6 addresses, we'll differentiate them later in the Resolver if gc.LocalIfaceString != "" { li, err := net.InterfaceByName(gc.LocalIfaceString) if err != nil { From fe07e257f313b762ba77882f6851c8ec71593079 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 13:40:14 -0400 Subject: [PATCH 05/59] CLI side done --- src/cli/config_validation.go | 4 ++-- src/cli/worker_manager.go | 21 ++++++++++++++++++--- src/internal/util/util.go | 23 +++++++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index befab603..181fbf5b 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -39,7 +39,7 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.Wrap(err, "client subnet did not pass validation") } - // local address - the user can enter both IPv4 and IPv6 addresses. We'll differentiate them later in the Resolver + // local address - the user can enter both IPv4 and IPv6 addresses. We'll differentiate them later if GC.LocalAddrString != "" { for _, la := range strings.Split(GC.LocalAddrString, ",") { ip := net.ParseIP(la) @@ -52,7 +52,7 @@ func populateNetworkingConfig(gc *CLIConf) error { gc.LocalAddrSpecified = true } - // local interface - same as local addresses, an interface could have both IPv4 and IPv6 addresses, we'll differentiate them later in the Resolver + // local interface - same as local addresses, an interface could have both IPv4 and IPv6 addresses, we'll differentiate them later if gc.LocalIfaceString != "" { li, err := net.InterfaceByName(gc.LocalIfaceString) if err != nil { diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 57ccba6c..d9cef0ba 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -170,9 +170,26 @@ func populateResolverConfig(gc *CLIConf, flags *pflag.FlagSet) *zdns.ResolverCon config.Timeout = time.Second * time.Duration(gc.Timeout) config.IterativeTimeout = time.Second * time.Duration(gc.IterationTimeout) // copy nameservers to resolver config - config.ExternalNameServers = gc.NameServers + ipv4NSes, ipv6NSes, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) + if err != nil { + log.Fatalf("unable to split IPv4 and IPv6 addresses (%s): %v", gc.NameServers, err) + } + config.ExternalNameServersV4 = ipv4NSes + // this will be empty if no IPv6 addresses are specified + config.ExternalNameServersV6 = ipv6NSes config.LookupAllNameServers = gc.LookupAllNameServers + // Local Addresses + for _, ip := range gc.LocalAddrs { + if ip.To4() != nil { + config.LocalAddrsV4 = append(config.LocalAddrsV4, ip) + } else if ip.To16() != nil { + config.LocalAddrsV6 = append(config.LocalAddrsV6, ip) + } else { + log.Fatalf("invalid local address: %s", ip.String()) + } + } + if gc.UseNSID { config.EdnsOptions = append(config.EdnsOptions, new(dns.EDNS0_NSID)) } @@ -185,8 +202,6 @@ func populateResolverConfig(gc *CLIConf, flags *pflag.FlagSet) *zdns.ResolverCon config.MaxDepth = gc.MaxDepth config.CheckingDisabledBit = gc.CheckingDisabled config.ShouldRecycleSockets = gc.RecycleSockets - config.ExternalNameServers = gc.NameServers - config.LocalAddrs = gc.LocalAddrs config.DNSSecEnabled = gc.Dnssec config.DNSConfigFilePath = gc.ConfigFilePath diff --git a/src/internal/util/util.go b/src/internal/util/util.go index 1f387cfe..453a2ab4 100644 --- a/src/internal/util/util.go +++ b/src/internal/util/util.go @@ -74,7 +74,30 @@ func SplitHostPort(inaddr string) (net.IP, int, error) { } return ip, portInt, nil +} +// SplitIPv4AndIPv6Addrs splits a list of IP addresses (either with port attached or not) into IPv4 and IPv6 addresses. +// Returns a slice of IPv4/IPv6 addresses that are guaranteed to be valid. If the port was attached, it'll be included. +func SplitIPv4AndIPv6Addrs(addrs []string) (ipv4 []string, ipv6 []string, err error) { + for _, addr := range addrs { + ip, _, err := SplitHostPort(addr) + if err != nil { + // addr may be an IP without a port + ip = net.ParseIP(addr) + } + if ip == nil { + return nil, nil, fmt.Errorf("invalid IP address: %s", addr) + } + // ip is valid, check if it's IPv4 or IPv6 + if ip.To4() != nil { + ipv4 = append(ipv4, addr) + } else if ip.To16() != nil { + ipv6 = append(ipv6, addr) + } else { + return nil, nil, fmt.Errorf("invalid IP address: %s", addr) + } + } + return ipv4, ipv6, nil } // Reference: https://github.com/carolynvs/stingoftheviper/blob/main/main.go From 7eb62b78e87a5360f2d1a3ee62f9fecd699bf7e2 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 13:43:11 -0400 Subject: [PATCH 06/59] init LocalAddr arrays in new RC and return error if ipv6only and we can't get an IPv6 address --- src/zdns/resolver.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index d3c0a749..e747077c 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -166,7 +166,14 @@ func (rc *ResolverConfig) populateLocalAddrs() error { // localAddr not set, so we need to find the default IPv6 address conn, err := net.Dial("udp", googleDNSResolverAddrV6) if err != nil { - return fmt.Errorf("unable to find default IPv6 address to open socket: %w", err) + if rc.IPVersionMode == IPv6Only { + // if user selected only IPv6 and we can't find a default IPv6 address, return an error + return errors.New("unable to find default IPv6 address to open socket") + } + // user didn't specify IPv6 only, so we'll just log the issue and continue with IPv4 + log.Info("unable to find default IPv6 address to open socket, using IPv4 only: ", err) + rc.IPVersionMode = IPv4Only + return nil } rc.LocalAddrsV6 = append(rc.LocalAddrsV6, conn.LocalAddr().(*net.UDPAddr).IP) // cleanup socket @@ -303,8 +310,8 @@ func NewResolverConfig() *ResolverConfig { Cache: c, Blacklist: blacklist.New(), - LocalAddrsV4: nil, - LocalAddrsV6: nil, + LocalAddrsV4: []net.IP{}, + LocalAddrsV6: []net.IP{}, TransportMode: defaultTransportMode, IPVersionMode: defaultIPVersionMode, From 92c51fd8f6be2722a0988ba857e1fcaf0e7b4e24 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 15:12:46 -0400 Subject: [PATCH 07/59] compiles with IPv6 support --- src/zdns/lookup.go | 65 +++++++++++----------- src/zdns/resolver.go | 126 +++++++++++++++++++++++++++---------------- 2 files changed, 114 insertions(+), 77 deletions(-) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 97ec1bed..1cdd6563 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -68,35 +68,7 @@ func (lc LookupClient) DoSingleDstServerLookup(r *Resolver, q Question, nameServ } func (r *Resolver) doSingleDstServerLookup(q Question, nameServer string, isIterative bool) (*SingleQueryResult, Trace, Status, error) { - // Check that nameserver isn't blacklisted - nameServerIPString, _, err := net.SplitHostPort(nameServer) - if err != nil { - return nil, nil, StatusIllegalInput, fmt.Errorf("could not split nameserver %s: %w", nameServer, err) - } - // nameserver is required - if nameServer == "" { - return nil, nil, StatusIllegalInput, errors.New("no nameserver specified") - } - // nameserver must be reachable from the local address - nameServerIP := net.ParseIP(nameServerIPString) - if nameServerIP == nil { - return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse nameserver IP: %s", nameServerIPString) - } - if nameServerIP.IsLoopback() != r.localAddr.IsLoopback() { - return nil, nil, StatusIllegalInput, fmt.Errorf("nameserver %s must be reachable from the local address %s, ie. both must be loopback or not loopback", nameServerIPString, r.localAddr.String()) - } - - // Stop if we hit a nameserver we don't want to hit - if r.blacklist != nil { - if blacklisted, blacklistedErr := r.blacklist.IsBlacklisted(nameServerIPString); blacklistedErr != nil { - var r SingleQueryResult - return &r, Trace{}, StatusError, fmt.Errorf("could not check blacklist for nameserver %s: %w", nameServer, err) - } else if blacklisted { - var r SingleQueryResult - return &r, Trace{}, StatusBlacklist, nil - } - } - + var err error if q.Type == dns.TypePTR { var qname string qname, err = dns.ReverseAddr(q.Name) @@ -302,14 +274,43 @@ func (r *Resolver) cachedRetryingLookup(ctx context.Context, q Question, nameSer // retryingLookup wraps around wireLookup to perform a DNS lookup with retries // Returns the result, status, number of tries, and error func (r *Resolver) retryingLookup(ctx context.Context, q Question, nameServer string, recursive bool) (SingleQueryResult, Status, int, error) { + // nameserver is required + if nameServer == "" { + return SingleQueryResult{}, StatusIllegalInput, 0, errors.New("no nameserver specified") + } + nameServerIP, _, err := util.SplitHostPort(nameServer) + if err != nil { + return SingleQueryResult{}, StatusError, 0, errors.Wrapf(err, "could not split nameserver %s to get IP", nameServer) + } + // Check that nameserver isn't blacklisted + + // Stop if we hit a nameserver we don't want to hit + if r.blacklist != nil { + if blacklisted, blacklistedErr := r.blacklist.IsBlacklisted(nameServerIP.String()); blacklistedErr != nil { + return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not check blacklist for nameserver %s: %w", nameServer, err) + } else if blacklisted { + return SingleQueryResult{}, StatusBlacklist, 0, nil + } + } + var connInfo *ConnectionInfo + if nameServerIP.To4() != nil { + connInfo = r.connInfoIPv4 + } else if nameServerIP.To16() != nil { + connInfo = r.connInfoIPv6 + } else { + return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not determine IP version of nameserver: %s", nameServer) + } + // check loopback consistency + if nameServerIP.IsLoopback() != connInfo.localAddr.IsLoopback() { + return SingleQueryResult{}, StatusIllegalInput, 0, fmt.Errorf("nameserver %s must be reachable from the local address %s, ie. both must be loopback or not loopback", nameServer, connInfo.localAddr.String()) + } r.verboseLog(1, "****WIRE LOOKUP*** ", dns.TypeToString[q.Type], " ", q.Name, " ", nameServer) for i := 0; i <= r.retries; i++ { // check context before going into wireLookup if util.HasCtxExpired(&ctx) { - var r SingleQueryResult - return r, StatusTimeout, i + 1, nil + return SingleQueryResult{}, StatusTimeout, i + 1, nil } - result, status, err := wireLookup(ctx, r.udpClient, r.tcpClient, r.conn, q, nameServer, recursive, r.ednsOptions, r.dnsSecEnabled, r.checkingDisabledBit) + result, status, err := wireLookup(ctx, connInfo.udpClient, connInfo.tcpClient, connInfo.conn, q, nameServer, recursive, r.ednsOptions, r.dnsSecEnabled, r.checkingDisabledBit) if status != StatusTimeout || i == r.retries { return result, status, i + 1, err } diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index e747077c..1ae9dee7 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -126,6 +126,20 @@ func (rc *ResolverConfig) PopulateAndValidate() error { return errors.Wrap(err, "could not validate loopback consistency") } + // If we're using IPv6, we need both a local IPv4 address and an IPv6 nameserver + if rc.IPVersionMode != IPv4Only && (len(rc.LocalAddrsV6) == 0 || len(rc.ExternalNameServersV6) == 0) { + if rc.IPVersionMode == IPv6Only { + return errors.New("IPv6 only mode requires both local IPv6 addresses and IPv6 nameservers") + } + log.Info("cannot use IPv6 only mode without both local IPv6 addresses and IPv6 nameservers, defaulting to IPv4 only") + rc.IPVersionMode = IPv4Only + } + + if rc.IPVersionMode == IPv4Only { + rc.LocalAddrsV6 = nil + rc.ExternalNameServersV6 = nil + } + return nil } @@ -330,6 +344,13 @@ func NewResolverConfig() *ResolverConfig { } } +type ConnectionInfo struct { + udpClient *dns.Client + tcpClient *dns.Client + conn *dns.Conn + localAddr net.IP +} + // Resolver is a struct that holds the state of a DNS resolver. It is used to perform DNS lookups. type Resolver struct { cache *Cache @@ -337,10 +358,8 @@ type Resolver struct { blacklist *blacklist.SafeBlacklist - udpClient *dns.Client - tcpClient *dns.Client - conn *dns.Conn - localAddr net.IP + connInfoIPv4 *ConnectionInfo + connInfoIPv6 *ConnectionInfo retries int logLevel log.Level @@ -401,41 +420,26 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { checkingDisabledBit: config.CheckingDisabledBit, } log.SetLevel(r.logLevel) - r.localAddr = config.LocalAddrs[rand.Intn(len(config.LocalAddrs))] - - if r.shouldRecycleSockets { - // create persistent connection - conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: r.localAddr}) + // create connection info for IPv4 + if config.IPVersionMode == IPv4Only || config.IPVersionMode == IPv4OrIPv6 { + connInfo, err := getConnectionInfo(config.LocalAddrsV4, config.TransportMode, config.Timeout, config.ShouldRecycleSockets) if err != nil { - return nil, fmt.Errorf("unable to create UDP connection: %w", err) - } - r.conn = new(dns.Conn) - r.conn.Conn = conn - } - - usingUDP := r.transportMode == UDPOrTCP || r.transportMode == UDPOnly - if usingUDP { - r.udpClient = new(dns.Client) - r.udpClient.Timeout = r.timeout - r.udpClient.Dialer = &net.Dialer{ - Timeout: r.timeout, - LocalAddr: &net.UDPAddr{IP: r.localAddr}, + return nil, fmt.Errorf("could not create connection info for IPv4: %w", err) } + r.connInfoIPv4 = connInfo } - usingTCP := r.transportMode == UDPOrTCP || r.transportMode == TCPOnly - if usingTCP { - r.tcpClient = new(dns.Client) - r.tcpClient.Net = "tcp" - r.tcpClient.Timeout = r.timeout - r.tcpClient.Dialer = &net.Dialer{ - Timeout: config.Timeout, - LocalAddr: &net.TCPAddr{IP: r.localAddr}, + // create connection info for IPv6 + if config.IPVersionMode == IPv6Only || config.IPVersionMode == IPv4OrIPv6 { + connInfo, err := getConnectionInfo(config.LocalAddrsV6, config.TransportMode, config.Timeout, config.ShouldRecycleSockets) + if err != nil { + return nil, fmt.Errorf("could not create connection info for IPv6: %w", err) } + r.connInfoIPv6 = connInfo } - r.externalNameServers = make([]string, len(config.ExternalNameServers)) + r.externalNameServers = make([]string, len(config.ExternalNameServersV4)+len(config.ExternalNameServersV6)) // deep copy external name servers from config to resolver - elemsCopied := copy(r.externalNameServers, config.ExternalNameServers) - if elemsCopied != len(config.ExternalNameServers) { + elemsCopied := copy(r.externalNameServers, append(config.ExternalNameServersV4, config.ExternalNameServersV6...)) + if elemsCopied != len(config.ExternalNameServersV4)+len(config.ExternalNameServersV6) { log.Fatal("failed to copy entire name servers list from config") } r.iterativeTimeout = config.IterativeTimeout @@ -445,6 +449,42 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { return r, nil } +func getConnectionInfo(localAddr []net.IP, transportMode transportMode, timeout time.Duration, shouldRecycleSockets bool) (*ConnectionInfo, error) { + connInfo := &ConnectionInfo{ + localAddr: localAddr[rand.Intn(len(localAddr))], + } + if shouldRecycleSockets { + // create persistent connection + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: connInfo.localAddr}) + if err != nil { + return nil, fmt.Errorf("unable to create UDP connection: %w", err) + } + connInfo.conn = new(dns.Conn) + connInfo.conn.Conn = conn + } + + usingUDP := transportMode == UDPOrTCP || transportMode == UDPOnly + if usingUDP { + connInfo.udpClient = new(dns.Client) + connInfo.udpClient.Timeout = timeout + connInfo.udpClient.Dialer = &net.Dialer{ + Timeout: timeout, + LocalAddr: &net.UDPAddr{IP: connInfo.localAddr}, + } + } + usingTCP := transportMode == UDPOrTCP || transportMode == TCPOnly + if usingTCP { + connInfo.tcpClient = new(dns.Client) + connInfo.tcpClient.Net = "tcp" + connInfo.tcpClient.Timeout = timeout + connInfo.tcpClient.Dialer = &net.Dialer{ + Timeout: timeout, + LocalAddr: &net.TCPAddr{IP: connInfo.localAddr}, + } + } + return connInfo, nil +} + // ExternalLookup performs a single lookup of a DNS question, q, against an external name server. // dstServer, (ex: '1.1.1.1:53') can be set to over-ride the nameservers defined in the ResolverConfig. // If dstServer is not specified (ie. is an empty string), a random external name server will be used from the resolver's list of external name servers. @@ -468,15 +508,6 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer string) (*SingleQueryRe if dstServer != dstServerWithPort { log.Info("no port provided for external lookup, using default port 53") } - dstServerIP, _, err := util.SplitHostPort(dstServerWithPort) - if err != nil { - return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w", dstServer, err) - } - // check that local address and dstServer's don't have a loopback mismatch - if r.localAddr.IsLoopback() != dstServerIP.IsLoopback() { - return nil, nil, StatusIllegalInput, errors.New("cannot mix loopback and non-loopback addresses") - - } // dstServer has been validated and has a port dstServer = dstServerWithPort lookup, trace, status, err := r.lookupClient.DoSingleDstServerLookup(r, *q, dstServer, false) @@ -499,9 +530,14 @@ func (r *Resolver) IterativeLookup(q *Question) (*SingleQueryResult, Trace, Stat // Close cleans up any resources used by the resolver. This should be called when the resolver is no longer needed. // Lookup will panic if called after Close. func (r *Resolver) Close() { - if r.conn != nil { - if err := r.conn.Close(); err != nil { - log.Errorf("error closing connection: %v", err) + if r.connInfoIPv4.conn != nil { + if err := r.connInfoIPv4.conn.Close(); err != nil { + log.Errorf("error closing IPv4 connection: %v", err) + } + } + if r.connInfoIPv6.conn != nil { + if err := r.connInfoIPv6.conn.Close(); err != nil { + log.Errorf("error closing IPv6 connection: %v", err) } } } From 01c6d374094deb2937882843e0c2d5d68b62ef65 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 15:13:24 -0400 Subject: [PATCH 08/59] remove ipv6 todo --- src/zdns/resolver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 1ae9dee7..add71cfe 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -31,7 +31,6 @@ import ( ) const ( - // TODO - we'll need to update this when we add IPv6 support LoopbackAddrString = "127.0.0.1" googleDNSResolverAddr = "8.8.8.8:53" googleDNSResolverAddrV6 = "2001:4860:4860::8888:53" From 8cc3bb432b5135e640c3fd3f3ce0f9a452dcbffb Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 15:20:15 -0400 Subject: [PATCH 09/59] fix bug in parsing IPv6 resolv.conf --- src/zdns/lookup.go | 5 ++++- src/zdns/resolver.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 1cdd6563..56fcb20b 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -44,7 +44,10 @@ func GetDNSServers(path string) (ipv4, ipv6 []string, err error) { ipv4 = make([]string, 0, len(servers)) ipv6 = make([]string, 0, len(servers)) for _, s := range servers { - ip := net.ParseIP(s) + ip, _, err := util.SplitHostPort(s) + if err != nil { + return []string{}, []string{}, fmt.Errorf("could not parse IP address (%s) from file: %w", s, err) + } if ip.To4() != nil { ipv4 = append(ipv4, s) } else if ip.To16() != nil { diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index add71cfe..81ac9eff 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -150,7 +150,7 @@ func (rc *ResolverConfig) populateResolverConfig() error { return errors.Wrap(err, "could not populate local addresses") } // if there is no IPv6 local addresses, we should not use IPv6 - if len(rc.LocalAddrsV6) == 0 { + if len(rc.LocalAddrsV6) == 0 && rc.IPVersionMode != IPv4Only { log.Warn("no IPv6 local addresses found, only using IPv4") rc.IPVersionMode = IPv4Only } @@ -213,7 +213,7 @@ func (rc *ResolverConfig) populateNameServers() error { nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) if err != nil { nsv4, nsv6 = util.GetDefaultResolvers() - log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(append(nsv4, nsv6...), ", ")) + log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(nsv4, nsv6...), ", ")) } rc.ExternalNameServersV4 = nsv4 rc.ExternalNameServersV6 = nsv6 From 54947087de1cfe3c98ccd94343151d1b33f8f6d4 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 15:29:40 -0400 Subject: [PATCH 10/59] populate root servers with IPv6 if applicable --- src/zdns/conf.go | 4 ++-- src/zdns/resolver.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/zdns/conf.go b/src/zdns/conf.go index 3264fb3b..cc77729d 100644 --- a/src/zdns/conf.go +++ b/src/zdns/conf.go @@ -53,7 +53,7 @@ const ( StatusNoAuth Status = "NOAUTH" ) -var RootServersV4 = [...]string{ +var RootServersV4 = []string{ "198.41.0.4:53", // A "170.247.170.2:53", // B - Changed several times, this is current as of July '24 "192.33.4.12:53", // C @@ -68,7 +68,7 @@ var RootServersV4 = [...]string{ "199.7.83.42:53", // L "202.12.27.33:53"} // M -var RootServersV6 = [...]string{ +var RootServersV6 = []string{ "[2001:503:ba3e::2:30]:53", // A "[2801:1b8:10::b]:53", // B "[2001:500:2::c]:53", // C diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 81ac9eff..edade9d6 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -417,6 +417,8 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { dnsSecEnabled: config.DNSSecEnabled, ednsOptions: config.EdnsOptions, checkingDisabledBit: config.CheckingDisabledBit, + + rootNameServers: []string{}, } log.SetLevel(r.logLevel) // create connection info for IPv4 @@ -444,6 +446,13 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { r.iterativeTimeout = config.IterativeTimeout r.maxDepth = config.MaxDepth // use the set of 13 root name servers + if r.ipVersionMode != IPv6Only { + r.rootNameServers = append(r.rootNameServers, RootServersV4...) + } + if r.ipVersionMode != IPv4Only { + r.rootNameServers = append(r.rootNameServers, RootServersV6...) + } + r.rootNameServers = RootServersV4[:] return r, nil } From b56781e854620bcc50d116182d4dfc5d7100bc60 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 29 Jul 2024 16:12:52 -0400 Subject: [PATCH 11/59] fixed population bug with ipv6 roots --- src/zdns/resolver.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index edade9d6..01b7fe3c 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -33,7 +33,7 @@ import ( const ( LoopbackAddrString = "127.0.0.1" googleDNSResolverAddr = "8.8.8.8:53" - googleDNSResolverAddrV6 = "2001:4860:4860::8888:53" + googleDNSResolverAddrV6 = "[2001:4860:4860::8888]:53" defaultTimeout = 15 * time.Second // timeout for resolving a single name defaultIterativeTimeout = 4 * time.Second // timeout for single iteration in an iterative query @@ -137,6 +137,9 @@ func (rc *ResolverConfig) PopulateAndValidate() error { if rc.IPVersionMode == IPv4Only { rc.LocalAddrsV6 = nil rc.ExternalNameServersV6 = nil + } else if rc.IPVersionMode == IPv6Only { + rc.LocalAddrsV4 = nil + rc.ExternalNameServersV4 = nil } return nil @@ -447,13 +450,14 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { r.maxDepth = config.MaxDepth // use the set of 13 root name servers if r.ipVersionMode != IPv6Only { + // add IPv4 root servers r.rootNameServers = append(r.rootNameServers, RootServersV4...) } if r.ipVersionMode != IPv4Only { + // add IPv6 root servers r.rootNameServers = append(r.rootNameServers, RootServersV6...) } - r.rootNameServers = RootServersV4[:] return r, nil } From fb96b8f6f7a96605af1a54a2a8e832bef8e14a75 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 10:51:35 -0400 Subject: [PATCH 12/59] fixed seg fault with a guard condition, need to fix root cause tho --- src/zdns/lookup.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 56fcb20b..72d8cae2 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -303,6 +303,10 @@ func (r *Resolver) retryingLookup(ctx context.Context, q Question, nameServer st } else { return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not determine IP version of nameserver: %s", nameServer) } + // check that our connection info is valid + if connInfo == nil { + return SingleQueryResult{}, StatusError, 0, fmt.Errorf("no connection info for nameserver: %s", nameServer) + } // check loopback consistency if nameServerIP.IsLoopback() != connInfo.localAddr.IsLoopback() { return SingleQueryResult{}, StatusIllegalInput, 0, fmt.Errorf("nameserver %s must be reachable from the local address %s, ie. both must be loopback or not loopback", nameServer, connInfo.localAddr.String()) From 0529f1234923bedf31277206993425acb525e8e2 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 12:31:18 -0400 Subject: [PATCH 13/59] added thread-saftey for PopulateAndValidate and fixed some lack of IPv6 support in authority/additional iteration --- src/zdns/lookup.go | 7 +++++-- src/zdns/resolver.go | 10 +++++++--- src/zdns/util.go | 13 ++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 72d8cae2..9fa2c681 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -489,7 +489,7 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, // Short circuit a lookup from the glue // Normally this would be handled by caching, but we want to support following glue // that would normally be cache poison. Because it's "ok" and quite common - res, status := checkGlue(server, result) + res, status := checkGlue(server, result, r.ipVersionMode) if status != StatusNoError { // Fall through to normal query var q Question @@ -508,9 +508,12 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, if !ok { continue } - if innerAns.Type == "A" { + if r.ipVersionMode != IPv6Only && innerAns.Type == "A" { server := strings.TrimSuffix(innerAns.Answer, ".") + ":53" return server, StatusNoError, layer, trace + } else if r.ipVersionMode != IPv4Only && innerAns.Type == "AAAA" { + server := "[" + strings.TrimSuffix(innerAns.Answer, ".") + "]:53" + return server, StatusNoError, layer, trace } } } diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 01b7fe3c..02b5f365 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -19,6 +19,7 @@ import ( "math/rand" "net" "strings" + "sync" "time" "github.com/pkg/errors" @@ -54,6 +55,7 @@ const ( // ResolverConfig is a struct that holds all the configuration options for a Resolver. It is used to create a new Resolver. type ResolverConfig struct { + sync.Mutex // lock for populateAndValidate Cache *Cache CacheSize int // don't use both cache and cacheSize LookupClient Lookuper // either a functional or mock Lookuper client for testing @@ -85,6 +87,9 @@ type ResolverConfig struct { // PopulateAndValidate checks if the ResolverConfig is valid and populates any missing fields with default values. func (rc *ResolverConfig) PopulateAndValidate() error { + // This is called in every InitResolver while re-using the config, so it needs to be thread-safe + rc.Lock() + defer rc.Unlock() // populate any missing values in resolver config if err := rc.populateResolverConfig(); err != nil { return errors.Wrap(err, "could not populate resolver config") @@ -164,7 +169,7 @@ func (rc *ResolverConfig) populateResolverConfig() error { // populateLocalAddrs populates/validates the local addresses for the resolver. // If no local addresses are set, it will find a IP address and IPv6 address, if applicable. func (rc *ResolverConfig) populateLocalAddrs() error { - if len(rc.LocalAddrsV4) == 0 { + if rc.IPVersionMode != IPv6Only && len(rc.LocalAddrsV4) == 0 { // localAddr not set, so we need to find the default IP address conn, err := net.Dial("udp", googleDNSResolverAddr) if err != nil { @@ -177,8 +182,7 @@ func (rc *ResolverConfig) populateLocalAddrs() error { } } - lookupIPv6 := rc.IPVersionMode != IPv4Only - if len(rc.LocalAddrsV6) == 0 && lookupIPv6 { + if rc.IPVersionMode != IPv4Only && len(rc.LocalAddrsV6) == 0 { // localAddr not set, so we need to find the default IPv6 address conn, err := net.Dial("udp", googleDNSResolverAddrV6) if err != nil { diff --git a/src/zdns/util.go b/src/zdns/util.go index 67d6d4a4..1abdd5b2 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -88,16 +88,23 @@ func nextAuthority(name, layer string) (string, error) { return next, nil } -func checkGlue(server string, result SingleQueryResult) (SingleQueryResult, Status) { +func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode) (SingleQueryResult, Status) { for _, additional := range result.Additional { ans, ok := additional.(Answer) if !ok { continue } - if ans.Type == "A" && strings.TrimSuffix(ans.Name, ".") == server { + if ipMode != IPv6Only && ans.Type == "A" && strings.TrimSuffix(ans.Name, ".") == server { var retv SingleQueryResult retv.Authorities = make([]interface{}, 0) - retv.Answers = make([]interface{}, 0) + retv.Answers = make([]interface{}, 0, 1) + retv.Additional = make([]interface{}, 0) + retv.Answers = append(retv.Answers, ans) + return retv, StatusNoError + } else if ipMode != IPv4Only && ans.Type == "AAAA" && strings.TrimSuffix(ans.Name, ".") == server { + var retv SingleQueryResult + retv.Authorities = make([]interface{}, 0) + retv.Answers = make([]interface{}, 0, 1) retv.Additional = make([]interface{}, 0) retv.Answers = append(retv.Answers, ans) return retv, StatusNoError From af3bd85201e9b82cfc87a5f990376cf5cebad8c3 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 14:20:53 -0400 Subject: [PATCH 14/59] added a few sanity tests for IPv6 --- testing/integration_tests.py | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index b3072644..9aa2d32f 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -39,7 +39,7 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "./zdns" + ZDNS_EXECUTABLE = "../zdns" def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE): flags = flags + " --threads=10" @@ -975,6 +975,54 @@ def test_timetamps_nanoseconds(self): # but it is very unlikely. (1 in 1,000,000). Python's datetime.date's smallest unit of time is microseconds, # so that's why we're using this in place of nanoseconds. It should not affect the test's validity. + def test_ipv6_unreachable(self): + c = "A --iterative --ipv6-lookup=true --ipv4-lookup=false" + name = "esrg.stanford.edu" + cmd, res = self.run_zdns(c, name) + # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, this will fail. + self.assertServFail(res, cmd) + + def test_ipv6_external_lookup_unreachable_nameserver(self): + c = "A --ipv6-lookup=true --ipv4-lookup=false --name-servers=1.1.1.1" + name = "zdns-testing.com" + try: + cmd, res = self.run_zdns(c, name) + except Exception as e: + return True + self.fail("Should have thrown an exception, shouldn't be able to reach any IPv4 servers while in IPv6 mode") + + def test_ipv4_external_lookup_unreachable_nameserver(self): + c = "A --ipv6-lookup=false --ipv4-lookup=true --name-servers=2606:4700:4700::1111" + name = "zdns-testing.com" + try: + cmd, res = self.run_zdns(c, name) + except Exception as e: + return True + self.fail("Should have thrown an exception, shouldn't be able to reach any IPv6 servers while in IPv4 mode") + + def test_ipv6_happy_path_external(self): + c = "A --ipv6-lookup=true" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def test_ipv6_happy_path_iterative(self): + c = "A --ipv6-lookup=true --iterative" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def test_ipv6_happy_path_no_ipv4_iterative(self): + c = "A --ipv6-lookup=true --ipv4-lookup=false --iterative" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + + if __name__ == "__main__": unittest.main() From c4ae1db5c1144d385156661a51d4767009fd9f37 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 14:55:23 -0400 Subject: [PATCH 15/59] fixed bug by not deleting unneeded local addrs and nameservers --- src/zdns/resolver.go | 23 +++++++++++++++-------- testing/integration_tests.py | 1 + 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 02b5f365..45e71ff7 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -130,7 +130,14 @@ func (rc *ResolverConfig) PopulateAndValidate() error { return errors.Wrap(err, "could not validate loopback consistency") } - // If we're using IPv6, we need both a local IPv4 address and an IPv6 nameserver + if rc.IPVersionMode == IPv4Only { + rc.LocalAddrsV6 = nil + rc.ExternalNameServersV6 = nil + } else if rc.IPVersionMode == IPv6Only { + rc.LocalAddrsV4 = nil + rc.ExternalNameServersV4 = nil + } + // If we're using IPv6, we need both a local IPv6 address and an IPv6 nameserver if rc.IPVersionMode != IPv4Only && (len(rc.LocalAddrsV6) == 0 || len(rc.ExternalNameServersV6) == 0) { if rc.IPVersionMode == IPv6Only { return errors.New("IPv6 only mode requires both local IPv6 addresses and IPv6 nameservers") @@ -138,13 +145,13 @@ func (rc *ResolverConfig) PopulateAndValidate() error { log.Info("cannot use IPv6 only mode without both local IPv6 addresses and IPv6 nameservers, defaulting to IPv4 only") rc.IPVersionMode = IPv4Only } - - if rc.IPVersionMode == IPv4Only { - rc.LocalAddrsV6 = nil - rc.ExternalNameServersV6 = nil - } else if rc.IPVersionMode == IPv6Only { - rc.LocalAddrsV4 = nil - rc.ExternalNameServersV4 = nil + // If we're using IPv4, we need both a local IPv4 address and an IPv4 nameserver + if rc.IPVersionMode != IPv6Only && (len(rc.LocalAddrsV4) == 0 || len(rc.ExternalNameServersV4) == 0) { + if rc.IPVersionMode == IPv4Only { + return errors.New("IPv4 only mode requires both local IPv4 addresses and IPv4 nameservers") + } + log.Info("cannot use IPv4 only mode without both local IPv4 addresses and IPv4 nameservers, defaulting to IPv6 only") + rc.IPVersionMode = IPv6Only } return nil diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 9aa2d32f..39c022fb 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -975,6 +975,7 @@ def test_timetamps_nanoseconds(self): # but it is very unlikely. (1 in 1,000,000). Python's datetime.date's smallest unit of time is microseconds, # so that's why we're using this in place of nanoseconds. It should not affect the test's validity. +# Test Cases for IPv6 def test_ipv6_unreachable(self): c = "A --iterative --ipv6-lookup=true --ipv4-lookup=false" name = "esrg.stanford.edu" From 23b0ae2ea99505efbb8d1eba5d88a0edae3185ff Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 14:58:11 -0400 Subject: [PATCH 16/59] fixed executable name for testing --- testing/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 39c022fb..e8f1ca87 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -39,7 +39,7 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "../zdns" + ZDNS_EXECUTABLE = "./zdns" def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE): flags = flags + " --threads=10" From 5408aec55699c6f65221be2da3a29df2f427c774 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 15:11:57 -0400 Subject: [PATCH 17/59] loopback handling in ExternalLoopback --- src/zdns/resolver.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 45e71ff7..e0b0bed8 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -531,6 +531,16 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer string) (*SingleQueryRe if dstServer != dstServerWithPort { log.Info("no port provided for external lookup, using default port 53") } + // Check for loopback mis-match + nsIP, _, err := util.SplitHostPort(dstServerWithPort) + if err != nil { + return nil, nil, StatusIllegalInput, fmt.Errorf("could not split host and port for name server: %w", err) + } + if nsIP.To4() != nil && r.connInfoIPv4.localAddr.IsLoopback() != nsIP.IsLoopback() { + return nil, nil, StatusIllegalInput, errors.New("nameserver (%s) and local address(%s) must be both loopback or non-loopback") + } else if nsIP.To16() != nil && nsIP.IsLoopback() { + return nil, nil, StatusIllegalInput, errors.New("cannot use IPv6 loopback nameserver") + } // dstServer has been validated and has a port dstServer = dstServerWithPort lookup, trace, status, err := r.lookupClient.DoSingleDstServerLookup(r, *q, dstServer, false) From 6274ac296b4d16821707aad0cea386cc48871cde Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 30 Jul 2024 15:19:17 -0400 Subject: [PATCH 18/59] fixed compile issues in tests --- src/modules/bindversion/bindversion_test.go | 4 +- src/modules/dmarc/dmarc_test.go | 4 +- src/modules/spf/spf_test.go | 4 +- src/zdns/lookup_test.go | 72 ++++++++++----------- src/zdns/resolver_test.go | 44 ++++++------- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/modules/bindversion/bindversion_test.go b/src/modules/bindversion/bindversion_test.go index ccdf796a..7f4c39f3 100644 --- a/src/modules/bindversion/bindversion_test.go +++ b/src/modules/bindversion/bindversion_test.go @@ -46,8 +46,8 @@ func (ml MockLookup) DoSingleDstServerLookup(r *zdns.Resolver, question zdns.Que func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1"}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"1.1.1.1"}, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/modules/dmarc/dmarc_test.go b/src/modules/dmarc/dmarc_test.go index facb355f..1dc750da 100644 --- a/src/modules/dmarc/dmarc_test.go +++ b/src/modules/dmarc/dmarc_test.go @@ -46,8 +46,8 @@ func (ml MockLookup) DoSingleDstServerLookup(r *zdns.Resolver, question zdns.Que func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53"}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"127.0.0.1:53"}, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/modules/spf/spf_test.go b/src/modules/spf/spf_test.go index 898acf99..b832ec51 100644 --- a/src/modules/spf/spf_test.go +++ b/src/modules/spf/spf_test.go @@ -47,8 +47,8 @@ func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) queries = make([]QueryRecord, 0) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53"}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"127.0.0.1:53"}, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go index 03b0649c..7cca1f28 100644 --- a/src/zdns/lookup_test.go +++ b/src/zdns/lookup_test.go @@ -58,7 +58,7 @@ func InitTest(t *testing.T) *ResolverConfig { mc := MockLookupClient{} config := NewResolverConfig() - config.ExternalNameServers = []string{"127.0.0.1"} + config.ExternalNameServersV4 = []string{"127.0.0.1"} config.LookupClient = mc return config @@ -623,7 +623,7 @@ func TestOneA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -651,7 +651,7 @@ func TestTwoA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -686,7 +686,7 @@ func TestQuadAWithoutFlag(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -722,7 +722,7 @@ func TestOnlyQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -752,7 +752,7 @@ func TestAandQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -788,7 +788,7 @@ func TestTwoQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -825,7 +825,7 @@ func TestNoResults(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -847,7 +847,7 @@ func TestCname(t *testing.T) { require.NoError(t, err) domain1 := "cname.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -893,7 +893,7 @@ func TestQuadAWithCname(t *testing.T) { require.NoError(t, err) domain1 := "cname.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -928,7 +928,7 @@ func TestUnexpectedMxOnly(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -962,7 +962,7 @@ func TestMxAndAdditionals(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1004,7 +1004,7 @@ func TestMismatchIpType(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1038,7 +1038,7 @@ func TestCnameLoops(t *testing.T) { require.NoError(t, err) domain1 := "cname1.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1089,7 +1089,7 @@ func TestExtendedRecursion(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] // Create a CNAME chain of length > 10 for i := 1; i < 12; i++ { domainNSRecord := domainNS{ @@ -1128,7 +1128,7 @@ func TestEmptyNonTerminal(t *testing.T) { require.NoError(t, err) domain1 := "leaf.intermediate.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1171,7 +1171,7 @@ func TestNXDomain(t *testing.T) { config := InitTest(t) resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] res, _, status, _ := resolver.DoTargetedLookup("nonexistent.example.com", ns1, IPv4OrIPv6, false) if status != StatusNXDomain { t.Errorf("Expected StatusNXDomain status, got %v", status) @@ -1189,7 +1189,7 @@ func TestAandQuadADedup(t *testing.T) { domain1 := "cname1.example.com" domain2 := "cname2.example.com" domain3 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} domainNS2 := domainNS{domain: domain2, ns: ns1} domainNS3 := domainNS{domain: domain3, ns: ns1} @@ -1285,7 +1285,7 @@ func TestServFail(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{} @@ -1322,7 +1322,7 @@ func TestNsAInAdditional(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1365,7 +1365,7 @@ func TestTwoNSInAdditional(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1426,7 +1426,7 @@ func TestAandQuadAInAdditional(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1476,7 +1476,7 @@ func TestNsMismatchIpType(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1526,7 +1526,7 @@ func TestAandQuadALookup(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1586,7 +1586,7 @@ func TestNsNXDomain(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] _, _, status, _ := resolver.DoNSLookup("nonexistentexample.com", ns1, false) @@ -1599,7 +1599,7 @@ func TestNsServFail(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{} @@ -1617,7 +1617,7 @@ func TestErrorInTargetedLookup(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1646,12 +1646,12 @@ func TestErrorInTargetedLookup(t *testing.T) { // Test One NS with one IP with only ipv4-lookup func TestAllNsLookupOneNs(t *testing.T) { config := InitTest(t) - config.LocalAddrs = []net.IP{net.ParseIP(LoopbackAddrString)} + config.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" @@ -1758,7 +1758,7 @@ func TestAllNsLookupOneNsMultipleIps(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" @@ -1881,7 +1881,7 @@ func TestAllNsLookupTwoNs(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" nsDomain2 := "ns2.example.com" @@ -1997,7 +1997,7 @@ func TestAllNsLookupErrorInOne(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" @@ -2097,7 +2097,7 @@ func TestAllNsLookupNXDomain(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] q := Question{ Type: dns.TypeNS, Class: dns.ClassINET, @@ -2117,7 +2117,7 @@ func TestAllNsLookupServFail(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" domainNS1 := domainNS{domain: domain1, ns: ns1} @@ -2138,8 +2138,8 @@ func TestAllNsLookupServFail(t *testing.T) { func TestInvalidInputsLookup(t *testing.T) { config := InitTest(t) - config.LocalAddrs = []net.IP{net.ParseIP("127.0.0.1")} - config.ExternalNameServers = []string{"127.0.0.1:53"} + config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")} + config.ExternalNameServersV4 = []string{"127.0.0.1:53"} resolver, err := InitResolver(config) require.NoError(t, err) q := Question{ diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index 290e9379..739c3d19 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -24,41 +24,41 @@ import ( func TestResolverConfig_PopulateAndValidate(t *testing.T) { t.Run("Using loopback nameserver and no specified local address", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, } err := rc.PopulateAndValidate() require.Nil(t, err, "Expected no error but got %v", err) - require.Equal(t, LoopbackAddrString, rc.LocalAddrs[0].String()) + require.Equal(t, LoopbackAddrString, rc.LocalAddrsV4[0].String()) }) t.Run("Using nameserver with no port", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1"}, + ExternalNameServersV4: []string{"1.1.1.1"}, } err := rc.PopulateAndValidate() require.Nil(t, err) - require.Equal(t, "1.1.1.1:53", rc.ExternalNameServers[0], "Expected port 53 to be appended to nameserver") + require.Equal(t, "1.1.1.1:53", rc.ExternalNameServersV4[0], "Expected port 53 to be appended to nameserver") }) t.Run("Using nameserver with port", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1:64"}, + ExternalNameServersV4: []string{"1.1.1.1:64"}, } err := rc.PopulateAndValidate() require.Nil(t, err) - require.Equal(t, "1.1.1.1:64", rc.ExternalNameServers[0], "Expected nameserver to remain unchanged") + require.Equal(t, "1.1.1.1:64", rc.ExternalNameServersV4[0], "Expected nameserver to remain unchanged") }) t.Run("Using local address with no port", func(t *testing.T) { rc := &ResolverConfig{ - LocalAddrs: []net.IP{net.ParseIP("192.168.1.1")}, + LocalAddrsV4: []net.IP{net.ParseIP("192.168.1.1")}, } err := rc.PopulateAndValidate() require.Nil(t, err) - require.Equal(t, "192.168.1.1", rc.LocalAddrs[0].String(), "Expected local address to be unchanged") + require.Equal(t, "192.168.1.1", rc.LocalAddrsV4[0].String(), "Expected local address to be unchanged") }) t.Run("Mixing loopback and non-loopback nameservers results in error", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53", "8.8.8.8:53"}, + ExternalNameServersV4: []string{"127.0.0.1:53", "8.8.8.8:53"}, } err := rc.PopulateAndValidate() require.NotNil(t, err, "Mixing loopback and non-loopback nameservers should result in an error") @@ -66,8 +66,8 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { t.Run("Using non-loopback nameservers with loopback local address results in nameserver being overwritten", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"8.8.8.8:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"8.8.8.8:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.PopulateAndValidate() require.NotNil(t, err, "Using non-loopback nameservers with loopback local address should result in an error") @@ -75,33 +75,33 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { t.Run("Using loopback nameservers with non-loopback local address results in local address being overwritten", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53"}, - LocalAddrs: []net.IP{net.ParseIP("192.168.0.1")}, + ExternalNameServersV4: []string{"127.0.0.1:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("192.168.0.1")}, } err := rc.PopulateAndValidate() require.Nil(t, err) - require.Equal(t, LoopbackAddrString, rc.LocalAddrs[0].String(), "Expected local address to be overwritten with loopback address") + require.Equal(t, LoopbackAddrString, rc.LocalAddrsV4[0].String(), "Expected local address to be overwritten with loopback address") }) t.Run("Valid non-loopback nameservers and local addresses", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"8.8.8.8:53", "8.8.4.4:53"}, - LocalAddrs: []net.IP{net.ParseIP("192.168.0.1"), net.ParseIP("192.168.0.2")}, + ExternalNameServersV4: []string{"8.8.8.8:53", "8.8.4.4:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("192.168.0.1"), net.ParseIP("192.168.0.2")}, } err := rc.PopulateAndValidate() require.Nil(t, err, "Valid non-loopback nameservers and local addresses should not result in an error") - require.Equal(t, "8.8.8.8:53", rc.ExternalNameServers[0], "Expected nameserver to remain unchanged") - require.Equal(t, "192.168.0.1", rc.LocalAddrs[0].String(), "Expected local address to remain unchanged") + require.Equal(t, "8.8.8.8:53", rc.ExternalNameServersV4[0], "Expected nameserver to remain unchanged") + require.Equal(t, "192.168.0.1", rc.LocalAddrsV4[0].String(), "Expected local address to remain unchanged") }) t.Run("Valid loopback nameservers and local addresses", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53", "127.0.0.2:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, + ExternalNameServersV4: []string{"127.0.0.1:53", "127.0.0.2:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, } err := rc.PopulateAndValidate() require.Nil(t, err, "Valid loopback nameservers and local addresses should not result in an error") - require.Equal(t, LoopbackAddrString, rc.LocalAddrs[0].String(), "Expected local address to be overwritten with loopback address") - require.Equal(t, "127.0.0.1:53", rc.ExternalNameServers[0], "Expected nameserver to remain unchanged") + require.Equal(t, LoopbackAddrString, rc.LocalAddrsV4[0].String(), "Expected local address to be overwritten with loopback address") + require.Equal(t, "127.0.0.1:53", rc.ExternalNameServersV4[0], "Expected nameserver to remain unchanged") }) } From daa12f7deb487f93c2fe0a87f0285e91f4bdad33 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 10:42:00 -0400 Subject: [PATCH 19/59] fixed bug with proper IPv6 detection and make populateAndValidate idempotent --- src/cli/worker_manager.go | 2 +- src/internal/util/util.go | 5 +++++ src/zdns/lookup.go | 2 +- src/zdns/resolver.go | 37 +++++++++++++++++++++---------------- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 4e3a441d..36371ac0 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -183,7 +183,7 @@ func populateResolverConfig(gc *CLIConf, flags *pflag.FlagSet) *zdns.ResolverCon for _, ip := range gc.LocalAddrs { if ip.To4() != nil { config.LocalAddrsV4 = append(config.LocalAddrsV4, ip) - } else if ip.To16() != nil { + } else if util.IsIPv6(&ip) { config.LocalAddrsV6 = append(config.LocalAddrsV6, ip) } else { log.Fatalf("invalid local address: %s", ip.String()) diff --git a/src/internal/util/util.go b/src/internal/util/util.go index 453a2ab4..9f34b63a 100644 --- a/src/internal/util/util.go +++ b/src/internal/util/util.go @@ -100,6 +100,11 @@ func SplitIPv4AndIPv6Addrs(addrs []string) (ipv4 []string, ipv6 []string, err er return ipv4, ipv6, nil } +// IsIPv6 checks if the given IP address is an IPv6 address. +func IsIPv6(ip *net.IP) bool { + return ip != nil && ip.To4() == nil && ip.To16() != nil +} + // Reference: https://github.com/carolynvs/stingoftheviper/blob/main/main.go // For how to make cobra/viper sync up, and still use custom struct // Bind each cobra flag to its associated viper configuration (config file and environment variable) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 67702f30..d47a2b6f 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -50,7 +50,7 @@ func GetDNSServers(path string) (ipv4, ipv6 []string, err error) { } if ip.To4() != nil { ipv4 = append(ipv4, s) - } else if ip.To16() != nil { + } else if util.IsIPv6(&ip) { ipv6 = append(ipv6, s) } else { return []string{}, []string{}, fmt.Errorf("could not parse IP address (%s) from file: %s", s, path) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index e0b0bed8..d6f4a942 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -130,13 +130,6 @@ func (rc *ResolverConfig) PopulateAndValidate() error { return errors.Wrap(err, "could not validate loopback consistency") } - if rc.IPVersionMode == IPv4Only { - rc.LocalAddrsV6 = nil - rc.ExternalNameServersV6 = nil - } else if rc.IPVersionMode == IPv6Only { - rc.LocalAddrsV4 = nil - rc.ExternalNameServersV4 = nil - } // If we're using IPv6, we need both a local IPv6 address and an IPv6 nameserver if rc.IPVersionMode != IPv4Only && (len(rc.LocalAddrsV6) == 0 || len(rc.ExternalNameServersV6) == 0) { if rc.IPVersionMode == IPv6Only { @@ -210,7 +203,7 @@ func (rc *ResolverConfig) populateLocalAddrs() error { nonLinkLocalIPv6 := make([]net.IP, 0, len(rc.LocalAddrsV6)) for _, ip := range rc.LocalAddrsV6 { - if ip != nil && ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { + if util.IsIPv6(&ip) && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { log.Debug("ignoring link-local IPv6 nameserver: ", ip) continue } @@ -258,7 +251,7 @@ func (rc *ResolverConfig) populateNameServers() error { if err != nil { return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) } - if ip.To4() == nil && ip.To16() != nil && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { + if util.IsIPv6(&ip) && (ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast()) { log.Debug("ignoring link-local IPv6 nameserver: ", ns) continue } @@ -451,12 +444,24 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { } r.connInfoIPv6 = connInfo } - r.externalNameServers = make([]string, len(config.ExternalNameServersV4)+len(config.ExternalNameServersV6)) - // deep copy external name servers from config to resolver - elemsCopied := copy(r.externalNameServers, append(config.ExternalNameServersV4, config.ExternalNameServersV6...)) - if elemsCopied != len(config.ExternalNameServersV4)+len(config.ExternalNameServersV6) { - log.Fatal("failed to copy entire name servers list from config") + ipv4Nameservers := make([]string, 0, len(config.ExternalNameServersV4)) + if config.IPVersionMode == IPv4Only || config.IPVersionMode == IPv4OrIPv6 { + // copy over IPv4 nameservers + elemsCopied := copy(ipv4Nameservers, config.ExternalNameServersV4) + if elemsCopied != len(config.ExternalNameServersV4) { + log.Fatal("failed to copy entire IPv4 name servers list from config") + } } + ipv6Nameservers := make([]string, 0, len(config.ExternalNameServersV6)) + if config.IPVersionMode == IPv6Only || config.IPVersionMode == IPv4OrIPv6 { + // copy over IPv6 nameservers + elemsCopied := copy(ipv6Nameservers, config.ExternalNameServersV6) + if elemsCopied != len(config.ExternalNameServersV6) { + log.Fatal("failed to copy entire IPv6 name servers list from config") + } + } + r.externalNameServers = append(ipv4Nameservers, ipv6Nameservers...) + // deep copy external name servers from config to resolver r.iterativeTimeout = config.IterativeTimeout r.maxDepth = config.MaxDepth // use the set of 13 root name servers @@ -536,9 +541,9 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer string) (*SingleQueryRe if err != nil { return nil, nil, StatusIllegalInput, fmt.Errorf("could not split host and port for name server: %w", err) } - if nsIP.To4() != nil && r.connInfoIPv4.localAddr.IsLoopback() != nsIP.IsLoopback() { + if nsIP.To4() != nil && r.connInfoIPv4 != nil && r.connInfoIPv4.localAddr.IsLoopback() != nsIP.IsLoopback() { return nil, nil, StatusIllegalInput, errors.New("nameserver (%s) and local address(%s) must be both loopback or non-loopback") - } else if nsIP.To16() != nil && nsIP.IsLoopback() { + } else if util.IsIPv6(&nsIP) && nsIP.IsLoopback() { return nil, nil, StatusIllegalInput, errors.New("cannot use IPv6 loopback nameserver") } // dstServer has been validated and has a port From a44af4fff6075ed4b3a063fcc3106d78d6de19a1 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 11:06:37 -0400 Subject: [PATCH 20/59] fixed issue with copying ns arrays in resolver init --- src/zdns/resolver.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index d6f4a942..eb5d72b8 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -135,7 +135,7 @@ func (rc *ResolverConfig) PopulateAndValidate() error { if rc.IPVersionMode == IPv6Only { return errors.New("IPv6 only mode requires both local IPv6 addresses and IPv6 nameservers") } - log.Info("cannot use IPv6 only mode without both local IPv6 addresses and IPv6 nameservers, defaulting to IPv4 only") + log.Info("cannot use IPv6 mode without both local IPv6 addresses and IPv6 nameservers, defaulting to IPv4 only") rc.IPVersionMode = IPv4Only } // If we're using IPv4, we need both a local IPv4 address and an IPv4 nameserver @@ -143,7 +143,7 @@ func (rc *ResolverConfig) PopulateAndValidate() error { if rc.IPVersionMode == IPv4Only { return errors.New("IPv4 only mode requires both local IPv4 addresses and IPv4 nameservers") } - log.Info("cannot use IPv4 only mode without both local IPv4 addresses and IPv4 nameservers, defaulting to IPv6 only") + log.Info("cannot use IPv4 mode without both local IPv4 addresses and IPv4 nameservers, defaulting to IPv6 only") rc.IPVersionMode = IPv6Only } @@ -444,23 +444,26 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { } r.connInfoIPv6 = connInfo } - ipv4Nameservers := make([]string, 0, len(config.ExternalNameServersV4)) + // need to deep-copy here so we're not reliant on the state of the resolver config post-resolver creation + r.externalNameServers = make([]string, 0) if config.IPVersionMode == IPv4Only || config.IPVersionMode == IPv4OrIPv6 { + ipv4Nameservers := make([]string, len(config.ExternalNameServersV4)) // copy over IPv4 nameservers elemsCopied := copy(ipv4Nameservers, config.ExternalNameServersV4) if elemsCopied != len(config.ExternalNameServersV4) { log.Fatal("failed to copy entire IPv4 name servers list from config") } + r.externalNameServers = append(r.externalNameServers, ipv4Nameservers...) } - ipv6Nameservers := make([]string, 0, len(config.ExternalNameServersV6)) + ipv6Nameservers := make([]string, len(config.ExternalNameServersV6)) if config.IPVersionMode == IPv6Only || config.IPVersionMode == IPv4OrIPv6 { // copy over IPv6 nameservers elemsCopied := copy(ipv6Nameservers, config.ExternalNameServersV6) if elemsCopied != len(config.ExternalNameServersV6) { log.Fatal("failed to copy entire IPv6 name servers list from config") } + r.externalNameServers = append(r.externalNameServers, ipv6Nameservers...) } - r.externalNameServers = append(ipv4Nameservers, ipv6Nameservers...) // deep copy external name servers from config to resolver r.iterativeTimeout = config.IterativeTimeout r.maxDepth = config.MaxDepth From b3c70da87ae9842595203db57bd17ed41e1f6805 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 11:22:40 -0400 Subject: [PATCH 21/59] added new --4 and --6 flags to disambiguate from the lookup A and AAAA flags --- src/cli/alookup.go | 4 ++-- src/cli/cli.go | 8 ++++++-- src/cli/config_validation.go | 4 ++++ src/cli/worker_manager.go | 15 ++++----------- testing/integration_tests.py | 12 ++++++------ 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/cli/alookup.go b/src/cli/alookup.go index 755fcb1e..0c079840 100644 --- a/src/cli/alookup.go +++ b/src/cli/alookup.go @@ -40,8 +40,8 @@ Specifically, alookup acts similar to nslookup and will follow CNAME records.`, func init() { rootCmd.AddCommand(alookupCmd) - alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each MX server") - alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each MX server") + alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each server") + alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each server") util.BindFlags(alookupCmd, viper.GetViper(), util.EnvPrefix) } diff --git a/src/cli/cli.go b/src/cli/cli.go index 68fe49a7..f15d2388 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -69,6 +69,8 @@ type CLIConf struct { LookupAllNameServers bool TCPOnly bool UDPOnly bool + IPv4Transport bool + IPv6Transport bool RecycleSockets bool LocalAddrSpecified bool LocalAddrs []net.IP @@ -163,6 +165,8 @@ func init() { rootCmd.PersistentFlags().StringVar(&GC.NameServersString, "name-servers", "", "List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53.") rootCmd.PersistentFlags().StringVar(&GC.LocalAddrString, "local-addr", "", "comma-delimited list of local addresses to use, serve as the source IP for outbound queries") rootCmd.PersistentFlags().StringVar(&GC.LocalIfaceString, "local-interface", "", "local interface to use") + rootCmd.PersistentFlags().BoolVar(&GC.IPv4Transport, "4", true, "utilize IPv4 query transport, must have an IPv4 local address") + rootCmd.PersistentFlags().BoolVar(&GC.IPv6Transport, "6", false, "utilize IPv6 query transport, must have an IPv6 local address") rootCmd.PersistentFlags().StringVar(&GC.ConfigFilePath, "conf-file", zdns.DefaultNameServerConfigFile, "config file for DNS servers") rootCmd.PersistentFlags().IntVar(&GC.Timeout, "timeout", 15, "timeout for resolving a individual name, in seconds") rootCmd.PersistentFlags().IntVar(&GC.IterationTimeout, "iteration-timeout", 4, "timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative") @@ -172,8 +176,8 @@ func init() { rootCmd.PersistentFlags().BoolVar(&GC.Dnssec, "dnssec", false, "Requests DNSSEC records by setting the DNSSEC OK (DO) bit") rootCmd.PersistentFlags().BoolVar(&GC.UseNSID, "nsid", false, "Request NSID.") - rootCmd.PersistentFlags().Bool("ipv4-lookup", false, "Perform an IPv4 Lookup in modules") - rootCmd.PersistentFlags().Bool("ipv6-lookup", false, "Perform an IPv6 Lookup in modules") + rootCmd.PersistentFlags().Bool("ipv4-lookup", false, "Perform an IPv4 Lookup (requests A records) in modules") + rootCmd.PersistentFlags().Bool("ipv6-lookup", false, "Perform an IPv6 Lookup (requests AAAA recoreds) in modules") rootCmd.PersistentFlags().StringVar(&GC.BlacklistFilePath, "blacklist-file", "", "blacklist file for servers to exclude from lookups") } diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 181fbf5b..7863b7ae 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -31,6 +31,10 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.New("--local-addr and --local-interface cannot both be specified") } + if !gc.IPv4Transport && !gc.IPv6Transport { + return errors.New("both IPv4 and IPv6 transport cannot both be disabled") + } + if err := populateNameServers(gc); err != nil { return errors.Wrap(err, "name servers did not pass validation") } diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 36371ac0..21ca782f 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -154,17 +154,10 @@ func populateCLIConfig(gc *CLIConf, flags *pflag.FlagSet) *CLIConf { return gc } -func populateResolverConfig(gc *CLIConf, flags *pflag.FlagSet) *zdns.ResolverConfig { +func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config := zdns.NewResolverConfig() - useIPv4, err := flags.GetBool("ipv4-lookup") - if err != nil { - log.Fatal("Unable to parse ipv4 flag: ", err) - } - useIPv6, err := flags.GetBool("ipv6-lookup") - if err != nil { - log.Fatal("Unable to parse ipv6 flag: ", err) - } - config.IPVersionMode = zdns.GetIPVersionMode(useIPv4, useIPv6) + + config.IPVersionMode = zdns.GetIPVersionMode(gc.IPv4Transport, gc.IPv6Transport) config.TransportMode = zdns.GetTransportMode(gc.UDPOnly, gc.TCPOnly) config.Timeout = time.Second * time.Duration(gc.Timeout) @@ -218,7 +211,7 @@ func populateResolverConfig(gc *CLIConf, flags *pflag.FlagSet) *zdns.ResolverCon func Run(gc CLIConf, flags *pflag.FlagSet) { gc = *populateCLIConfig(&gc, flags) - resolverConfig := populateResolverConfig(&gc, flags) + resolverConfig := populateResolverConfig(&gc) err := resolverConfig.PopulateAndValidate() if err != nil { log.Fatal("could not populate defaults and validate resolver config: ", err) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index e8f1ca87..e978aae0 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -977,14 +977,14 @@ def test_timetamps_nanoseconds(self): # Test Cases for IPv6 def test_ipv6_unreachable(self): - c = "A --iterative --ipv6-lookup=true --ipv4-lookup=false" + c = "A --iterative --6=true --4=false" name = "esrg.stanford.edu" cmd, res = self.run_zdns(c, name) # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, this will fail. self.assertServFail(res, cmd) def test_ipv6_external_lookup_unreachable_nameserver(self): - c = "A --ipv6-lookup=true --ipv4-lookup=false --name-servers=1.1.1.1" + c = "A --6=true --4=false --name-servers=1.1.1.1" name = "zdns-testing.com" try: cmd, res = self.run_zdns(c, name) @@ -993,7 +993,7 @@ def test_ipv6_external_lookup_unreachable_nameserver(self): self.fail("Should have thrown an exception, shouldn't be able to reach any IPv4 servers while in IPv6 mode") def test_ipv4_external_lookup_unreachable_nameserver(self): - c = "A --ipv6-lookup=false --ipv4-lookup=true --name-servers=2606:4700:4700::1111" + c = "A --6=false --4=true --name-servers=2606:4700:4700::1111" name = "zdns-testing.com" try: cmd, res = self.run_zdns(c, name) @@ -1002,21 +1002,21 @@ def test_ipv4_external_lookup_unreachable_nameserver(self): self.fail("Should have thrown an exception, shouldn't be able to reach any IPv6 servers while in IPv4 mode") def test_ipv6_happy_path_external(self): - c = "A --ipv6-lookup=true" + c = "A --6=true" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) def test_ipv6_happy_path_iterative(self): - c = "A --ipv6-lookup=true --iterative" + c = "A --6=true --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) def test_ipv6_happy_path_no_ipv4_iterative(self): - c = "A --ipv6-lookup=true --ipv4-lookup=false --iterative" + c = "A --6=true --4=false --iterative" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) self.assertSuccess(res, cmd) From 78e11875a43084f3b6dcf01f5461cad120708407 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 12:25:16 -0400 Subject: [PATCH 22/59] fix up unit tests, messed them up by misunderstanding the lookup-ipv4 flag --- src/cli/config_validation_test.go | 3 + src/modules/alookup/a_lookup.go | 2 +- src/modules/mxlookup/mx_lookup.go | 2 +- src/modules/nslookup/ns_lookup.go | 2 +- src/zdns/alookup.go | 12 ++- src/zdns/lookup.go | 9 +- src/zdns/lookup_test.go | 142 ++++++++++++++++-------------- src/zdns/nslookup.go | 14 +-- 8 files changed, 100 insertions(+), 86 deletions(-) diff --git a/src/cli/config_validation_test.go b/src/cli/config_validation_test.go index efda52ca..53310f01 100644 --- a/src/cli/config_validation_test.go +++ b/src/cli/config_validation_test.go @@ -24,6 +24,7 @@ func TestValidateNetworkingConfig(t *testing.T) { gc := &CLIConf{ LocalAddrString: "1.1.1.1", LocalIfaceString: "eth0", + IPv4Transport: true, } err := populateNetworkingConfig(gc) require.NotNil(t, err, "Expected an error but got nil") @@ -31,6 +32,7 @@ func TestValidateNetworkingConfig(t *testing.T) { t.Run("Using invalid interface", func(t *testing.T) { gc := &CLIConf{ LocalIfaceString: "invalid_interface", + IPv4Transport: true, } err := populateNetworkingConfig(gc) require.NotNil(t, err, "Expected an error but got nil") @@ -38,6 +40,7 @@ func TestValidateNetworkingConfig(t *testing.T) { t.Run("Using nameserver with port", func(t *testing.T) { gc := &CLIConf{ NameServersString: "127.0.0.1:53", + IPv4Transport: true, } err := populateNetworkingConfig(gc) require.Nil(t, err, "Expected no error but got %v", err) diff --git a/src/modules/alookup/a_lookup.go b/src/modules/alookup/a_lookup.go index 44d557f8..41dbc77d 100644 --- a/src/modules/alookup/a_lookup.go +++ b/src/modules/alookup/a_lookup.go @@ -58,7 +58,7 @@ func (aMod *ALookupModule) Init(ipv4Lookup bool, ipv6Lookup bool) { } func (aMod *ALookupModule) Lookup(r *zdns.Resolver, lookupName, nameServer string) (interface{}, zdns.Trace, zdns.Status, error) { - ipResult, trace, status, err := r.DoTargetedLookup(lookupName, nameServer, zdns.GetIPVersionMode(aMod.IPv4Lookup, aMod.IPv6Lookup), aMod.baseModule.IsIterative) + ipResult, trace, status, err := r.DoTargetedLookup(lookupName, nameServer, aMod.baseModule.IsIterative, aMod.IPv4Lookup, aMod.IPv6Lookup) return ipResult, trace, status, err } diff --git a/src/modules/mxlookup/mx_lookup.go b/src/modules/mxlookup/mx_lookup.go index 75a255b6..7a251df3 100644 --- a/src/modules/mxlookup/mx_lookup.go +++ b/src/modules/mxlookup/mx_lookup.go @@ -110,7 +110,7 @@ func (mxMod *MXLookupModule) lookupIPs(r *zdns.Resolver, name, nameServer string return res.(CachedAddresses), zdns.Trace{} } retv := CachedAddresses{} - result, trace, status, _ := r.DoTargetedLookup(name, nameServer, ipMode, mxMod.IsIterative) + result, trace, status, _ := r.DoTargetedLookup(name, nameServer, mxMod.IsIterative, mxMod.IPv4Lookup, mxMod.IPv6Lookup) if status == zdns.StatusNoError && result != nil { retv.IPv4Addresses = result.IPv4Addresses retv.IPv6Addresses = result.IPv6Addresses diff --git a/src/modules/nslookup/ns_lookup.go b/src/modules/nslookup/ns_lookup.go index b79c2d93..71c2b01d 100644 --- a/src/modules/nslookup/ns_lookup.go +++ b/src/modules/nslookup/ns_lookup.go @@ -73,7 +73,7 @@ func (nsMod *NSLookupModule) Lookup(r *zdns.Resolver, lookupName string, nameSer log.Warn("iterative lookup requested with lookupName server, ignoring lookupName server") } - res, trace, status, err := r.DoNSLookup(lookupName, nameServer, nsMod.IsIterative) + res, trace, status, err := r.DoNSLookup(lookupName, nameServer, nsMod.IsIterative, nsMod.IPv4Lookup, nsMod.IPv6Lookup) if trace == nil { trace = zdns.Trace{} } diff --git a/src/zdns/alookup.go b/src/zdns/alookup.go index fc7a4909..e4b93b61 100644 --- a/src/zdns/alookup.go +++ b/src/zdns/alookup.go @@ -22,9 +22,7 @@ import ( // DoTargetedLookup performs a lookup of the given domain name against the given nameserver, looking up both IPv4 and IPv6 addresses // Will follow CNAME records as well as A/AAAA records to get IP addresses -func (r *Resolver) DoTargetedLookup(name, nameServer string, ipMode IPVersionMode, isIterative bool) (*IPResult, Trace, Status, error) { - lookupIPv4 := ipMode == IPv4Only || ipMode == IPv4OrIPv6 - lookupIPv6 := ipMode == IPv6Only || ipMode == IPv4OrIPv6 +func (r *Resolver) DoTargetedLookup(name, nameServer string, isIterative, lookupA, lookupAAAA bool) (*IPResult, Trace, Status, error) { name = strings.ToLower(name) res := IPResult{} candidateSet := map[string][]Answer{} @@ -36,7 +34,7 @@ func (r *Resolver) DoTargetedLookup(name, nameServer string, ipMode IPVersionMod var ipv4status Status var ipv6status Status - if lookupIPv4 { + if lookupA { ipv4, ipv4Trace, ipv4status, _ = recursiveIPLookup(r, name, nameServer, dns.TypeA, candidateSet, cnameSet, name, 0, isIterative) if len(ipv4) > 0 { ipv4 = Unique(ipv4) @@ -46,7 +44,7 @@ func (r *Resolver) DoTargetedLookup(name, nameServer string, ipMode IPVersionMod } candidateSet = map[string][]Answer{} cnameSet = map[string][]Answer{} - if lookupIPv6 { + if lookupAAAA { ipv6, ipv6Trace, ipv6status, _ = recursiveIPLookup(r, name, nameServer, dns.TypeAAAA, candidateSet, cnameSet, name, 0, isIterative) if len(ipv6) > 0 { ipv6 = Unique(ipv6) @@ -60,9 +58,9 @@ func (r *Resolver) DoTargetedLookup(name, nameServer string, ipMode IPVersionMod // In case we get no IPs and a non-NOERROR status from either // IPv4 or IPv6 lookup, we return that status. if len(res.IPv4Addresses) == 0 && len(res.IPv6Addresses) == 0 { - if lookupIPv4 && !SafeStatus(ipv4status) { + if lookupA && !SafeStatus(ipv4status) { return nil, combinedTrace, ipv4status, nil - } else if lookupIPv6 && !SafeStatus(ipv6status) { + } else if lookupAAAA && !SafeStatus(ipv6status) { return nil, combinedTrace, ipv6status, nil } else { return &res, combinedTrace, StatusNoError, nil diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index d47a2b6f..0d2b1720 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -119,7 +119,7 @@ func (r *Resolver) LookupAllNameservers(q *Question, nameServer string) (*Combin var curServer string // Lookup both ipv4 and ipv6 addresses of nameservers. - nsResults, nsTrace, nsStatus, nsError := r.DoNSLookup(q.Name, nameServer, false) + nsResults, nsTrace, nsStatus, nsError := r.DoNSLookup(q.Name, nameServer, false, true, true) // Terminate early if nameserver lookup also failed if nsStatus != StatusNoError { @@ -140,7 +140,12 @@ func (r *Resolver) LookupAllNameservers(q *Question, nameServer string) (*Combin ips := append(nserver.IPv4Addresses, nserver.IPv6Addresses...) for _, ip := range ips { curServer = net.JoinHostPort(ip, "53") - res, trace, status, _ := r.ExternalLookup(q, curServer) + res, trace, status, err := r.ExternalLookup(q, curServer) + if err != nil { + // log and move on + log.Errorf("lookup for domain %s to nameserver %s failed with error %s. Continueing to next nameserver", q.Name, curServer, err) + continue + } fullTrace = append(fullTrace, trace...) extendedResult := ExtendedResult{ diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go index 7cca1f28..effe31f3 100644 --- a/src/zdns/lookup_test.go +++ b/src/zdns/lookup_test.go @@ -639,7 +639,7 @@ func TestOneA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -674,7 +674,7 @@ func TestTwoA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1", "192.0.2.2"}, nil) } @@ -710,7 +710,7 @@ func TestQuadAWithoutFlag(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -739,7 +739,7 @@ func TestOnlyQuadA(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, false, true) assert.NotNil(t, res) verifyResult(t, *res, nil, []string{"2001:db8::1"}) } @@ -775,7 +775,7 @@ func TestAandQuadA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, true) assert.NotNil(t, res) verifyResult(t, *res, []string{"192.0.2.1"}, []string{"2001:db8::1"}) } @@ -811,7 +811,7 @@ func TestTwoQuadA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, false, true) assert.NotNil(t, res) verifyResult(t, *res, nil, []string{"2001:db8::1", "2001:db8::2"}) } @@ -835,7 +835,7 @@ func TestNoResults(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, false) verifyResult(t, *res, nil, nil) } @@ -881,7 +881,7 @@ func TestCname(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -916,7 +916,7 @@ func TestQuadAWithCname(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, false, false, true) verifyResult(t, *res, nil, []string{"2001:db8::3"}) } @@ -945,7 +945,7 @@ func TestUnexpectedMxOnly(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -992,7 +992,7 @@ func TestMxAndAdditionals(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) verifyResult(t, *res, []string{"192.0.2.3"}, []string{"2001:db8::4"}) } @@ -1021,7 +1021,7 @@ func TestMismatchIpType(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1073,7 +1073,7 @@ func TestCnameLoops(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1111,7 +1111,7 @@ func TestExtendedRecursion(t *testing.T) { } } - res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1157,11 +1157,11 @@ func TestEmptyNonTerminal(t *testing.T) { Flags: DNSFlags{}, } // Verify leaf returns correctly - res, _, _, _ := resolver.DoTargetedLookup("leaf.intermediate.example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("leaf.intermediate.example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.3"}, nil) // Verify empty non-terminal returns no answer - res, _, _, _ = resolver.DoTargetedLookup("intermediate.example.com", ns1, IPv4OrIPv6, false) + res, _, _, _ = resolver.DoTargetedLookup("intermediate.example.com", ns1, false, true, true) verifyResult(t, *res, nil, nil) } @@ -1172,7 +1172,7 @@ func TestNXDomain(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) ns1 := config.ExternalNameServersV4[0] - res, _, status, _ := resolver.DoTargetedLookup("nonexistent.example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("nonexistent.example.com", ns1, false, true, true) if status != StatusNXDomain { t.Errorf("Expected StatusNXDomain status, got %v", status) } else if res != nil { @@ -1272,7 +1272,7 @@ func TestAandQuadADedup(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, true) assert.NotNil(t, res) verifyResult(t, *res, []string{"192.0.2.1"}, []string{"2001:db8::3"}) } @@ -1292,7 +1292,7 @@ func TestServFail(t *testing.T) { name := "example.com" protocolStatus[domainNS1] = StatusServFail - res, _, finalStatus, _ := resolver.DoTargetedLookup(name, ns1, IPv4OrIPv6, false) + res, _, finalStatus, _ := resolver.DoTargetedLookup(name, ns1, false, true, true) if finalStatus != protocolStatus[domainNS1] { t.Errorf("Expected %v status, got %v", protocolStatus, finalStatus) @@ -1315,9 +1315,52 @@ func verifyResult(t *testing.T, res IPResult, ipv4 []string, ipv6 []string) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // /*NS lookup tests*/ +func TestNoAOrAAAA(t *testing.T) { + config := InitTest(t) + resolver, err := InitResolver(config) + require.NoError(t, err) + + domain1 := "example.com" + ns1 := config.ExternalNameServersV4[0] + domainNS1 := domainNS{domain: domain1, ns: ns1} + + mockResults[domainNS1] = SingleQueryResult{ + Answers: []interface{}{ + Answer{ + TTL: 3600, + Type: "NS", + Class: "IN", + Name: "example.com.", + Answer: "ns1.example.com.", + }, + }, + Additional: []interface{}{ + Answer{ + TTL: 3600, + Type: "A", + Class: "IN", + Name: "ns1.example.com.", + Answer: "192.0.2.3", + }, + }, + Authorities: nil, + Protocol: "", + Flags: DNSFlags{}, + } + + expectedServersMap := make(map[string]IPResult) + expectedServersMap["ns1.example.com"] = IPResult{ + IPv4Addresses: []string{"192.0.2.3"}, + IPv6Addresses: nil, + } + // must either request A or AAAA or both + res, _, _, err := resolver.DoNSLookup("example.com", ns1, false, false, false) + assert.Error(t, err, "shouldn't be able to perform NSLookup with both A and AAAA lookups as false") + assert.Nil(t, res) + +} func TestNsAInAdditional(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4Only resolver, err := InitResolver(config) require.NoError(t, err) @@ -1354,13 +1397,12 @@ func TestNsAInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) verifyNsResult(t, res.Servers, expectedServersMap) } func TestTwoNSInAdditional(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4Only resolver, err := InitResolver(config) require.NoError(t, err) @@ -1415,13 +1457,12 @@ func TestTwoNSInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.4"}, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) verifyNsResult(t, res.Servers, expectedServersMap) } func TestAandQuadAInAdditional(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) @@ -1465,13 +1506,12 @@ func TestAandQuadAInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: []string{"2001:db8::4"}, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } func TestNsMismatchIpType(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) @@ -1515,13 +1555,12 @@ func TestNsMismatchIpType(t *testing.T) { IPv4Addresses: nil, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } func TestAandQuadALookup(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) @@ -1577,7 +1616,7 @@ func TestAandQuadALookup(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: []string{"2001:db8::4"}, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } @@ -1588,7 +1627,7 @@ func TestNsNXDomain(t *testing.T) { ns1 := config.ExternalNameServersV4[0] - _, _, status, _ := resolver.DoNSLookup("nonexistentexample.com", ns1, false) + _, _, status, _ := resolver.DoNSLookup("nonexistentexample.com", ns1, false, true, true) assert.Equal(t, StatusNXDomain, status) } @@ -1605,7 +1644,7 @@ func TestNsServFail(t *testing.T) { mockResults[domainNS1] = SingleQueryResult{} protocolStatus[domainNS1] = StatusServFail - res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) assert.Equal(t, status, protocolStatus[domainNS1]) assert.Empty(t, res.Servers) @@ -1638,16 +1677,17 @@ func TestErrorInTargetedLookup(t *testing.T) { protocolStatus[domainNS1] = StatusError - res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) assert.Empty(t, len(res.Servers), 0) assert.Equal(t, status, protocolStatus[domainNS1]) } // Test One NS with one IP with only ipv4-lookup +// Cannot test IPv6 since you can't assign a loopback local address as a nameserver +// And you can't assign a fake IPv6 address since we won't be able to bind to it func TestAllNsLookupOneNs(t *testing.T) { config := InitTest(t) - config.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} - config.IPVersionMode = IPv4OrIPv6 + config.IPVersionMode = IPv4Only resolver, err := InitResolver(config) require.NoError(t, err) @@ -1655,7 +1695,6 @@ func TestAllNsLookupOneNs(t *testing.T) { domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" - ipv6_1 := "::1" domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1676,13 +1715,6 @@ func TestAllNsLookupOneNs(t *testing.T) { Name: nsDomain1 + ".", Answer: ipv4_1, }, - Answer{ - TTL: 3600, - Type: "AAAA", - Class: "IN", - Name: nsDomain1 + ".", - Answer: ipv6_1, - }, }, Authorities: nil, Protocol: "", @@ -1691,7 +1723,7 @@ func TestAllNsLookupOneNs(t *testing.T) { ns2 := net.JoinHostPort(ipv4_1, "53") domainNS2 := domainNS{domain: domain1, ns: ns2} - ipv4_2 := "127.0.0.3" + ipv4_2 := "9.8.7.6.5" mockResults[domainNS2] = SingleQueryResult{ Answers: []interface{}{ Answer{ @@ -1708,36 +1740,12 @@ func TestAllNsLookupOneNs(t *testing.T) { Flags: DNSFlags{}, } - ns3 := net.JoinHostPort(ipv6_1, "53") - domainNS3 := domainNS{domain: domain1, ns: ns3} - ipv4_3 := "127.0.0.4" - mockResults[domainNS3] = SingleQueryResult{ - Answers: []interface{}{ - Answer{ - TTL: 3600, - Type: "A", - Class: "IN", - Name: "example.com.", - Answer: ipv4_3, - }, - }, - Additional: nil, - Authorities: nil, - Protocol: "", - Flags: DNSFlags{}, - } - expectedRes := []ExtendedResult{ { Nameserver: nsDomain1, Status: StatusNoError, Res: mockResults[domainNS2], }, - { - Nameserver: nsDomain1, - Status: StatusNoError, - Res: mockResults[domainNS3], - }, } q := Question{ Type: dns.TypeNS, diff --git a/src/zdns/nslookup.go b/src/zdns/nslookup.go index 3a306f97..a45c603e 100644 --- a/src/zdns/nslookup.go +++ b/src/zdns/nslookup.go @@ -39,10 +39,13 @@ type NSResult struct { } // DoNSLookup performs a DNS NS lookup on the given name against the given name server. -func (r *Resolver) DoNSLookup(lookupName, nameServer string, isIterative bool) (*NSResult, Trace, Status, error) { +func (r *Resolver) DoNSLookup(lookupName, nameServer string, isIterative, lookupA, lookupAAAA bool) (*NSResult, Trace, Status, error) { if len(lookupName) == 0 { return nil, nil, "", errors.New("no name provided for NS lookup") } + if !lookupA && !lookupAAAA { + return nil, nil, "", errors.New("must lookup either A or AAAA") + } var trace Trace var ns *SingleQueryResult @@ -93,17 +96,14 @@ func (r *Resolver) DoNSLookup(lookupName, nameServer string, isIterative bool) ( var findIpv4 = false var findIpv6 = false - lookupIPv4 := r.ipVersionMode == IPv4Only || r.ipVersionMode == IPv4OrIPv6 - lookupIPv6 := r.ipVersionMode == IPv6Only || r.ipVersionMode == IPv4OrIPv6 - - if lookupIPv4 { + if lookupA { if ips, ok := ipv4s[rec.Name]; ok { rec.IPv4Addresses = ips } else { findIpv4 = true } } - if lookupIPv6 { + if lookupAAAA { if ips, ok := ipv6s[rec.Name]; ok { rec.IPv6Addresses = ips } else { @@ -111,7 +111,7 @@ func (r *Resolver) DoNSLookup(lookupName, nameServer string, isIterative bool) ( } } if findIpv4 || findIpv6 { - res, nextTrace, _, _ := r.DoTargetedLookup(rec.Name, nameServer, r.ipVersionMode, false) + res, nextTrace, _, _ := r.DoTargetedLookup(rec.Name, nameServer, false, lookupA, lookupAAAA) if res != nil { if findIpv4 { rec.IPv4Addresses = res.IPv4Addresses From 2099ba73d11b6ff525afeecfa6f9bc9c131819ec Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 12:56:41 -0400 Subject: [PATCH 23/59] fix up IPv6 test so it doesn't run on non-IPv6 supported hosts --- testing/integration_tests.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index e978aae0..0a79a8fc 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import copy +import socket import subprocess import json import unittest @@ -39,7 +40,7 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "./zdns" + ZDNS_EXECUTABLE = "../zdns" def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE): flags = flags + " --threads=10" @@ -546,8 +547,21 @@ def test_a(self): self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def check_ipv6_support(self): + try: + # Attempt to create an IPv6 socket + socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + return True + except OSError: + return False + def test_a_ipv6(self): - c = "A --name-servers=2001:4860:4860::8888:53" + if not self.check_ipv6_support(): + # can only run this integration test if we have IPv6 support on this host + print("IPv6 not supported on this host, test passing by default since we can't test") + return True + c = "A --6=true --name-servers=[2001:4860:4860::8888]:53" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) self.assertSuccess(res, cmd) From 82a983455db768648be56e1a5618462e861ab989 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 12:57:56 -0400 Subject: [PATCH 24/59] forgot to rename the ZDNS exe back --- testing/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 0a79a8fc..74556ab1 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -40,7 +40,7 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "../zdns" + ZDNS_EXECUTABLE = "./zdns" def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE): flags = flags + " --threads=10" From 64e58c63f39c921cc31fb0ab9aa1f6ddc059068b Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 13:52:41 -0400 Subject: [PATCH 25/59] moved ipv6 tests into their own workflow --- makefile | 5 ++ testing/integration_tests.py | 68 ------------------ testing/ipv6_tests.py | 132 +++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 68 deletions(-) create mode 100644 testing/ipv6_tests.py diff --git a/makefile b/makefile index 52157729..ef12dc53 100644 --- a/makefile +++ b/makefile @@ -17,6 +17,11 @@ integration-tests: zdns python3 testing/integration_tests.py python3 testing/large_scan_integration/large_scan_integration_tests.py +# Not all hosts support this, so this will be a custom make target +ipv6-tests: zdns + pip3 install -r testing/requirements.txt + python3 testing/ipv6_tests.py + lint: goimports -w -local "github.com/zmap/zdns" ./ gofmt -s -w ./ diff --git a/testing/integration_tests.py b/testing/integration_tests.py index 74556ab1..cd400871 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -548,25 +548,6 @@ def test_a(self): self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - def check_ipv6_support(self): - try: - # Attempt to create an IPv6 socket - socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - return True - except OSError: - return False - - def test_a_ipv6(self): - if not self.check_ipv6_support(): - # can only run this integration test if we have IPv6 support on this host - print("IPv6 not supported on this host, test passing by default since we can't test") - return True - c = "A --6=true --name-servers=[2001:4860:4860::8888]:53" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - def test_cname(self): c = "CNAME" name = "www.zdns-testing.com" @@ -989,55 +970,6 @@ def test_timetamps_nanoseconds(self): # but it is very unlikely. (1 in 1,000,000). Python's datetime.date's smallest unit of time is microseconds, # so that's why we're using this in place of nanoseconds. It should not affect the test's validity. -# Test Cases for IPv6 - def test_ipv6_unreachable(self): - c = "A --iterative --6=true --4=false" - name = "esrg.stanford.edu" - cmd, res = self.run_zdns(c, name) - # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, this will fail. - self.assertServFail(res, cmd) - - def test_ipv6_external_lookup_unreachable_nameserver(self): - c = "A --6=true --4=false --name-servers=1.1.1.1" - name = "zdns-testing.com" - try: - cmd, res = self.run_zdns(c, name) - except Exception as e: - return True - self.fail("Should have thrown an exception, shouldn't be able to reach any IPv4 servers while in IPv6 mode") - - def test_ipv4_external_lookup_unreachable_nameserver(self): - c = "A --6=false --4=true --name-servers=2606:4700:4700::1111" - name = "zdns-testing.com" - try: - cmd, res = self.run_zdns(c, name) - except Exception as e: - return True - self.fail("Should have thrown an exception, shouldn't be able to reach any IPv6 servers while in IPv4 mode") - - def test_ipv6_happy_path_external(self): - c = "A --6=true" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - - def test_ipv6_happy_path_iterative(self): - c = "A --6=true --iterative" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - - def test_ipv6_happy_path_no_ipv4_iterative(self): - c = "A --6=true --4=false --iterative" - name = "zdns-testing.com" - cmd, res = self.run_zdns(c, name) - self.assertSuccess(res, cmd) - self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - - - if __name__ == "__main__": unittest.main() diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py new file mode 100644 index 00000000..364eeebd --- /dev/null +++ b/testing/ipv6_tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +import copy +import socket +import subprocess +import json +import unittest +import tempfile +import datetime +from dateutil import parser +from ipaddress import ip_address + + +def recursiveSort(obj): + def listSort(l): + assert (type(l) == type(list())) + new_list = [] + for item in l: + item = recursiveSort(item) + new_list.append(item) + if len(new_list) > 0 and type(new_list[0]) == dict: + return sorted(new_list, key=lambda x: x["name"]) + else: + return sorted(new_list) + + def dictSort(d): + assert (type(d) == type(dict())) + for key in d: + d[key] = recursiveSort(d[key]) + return d + + if type(obj) == list: + return listSort(obj) + + elif type(obj) == dict: + return dictSort(obj) + else: + return obj + + +class Tests(unittest.TestCase): + maxDiff = None + ZDNS_EXECUTABLE = "../zdns" + + ROOT_A = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} + + ROOT_A_ANSWERS = [{"type": "A", "class": "IN", "answer": x, + "name": "zdns-testing.com"} for x in ROOT_A] + + def run_zdns(self, flags, name, executable=ZDNS_EXECUTABLE): + flags = flags + " --threads=10" + c = f"echo '{name}' | {executable} {flags}" + o = subprocess.check_output(c, shell=True) + return c, json.loads(o.rstrip()) + + def assertSuccess(self, res, cmd): + self.assertEqual(res["status"], "NOERROR", cmd) + + def assertServFail(self, res, cmd): + self.assertEqual(res["status"], "SERVFAIL", cmd) + + def assertEqualAnswers(self, res, correct, cmd, key="answer"): + self.assertIn("answers", res["data"]) + for answer in res["data"]["answers"]: + del answer["ttl"] + a = sorted(res["data"]["answers"], key=lambda x: x[key]) + b = sorted(correct, key=lambda x: x[key]) + helptext = "%s\nExpected:\n%s\n\nActual:\n%s" % (cmd, + json.dumps(b, indent=4), json.dumps(a, indent=4)) + + def test_a_ipv6(self): + c = "A --6=true --name-servers=[2001:4860:4860::8888]:53" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def test_ipv6_unreachable(self): + c = "A --iterative --6=true --4=false" + name = "esrg.stanford.edu" + cmd, res = self.run_zdns(c, name) + # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, this will fail. + self.assertServFail(res, cmd) + + def test_ipv6_external_lookup_unreachable_nameserver(self): + c = "A --6=true --4=false --name-servers=1.1.1.1" + name = "zdns-testing.com" + try: + cmd, res = self.run_zdns(c, name) + except Exception as e: + return True + self.fail("Should have thrown an exception, shouldn't be able to reach any IPv4 servers while in IPv6 mode") + + def test_ipv4_external_lookup_unreachable_nameserver(self): + c = "A --6=false --4=true --name-servers=2606:4700:4700::1111" + name = "zdns-testing.com" + try: + cmd, res = self.run_zdns(c, name) + except Exception as e: + return True + self.fail("Should have thrown an exception, shouldn't be able to reach any IPv6 servers while in IPv4 mode") + + def test_ipv6_happy_path_external(self): + c = "A --6=true" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def test_ipv6_happy_path_iterative(self): + c = "A --6=true --iterative" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + def test_ipv6_happy_path_no_ipv4_iterative(self): + c = "A --6=true --4=false --iterative" + name = "zdns-testing.com" + cmd, res = self.run_zdns(c, name) + self.assertSuccess(res, cmd) + self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) + + +if __name__ == "__main__": + try: + # Attempt to create an IPv6 socket + socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + except OSError: + print("Error: no IPv6 support on this machine, cannot test IPv6 functionality") + exit(1) + unittest.main() From 364b34ec0ce9c5a6a065a9d55770dc64feda1ea6 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 13:53:51 -0400 Subject: [PATCH 26/59] renamed zdns exe --- testing/ipv6_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index 364eeebd..33048244 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -40,7 +40,7 @@ def dictSort(d): class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "../zdns" + ZDNS_EXECUTABLE = "./zdns" ROOT_A = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} From b956318e85141215fe267cc59644fcbb81f23906 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 13:55:07 -0400 Subject: [PATCH 27/59] cleaned up ipv6 tests --- testing/ipv6_tests.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index 33048244..d4cfc628 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -1,41 +1,9 @@ #!/usr/bin/env python3 -import copy import socket import subprocess import json import unittest -import tempfile -import datetime -from dateutil import parser -from ipaddress import ip_address - - -def recursiveSort(obj): - def listSort(l): - assert (type(l) == type(list())) - new_list = [] - for item in l: - item = recursiveSort(item) - new_list.append(item) - if len(new_list) > 0 and type(new_list[0]) == dict: - return sorted(new_list, key=lambda x: x["name"]) - else: - return sorted(new_list) - - def dictSort(d): - assert (type(d) == type(dict())) - for key in d: - d[key] = recursiveSort(d[key]) - return d - - if type(obj) == list: - return listSort(obj) - - elif type(obj) == dict: - return dictSort(obj) - else: - return obj class Tests(unittest.TestCase): From 9df8ad83ea6ef70cdd0871f1ad16b49a0f035ceb Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 16:16:54 -0400 Subject: [PATCH 28/59] added loopback test for ipv6 --- testing/ipv6_tests.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index d4cfc628..f44c61b1 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -8,7 +8,7 @@ class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "./zdns" + ZDNS_EXECUTABLE = "../zdns" ROOT_A = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} @@ -68,6 +68,15 @@ def test_ipv4_external_lookup_unreachable_nameserver(self): return True self.fail("Should have thrown an exception, shouldn't be able to reach any IPv6 servers while in IPv4 mode") + def test_ipv6_external_lookup_loopback_nameserver(self): + c = "A --6=true --4=false --name-servers=[::1]:53" + name = "zdns-testing.com" + try: + cmd, res = self.run_zdns(c, name) + except Exception as e: + return True + self.fail("Should have thrown an exception, shouldn't be able to use a loopback address as a nameserver in IPv6") + def test_ipv6_happy_path_external(self): c = "A --6=true" name = "zdns-testing.com" From 6ea918e8103be6bf439488be05282edf622e75be Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 31 Jul 2024 16:40:32 -0400 Subject: [PATCH 29/59] elevate warning msg about not being able to find IPv6 socket to warning or else it's hard to identify --- src/zdns/resolver.go | 2 +- testing/ipv6_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index ace0741d..536ca56b 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -193,7 +193,7 @@ func (rc *ResolverConfig) populateLocalAddrs() error { return errors.New("unable to find default IPv6 address to open socket") } // user didn't specify IPv6 only, so we'll just log the issue and continue with IPv4 - log.Info("unable to find default IPv6 address to open socket, using IPv4 only: ", err) + log.Warn("unable to find default IPv6 address to open socket, using IPv4 only: ", err) rc.IPVersionMode = IPv4Only return nil } diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index f44c61b1..2de75f8c 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -8,7 +8,7 @@ class Tests(unittest.TestCase): maxDiff = None - ZDNS_EXECUTABLE = "../zdns" + ZDNS_EXECUTABLE = "./zdns" ROOT_A = {"1.2.3.4", "2.3.4.5", "3.4.5.6"} From 844890f69d0e7a8212d69f6d63bc7a1d0cd404dd Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Thu, 1 Aug 2024 09:50:52 -0400 Subject: [PATCH 30/59] add fix for using root servers if in iterative mode in CLI --- src/cli/config_validation.go | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 7863b7ae..ef0cb18b 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -23,6 +23,9 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/zmap/dns" + + "github.com/zmap/zdns/src/internal/util" + "github.com/zmap/zdns/src/zdns" ) func populateNetworkingConfig(gc *CLIConf) error { @@ -77,9 +80,50 @@ func populateNetworkingConfig(gc *CLIConf) error { } log.Info("using local interface: ", gc.LocalIfaceString) } + + // If we're in iterative mode, we always start the DNS resolution iterative process at the root DNS servers. + // However, the ZDNS resolver library we'll create doesn't know that all queries will be iterative, it's designed to be able to do + // both iterative queries and use a recursive resolver with the same config. While usually fine, there's an edge case here + // if it is the case that we're only doing iterative queries AND the OS' configured NS's are loopback, ZDNS library + // will set the local address to a loopback address so the NS's are reachable. + // Unfortunately, this will cause the iterative queries to fail, as the root servers are not reachable from the loopback address. + // + // To prevent this, we'll check if we're in iterative mode, the user hasn't passed in the local addr/nameservers directly to ZDNS, + // and the OS' configured NS's are loopback. If so, we'll set the nameservers to be our default non-loopback recursive resolvers. + // This prevents the edge case described above and has no effect on iterative queries since we just use the root nameservers. + if gc.IterativeResolution && !gc.LocalAddrSpecified && areOSNameserversLoopback(gc) { + log.Debug("OS external resolution nameservers are loopback and iterative mode is enabled. " + + "Using default non-loopback nameservers to prevent resolution failure edge case") + ipv4NS, ipv6NS := util.GetDefaultResolvers() + gc.NameServers = append(ipv4NS, ipv6NS...) + } return nil } +// areOSNameserversLoopback returns true if the OS' configured nameservers (in /etc/resolv.conf by default) are loopback addresses +func areOSNameserversLoopback(gc *CLIConf) bool { + nsesIPv4, nsesIPv6, err := zdns.GetDNSServers(gc.ConfigFilePath) + if err != nil { + log.Fatalf("Error getting OS nameservers: %s", err.Error()) + } + for _, ns := range append(nsesIPv4, nsesIPv6...) { + ipString, _, err := net.SplitHostPort(ns) + if err != nil { + // might be missing a port + ipString = ns + } + ip := net.ParseIP(ipString) + if ip == nil { + log.Fatalf("Error parsing OS nameserver IP: %s", ns) + } + if ip.IsLoopback() { + return true + } + + } + return false +} + func validateClientSubnetString(gc *CLIConf) error { if gc.ClientSubnetString != "" { parts := strings.Split(gc.ClientSubnetString, "/") From 05e9a8e71dcb87515cf70d7e893f75a76e7a422c Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Thu, 1 Aug 2024 09:53:32 -0400 Subject: [PATCH 31/59] added details to loopback warning msg --- src/zdns/resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 536ca56b..5208146c 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -308,7 +308,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { // Both nameservers and local addresses are completely loopback or non-loopback // if using loopback nameservers, override local addresses to be loopback and warn user if allNameserversLoopback && noneLocalAddrsLoopback { - log.Warn("nameservers are loopback, setting local address to loopback to match") + log.Warnf("nameservers (%s) are loopback, setting local address to loopback (%s) to match", allNameServers, LoopbackAddrString) rc.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} // we ignore link-local local addresses, so nothing to be done for IPv6 } else if noneNameserversLoopback && allLocalAddrsLoopback { From c60734ba6cb670051c30471da16c2951d5f7d2bc Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Thu, 1 Aug 2024 10:34:53 -0400 Subject: [PATCH 32/59] lookup AAAA for nameservers in extract authority --- src/zdns/lookup.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 30c5b228..7f496734 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -649,6 +649,13 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, q.Name = server q.Type = dns.TypeA q.Class = dns.ClassINET + // TODO This logic prefers IPv4 for IPv4OrIPv6 lookups. + // When we address the allNameservers not actually trying allNameservers (#352), we should address this + // Likely, we need to make both A/AAAA lookups and combine the results + if r.ipVersionMode == IPv6Only { + // If we're only looking for IPv6, we need to request the AAAA record so we can get the IP of this authority + q.Type = dns.TypeAAAA + } res, trace, status, _ = r.iterativeLookup(ctx, q, r.randomRootNameServer(), depth+1, ".", trace) } if status == StatusIterTimeout { From 3b5794d034db9aa41c17690a775a615317a48f7f Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Thu, 1 Aug 2024 15:42:23 -0400 Subject: [PATCH 33/59] added prefer ipv4 and ipv6 options --- src/cli/cli.go | 4 ++++ src/cli/config_validation.go | 7 ++++++ src/cli/worker_manager.go | 1 + src/zdns/lookup.go | 11 ++++----- src/zdns/resolver.go | 42 ++++++++++++++++++++++------------ src/zdns/types.go | 24 ++++++++++++++++++++ src/zdns/util.go | 44 +++++++++++++++++++++++++++--------- 7 files changed, 100 insertions(+), 33 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 728a4001..d49b83d0 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -71,6 +71,8 @@ type CLIConf struct { UDPOnly bool IPv4Transport bool IPv6Transport bool + PreferIPv4Iteration bool // Prefer IPv4/A record lookups during iterative resolution + PreferIPv6Iteration bool // Prefer IPv6/AAAA record lookups during iterative resolution RecycleSockets bool LocalAddrSpecified bool LocalAddrs []net.IP @@ -169,6 +171,8 @@ func init() { rootCmd.PersistentFlags().StringVar(&GC.LocalIfaceString, "local-interface", "", "local interface to use") rootCmd.PersistentFlags().BoolVar(&GC.IPv4Transport, "4", true, "utilize IPv4 query transport, must have an IPv4 local address") rootCmd.PersistentFlags().BoolVar(&GC.IPv6Transport, "6", false, "utilize IPv6 query transport, must have an IPv6 local address") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", true, "Prefer IPv4/A record lookups during iterative resolution") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution") rootCmd.PersistentFlags().StringVar(&GC.ConfigFilePath, "conf-file", zdns.DefaultNameServerConfigFile, "config file for DNS servers") rootCmd.PersistentFlags().IntVar(&GC.Timeout, "timeout", 15, "timeout for resolving a individual name, in seconds") rootCmd.PersistentFlags().IntVar(&GC.IterationTimeout, "iteration-timeout", 4, "timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative") diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index ef0cb18b..957ccbc7 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -38,6 +38,13 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.New("both IPv4 and IPv6 transport cannot both be disabled") } + if gc.IPv4Transport && gc.IPv6Transport && !gc.PreferIPv4Iteration && !gc.PreferIPv6Iteration { + return errors.New("both prefer IPv4 and prefer IPv6 iteration cannot both be disabled when using both IPv4 and IPv6 transport") + } + if gc.IPv4Transport && gc.IPv6Transport && gc.PreferIPv4Iteration && gc.PreferIPv6Iteration { + return errors.New("both prefer IPv4 and prefer IPv6 iteration cannot both be enabled when using both IPv4 and IPv6 transport") + } + if err := populateNameServers(gc); err != nil { return errors.Wrap(err, "name servers did not pass validation") } diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index ee1b3382..1b1b7616 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -159,6 +159,7 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config.IPVersionMode = zdns.GetIPVersionMode(gc.IPv4Transport, gc.IPv6Transport) config.TransportMode = zdns.GetTransportMode(gc.UDPOnly, gc.TCPOnly) + config.IterationIPPreference = zdns.GetIterationIPPreference(gc.PreferIPv4Iteration, gc.PreferIPv6Iteration) config.Timeout = time.Second * time.Duration(gc.Timeout) config.IterativeTimeout = time.Second * time.Duration(gc.IterationTimeout) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 7f496734..bbed59a9 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -642,19 +642,16 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, // Short circuit a lookup from the glue // Normally this would be handled by caching, but we want to support following glue // that would normally be cache poison. Because it's "ok" and quite common - res, status := checkGlue(server, *result, r.ipVersionMode) + res, status := checkGlue(server, *result, r.ipVersionMode, r.iterationIPPreference) if status != StatusNoError { // Fall through to normal query var q Question q.Name = server - q.Type = dns.TypeA q.Class = dns.ClassINET - // TODO This logic prefers IPv4 for IPv4OrIPv6 lookups. - // When we address the allNameservers not actually trying allNameservers (#352), we should address this - // Likely, we need to make both A/AAAA lookups and combine the results - if r.ipVersionMode == IPv6Only { - // If we're only looking for IPv6, we need to request the AAAA record so we can get the IP of this authority + if r.ipVersionMode != IPv4Only && r.iterationIPPreference == PreferIPv6 { q.Type = dns.TypeAAAA + } else { + q.Type = dns.TypeA } res, trace, status, _ = r.iterativeLookup(ctx, q, r.randomRootNameServer(), depth+1, ".", trace) } diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 5208146c..2589be55 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -50,6 +50,7 @@ const ( defaultShouldTrace = false defaultDNSSECEnabled = false defaultIPVersionMode = IPv4Only + defaultIterationIPPreference = PreferIPv4 DefaultNameServerConfigFile = "/etc/resolv.conf" defaultLookupAllNameServers = false ) @@ -69,9 +70,10 @@ type ResolverConfig struct { Retries int LogLevel log.Level - TransportMode transportMode - IPVersionMode IPVersionMode - ShouldRecycleSockets bool + TransportMode transportMode + IPVersionMode IPVersionMode + IterationIPPreference IterationIPPreference // preference for IPv4 or IPv6 lookups in iterative queries + ShouldRecycleSockets bool IterativeTimeout time.Duration // applicable to iterative queries only, timeout for a single iteration step Timeout time.Duration // timeout for the resolution of a single name @@ -149,6 +151,13 @@ func (rc *ResolverConfig) PopulateAndValidate() error { rc.IPVersionMode = IPv6Only } + if rc.IterationIPPreference == PreferIPv6 && rc.IPVersionMode == IPv4Only { + return errors.New("cannot prefer IPv6 in iterative queries with IPv4 only mode") + } + if rc.IterationIPPreference == PreferIPv4 && rc.IPVersionMode == IPv6Only { + return errors.New("cannot prefer IPv4 in iterative queries with IPv6 only mode") + } + return nil } @@ -335,11 +344,12 @@ func NewResolverConfig() *ResolverConfig { LocalAddrsV4: []net.IP{}, LocalAddrsV6: []net.IP{}, - TransportMode: defaultTransportMode, - IPVersionMode: defaultIPVersionMode, - ShouldRecycleSockets: defaultShouldRecycleSockets, - LookupAllNameServers: false, - FollowCNAMEs: defaultFollowCNAMEs, + TransportMode: defaultTransportMode, + IPVersionMode: defaultIPVersionMode, + IterationIPPreference: defaultIterationIPPreference, + ShouldRecycleSockets: defaultShouldRecycleSockets, + LookupAllNameServers: false, + FollowCNAMEs: defaultFollowCNAMEs, Retries: defaultRetries, LogLevel: defaultLogVerbosity, @@ -373,9 +383,10 @@ type Resolver struct { retries int logLevel log.Level - transportMode transportMode - ipVersionMode IPVersionMode - shouldRecycleSockets bool + transportMode transportMode + ipVersionMode IPVersionMode + iterationIPPreference IterationIPPreference + shouldRecycleSockets bool iterativeTimeout time.Duration timeout time.Duration // timeout for the network conns @@ -419,10 +430,11 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { logLevel: config.LogLevel, lookupAllNameServers: config.LookupAllNameServers, - transportMode: config.TransportMode, - ipVersionMode: config.IPVersionMode, - shouldRecycleSockets: config.ShouldRecycleSockets, - followCNAMEs: config.FollowCNAMEs, + transportMode: config.TransportMode, + ipVersionMode: config.IPVersionMode, + iterationIPPreference: config.IterationIPPreference, + shouldRecycleSockets: config.ShouldRecycleSockets, + followCNAMEs: config.FollowCNAMEs, timeout: config.Timeout, diff --git a/src/zdns/types.go b/src/zdns/types.go index 6fdb06ff..34fdd7d6 100644 --- a/src/zdns/types.go +++ b/src/zdns/types.go @@ -68,3 +68,27 @@ func (ivm IPVersionMode) IsValid() (bool, string) { } return true, "" } + +type IterationIPPreference int + +const ( + PreferIPv4 IterationIPPreference = iota + PreferIPv6 +) + +func GetIterationIPPreference(preferIPv4, preferIPv6 bool) IterationIPPreference { + if preferIPv4 { + return PreferIPv4 + } else if preferIPv6 { + return PreferIPv6 + } + return PreferIPv4 +} + +func (iip IterationIPPreference) IsValid() (bool, string) { + isValid := iip >= 0 && iip <= 1 + if !isValid { + return false, fmt.Sprintf("invalid iteration ip preference: %d", iip) + } + return true, "" +} diff --git a/src/zdns/util.go b/src/zdns/util.go index fdb7ca7b..9e9035a5 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -16,6 +16,7 @@ package zdns import ( "fmt" + log "github.com/sirupsen/logrus" "net" "strings" @@ -88,20 +89,42 @@ func nextAuthority(name, layer string) (string, error) { return next, nil } -func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode) (SingleQueryResult, Status) { +func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode, ipPreference IterationIPPreference) (SingleQueryResult, Status) { + var ansType string + if ipMode == IPv4Only { + ansType = "A" + } else if ipMode == IPv6Only { + ansType = "AAAA" + } else if ipPreference == PreferIPv4 { + // msut be using either IPv4 or IPv6 + ansType = "A" + } else if ipPreference == PreferIPv6 { + // msut be using either IPv4 or IPv6 + ansType = "AAAA" + } else { + log.Fatal("should never hit this case in check glue: ", ipMode, ipPreference) + } + res, status := checkGlueHelper(server, ansType, result) + if status == StatusNoError || ipMode != IPv4OrIPv6 { + // If we have a valid answer, or we're not looking for both A and AAAA records, return + return res, status + } + // If we're looking for both A and AAAA records, and we didn't find an answer, try the other type + if ansType == "A" { + ansType = "AAAA" + } else { + ansType = "A" + } + return checkGlueHelper(server, ansType, result) +} + +func checkGlueHelper(server, ansType string, result SingleQueryResult) (SingleQueryResult, Status) { for _, additional := range result.Additional { ans, ok := additional.(Answer) if !ok { continue } - if ipMode != IPv6Only && ans.Type == "A" && strings.TrimSuffix(ans.Name, ".") == server { - var retv SingleQueryResult - retv.Authorities = make([]interface{}, 0) - retv.Answers = make([]interface{}, 0, 1) - retv.Additional = make([]interface{}, 0) - retv.Answers = append(retv.Answers, ans) - return retv, StatusNoError - } else if ipMode != IPv4Only && ans.Type == "AAAA" && strings.TrimSuffix(ans.Name, ".") == server { + if ans.Type == ansType && strings.TrimSuffix(ans.Name, ".") == server { var retv SingleQueryResult retv.Authorities = make([]interface{}, 0) retv.Answers = make([]interface{}, 0, 1) @@ -110,8 +133,7 @@ func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode) (S return retv, StatusNoError } } - var r SingleQueryResult - return r, StatusError + return SingleQueryResult{}, StatusError } func makeVerbosePrefix(depth int) string { From 9499bfbd5e2fdbe97705a2f4f941dbdd42e74ba5 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 2 Aug 2024 11:10:19 -0400 Subject: [PATCH 34/59] better handling around the ipv4 preference validation, better UX, more tests --- src/cli/cli.go | 7 ++++--- src/cli/config_validation.go | 7 ------- src/cli/worker_manager.go | 14 +++++++++++++- src/zdns/resolver_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index d49b83d0..31c2889d 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -169,10 +169,11 @@ func init() { rootCmd.PersistentFlags().StringVar(&GC.NameServersString, "name-servers", "", "List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53.") rootCmd.PersistentFlags().StringVar(&GC.LocalAddrString, "local-addr", "", "comma-delimited list of local addresses to use, serve as the source IP for outbound queries") rootCmd.PersistentFlags().StringVar(&GC.LocalIfaceString, "local-interface", "", "local interface to use") - rootCmd.PersistentFlags().BoolVar(&GC.IPv4Transport, "4", true, "utilize IPv4 query transport, must have an IPv4 local address") + rootCmd.PersistentFlags().BoolVar(&GC.IPv4Transport, "4", false, "utilize IPv4 query transport, must have an IPv4 local address") rootCmd.PersistentFlags().BoolVar(&GC.IPv6Transport, "6", false, "utilize IPv6 query transport, must have an IPv6 local address") - rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", true, "Prefer IPv4/A record lookups during iterative resolution") - rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", false, "Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both --4 and --6") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both --4 and --6") + rootCmd.PersistentFlags().StringVar(&GC.ConfigFilePath, "conf-file", zdns.DefaultNameServerConfigFile, "config file for DNS servers") rootCmd.PersistentFlags().IntVar(&GC.Timeout, "timeout", 15, "timeout for resolving a individual name, in seconds") rootCmd.PersistentFlags().IntVar(&GC.IterationTimeout, "iteration-timeout", 4, "timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative") diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 957ccbc7..ef20314d 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -34,13 +34,6 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.New("--local-addr and --local-interface cannot both be specified") } - if !gc.IPv4Transport && !gc.IPv6Transport { - return errors.New("both IPv4 and IPv6 transport cannot both be disabled") - } - - if gc.IPv4Transport && gc.IPv6Transport && !gc.PreferIPv4Iteration && !gc.PreferIPv6Iteration { - return errors.New("both prefer IPv4 and prefer IPv6 iteration cannot both be disabled when using both IPv4 and IPv6 transport") - } if gc.IPv4Transport && gc.IPv6Transport && gc.PreferIPv4Iteration && gc.PreferIPv6Iteration { return errors.New("both prefer IPv4 and prefer IPv6 iteration cannot both be enabled when using both IPv4 and IPv6 transport") } diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 1b1b7616..3b2ff2ce 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -158,8 +158,20 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config := zdns.NewResolverConfig() config.IPVersionMode = zdns.GetIPVersionMode(gc.IPv4Transport, gc.IPv6Transport) + // if we're in IPv4 or IPv6 only mode, set the iteration preference to match + // This is used in extractAuthorities where we need to know whether to request A or AAAA records to continue iteration + if config.IPVersionMode == zdns.IPv4Only { + config.IterationIPPreference = zdns.PreferIPv4 + } else if config.IPVersionMode == zdns.IPv6Only { + config.IterationIPPreference = zdns.PreferIPv6 + } else if config.IPVersionMode == zdns.IPv4OrIPv6 && !gc.PreferIPv4Iteration && !gc.PreferIPv6Iteration { + // need to specify some type of preference, we'll default to IPv4 and inform the user + log.Info("No iteration IP preference specified, defaulting to IPv4 preferred. See --prefer-ipv4-iteration and --prefer-ipv6-iteration for more info") + config.IterationIPPreference = zdns.PreferIPv4 + } else { + config.IterationIPPreference = zdns.GetIterationIPPreference(gc.PreferIPv4Iteration, gc.PreferIPv6Iteration) + } config.TransportMode = zdns.GetTransportMode(gc.UDPOnly, gc.TCPOnly) - config.IterationIPPreference = zdns.GetIterationIPPreference(gc.PreferIPv4Iteration, gc.PreferIPv6Iteration) config.Timeout = time.Second * time.Duration(gc.Timeout) config.IterativeTimeout = time.Second * time.Duration(gc.IterationTimeout) diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index 739c3d19..09003eb0 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -104,4 +104,31 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { require.Equal(t, LoopbackAddrString, rc.LocalAddrsV4[0].String(), "Expected local address to be overwritten with loopback address") require.Equal(t, "127.0.0.1:53", rc.ExternalNameServersV4[0], "Expected nameserver to remain unchanged") }) + + t.Run("Test invalid IPv4/IPv6 configuration", func(t *testing.T) { + rc := &ResolverConfig{ + IPVersionMode: IPv4Only, + IterationIPPreference: PreferIPv6, + } + err := rc.PopulateAndValidate() + require.NotNil(t, err, "Expected error but got nil") + rc = &ResolverConfig{ + IPVersionMode: IPv6Only, + IterationIPPreference: PreferIPv4, + } + err = rc.PopulateAndValidate() + require.NotNil(t, err, "Expected error but got nil") + rc = &ResolverConfig{ + IPVersionMode: IPv4OrIPv6, + IterationIPPreference: PreferIPv4, + } + err = rc.PopulateAndValidate() + require.Nil(t, err, "This is valid") + rc = &ResolverConfig{ + IPVersionMode: IPv4OrIPv6, + IterationIPPreference: PreferIPv6, + } + err = rc.PopulateAndValidate() + require.Nil(t, err, "This is Valid") + }) } From 77afc83a6f1851eb359e67aaed0cf690f2e9c486 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 2 Aug 2024 11:26:22 -0400 Subject: [PATCH 35/59] lint --- src/zdns/util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zdns/util.go b/src/zdns/util.go index 9e9035a5..b788c5bd 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -16,10 +16,11 @@ package zdns import ( "fmt" - log "github.com/sirupsen/logrus" "net" "strings" + log "github.com/sirupsen/logrus" + "github.com/pkg/errors" "github.com/zmap/dns" From 3dabf8d8946d7a6d2370ed9259c4794449c824a3 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 2 Aug 2024 11:44:53 -0400 Subject: [PATCH 36/59] hack to get IPv6 test to pass on hosts that don't support IPv6 --- src/zdns/resolver_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index 09003eb0..e0c483a0 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -114,18 +114,21 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { require.NotNil(t, err, "Expected error but got nil") rc = &ResolverConfig{ IPVersionMode: IPv6Only, + LocalAddrsV6: []net.IP{net.ParseIP("::1")}, IterationIPPreference: PreferIPv4, } err = rc.PopulateAndValidate() require.NotNil(t, err, "Expected error but got nil") rc = &ResolverConfig{ IPVersionMode: IPv4OrIPv6, + LocalAddrsV6: []net.IP{net.ParseIP("::1")}, IterationIPPreference: PreferIPv4, } err = rc.PopulateAndValidate() require.Nil(t, err, "This is valid") rc = &ResolverConfig{ IPVersionMode: IPv4OrIPv6, + LocalAddrsV6: []net.IP{net.ParseIP("::1")}, IterationIPPreference: PreferIPv6, } err = rc.PopulateAndValidate() From e3b4166ada805634764088a05ce4fa545a22ef62 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 2 Aug 2024 11:48:16 -0400 Subject: [PATCH 37/59] a lil more hack --- src/zdns/resolver_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index e0c483a0..cf0a4293 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -121,14 +121,20 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { require.NotNil(t, err, "Expected error but got nil") rc = &ResolverConfig{ IPVersionMode: IPv4OrIPv6, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, LocalAddrsV6: []net.IP{net.ParseIP("::1")}, + ExternalNameServersV4: []string{"127.0.0.1"}, + ExternalNameServersV6: []string{"::1"}, IterationIPPreference: PreferIPv4, } err = rc.PopulateAndValidate() require.Nil(t, err, "This is valid") rc = &ResolverConfig{ IPVersionMode: IPv4OrIPv6, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, LocalAddrsV6: []net.IP{net.ParseIP("::1")}, + ExternalNameServersV4: []string{"127.0.0.1"}, + ExternalNameServersV6: []string{"::1"}, IterationIPPreference: PreferIPv6, } err = rc.PopulateAndValidate() From 644b544e10245f06a0252155e3d0d4192ecd7807 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 2 Aug 2024 11:49:41 -0400 Subject: [PATCH 38/59] added comment to explain hack in unit tests --- src/zdns/resolver_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index cf0a4293..d6401128 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -120,7 +120,10 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { err = rc.PopulateAndValidate() require.NotNil(t, err, "Expected error but got nil") rc = &ResolverConfig{ - IPVersionMode: IPv4OrIPv6, + IPVersionMode: IPv4OrIPv6, + // Hack - we don't technically support loopback nameservers, but if we don't have this + // The PopulateAndValidate function will try to setup an IPv6 connection which will fail on hosts without IPv6 + // This is a hack to get around that LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, LocalAddrsV6: []net.IP{net.ParseIP("::1")}, ExternalNameServersV4: []string{"127.0.0.1"}, From 6cb473d33127db220f795da4b07bd690538d5c52 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 11:59:28 -0400 Subject: [PATCH 39/59] fixed compile issues in non-test code --- src/cli/worker_manager.go | 30 +++++---- src/zdns/resolver.go | 136 +++++++++++++++++++++++--------------- 2 files changed, 98 insertions(+), 68 deletions(-) diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 7a8d6b64..bf3bf96b 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -176,23 +176,25 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config.Timeout = time.Second * time.Duration(gc.Timeout) config.IterativeTimeout = time.Second * time.Duration(gc.IterationTimeout) // copy nameservers to resolver config - // TODO - add IPv6 support here if gc.IterativeResolution && len(gc.NameServers) != 0 { - config.RootNameServers = gc.NameServers + ipv4NSs, ipv6NSs, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) + if err != nil { + log.Fatalf("unable to split IPv4 and IPv6 name-server addresses (%s): %v", gc.NameServers, err) + } + config.RootNameServersV4 = ipv4NSs + config.RootNameServersV6 = ipv6NSs } else if gc.IterativeResolution { - config.RootNameServers = zdns.RootServersV4[:] + config.RootNameServersV4 = zdns.RootServersV4[:] + config.RootNameServersV6 = zdns.RootServersV6[:] } else if !gc.IterativeResolution && len(gc.NameServers) != 0 { - config.ExternalNameServers = gc.NameServers - } - // TODO - this was the old IPv6 code - // ipv4NSes, ipv6NSes, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) - // if err != nil { - // log.Fatalf("unable to split IPv4 and IPv6 addresses (%s): %v", gc.NameServers, err) - // } - // config.ExternalNameServersV4 = ipv4NSes - // // this will be empty if no IPv6 addresses are specified - // config.ExternalNameServersV6 = ipv6NSes - // Else: Resolver will populate the external name servers with either the OS default or the ZDNS default if none exist + ipv4NSs, ipv6NSs, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) + if err != nil { + log.Fatalf("unable to split IPv4 and IPv6 name-server addresses (%s): %v", gc.NameServers, err) + } + config.ExternalNameServersV4 = ipv4NSs + config.ExternalNameServersV6 = ipv6NSs + } + // Else: resolver will populate the external name servers with either the OS default or the ZDNS default if none exist config.LookupAllNameServers = gc.LookupAllNameServers config.FollowCNAMEs = gc.FollowCNAMEs diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 7c39b46b..5f7b154e 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -80,8 +80,8 @@ type ResolverConfig struct { MaxDepth int ExternalNameServersV4 []string // v4 name servers used for external lookups ExternalNameServersV6 []string // v6 name servers used for external lookups - RootNameServersV4 []string // v4 root servers used for iterative lookups - RootNameServersV6 []string // v6 root servers used for iterative lookups + RootNameServersV4 []string // v4 root servers used for iterative lookups + RootNameServersV6 []string // v6 root servers used for iterative lookups LookupAllNameServers bool // perform the lookup via all the nameservers for the domain FollowCNAMEs bool // whether iterative lookups should follow CNAMEs/DNAMEs DNSConfigFilePath string // path to the DNS config file, ex: /etc/resolv.conf @@ -125,7 +125,7 @@ func (rc *ResolverConfig) PopulateAndValidate() error { return fmt.Errorf("invalid external name server: %s", ns) } } - for _, ns := range rc.RootNameServers { + for _, ns := range append(append([]string{}, rc.RootNameServersV4...), rc.RootNameServersV6...) { if _, _, err := net.SplitHostPort(ns); err != nil { return fmt.Errorf("invalid root name server: %s", ns) } @@ -172,8 +172,6 @@ func (rc *ResolverConfig) PopulateAndValidate() error { } func (rc *ResolverConfig) populateResolverConfig() error { - // External Nameservers - if len(rc.ExternalNameServers) == 0 { if err := rc.populateNameServers(); err != nil { return errors.Wrap(err, "could not populate name servers") } @@ -237,22 +235,37 @@ func (rc *ResolverConfig) populateLocalAddrs() error { return nil } +// populateNameServers populates the name servers (external and root) for the resolver. +// Check individual functions for more details. func (rc *ResolverConfig) populateNameServers() error { + if err := rc.populateExternalNameServers(); err != nil { + return errors.Wrap(err, "could not populate external name servers") + } + if err := rc.populateRootNameServers(); err != nil { + return errors.Wrap(err, "could not populate root name servers") + } + return nil +} + +// populateExternalNameServers populates the external name servers for the resolver if they're not set +// Also, validates the nameservers and adds a default port if necessary +// IPv6 note: link-local IPv6 nameservers are ignored +func (rc *ResolverConfig) populateExternalNameServers() error { if len(rc.ExternalNameServersV4) == 0 && len(rc.ExternalNameServersV6) == 0 { // if nameservers aren't set, use OS' default nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) if err != nil { + // if can't retrieve OS defaults, use hard-coded ZDNS defaults nsv4, nsv6 = util.GetDefaultResolvers() - log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(nsv4, nsv6...), ", ")) + log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(append([]string{}, nsv4...), nsv6...), ", ")) } rc.ExternalNameServersV4 = nsv4 rc.ExternalNameServersV6 = nsv6 } - // ensure port is set for all nameservers + // check that the nameservers have a port and append one if necessary portValidatedNSsV4 := make([]string, 0, len(rc.ExternalNameServersV4)) portValidatedNSsV6 := make([]string, 0, len(rc.ExternalNameServersV6)) - // check that the nameservers have a port and append one if necessary for _, ns := range rc.ExternalNameServersV4 { portNS, err := util.AddDefaultPortToDNSServerName(ns) if err != nil { @@ -265,20 +278,7 @@ func (rc *ResolverConfig) populateNameServers() error { if err != nil { return fmt.Errorf("could not parse name server: %s", ns) } - rc.ExternalNameServers = portValidatedNSs - } - if len(rc.RootNameServers) != 0 { - portValidatedNSs := make([]string, 0, len(rc.RootNameServers)) - // check that the nameservers have a port and append one if necessary - for _, ns := range rc.RootNameServers { - portNS, err := util.AddDefaultPortToDNSServerName(ns) - if err != nil { - return fmt.Errorf("could not parse root name server: %s", ns) - } - portValidatedNSs = append(portValidatedNSs, portNS) - } - rc.RootNameServers = portValidatedNSs - + portValidatedNSsV6 = append(portValidatedNSsV6, portNS) } // remove link-local IPv6 nameservers nonLinkLocalIPv6NSs := make([]string, 0, len(portValidatedNSsV6)) @@ -295,7 +295,49 @@ func (rc *ResolverConfig) populateNameServers() error { } rc.ExternalNameServersV4 = portValidatedNSsV4 rc.ExternalNameServersV6 = nonLinkLocalIPv6NSs + return nil +} +// populateRootNameServers populates the root name servers for the resolver if they're not set +// Also, validates the nameservers and adds a default port if necessary +// Link-local IPv6 root nameservers are not allowed +func (rc *ResolverConfig) populateRootNameServers() error { + if len(rc.RootNameServersV4) == 0 && len(rc.RootNameServersV6) == 0 { + // if nameservers aren't set, use the set of 13 root name servers + rc.RootNameServersV4 = RootServersV4[:] + rc.RootNameServersV6 = RootServersV4[:] + return nil + } + // check that the nameservers have a port and append one if necessary + portValidatedNSsV4 := make([]string, 0, len(rc.RootNameServersV4)) + portValidatedNSsV6 := make([]string, 0, len(rc.RootNameServersV6)) + for _, ns := range rc.RootNameServersV4 { + portNS, err := util.AddDefaultPortToDNSServerName(ns) + if err != nil { + return fmt.Errorf("could not parse name server: %s", ns) + } + portValidatedNSsV4 = append(portValidatedNSsV4, portNS) + } + for _, ns := range rc.RootNameServersV6 { + portNS, err := util.AddDefaultPortToDNSServerName(ns) + if err != nil { + return fmt.Errorf("could not parse name server: %s", ns) + } + portValidatedNSsV6 = append(portValidatedNSsV6, portNS) + } + // all root nameservers should be non-link-local + // appending like this so we don't change the underlying array + for _, ns := range portValidatedNSsV6 { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) + } + if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return fmt.Errorf("link-local IPv6 root nameservers are not supported: %s", ns) + } + } + rc.RootNameServersV4 = portValidatedNSsV4 + rc.RootNameServersV6 = portValidatedNSsV6 return nil } @@ -304,7 +346,7 @@ func (rc *ResolverConfig) populateNameServers() error { // - if using a loopback local address, all local addresses are loopback and vice-versa // - either all nameservers AND all local addresses are loopback, or none are func (rc *ResolverConfig) validateLoopbackConsistency() error { - allNameServers := append(rc.ExternalNameServersV4, rc.ExternalNameServersV6...) + allNameServers := append(append(append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...), rc.RootNameServersV4...), rc.ExternalNameServersV6...) // check if all nameservers are loopback or non-loopback allNameserversLoopback := true noneNameserversLoopback := true @@ -319,22 +361,9 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { allNameserversLoopback = false } } - for _, ns := range rc.RootNameServers { - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) - } - if ip.IsLoopback() { - noneNameserversLoopback = false - } else { - allNameserversLoopback = false - } - } - // TODO this needs to be fixed for the new two root nameservers slices. - // TODO keep in mind the whole append changing underlying array trick (append([]slice{}, realSlice)) loopbackNameserverMismatch := allNameserversLoopback == noneNameserversLoopback - if len(rc.ExternalNameServers) > 0 && loopbackNameserverMismatch { - return fmt.Errorf("cannot mix loopback and non-loopback nameservers: %v", rc.ExternalNameServers) + if len(allNameServers) > 0 && loopbackNameserverMismatch { + return errors.New("cannot mix loopback and non-loopback nameservers") } allLocalAddrsLoopback := true @@ -367,9 +396,9 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } func (rc *ResolverConfig) PrintInfo() { - log.Infof("using local addresses: %v", rc.LocalAddrs) - log.Infof("for non-iterative lookups, using external nameservers: %s", strings.Join(rc.ExternalNameServers, ", ")) - log.Infof("for iterative lookups, using nameservers: %s", strings.Join(rc.RootNameServers, ", ")) + log.Infof("using local addresses: %v", append(append([]net.IP{}, rc.LocalAddrsV4...), rc.LocalAddrsV6...)) + log.Infof("for non-iterative lookups, using external nameservers: %s", strings.Join(append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...), ", ")) + log.Infof("for iterative lookups, using nameservers: %s", strings.Join(append(append([]string{}, rc.RootNameServersV4...), rc.RootNameServersV6...), ", ")) } // NewResolverConfig creates a new ResolverConfig with default values. @@ -524,25 +553,24 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { // deep copy external name servers from config to resolver r.iterativeTimeout = config.IterativeTimeout r.maxDepth = config.MaxDepth - // TODO Fix this - // use the set of 13 root name servers - if r.ipVersionMode != IPv6Only { + if r.ipVersionMode != IPv6Only && len(config.RootNameServersV4) == 0 { // add IPv4 root servers r.rootNameServers = append(r.rootNameServers, RootServersV4...) + } else if r.ipVersionMode != IPv6Only { + // deep copy root name servers from config to resolver + elemsCopied := copy(r.rootNameServers, config.RootNameServersV4) + if elemsCopied != len(config.RootNameServersV4) { + log.Fatal("failed to copy entire v4 root name servers list from config") + } } - if r.ipVersionMode != IPv4Only { + if r.ipVersionMode != IPv4Only && len(config.RootNameServersV6) == 0 { // add IPv6 root servers r.rootNameServers = append(r.rootNameServers, RootServersV6...) - } - // TODO And this - if len(config.RootNameServers) == 0 { - r.rootNameServers = RootServersV4[:] - } else { - r.rootNameServers = make([]string, len(config.RootNameServers)) + } else if r.ipVersionMode != IPv4Only { // deep copy root name servers from config to resolver - elemsCopied = copy(r.rootNameServers, config.RootNameServers) - if elemsCopied != len(config.RootNameServers) { - log.Fatal("failed to copy entire root name servers list from config") + elemsCopied := copy(r.rootNameServers, config.RootNameServersV6) + if elemsCopied != len(config.RootNameServersV6) { + log.Fatal("failed to copy entire v6 root name servers list from config") } } return r, nil From 28377db436fd6a3b1816a035b4c20e7f3e049aaa Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 13:11:06 -0400 Subject: [PATCH 40/59] tests passing --- src/cli/config_validation.go | 12 ++++++++++++ src/zdns/resolver.go | 19 ++++++------------- src/zdns/resolver_test.go | 19 +++++-------------- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index b4dcd965..0e065c4f 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -180,6 +180,18 @@ func populateNameServers(gc *CLIConf) error { } else { ns = strings.Split(gc.NameServersString, ",") } + // in --iterative mode, we don't allow loopback nameservers + if gc.IterativeResolution { + for _, ns := range ns { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return err + } + if ip.IsLoopback() { + return fmt.Errorf("loopback nameservers not allowed in iterative mode: %s", ns) + } + } + } gc.NameServers = ns } return nil diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 5f7b154e..9257cf49 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -305,7 +305,7 @@ func (rc *ResolverConfig) populateRootNameServers() error { if len(rc.RootNameServersV4) == 0 && len(rc.RootNameServersV6) == 0 { // if nameservers aren't set, use the set of 13 root name servers rc.RootNameServersV4 = RootServersV4[:] - rc.RootNameServersV6 = RootServersV4[:] + rc.RootNameServersV6 = RootServersV6[:] return nil } // check that the nameservers have a port and append one if necessary @@ -346,8 +346,8 @@ func (rc *ResolverConfig) populateRootNameServers() error { // - if using a loopback local address, all local addresses are loopback and vice-versa // - either all nameservers AND all local addresses are loopback, or none are func (rc *ResolverConfig) validateLoopbackConsistency() error { - allNameServers := append(append(append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...), rc.RootNameServersV4...), rc.ExternalNameServersV6...) - // check if all nameservers are loopback or non-loopback + allNameServers := append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...) + // check if external nameservers are loopback or non-loopback allNameserversLoopback := true noneNameserversLoopback := true for _, ns := range allNameServers { @@ -553,25 +553,18 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { // deep copy external name servers from config to resolver r.iterativeTimeout = config.IterativeTimeout r.maxDepth = config.MaxDepth + r.rootNameServers = make([]string, 0, len(config.RootNameServersV4)+len(config.RootNameServersV6)) if r.ipVersionMode != IPv6Only && len(config.RootNameServersV4) == 0 { // add IPv4 root servers r.rootNameServers = append(r.rootNameServers, RootServersV4...) } else if r.ipVersionMode != IPv6Only { - // deep copy root name servers from config to resolver - elemsCopied := copy(r.rootNameServers, config.RootNameServersV4) - if elemsCopied != len(config.RootNameServersV4) { - log.Fatal("failed to copy entire v4 root name servers list from config") - } + r.rootNameServers = append(r.rootNameServers, config.RootNameServersV4...) } if r.ipVersionMode != IPv4Only && len(config.RootNameServersV6) == 0 { // add IPv6 root servers r.rootNameServers = append(r.rootNameServers, RootServersV6...) } else if r.ipVersionMode != IPv4Only { - // deep copy root name servers from config to resolver - elemsCopied := copy(r.rootNameServers, config.RootNameServersV6) - if elemsCopied != len(config.RootNameServersV6) { - log.Fatal("failed to copy entire v6 root name servers list from config") - } + r.rootNameServers = append(r.rootNameServers, config.RootNameServersV6...) } return r, nil } diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index a5aa2462..c0648724 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -145,18 +145,9 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { require.Nil(t, err, "This is Valid") }) - t.Run("Valid loobpack root nameservers and valid non-loopback external nameservers is not allowed", func(t *testing.T) { - rc := &ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1"}, - RootNameServers: []string{"127.0.0.1"}, - } - err := rc.PopulateAndValidate() - require.NotNil(t, err, "cannot mix loopback root nameservers with non-loopback external nameservers") - }) - t.Run("Invalid Root NS", func(t *testing.T) { rc := &ResolverConfig{ - RootNameServers: []string{"1.2.3"}, + RootNameServersV4: []string{"1.2.3"}, } err := rc.PopulateAndValidate() require.NotNil(t, err, "Expected error for invalid root nameserver") @@ -164,16 +155,16 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { t.Run("Validate Port-appended Root NS", func(t *testing.T) { rc := &ResolverConfig{ - RootNameServers: []string{"1.2.3.4:49"}, + RootNameServersV4: []string{"1.2.3.4:49"}, } err := rc.PopulateAndValidate() require.Nil(t, err, "Expected no error for valid root nameserver") - require.Equal(t, "49", strings.Split(rc.RootNameServers[0], ":")[1], "Expected port to be unchanged") + require.Equal(t, "49", strings.Split(rc.RootNameServersV4[0], ":")[1], "Expected port to be unchanged") rc = &ResolverConfig{ - RootNameServers: []string{"1.2.3.4"}, + RootNameServersV4: []string{"1.2.3.4"}, } err = rc.PopulateAndValidate() require.Nil(t, err, "Expected no error for valid root nameserver") - require.Equal(t, "53", strings.Split(rc.RootNameServers[0], ":")[1], "Expected port to be populated") + require.Equal(t, "53", strings.Split(rc.RootNameServersV4[0], ":")[1], "Expected port to be populated") }) } From a082f1420b7a466163171f4c915cd9d29384ec07 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 13:11:19 -0400 Subject: [PATCH 41/59] lint --- src/zdns/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zdns/util.go b/src/zdns/util.go index 92d2f6d5..ab08b297 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -57,7 +57,6 @@ func nameIsBeneath(name, layer string) (bool, string) { return false, "" } - func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode, ipPreference IterationIPPreference) (SingleQueryResult, Status) { var ansType string if ipMode == IPv4Only { From a843f6aa0d0a200721b789acb53892383fe13844 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 13:25:54 -0400 Subject: [PATCH 42/59] remove tests with loopback IPv6, we can't have IPv6 loopback NSs --- src/zdns/resolver.go | 19 +++++++++++++++---- src/zdns/resolver_test.go | 39 --------------------------------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 9257cf49..eaa01cbd 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -346,11 +346,10 @@ func (rc *ResolverConfig) populateRootNameServers() error { // - if using a loopback local address, all local addresses are loopback and vice-versa // - either all nameservers AND all local addresses are loopback, or none are func (rc *ResolverConfig) validateLoopbackConsistency() error { - allNameServers := append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...) // check if external nameservers are loopback or non-loopback allNameserversLoopback := true noneNameserversLoopback := true - for _, ns := range allNameServers { + for _, ns := range rc.ExternalNameServersV4 { ip, _, err := util.SplitHostPort(ns) if err != nil { return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) @@ -362,10 +361,22 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } } loopbackNameserverMismatch := allNameserversLoopback == noneNameserversLoopback - if len(allNameServers) > 0 && loopbackNameserverMismatch { + if len(rc.ExternalNameServersV4) > 0 && loopbackNameserverMismatch { return errors.New("cannot mix loopback and non-loopback nameservers") } + // Loopback IPv6 addresses are not allowed + allIPv6Nameservers := append(append([]string{}, rc.ExternalNameServersV6...), rc.RootNameServersV6...) + for _, ns := range allIPv6Nameservers { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) + } + if ip.IsLoopback() { + return fmt.Errorf("loopback IPv6 nameservers are not supported: %s", ns) + } + } + allLocalAddrsLoopback := true noneLocalAddrsLoopback := true // we don't want to change the underlying slice with append, so we create a new slice @@ -385,7 +396,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { // Both nameservers and local addresses are completely loopback or non-loopback // if using loopback nameservers, override local addresses to be loopback and warn user if allNameserversLoopback && noneLocalAddrsLoopback { - log.Warnf("nameservers (%s) are loopback, setting local address to loopback (%s) to match", allNameServers, LoopbackAddrString) + log.Warnf("nameservers (%s) are loopback, setting local address to loopback (%s) to match", rc.ExternalNameServersV4, LoopbackAddrString) rc.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} // we ignore link-local local addresses, so nothing to be done for IPv6 } else if noneNameserversLoopback && allLocalAddrsLoopback { diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index c0648724..c65bb4f2 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -106,45 +106,6 @@ func TestResolverConfig_PopulateAndValidate(t *testing.T) { require.Equal(t, "127.0.0.1:53", rc.ExternalNameServersV4[0], "Expected nameserver to remain unchanged") }) - t.Run("Test invalid IPv4/IPv6 configuration", func(t *testing.T) { - rc := &ResolverConfig{ - IPVersionMode: IPv4Only, - IterationIPPreference: PreferIPv6, - } - err := rc.PopulateAndValidate() - require.NotNil(t, err, "Expected error but got nil") - rc = &ResolverConfig{ - IPVersionMode: IPv6Only, - LocalAddrsV6: []net.IP{net.ParseIP("::1")}, - IterationIPPreference: PreferIPv4, - } - err = rc.PopulateAndValidate() - require.NotNil(t, err, "Expected error but got nil") - rc = &ResolverConfig{ - IPVersionMode: IPv4OrIPv6, - // Hack - we don't technically support loopback nameservers, but if we don't have this - // The PopulateAndValidate function will try to setup an IPv6 connection which will fail on hosts without IPv6 - // This is a hack to get around that - LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, - LocalAddrsV6: []net.IP{net.ParseIP("::1")}, - ExternalNameServersV4: []string{"127.0.0.1"}, - ExternalNameServersV6: []string{"::1"}, - IterationIPPreference: PreferIPv4, - } - err = rc.PopulateAndValidate() - require.Nil(t, err, "This is valid") - rc = &ResolverConfig{ - IPVersionMode: IPv4OrIPv6, - LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, - LocalAddrsV6: []net.IP{net.ParseIP("::1")}, - ExternalNameServersV4: []string{"127.0.0.1"}, - ExternalNameServersV6: []string{"::1"}, - IterationIPPreference: PreferIPv6, - } - err = rc.PopulateAndValidate() - require.Nil(t, err, "This is Valid") - }) - t.Run("Invalid Root NS", func(t *testing.T) { rc := &ResolverConfig{ RootNameServersV4: []string{"1.2.3"}, From 5540069fbf07bd276922f759e6f0b852dda3b85f Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 13:29:18 -0400 Subject: [PATCH 43/59] only add name servers if IP mode supports it --- src/zdns/resolver.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index eaa01cbd..b773d85e 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -293,8 +293,12 @@ func (rc *ResolverConfig) populateExternalNameServers() error { } nonLinkLocalIPv6NSs = append(nonLinkLocalIPv6NSs, ns) } - rc.ExternalNameServersV4 = portValidatedNSsV4 - rc.ExternalNameServersV6 = nonLinkLocalIPv6NSs + if rc.IPVersionMode != IPv4Only { + rc.ExternalNameServersV6 = nonLinkLocalIPv6NSs + } + if rc.IPVersionMode != IPv6Only { + rc.ExternalNameServersV4 = portValidatedNSsV4 + } return nil } @@ -336,8 +340,12 @@ func (rc *ResolverConfig) populateRootNameServers() error { return fmt.Errorf("link-local IPv6 root nameservers are not supported: %s", ns) } } - rc.RootNameServersV4 = portValidatedNSsV4 - rc.RootNameServersV6 = portValidatedNSsV6 + if rc.IPVersionMode != IPv4Only { + rc.RootNameServersV6 = portValidatedNSsV6 + } + if rc.IPVersionMode != IPv6Only { + rc.RootNameServersV4 = portValidatedNSsV4 + } return nil } From 2f10f2dbb092d284199bc3685f87f2780e2d86a3 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 15:51:58 -0400 Subject: [PATCH 44/59] cleanup and fixed some bugs from merge --- src/cli/config_validation.go | 58 ------------------------------------ src/internal/util/util.go | 6 ---- src/zdns/conf.go | 14 +++++++++ src/zdns/resolver.go | 44 ++++++++++++++++++++------- 4 files changed, 47 insertions(+), 75 deletions(-) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 0e065c4f..beeb56ce 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -23,9 +23,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/zmap/dns" - - "github.com/zmap/zdns/src/internal/util" - "github.com/zmap/zdns/src/zdns" ) func populateNetworkingConfig(gc *CLIConf) error { @@ -81,52 +78,9 @@ func populateNetworkingConfig(gc *CLIConf) error { log.Info("using local interface: ", gc.LocalIfaceString) } - // If we're in iterative mode, we always start the DNS resolution iterative process at the root DNS servers. - // However, the ZDNS resolver library we'll create doesn't know that all queries will be iterative, it's designed to be able to do - // both iterative queries and use a recursive resolver with the same config. While usually fine, there's an edge case here - // if it is the case that we're only doing iterative queries AND the OS' configured NS's are loopback, ZDNS library - // will set the local address to a loopback address so the NS's are reachable. - // Unfortunately, this will cause the iterative queries to fail, as the root servers are not reachable from the loopback address. - // - // To prevent this, we'll check if we're in iterative mode, the user hasn't passed in the local addr/nameservers directly to ZDNS, - // and the OS' configured NS's are loopback. If so, we'll set the nameservers to be our default non-loopback recursive resolvers. - // This prevents the edge case described above and has no effect on iterative queries since we just use the root nameservers. - if gc.IterativeResolution && !gc.LocalAddrSpecified && areOSNameserversLoopback(gc) && len(gc.NameServersString) == 0 { - log.Debug("OS external resolution nameservers are loopback and iterative mode is enabled. " + - "Using default non-loopback nameservers to prevent resolution failure edge case") - ipv4NS, ipv6NS := util.GetDefaultResolvers() - // we don't want to change the underlying slice with append, so we create a new slice - gc.NameServers = append(append([]string{}, ipv4NS...), ipv6NS...) - } - return nil } -// areOSNameserversLoopback returns true if the OS' configured nameservers (in /etc/resolv.conf by default) are loopback addresses -func areOSNameserversLoopback(gc *CLIConf) bool { - nsesIPv4, nsesIPv6, err := zdns.GetDNSServers(gc.ConfigFilePath) - if err != nil { - log.Fatalf("Error getting OS nameservers: %s", err.Error()) - } - // we don't want to change the underlying slice with append, so we create a new slice - for _, ns := range append(append([]string{}, nsesIPv4...), nsesIPv6...) { - ipString, _, err := net.SplitHostPort(ns) - if err != nil { - // might be missing a port - ipString = ns - } - ip := net.ParseIP(ipString) - if ip == nil { - log.Fatalf("Error parsing OS nameserver IP: %s", ns) - } - if ip.IsLoopback() { - return true - } - - } - return false -} - func validateClientSubnetString(gc *CLIConf) error { if gc.ClientSubnetString != "" { parts := strings.Split(gc.ClientSubnetString, "/") @@ -180,18 +134,6 @@ func populateNameServers(gc *CLIConf) error { } else { ns = strings.Split(gc.NameServersString, ",") } - // in --iterative mode, we don't allow loopback nameservers - if gc.IterativeResolution { - for _, ns := range ns { - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return err - } - if ip.IsLoopback() { - return fmt.Errorf("loopback nameservers not allowed in iterative mode: %s", ns) - } - } - } gc.NameServers = ns } return nil diff --git a/src/internal/util/util.go b/src/internal/util/util.go index 9f34b63a..ea92a7f1 100644 --- a/src/internal/util/util.go +++ b/src/internal/util/util.go @@ -131,12 +131,6 @@ func BindFlags(cmd *cobra.Command, v *viper.Viper, envPrefix string) { }) } -// GetDefaultResolvers returns a slice of default DNS resolvers to be used when no system resolvers could be discovered. -// Returns IPv4 and IPv6 resolvers. -func GetDefaultResolvers() ([]string, []string) { - return []string{"8.8.8.8:53", "8.8.4.4:53", "1.1.1.1:53", "1.0.0.1:53"}, []string{"2001:4860:4860::8888:53", "2001:4860:4860::8844:53", "2606:4700:4700::1111:53", "2606:4700:4700::1001:53"} -} - // IsStringValidDomainName checks if the given string is a valid domain name using regex func IsStringValidDomainName(domain string) bool { var domainRegex = regexp.MustCompile(`^(?i)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*\.[a-z]{2,}$`) diff --git a/src/zdns/conf.go b/src/zdns/conf.go index cc77729d..c939bcb3 100644 --- a/src/zdns/conf.go +++ b/src/zdns/conf.go @@ -83,3 +83,17 @@ var RootServersV6 = []string{ "[2001:500:9f::42]:53", // L "[2001:dc3::35]:53", // M } + +var DefaultExternalResolversV4 = []string{ + "8.8.8.8:53", + "8.8.4.4:53", + "1.1.1.1:53", + "1.0.0.1:53", +} + +var DefaultExternalResolversV6 = []string{ + "[2001:4860:4860::8888]:53", + "[2001:4860:4860::8844]:53", + "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1001]:53", +} diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index b773d85e..04a73186 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -251,15 +251,25 @@ func (rc *ResolverConfig) populateNameServers() error { // Also, validates the nameservers and adds a default port if necessary // IPv6 note: link-local IPv6 nameservers are ignored func (rc *ResolverConfig) populateExternalNameServers() error { - if len(rc.ExternalNameServersV4) == 0 && len(rc.ExternalNameServersV6) == 0 { - // if nameservers aren't set, use OS' default - nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) - if err != nil { - // if can't retrieve OS defaults, use hard-coded ZDNS defaults - nsv4, nsv6 = util.GetDefaultResolvers() - log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(append([]string{}, nsv4...), nsv6...), ", ")) + nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) + if err != nil { + // if can't retrieve OS defaults, use hard-coded ZDNS defaults + nsv4, nsv6 = DefaultExternalResolversV4, DefaultExternalResolversV6 + log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(append([]string{}, nsv4...), nsv6...), ", ")) + } else { + if rc.IPVersionMode != IPv4Only && len(nsv6) == 0 { + log.Fatal("no IPv6 nameservers found in OS configuration and IPv6 mode is enabled, please specify IPv6 nameservers") + } + if rc.IPVersionMode != IPv6Only && len(nsv4) == 0 { + log.Fatal("no IPv4 nameservers found in OS configuration and IPv4 mode is enabled, please specify IPv4 nameservers") } + } + if rc.IPVersionMode != IPv6Only && len(rc.ExternalNameServersV4) == 0 { + // if IPv4 nameservers aren't set, use OS' default rc.ExternalNameServersV4 = nsv4 + } + if rc.IPVersionMode != IPv4Only && len(rc.ExternalNameServersV6) == 0 { + // if IPv6 nameservers aren't set, use OS' default rc.ExternalNameServersV6 = nsv6 } @@ -374,14 +384,26 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } // Loopback IPv6 addresses are not allowed - allIPv6Nameservers := append(append([]string{}, rc.ExternalNameServersV6...), rc.RootNameServersV6...) - for _, ns := range allIPv6Nameservers { + for _, ns := range rc.ExternalNameServersV6 { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) + } + if ip.IsLoopback() { + rc.ExternalNameServersV6 = DefaultExternalResolversV6 + log.Warnf("loopback external IPv6 nameservers are not supported: %s, using ZDNS defaults: %v", ns, rc.ExternalNameServersV6) + break + } + } + for _, ns := range rc.RootNameServersV6 { ip, _, err := util.SplitHostPort(ns) if err != nil { return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) } if ip.IsLoopback() { - return fmt.Errorf("loopback IPv6 nameservers are not supported: %s", ns) + rc.RootNameServersV6 = RootServersV6 + log.Warnf("loopback root IPv6 nameservers are not supported: %s, using ZDNS defaults: %v", ns, rc.RootNameServersV6) + break } } @@ -403,7 +425,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { // Both nameservers and local addresses are completely loopback or non-loopback // if using loopback nameservers, override local addresses to be loopback and warn user - if allNameserversLoopback && noneLocalAddrsLoopback { + if allNameserversLoopback && noneLocalAddrsLoopback && rc.IPVersionMode != IPv6Only { log.Warnf("nameservers (%s) are loopback, setting local address to loopback (%s) to match", rc.ExternalNameServersV4, LoopbackAddrString) rc.LocalAddrsV4 = []net.IP{net.ParseIP(LoopbackAddrString)} // we ignore link-local local addresses, so nothing to be done for IPv6 From f0c53d1708775e814d25eb6cb1d874aab3dad5a3 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 17:15:57 -0400 Subject: [PATCH 45/59] loopback cli hack and added more string sanitization on listing --name-servers --- src/cli/config_validation.go | 13 +++++++++---- src/cli/worker_manager.go | 18 ++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index beeb56ce..aa67c7b0 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -120,7 +120,7 @@ func populateNameServers(gc *CLIConf) error { if gc.NameServerMode { log.Fatal("name servers cannot be specified on command line in --name-server-mode") } - var ns []string + var nses []string if (gc.NameServersString)[0] == '@' { filepath := (gc.NameServersString)[1:] f, err := os.ReadFile(filepath) @@ -130,11 +130,16 @@ func populateNameServers(gc *CLIConf) error { if len(f) == 0 { log.Fatalf("Empty file (%s)", filepath) } - ns = strings.Split(strings.Trim(string(f), "\n"), "\n") + nses = strings.Split(strings.Trim(string(f), "\n"), "\n") } else { - ns = strings.Split(gc.NameServersString, ",") + nses = strings.Split(gc.NameServersString, ",") + trimmedNSes := make([]string, 0, len(nses)) + for _, ns := range nses { + trimmedNSes = append(trimmedNSes, strings.TrimSpace(ns)) + } + nses = trimmedNSes } - gc.NameServers = ns + gc.NameServers = nses } return nil } diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index bf3bf96b..b6e91c67 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -176,23 +176,25 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config.Timeout = time.Second * time.Duration(gc.Timeout) config.IterativeTimeout = time.Second * time.Duration(gc.IterationTimeout) // copy nameservers to resolver config - if gc.IterativeResolution && len(gc.NameServers) != 0 { + if len(gc.NameServers) != 0 { ipv4NSs, ipv6NSs, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) if err != nil { log.Fatalf("unable to split IPv4 and IPv6 name-server addresses (%s): %v", gc.NameServers, err) } + // While this is a bit of a hack to set both the root and external name servers to the same values, the CLI + // can only be used in either recursive or iterative mode. If we don't do this and leave one or the other empty, + // the resolver will attempt to auto-populate with the OS/ZDNS defaults. If these defaults and the user-provided + // values have a loopback mismatch (some are loopback, others aren't), this causes issues. + // By setting them both here, we prevent that auto-populate. config.RootNameServersV4 = ipv4NSs config.RootNameServersV6 = ipv6NSs + config.ExternalNameServersV4 = ipv4NSs + config.ExternalNameServersV6 = ipv6NSs } else if gc.IterativeResolution { config.RootNameServersV4 = zdns.RootServersV4[:] config.RootNameServersV6 = zdns.RootServersV6[:] - } else if !gc.IterativeResolution && len(gc.NameServers) != 0 { - ipv4NSs, ipv6NSs, err := util.SplitIPv4AndIPv6Addrs(gc.NameServers) - if err != nil { - log.Fatalf("unable to split IPv4 and IPv6 name-server addresses (%s): %v", gc.NameServers, err) - } - config.ExternalNameServersV4 = ipv4NSs - config.ExternalNameServersV6 = ipv6NSs + config.ExternalNameServersV4 = zdns.RootServersV4[:] + config.ExternalNameServersV6 = zdns.RootServersV6[:] } // Else: resolver will populate the external name servers with either the OS default or the ZDNS default if none exist config.LookupAllNameServers = gc.LookupAllNameServers From a65c9752eda2aa4bed67d7cb3dd0f982cc19ea62 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 17:27:44 -0400 Subject: [PATCH 46/59] don't overwrite cli provided external NS' --- src/zdns/resolver.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index 04a73186..e2c4444f 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -251,19 +251,23 @@ func (rc *ResolverConfig) populateNameServers() error { // Also, validates the nameservers and adds a default port if necessary // IPv6 note: link-local IPv6 nameservers are ignored func (rc *ResolverConfig) populateExternalNameServers() error { - nsv4, nsv6, err := GetDNSServers(rc.DNSConfigFilePath) - if err != nil { - // if can't retrieve OS defaults, use hard-coded ZDNS defaults - nsv4, nsv6 = DefaultExternalResolversV4, DefaultExternalResolversV6 - log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(append([]string{}, nsv4...), nsv6...), ", ")) - } else { - if rc.IPVersionMode != IPv4Only && len(nsv6) == 0 { - log.Fatal("no IPv6 nameservers found in OS configuration and IPv6 mode is enabled, please specify IPv6 nameservers") - } - if rc.IPVersionMode != IPv6Only && len(nsv4) == 0 { - log.Fatal("no IPv4 nameservers found in OS configuration and IPv4 mode is enabled, please specify IPv4 nameservers") + var nsv4 []string + var nsv6 []string + var err error + if len(rc.ExternalNameServersV4) == 0 && len(rc.ExternalNameServersV6) == 0 { + nsv4, nsv6, err = GetDNSServers(rc.DNSConfigFilePath) + if err != nil { + // if can't retrieve OS defaults, use hard-coded ZDNS defaults + nsv4, nsv6 = DefaultExternalResolversV4, DefaultExternalResolversV6 + log.Warnf("Unable to parse resolvers file (%v). Using ZDNS defaults: %s", err, strings.Join(append(append([]string{}, nsv4...), nsv6...), ", ")) } } + if rc.IPVersionMode != IPv4Only && len(nsv6) == 0 { + log.Fatal("no IPv6 nameservers found in OS configuration and IPv6 mode is enabled, please specify IPv6 nameservers") + } + if rc.IPVersionMode != IPv6Only && len(nsv4) == 0 { + log.Fatal("no IPv4 nameservers found in OS configuration and IPv4 mode is enabled, please specify IPv4 nameservers") + } if rc.IPVersionMode != IPv6Only && len(rc.ExternalNameServersV4) == 0 { // if IPv4 nameservers aren't set, use OS' default rc.ExternalNameServersV4 = nsv4 From a805bfaee4614e28d176774decc63cd610319b93 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 7 Aug 2024 17:29:24 -0400 Subject: [PATCH 47/59] don't overwrite cli provided external NS bug --- src/zdns/resolver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index e2c4444f..6d308700 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -251,10 +251,10 @@ func (rc *ResolverConfig) populateNameServers() error { // Also, validates the nameservers and adds a default port if necessary // IPv6 note: link-local IPv6 nameservers are ignored func (rc *ResolverConfig) populateExternalNameServers() error { - var nsv4 []string - var nsv6 []string + nsv4 := rc.ExternalNameServersV4 + nsv6 := rc.ExternalNameServersV6 var err error - if len(rc.ExternalNameServersV4) == 0 && len(rc.ExternalNameServersV6) == 0 { + if len(nsv4) == 0 && len(nsv6) == 0 { nsv4, nsv6, err = GetDNSServers(rc.DNSConfigFilePath) if err != nil { // if can't retrieve OS defaults, use hard-coded ZDNS defaults From d38e5feb696e2309858a7b6ef5f5e94b5ac13ddc Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Fri, 9 Aug 2024 14:24:47 -0400 Subject: [PATCH 48/59] compiling --- src/cli/worker_manager.go | 134 +++++++++++++++++++---------------- src/internal/util/util.go | 5 -- src/zdns/resolver.go | 143 ++++++++++++++++++++------------------ 3 files changed, 152 insertions(+), 130 deletions(-) diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 876a47e6..8e4a373e 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -39,7 +39,9 @@ import ( ) const ( - googleDNSResolverAddr = "8.8.8.8:53" + googleDNSResolverAddr = "8.8.8.8:53" + googleDNSResolverAddrV6 = "[2001:4860:4860::8888]:53" + loopbackIPv4Addr = "127.0.0.1" ) type routineMetadata struct { @@ -203,18 +205,21 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { if gc.BlacklistFilePath != "" { config.Blacklist = blacklist.New() - if err = config.Blacklist.ParseFromFile(gc.BlacklistFilePath); err != nil { + if err := config.Blacklist.ParseFromFile(gc.BlacklistFilePath); err != nil { log.Fatal("unable to parse blacklist file: ", err) } } // This must occur after setting the DNSConfigFilePath above, so that ZDNS knows where to fetch the DNS Config - config, err = populateNameServers(gc, config) + config, err := populateNameServers(gc, config) if err != nil { log.Fatal("could not populate name servers: ", err) } // User/OS defaults could contain duplicates, remove - config.ExternalNameServers = util.RemoveDuplicates(config.ExternalNameServers) - config.RootNameServers = util.RemoveDuplicates(config.RootNameServers) + config.ExternalNameServersV4 = util.RemoveDuplicates(config.ExternalNameServersV4) + config.RootNameServersV4 = util.RemoveDuplicates(config.RootNameServersV4) + config.ExternalNameServersV6 = util.RemoveDuplicates(config.ExternalNameServersV6) + config.RootNameServersV6 = util.RemoveDuplicates(config.RootNameServersV6) + config, err = populateLocalAddresses(gc, config) if err != nil { log.Fatal("could not populate local addresses: ", err) @@ -231,6 +236,7 @@ func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Resolv // Additionally, both Root and External nameservers must be populated, since the Resolver doesn't know we'll only // be performing either iterative or recursive lookups, not both. + // IPv4 Name Servers/Local Address only needs to be populated if we're doing IPv4 lookups, same for IPv6 if len(gc.NameServers) != 0 { // User provided name servers, use them. // Check that the nameservers have a port and append one if necessary @@ -239,44 +245,47 @@ func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Resolv for _, ns := range gc.NameServers { portNS, err := util.AddDefaultPortToDNSServerName(ns) if err != nil { - // TODO Update error msg when we add IPv6 - return nil, fmt.Errorf("could not parse name server: %s. Correct IPv4 format: 1.1.1.1:53", ns) + return nil, fmt.Errorf("could not parse name server: %s. Correct IPv4 format: 1.1.1.1:53 or IPv6 format: [::1]:53", ns) } portValidatedNSs = append(portValidatedNSs, portNS) } - config.ExternalNameServers = portValidatedNSs - config.RootNameServers = portValidatedNSs + v4NameServers, v6NameServers, err := util.SplitIPv4AndIPv6Addrs(portValidatedNSs) + if err != nil { + return nil, errors.Wrap(err, "could not split IPv4 and IPv6 addresses for nameservers") + } + if config.IPVersionMode != zdns.IPv6Only { + config.ExternalNameServersV4 = v4NameServers + config.RootNameServersV4 = v4NameServers + } + if config.IPVersionMode != zdns.IPv4Only { + config.ExternalNameServersV6 = v6NameServers + config.RootNameServersV6 = v6NameServers + } return config, nil } // User did not provide nameservers if !gc.IterativeResolution { // Try to get the OS' default recursive resolver nameservers - ns, err := zdns.GetDNSServers(config.DNSConfigFilePath) + v4NameServers, v6NameServers, err := zdns.GetDNSServers(config.DNSConfigFilePath) if err != nil { - ns = util.GetDefaultResolvers() - log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(ns, ", ")) + v4NameServers, v6NameServers = zdns.DefaultExternalResolversV4, zdns.DefaultExternalResolversV6 + log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(util.Concat(v4NameServers, v6NameServers), ", ")) } - // TODO remove when we add IPv6 support, without this if the user's OS defaults contain both IPv4/6 - // It'll fail validation since we can't currently handle those - ipv4NameServers := make([]string, 0, len(ns)) - for _, addr := range ns { - ip, _, err := util.SplitHostPort(addr) - if err != nil { - return nil, errors.Wrapf(err, "could not split host and port for nameserver: %s", addr) - } - if ip.To4() == nil { - log.Infof("Ignoring non-IPv4 nameserver: %s", ip.String()) - continue - } - ipv4NameServers = append(ipv4NameServers, addr) + if config.IPVersionMode != zdns.IPv6Only { + config.ExternalNameServersV4 = v4NameServers + config.RootNameServersV4 = v4NameServers + } + if config.IPVersionMode != zdns.IPv4Only { + config.ExternalNameServersV6 = v6NameServers + config.RootNameServersV6 = v6NameServers } - config.ExternalNameServers = ipv4NameServers - config.RootNameServers = ipv4NameServers return config, nil } // User did not provide nameservers and we're doing iterative resolution, use ZDNS defaults - config.ExternalNameServers = zdns.RootServersV4[:] - config.RootNameServers = zdns.RootServersV4[:] + config.ExternalNameServersV4 = zdns.RootServersV4[:] + config.RootNameServersV4 = zdns.RootServersV4[:] + config.ExternalNameServersV6 = zdns.RootServersV6[:] + config.RootNameServersV6 = zdns.RootServersV6[:] return config, nil } @@ -285,28 +294,33 @@ func populateLocalAddresses(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Res // 1. If user provided local addresses, use those // 2. If the config's nameservers are loopback, use the local loopback address // 3. Otherwise, try to connect to Google's recursive resolver and take the IP address we use for the connection + + // IPv4 local addresses are required for IPv4 lookups, same for IPv6 if len(gc.LocalAddrs) != 0 { // if user provided a local address(es), that takes precedent - // TODO remove when we add IPv6 support, without this if the user provides --local-interface with both IPv4/6 - // It'll fail validation since we can't currently handle those - ipv4LocalAddrs := make([]net.IP, 0, len(gc.LocalAddrs)) + config.LocalAddrsV4, config.LocalAddrsV6 = []net.IP{}, []net.IP{} for _, addr := range gc.LocalAddrs { - if addr.To4() == nil { - log.Infof("Ignoring non-IPv4 local address: %s", addr.String()) - continue + if addr == nil { + return nil, errors.New("invalid nil local address") + } + if addr.To4() != nil { + config.LocalAddrsV4 = append(config.LocalAddrsV4, addr) + } else if util.IsIPv6(&addr) { + config.LocalAddrsV6 = append(config.LocalAddrsV6, addr) + } else { + return nil, fmt.Errorf("invalid local address: %s", addr.String()) } - ipv4LocalAddrs = append(ipv4LocalAddrs, addr) } - config.LocalAddrs = ipv4LocalAddrs return config, nil } // if the nameservers are loopback, use the loopback address - if len(config.ExternalNameServers) == 0 { + allNameServers := util.Concat(config.ExternalNameServersV4, config.ExternalNameServersV6, config.RootNameServersV4, config.RootNameServersV6) + if len(allNameServers) == 0 { // this shouldn't happen return nil, errors.New("name servers must be set before populating local addresses") } anyNameServersLoopack := false - for _, ns := range util.Concat(config.ExternalNameServers, config.RootNameServers) { + for _, ns := range allNameServers { ip, _, err := util.SplitHostPort(ns) if err != nil { return nil, errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) @@ -316,33 +330,37 @@ func populateLocalAddresses(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Res break } } + if anyNameServersLoopack { // set local address so name servers are reachable - config.LocalAddrs = []net.IP{net.ParseIP(zdns.LoopbackAddrString)} + config.LocalAddrsV4 = []net.IP{net.ParseIP(loopbackIPv4Addr)} + // loopback nameservers not supported for IPv6, we'll let Resolver validation take care of this } else { // localAddr not set, so we need to find the default IP address - conn, err := net.Dial("udp", googleDNSResolverAddr) - if err != nil { - return nil, fmt.Errorf("unable to find default IP address to open socket: %w", err) + if config.IPVersionMode != zdns.IPv6Only { + conn, err := net.Dial("udp", googleDNSResolverAddr) + if err != nil { + return nil, fmt.Errorf("unable to find default IP address to open socket: %w", err) + } + config.LocalAddrsV4 = []net.IP{conn.LocalAddr().(*net.UDPAddr).IP} + // cleanup socket + if err = conn.Close(); err != nil { + log.Error("unable to close test connection to Google public DNS: ", err) + } } - config.LocalAddrs = []net.IP{conn.LocalAddr().(*net.UDPAddr).IP} - // cleanup socket - if err = conn.Close(); err != nil { - log.Error("unable to close test connection to Google public DNS: ", err) + if config.IPVersionMode != zdns.IPv4Only { + conn, err := net.Dial("udp", googleDNSResolverAddrV6) + if err != nil { + return nil, fmt.Errorf("unable to find default IP address to open socket: %w", err) + } + config.LocalAddrsV6 = []net.IP{conn.LocalAddr().(*net.UDPAddr).IP} + // cleanup socket + if err = conn.Close(); err != nil { + log.Error("unable to close test connection to Google public IPv6 DNS: ", err) + } } } return config, nil - // TODO - from main - // // Local Addresses - // for _, ip := range gc.LocalAddrs { - // if ip.To4() != nil { - // config.LocalAddrsV4 = append(config.LocalAddrsV4, ip) - // } else if util.IsIPv6(&ip) { - // config.LocalAddrsV6 = append(config.LocalAddrsV6, ip) - // } else { - // log.Fatalf("invalid local address: %s", ip.String()) - // } - // } } func Run(gc CLIConf, flags *pflag.FlagSet) { diff --git a/src/internal/util/util.go b/src/internal/util/util.go index bd85485a..cbafcada 100644 --- a/src/internal/util/util.go +++ b/src/internal/util/util.go @@ -100,11 +100,6 @@ func SplitIPv4AndIPv6Addrs(addrs []string) (ipv4 []string, ipv6 []string, err er return ipv4, ipv6, nil } -// IsIPv6 checks if the given IP address is an IPv6 address. -func IsIPv6(ip *net.IP) bool { - return ip != nil && ip.To4() == nil && ip.To16() != nil -} - // Reference: https://github.com/carolynvs/stingoftheviper/blob/main/main.go // For how to make cobra/viper sync up, and still use custom struct // Bind each cobra flag to its associated viper configuration (config file and environment variable) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index bec48a3b..ae468fec 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -99,64 +99,58 @@ func (rc *ResolverConfig) Validate() error { } // External Nameservers - if len(rc.ExternalNameServers) == 0 { - return errors.New("must have at least one external name server") + if rc.IPVersionMode != IPv6Only && len(rc.ExternalNameServersV4) == 0 { + // If IPv4 is supported, we require at least one IPv4 external nameserver + return errors.New("must have at least one external IPv4 name server if IPv4 mode is enabled") + } + if rc.IPVersionMode != IPv4Only && len(rc.ExternalNameServersV6) == 0 { + // If IPv6 is supported, we require at least one IPv6 external nameserver + return errors.New("must have at least one external IPv6 name server if IPv6 mode is enabled") } - for _, ns := range rc.ExternalNameServers { + // Validate all nameservers have ports and are valid IPs + for _, ns := range util.Concat(rc.ExternalNameServersV4, rc.ExternalNameServersV6) { ipString, _, err := net.SplitHostPort(ns) if err != nil { - return fmt.Errorf("could not parse external name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53", ns) + return fmt.Errorf("could not parse external name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53 or [::1]:53", ns) } ip := net.ParseIP(ipString) if ip == nil { - return fmt.Errorf("could not parse external name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53", ns) + return fmt.Errorf("could not parse external name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53 or [::1]:53", ns) } } - // Check Root Servers - if len(rc.RootNameServers) == 0 { - return errors.New("must have at least one root name server") + // Root Nameservers + if rc.IPVersionMode != IPv6Only && len(rc.RootNameServersV4) == 0 { + // If IPv4 is supported, we require at least one IPv4 root nameserver + return errors.New("must have at least one root IPv4 name server if IPv4 mode is enabled") + } + if rc.IPVersionMode != IPv4Only && len(rc.RootNameServersV6) == 0 { + // If IPv6 is supported, we require at least one IPv6 root nameserver + return errors.New("must have at least one root IPv6 name server if IPv6 mode is enabled") } - for _, ns := range rc.RootNameServers { + + // Validate all nameservers have ports and are valid IPs + for _, ns := range util.Concat(rc.RootNameServersV4, rc.RootNameServersV6) { ipString, _, err := net.SplitHostPort(ns) if err != nil { - return fmt.Errorf("could not parse root name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53", ns) + return fmt.Errorf("could not parse root name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53 or [::1]:53", ns) } ip := net.ParseIP(ipString) if ip == nil { - return fmt.Errorf("could not parse root name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53", ns) - } - } - - // TODO - Remove when we add IPv6 support - for _, ns := range rc.RootNameServers { - // we know ns passed validation above - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return errors.Wrapf(err, "could not split host and port for root nameserver: %s", ns) - } - if util.IsIPv6(&ip) { - return fmt.Errorf("IPv6 root nameservers are not supported: %s", ns) + return fmt.Errorf("could not parse root name server (%s), must be valid IP and have port appended, ex: 1.2.3.4:53 or [::1]:53", ns) } } - for _, ns := range rc.ExternalNameServers { - // we know ns passed validation above - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return errors.Wrapf(err, "could not split host and port for external nameserver: %s", ns) - } - if util.IsIPv6(&ip) { - return fmt.Errorf("IPv6 extenral nameservers are not supported: %s", ns) - } - } - // TODO end IPv6 section // Local Addresses - if len(rc.LocalAddrs) == 0 { - return errors.New("must have a local address to send traffic from") + if rc.IPVersionMode != IPv6Only && len(rc.LocalAddrsV4) == 0 { + return errors.New("must have a local IPv4 address to send traffic from") + } + if rc.IPVersionMode != IPv4Only && len(rc.LocalAddrsV6) == 0 { + return errors.New("must have a local IPv6 address to send traffic from") } - for _, ip := range rc.LocalAddrs { + // Validate all local addresses are valid IPs + for _, ip := range util.Concat(rc.LocalAddrsV4, rc.LocalAddrsV6) { if ip == nil { return errors.New("local address cannot be nil") } @@ -165,17 +159,37 @@ func (rc *ResolverConfig) Validate() error { } } - // TODO - Remove when we add IPv6 support - for _, addr := range rc.LocalAddrs { - if util.IsIPv6(&addr) { - return fmt.Errorf("IPv6 local addresses are not supported: %v", rc.LocalAddrs) + // Validate IPv4 local addresses are IPv4 + for _, ip := range rc.LocalAddrsV4 { + if ip.To4() == nil { + return fmt.Errorf("local address is not IPv4: %v", ip) + } + } + + // Validate IPv6 local addresses are IPv6 + for _, ip := range rc.LocalAddrsV6 { + if !util.IsIPv6(&ip) { + return fmt.Errorf("IPv6 local address (%v) is not IPv6", ip) + } + } + + // Ensure no IPv6 link-local/multicast local addresses are used + for _, ip := range rc.LocalAddrsV6 { + if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return fmt.Errorf("link-local IPv6 local addresses are not supported: %v", ip) + } + } + + // Ensure no IPv6 link-local/multicast external/root nameservers are used + for _, ns := range util.Concat(rc.ExternalNameServersV6, rc.RootNameServersV6) { + ip, _, err := util.SplitHostPort(ns) + if err != nil { + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) } - // TODO - remember, no link-local/multicast IPv6 addrs if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return fmt.Errorf("link-local IPv6 root nameservers are not supported: %s", ns) + return fmt.Errorf("link-local IPv6 external/root nameservers are not supported: %v", ip) } } - // TODO end IPv6 section if err := rc.validateLoopbackConsistency(); err != nil { return errors.Wrap(err, "could not validate loopback consistency") @@ -187,20 +201,16 @@ func (rc *ResolverConfig) Validate() error { // validateLoopbackConsistency checks that the following is true // - either all nameservers AND all local addresses are loopback, or none are func (rc *ResolverConfig) validateLoopbackConsistency() error { - allIPsLength := len(rc.LocalAddrs) + len(rc.RootNameServers) + len(rc.ExternalNameServers) + allLocalAddrs := util.Concat(rc.LocalAddrsV4, rc.LocalAddrsV6) + allExternalNameServers := util.Concat(rc.ExternalNameServersV4, rc.ExternalNameServersV6) + allRootNameServers := util.Concat(rc.RootNameServersV4, rc.RootNameServersV6) + allIPsLength := len(allLocalAddrs) + len(allExternalNameServers) + len(allRootNameServers) allIPs := make([]net.IP, 0, allIPsLength) - allIPs = append(allIPs, rc.LocalAddrs...) - for _, ns := range rc.ExternalNameServers { + allIPs = append(allIPs, allLocalAddrs...) + for _, ns := range util.Concat(allExternalNameServers, allRootNameServers) { ip, _, err := util.SplitHostPort(ns) if err != nil { - return errors.Wrapf(err, "could not split host and port for external nameserver: %s", ns) - } - allIPs = append(allIPs, ip) - } - for _, ns := range rc.RootNameServers { - ip, _, err := util.SplitHostPort(ns) - if err != nil { - return errors.Wrapf(err, "could not split host and port for root nameserver: %s", ns) + return errors.Wrapf(err, "could not split host and port for nameserver: %s", ns) } allIPs = append(allIPs, ip) } @@ -214,7 +224,7 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } } if allIPsLoopback == noneIPsLoopback { - return fmt.Errorf("cannot mix loopback and non-loopback local addresses (%v) and name servers (%v)", rc.LocalAddrs, util.Concat(rc.ExternalNameServers, rc.RootNameServers)) + return fmt.Errorf("cannot mix loopback and non-loopback local addresses (%v) and name servers (%v)", allLocalAddrs, util.Concat(allExternalNameServers, allRootNameServers)) } return nil } @@ -336,16 +346,16 @@ func InitResolver(config *ResolverConfig) (*Resolver, error) { checkingDisabledBit: config.CheckingDisabledBit, } log.SetLevel(r.logLevel) - // create connection info for IPv4 - if config.IPVersionMode == IPv4Only || config.IPVersionMode == IPv4OrIPv6 { + if config.IPVersionMode != IPv6Only { + // create connection info for IPv4 connInfo, err := getConnectionInfo(config.LocalAddrsV4, config.TransportMode, config.Timeout, config.ShouldRecycleSockets) if err != nil { return nil, fmt.Errorf("could not create connection info for IPv4: %w", err) } r.connInfoIPv4 = connInfo } - // create connection info for IPv6 - if config.IPVersionMode == IPv6Only || config.IPVersionMode == IPv4OrIPv6 { + if config.IPVersionMode != IPv4Only { + // create connection info for IPv6 connInfo, err := getConnectionInfo(config.LocalAddrsV6, config.TransportMode, config.Timeout, config.ShouldRecycleSockets) if err != nil { return nil, fmt.Errorf("could not create connection info for IPv6: %w", err) @@ -445,23 +455,22 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer string) (*SingleQueryRe } dstServerWithPort, err := util.AddDefaultPortToDNSServerName(dstServer) if err != nil { - // TODO update below when adding IPv6, add ex. IPv6 - return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w. Correct format IPv4 (1.1.1.1:53)", dstServer, err) + return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w. Correct format IPv4 1.1.1.1:53 or IPv6 [::1]:53", dstServer, err) } if dstServer != dstServerWithPort { log.Info("no port provided for external lookup, using default port 53") } dstServerIP, _, err := util.SplitHostPort(dstServerWithPort) if err != nil { - // TODO update below when adding IPv6, add ex. IPv6 - return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w. Correct format IPv4 (1.1.1.1:53)", dstServer, err) + return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w. Correct format IPv4 1.1.1.1:53 or IPv6 [::1]:53", dstServer, err) } // check that local address and dstServer's don't have a loopback mismatch - if r.localAddr.IsLoopback() != dstServerIP.IsLoopback() { + if dstServerIP.To4() != nil && r.connInfoIPv4.localAddr.IsLoopback() != dstServerIP.IsLoopback() { + return nil, nil, StatusIllegalInput, errors.New("cannot mix loopback and non-loopback addresses") + } else if util.IsIPv6(&dstServerIP) && r.connInfoIPv6.localAddr.IsLoopback() != dstServerIP.IsLoopback() { return nil, nil, StatusIllegalInput, errors.New("cannot mix loopback and non-loopback addresses") - } - // dstServer has been validated and has a port + // dstServer has been validated and has a port, continue with lookup dstServer = dstServerWithPort lookup, trace, status, err := r.lookupClient.DoSingleDstServerLookup(r, *q, dstServer, false) return lookup, trace, status, err From 50fef992ec8577d8ed30e8355e4545468e99a845 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 12 Aug 2024 13:14:31 -0400 Subject: [PATCH 49/59] fixed up tests and added null checks to ExternalLookup. Removed test trying to use IPv6 loopback nameserver, since we don't support that --- src/modules/axfr/axfr_test.go | 7 +- src/modules/bindversion/bindversion_test.go | 9 +- src/modules/dmarc/dmarc_test.go | 9 +- src/modules/spf/spf_test.go | 9 +- src/zdns/lookup_test.go | 170 ++++++++------------ src/zdns/resolver.go | 5 + src/zdns/resolver_test.go | 54 +++---- 7 files changed, 120 insertions(+), 143 deletions(-) diff --git a/src/modules/axfr/axfr_test.go b/src/modules/axfr/axfr_test.go index 394f8c96..59263b27 100644 --- a/src/modules/axfr/axfr_test.go +++ b/src/modules/axfr/axfr_test.go @@ -93,9 +93,10 @@ func InitTest() (*AxfrLookupModule, *zdns.Resolver) { cc := new(cli.CLIConf) rc := new(zdns.ResolverConfig) - rc.RootNameServers = []string{"127.0.0.53:53"} - rc.ExternalNameServers = []string{"127.0.0.53:53"} - rc.LocalAddrs = []net.IP{net.ParseIP("127.0.0.1")} + rc.RootNameServersV4 = []string{"127.0.0.53:53"} + rc.ExternalNameServersV4 = []string{"127.0.0.53:53"} + rc.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")} + rc.IPVersionMode = zdns.IPv4Only flagSet := new(pflag.FlagSet) flagSet.Bool("ipv4-lookup", false, "Use IPv4") diff --git a/src/modules/bindversion/bindversion_test.go b/src/modules/bindversion/bindversion_test.go index c7508a12..988efd5f 100644 --- a/src/modules/bindversion/bindversion_test.go +++ b/src/modules/bindversion/bindversion_test.go @@ -47,10 +47,11 @@ func (ml MockLookup) DoSingleDstServerLookup(r *zdns.Resolver, question zdns.Que func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1:53"}, - RootNameServers: []string{"1.1.1.1:53"}, - LocalAddrs: []net.IP{net.ParseIP("192.168.1.1")}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"1.1.1.1:53"}, + RootNameServersV4: []string{"1.1.1.1:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("192.168.1.1")}, + IPVersionMode: zdns.IPv4Only, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/modules/dmarc/dmarc_test.go b/src/modules/dmarc/dmarc_test.go index 9630b612..b217345d 100644 --- a/src/modules/dmarc/dmarc_test.go +++ b/src/modules/dmarc/dmarc_test.go @@ -47,10 +47,11 @@ func (ml MockLookup) DoSingleDstServerLookup(r *zdns.Resolver, question zdns.Que func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"127.0.0.1:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, + IPVersionMode: zdns.IPv4Only, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/modules/spf/spf_test.go b/src/modules/spf/spf_test.go index bc215c6e..55c0ed79 100644 --- a/src/modules/spf/spf_test.go +++ b/src/modules/spf/spf_test.go @@ -48,10 +48,11 @@ func InitTest(t *testing.T) *zdns.Resolver { mockResults = make(map[string]*zdns.SingleQueryResult) queries = make([]QueryRecord, 0) rc := zdns.ResolverConfig{ - ExternalNameServers: []string{"127.0.0.1:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, - LookupClient: MockLookup{}} + ExternalNameServersV4: []string{"127.0.0.1:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, + IPVersionMode: zdns.IPv4Only, + LookupClient: MockLookup{}} r, err := zdns.InitResolver(&rc) assert.NilError(t, err) diff --git a/src/zdns/lookup_test.go b/src/zdns/lookup_test.go index 07aa31a7..f6780b9f 100644 --- a/src/zdns/lookup_test.go +++ b/src/zdns/lookup_test.go @@ -58,9 +58,10 @@ func InitTest(t *testing.T) *ResolverConfig { mc := MockLookupClient{} config := NewResolverConfig() - config.ExternalNameServers = []string{"127.0.0.1:53"} - config.RootNameServers = []string{"127.0.0.1:53"} - config.LocalAddrs = []net.IP{net.ParseIP("127.0.0.1")} + config.ExternalNameServersV4 = []string{"127.0.0.1:53"} + config.RootNameServersV4 = []string{"127.0.0.1:53"} + config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")} + config.IPVersionMode = IPv4Only config.LookupClient = mc return config @@ -625,7 +626,7 @@ func TestOneA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -641,7 +642,7 @@ func TestOneA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -653,7 +654,7 @@ func TestTwoA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -676,7 +677,7 @@ func TestTwoA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1", "192.0.2.2"}, nil) } @@ -688,7 +689,7 @@ func TestQuadAWithoutFlag(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -712,7 +713,7 @@ func TestQuadAWithoutFlag(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -724,7 +725,7 @@ func TestOnlyQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -741,7 +742,7 @@ func TestOnlyQuadA(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, false, true) assert.NotNil(t, res) verifyResult(t, *res, nil, []string{"2001:db8::1"}) } @@ -754,7 +755,7 @@ func TestAandQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -777,7 +778,7 @@ func TestAandQuadA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, true) assert.NotNil(t, res) verifyResult(t, *res, []string{"192.0.2.1"}, []string{"2001:db8::1"}) } @@ -790,7 +791,7 @@ func TestTwoQuadA(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -813,7 +814,7 @@ func TestTwoQuadA(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, false, true) assert.NotNil(t, res) verifyResult(t, *res, nil, []string{"2001:db8::1", "2001:db8::2"}) } @@ -827,7 +828,7 @@ func TestNoResults(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -837,7 +838,7 @@ func TestNoResults(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, false) verifyResult(t, *res, nil, nil) } @@ -849,7 +850,7 @@ func TestCname(t *testing.T) { require.NoError(t, err) domain1 := "cname.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -883,7 +884,7 @@ func TestCname(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.1"}, nil) } @@ -895,7 +896,7 @@ func TestQuadAWithCname(t *testing.T) { require.NoError(t, err) domain1 := "cname.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -918,7 +919,7 @@ func TestQuadAWithCname(t *testing.T) { Protocol: "", Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, IPv6Only, false) + res, _, _, _ := resolver.DoTargetedLookup("cname.example.com", ns1, false, false, true) verifyResult(t, *res, nil, []string{"2001:db8::3"}) } @@ -930,7 +931,7 @@ func TestUnexpectedMxOnly(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -947,7 +948,7 @@ func TestUnexpectedMxOnly(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -964,7 +965,7 @@ func TestMxAndAdditionals(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -994,7 +995,7 @@ func TestMxAndAdditionals(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) verifyResult(t, *res, []string{"192.0.2.3"}, []string{"2001:db8::4"}) } @@ -1006,7 +1007,7 @@ func TestMismatchIpType(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1023,7 +1024,7 @@ func TestMismatchIpType(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1040,7 +1041,7 @@ func TestCnameLoops(t *testing.T) { require.NoError(t, err) domain1 := "cname1.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1075,7 +1076,7 @@ func TestCnameLoops(t *testing.T) { Flags: DNSFlags{}, } - res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1091,7 +1092,7 @@ func TestExtendedRecursion(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] // Create a CNAME chain of length > 10 for i := 1; i < 12; i++ { domainNSRecord := domainNS{ @@ -1113,7 +1114,7 @@ func TestExtendedRecursion(t *testing.T) { } } - res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, IPv4OrIPv6, false) + res, _, status, _ := resolver.DoTargetedLookup("cname1.example.com", ns1, false, true, true) if status != StatusError { t.Errorf("Expected ERROR status, got %v", status) @@ -1130,7 +1131,7 @@ func TestEmptyNonTerminal(t *testing.T) { require.NoError(t, err) domain1 := "leaf.intermediate.example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1159,11 +1160,11 @@ func TestEmptyNonTerminal(t *testing.T) { Flags: DNSFlags{}, } // Verify leaf returns correctly - res, _, _, _ := resolver.DoTargetedLookup("leaf.intermediate.example.com", ns1, IPv4Only, false) + res, _, _, _ := resolver.DoTargetedLookup("leaf.intermediate.example.com", ns1, false, true, false) verifyResult(t, *res, []string{"192.0.2.3"}, nil) // Verify empty non-terminal returns no answer - res, _, _, _ = resolver.DoTargetedLookup("intermediate.example.com", ns1, IPv4OrIPv6, false) + res, _, _, _ = resolver.DoTargetedLookup("intermediate.example.com", ns1, false, true, true) verifyResult(t, *res, nil, nil) } @@ -1173,8 +1174,8 @@ func TestNXDomain(t *testing.T) { config := InitTest(t) resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] - res, _, status, _ := resolver.DoTargetedLookup("nonexistent.example.com", ns1, IPv4OrIPv6, false) + ns1 := config.ExternalNameServersV4[0] + res, _, status, _ := resolver.DoTargetedLookup("nonexistent.example.com", ns1, false, true, true) if status != StatusNXDomain { t.Errorf("Expected StatusNXDomain status, got %v", status) } else if res != nil { @@ -1191,7 +1192,7 @@ func TestAandQuadADedup(t *testing.T) { domain1 := "cname1.example.com" domain2 := "cname2.example.com" domain3 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} domainNS2 := domainNS{domain: domain2, ns: ns1} domainNS3 := domainNS{domain: domain3, ns: ns1} @@ -1274,7 +1275,7 @@ func TestAandQuadADedup(t *testing.T) { Flags: DNSFlags{}, } - res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, IPv4OrIPv6, false) + res, _, _, _ := resolver.DoTargetedLookup(domain1, ns1, false, true, true) assert.NotNil(t, res) verifyResult(t, *res, []string{"192.0.2.1"}, []string{"2001:db8::3"}) } @@ -1287,14 +1288,14 @@ func TestServFail(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{} name := "example.com" protocolStatus[domainNS1] = StatusServFail - res, _, finalStatus, _ := resolver.DoTargetedLookup(name, ns1, IPv4OrIPv6, false) + res, _, finalStatus, _ := resolver.DoTargetedLookup(name, ns1, false, true, true) if finalStatus != protocolStatus[domainNS1] { t.Errorf("Expected %v status, got %v", protocolStatus, finalStatus) @@ -1324,7 +1325,7 @@ func TestNsAInAdditional(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1356,7 +1357,7 @@ func TestNsAInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) verifyNsResult(t, res.Servers, expectedServersMap) } @@ -1367,7 +1368,7 @@ func TestTwoNSInAdditional(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1417,18 +1418,18 @@ func TestTwoNSInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.4"}, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) verifyNsResult(t, res.Servers, expectedServersMap) } func TestAandQuadAInAdditional(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 + //config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1467,18 +1468,18 @@ func TestAandQuadAInAdditional(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: []string{"2001:db8::4"}, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } func TestNsMismatchIpType(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 + //config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1517,18 +1518,18 @@ func TestNsMismatchIpType(t *testing.T) { IPv4Addresses: nil, IPv6Addresses: nil, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } func TestAandQuadALookup(t *testing.T) { config := InitTest(t) - config.IPVersionMode = IPv4OrIPv6 + //config.IPVersionMode = IPv4OrIPv6 resolver, err := InitResolver(config) require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1579,7 +1580,7 @@ func TestAandQuadALookup(t *testing.T) { IPv4Addresses: []string{"192.0.2.3"}, IPv6Addresses: []string{"2001:db8::4"}, } - res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, _, _ := resolver.DoNSLookup("example.com", ns1, false, true, true) verifyNsResult(t, res.Servers, expectedServersMap) } @@ -1588,9 +1589,9 @@ func TestNsNXDomain(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] - _, _, status, _ := resolver.DoNSLookup("nonexistentexample.com", ns1, false) + _, _, status, _ := resolver.DoNSLookup("nonexistentexample.com", ns1, false, true, true) assert.Equal(t, StatusNXDomain, status) } @@ -1601,13 +1602,13 @@ func TestNsServFail(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{} protocolStatus[domainNS1] = StatusServFail - res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) assert.Equal(t, status, protocolStatus[domainNS1]) assert.Empty(t, res.Servers) @@ -1619,7 +1620,7 @@ func TestErrorInTargetedLookup(t *testing.T) { require.NoError(t, err) domain1 := "example.com" - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1640,7 +1641,7 @@ func TestErrorInTargetedLookup(t *testing.T) { protocolStatus[domainNS1] = StatusError - res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false) + res, _, status, _ := resolver.DoNSLookup("example.com", ns1, false, true, false) assert.Empty(t, len(res.Servers), 0) assert.Equal(t, status, protocolStatus[domainNS1]) } @@ -1648,16 +1649,14 @@ func TestErrorInTargetedLookup(t *testing.T) { // Test One NS with one IP with only ipv4-lookup func TestAllNsLookupOneNs(t *testing.T) { config := InitTest(t) - config.LocalAddrs = []net.IP{net.ParseIP(LoopbackAddrString)} - config.IPVersionMode = IPv4OrIPv6 + config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")} resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" - ipv6_1 := "::1" domainNS1 := domainNS{domain: domain1, ns: ns1} mockResults[domainNS1] = SingleQueryResult{ @@ -1678,13 +1677,6 @@ func TestAllNsLookupOneNs(t *testing.T) { Name: nsDomain1 + ".", Answer: ipv4_1, }, - Answer{ - TTL: 3600, - Type: "AAAA", - Class: "IN", - Name: nsDomain1 + ".", - Answer: ipv6_1, - }, }, Authorities: nil, Protocol: "", @@ -1710,36 +1702,12 @@ func TestAllNsLookupOneNs(t *testing.T) { Flags: DNSFlags{}, } - ns3 := net.JoinHostPort(ipv6_1, "53") - domainNS3 := domainNS{domain: domain1, ns: ns3} - ipv4_3 := "127.0.0.4" - mockResults[domainNS3] = SingleQueryResult{ - Answers: []interface{}{ - Answer{ - TTL: 3600, - Type: "A", - Class: "IN", - Name: "example.com.", - Answer: ipv4_3, - }, - }, - Additional: nil, - Authorities: nil, - Protocol: "", - Flags: DNSFlags{}, - } - expectedRes := []ExtendedResult{ { Nameserver: nsDomain1, Status: StatusNoError, Res: mockResults[domainNS2], }, - { - Nameserver: nsDomain1, - Status: StatusNoError, - Res: mockResults[domainNS3], - }, } q := Question{ Type: dns.TypeNS, @@ -1760,7 +1728,7 @@ func TestAllNsLookupOneNsMultipleIps(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" @@ -1883,7 +1851,7 @@ func TestAllNsLookupTwoNs(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" nsDomain2 := "ns2.example.com" @@ -1999,7 +1967,7 @@ func TestAllNsLookupErrorInOne(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" nsDomain1 := "ns1.example.com" ipv4_1 := "127.0.0.2" @@ -2099,7 +2067,7 @@ func TestAllNsLookupNXDomain(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] q := Question{ Type: dns.TypeNS, Class: dns.ClassINET, @@ -2119,7 +2087,7 @@ func TestAllNsLookupServFail(t *testing.T) { resolver, err := InitResolver(config) require.NoError(t, err) - ns1 := config.ExternalNameServers[0] + ns1 := config.ExternalNameServersV4[0] domain1 := "example.com" domainNS1 := domainNS{domain: domain1, ns: ns1} @@ -2140,8 +2108,8 @@ func TestAllNsLookupServFail(t *testing.T) { func TestInvalidInputsLookup(t *testing.T) { config := InitTest(t) - config.LocalAddrs = []net.IP{net.ParseIP("127.0.0.1")} - config.ExternalNameServers = []string{"127.0.0.1:53"} + config.LocalAddrsV4 = []net.IP{net.ParseIP("127.0.0.1")} + config.ExternalNameServersV4 = []string{"127.0.0.1:53"} resolver, err := InitResolver(config) require.NoError(t, err) q := Question{ diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index ae468fec..aa598b41 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -464,6 +464,11 @@ func (r *Resolver) ExternalLookup(q *Question, dstServer string) (*SingleQueryRe if err != nil { return nil, nil, StatusIllegalInput, fmt.Errorf("could not parse name server (%s): %w. Correct format IPv4 1.1.1.1:53 or IPv6 [::1]:53", dstServer, err) } + if util.IsIPv6(&dstServerIP) && r.connInfoIPv6 == nil { + return nil, nil, StatusIllegalInput, fmt.Errorf("IPv6 external lookup requested for domain %s but no IPv6 local addresses provided to resolver", q.Name) + } else if dstServerIP.To4() != nil && r.connInfoIPv4 == nil { + return nil, nil, StatusIllegalInput, fmt.Errorf("IPv4 external lookup requested for domain %s but no IPv4 local addresses provided to resolver", q.Name) + } // check that local address and dstServer's don't have a loopback mismatch if dstServerIP.To4() != nil && r.connInfoIPv4.localAddr.IsLoopback() != dstServerIP.IsLoopback() { return nil, nil, StatusIllegalInput, errors.New("cannot mix loopback and non-loopback addresses") diff --git a/src/zdns/resolver_test.go b/src/zdns/resolver_test.go index 0e412351..3435aa7e 100644 --- a/src/zdns/resolver_test.go +++ b/src/zdns/resolver_test.go @@ -24,51 +24,51 @@ import ( func TestResolverConfig_Validate(t *testing.T) { t.Run("Valid config with external/root name servers and local addr", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.Nil(t, err, "Expected no error but got %v", err) }) t.Run("Using external nameserver with no port", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"127.0.0.53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Using root nameserver with no port", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, - RootNameServers: []string{"127.0.0.53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, + RootNameServersV4: []string{"127.0.0.53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Missing external nameserver", func(t *testing.T) { rc := &ResolverConfig{ - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Missing root nameserver", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Missing local addr", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, - RootNameServers: []string{"127.0.0.53:53"}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, } err := rc.Validate() require.NotNil(t, err) @@ -76,36 +76,36 @@ func TestResolverConfig_Validate(t *testing.T) { t.Run("Cannot mix loopback addresses in nameservers", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53, 1.1.1.1:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"127.0.0.53:53, 1.1.1.1:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Cannot mix loopback addresses among nameservers", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"1.1.1.1:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Cannot reach loopback NSes from non-loopback local address", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"127.0.0.53:53"}, - RootNameServers: []string{"127.0.0.53:53"}, - LocalAddrs: []net.IP{net.ParseIP("192.168.1.2")}, + ExternalNameServersV4: []string{"127.0.0.53:53"}, + RootNameServersV4: []string{"127.0.0.53:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("192.168.1.2")}, } err := rc.Validate() require.NotNil(t, err) }) t.Run("Cannot reach non-loopback NSes from loopback local address", func(t *testing.T) { rc := &ResolverConfig{ - ExternalNameServers: []string{"1.1.1.1:53"}, - RootNameServers: []string{"1.1.1.1:53"}, - LocalAddrs: []net.IP{net.ParseIP("127.0.0.1")}, + ExternalNameServersV4: []string{"1.1.1.1:53"}, + RootNameServersV4: []string{"1.1.1.1:53"}, + LocalAddrsV4: []net.IP{net.ParseIP("127.0.0.1")}, } err := rc.Validate() require.NotNil(t, err) From 536e9c2f60af08189cb9174fa379d8446bc8649d Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 12 Aug 2024 14:15:56 -0400 Subject: [PATCH 50/59] cleaned up unneeded changes in integration_tests --- testing/integration_tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/integration_tests.py b/testing/integration_tests.py index d6b9e843..62b9064f 100755 --- a/testing/integration_tests.py +++ b/testing/integration_tests.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import copy -import socket import subprocess import json import unittest @@ -582,7 +581,6 @@ def test_a(self): self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) - def test_cname(self): c = "CNAME" name = "www.zdns-testing.com" From 970aa632b4d0323fb2d53f4e8200dc27d1bcd6cc Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 12 Aug 2024 14:16:59 -0400 Subject: [PATCH 51/59] spelling --- src/zdns/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zdns/util.go b/src/zdns/util.go index ab08b297..4b4ae79e 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -64,10 +64,10 @@ func checkGlue(server string, result SingleQueryResult, ipMode IPVersionMode, ip } else if ipMode == IPv6Only { ansType = "AAAA" } else if ipPreference == PreferIPv4 { - // msut be using either IPv4 or IPv6 + // must be using either IPv4 or IPv6 ansType = "A" } else if ipPreference == PreferIPv6 { - // msut be using either IPv4 or IPv6 + // must be using either IPv4 or IPv6 ansType = "AAAA" } else { log.Fatal("should never hit this case in check glue: ", ipMode, ipPreference) From f11791f6174387564994d2ae9d42461e32f57122 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 12 Aug 2024 14:22:42 -0400 Subject: [PATCH 52/59] use new concat --- src/zdns/resolver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/zdns/resolver.go b/src/zdns/resolver.go index aa598b41..3af74be8 100644 --- a/src/zdns/resolver.go +++ b/src/zdns/resolver.go @@ -230,9 +230,9 @@ func (rc *ResolverConfig) validateLoopbackConsistency() error { } func (rc *ResolverConfig) PrintInfo() { - log.Infof("using local addresses: %v", append(append([]net.IP{}, rc.LocalAddrsV4...), rc.LocalAddrsV6...)) - log.Infof("for non-iterative lookups, using external nameservers: %s", strings.Join(append(append([]string{}, rc.ExternalNameServersV4...), rc.ExternalNameServersV6...), ", ")) - log.Infof("for iterative lookups, using nameservers: %s", strings.Join(append(append([]string{}, rc.RootNameServersV4...), rc.RootNameServersV6...), ", ")) + log.Infof("using local addresses: %v", util.Concat(rc.LocalAddrsV4, rc.LocalAddrsV6)) + log.Infof("for non-iterative lookups, using external nameservers: %s", strings.Join(util.Concat(rc.ExternalNameServersV4, rc.ExternalNameServersV6), ", ")) + log.Infof("for iterative lookups, using nameservers: %s", strings.Join(util.Concat(rc.RootNameServersV4, rc.RootNameServersV6), ", ")) } // NewResolverConfig creates a new ResolverConfig with default values. From 77784dbfb86d9a0e692b5fa6afdb949dc292db23 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Mon, 12 Aug 2024 14:30:41 -0400 Subject: [PATCH 53/59] avoid redundent check and remove todo --- src/zdns/lookup.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index bc56e4b9..8c16e8f3 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -72,7 +72,6 @@ func (lc LookupClient) DoSingleDstServerLookup(r *Resolver, q Question, nameServ func (r *Resolver) doSingleDstServerLookup(q Question, nameServer string, isIterative bool) (*SingleQueryResult, Trace, Status, error) { var err error - // TODO check that the next 20 lines aren't located somewhere else, avoid duplicate checks // Check that nameserver isn't blacklisted nameServerIPString, _, err := net.SplitHostPort(nameServer) if err != nil { @@ -428,7 +427,6 @@ func (r *Resolver) cachedRetryingLookup(ctx context.Context, q Question, nameSer // retryingLookup wraps around wireLookup to perform a DNS lookup with retries // Returns the result, status, number of tries, and error func (r *Resolver) retryingLookup(ctx context.Context, q Question, nameServer string, recursive bool) (SingleQueryResult, Status, int, error) { - // TODO - think we're duplicating this logic // nameserver is required if nameServer == "" { return SingleQueryResult{}, StatusIllegalInput, 0, errors.New("no nameserver specified") @@ -437,16 +435,6 @@ func (r *Resolver) retryingLookup(ctx context.Context, q Question, nameServer st if err != nil { return SingleQueryResult{}, StatusError, 0, errors.Wrapf(err, "could not split nameserver %s to get IP", nameServer) } - // Check that nameserver isn't blacklisted - - // Stop if we hit a nameserver we don't want to hit - if r.blacklist != nil { - if blacklisted, blacklistedErr := r.blacklist.IsBlacklisted(nameServerIP.String()); blacklistedErr != nil { - return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not check blacklist for nameserver %s: %w", nameServer, err) - } else if blacklisted { - return SingleQueryResult{}, StatusBlacklist, 0, nil - } - } var connInfo *ConnectionInfo if nameServerIP.To4() != nil { connInfo = r.connInfoIPv4 From c4ca392198b7c219181df6e8643370ca2db7e208 Mon Sep 17 00:00:00 2001 From: Phillip Stephens Date: Tue, 13 Aug 2024 15:15:53 -0400 Subject: [PATCH 54/59] Stop handling a domain if all nameservers don't provide sufficient glue records when they should (#417) * stop handling a domain if the nameserver that should provide glue records doesn't * add rfc comment * lint * updated ipv6 integration test * update comment --- src/zdns/conf.go | 1 + src/zdns/lookup.go | 15 +++++++++++---- src/zdns/util.go | 8 ++++++-- testing/ipv6_tests.py | 4 ++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/zdns/conf.go b/src/zdns/conf.go index c939bcb3..b9fbea1e 100644 --- a/src/zdns/conf.go +++ b/src/zdns/conf.go @@ -51,6 +51,7 @@ const ( StatusTimeout Status = "TIMEOUT" StatusIterTimeout Status = "ITERATIVE_TIMEOUT" StatusNoAuth Status = "NOAUTH" + StatusNoNeededGlue Status = "NONEEDEDGLUE" // When a nameserver is authoritative for itself and the parent nameserver doesn't provide the glue to look it up ) var RootServersV4 = []string{ diff --git a/src/zdns/lookup.go b/src/zdns/lookup.go index 8c16e8f3..75e6366c 100644 --- a/src/zdns/lookup.go +++ b/src/zdns/lookup.go @@ -572,7 +572,6 @@ func (r *Resolver) iterateOnAuthorities(ctx context.Context, q Question, depth i if nsStatus != StatusNoError { var err error newStatus, err := handleStatus(nsStatus, err) - // default case we continue if err == nil { if i+1 == len(result.Authorities) { r.verboseLog((depth + 2), "--> Auth find Failed. Unknown error. No more authorities to try, terminating: ", nsStatus) @@ -596,8 +595,11 @@ func (r *Resolver) iterateOnAuthorities(ctx context.Context, q Question, depth i } } iterateResult, newTrace, status, err := r.iterativeLookup(ctx, q, ns, depth+1, newLayer, newTrace) - if isStatusAnswer(status) { - r.verboseLog((depth + 1), "--> Auth Resolution success: ", status) + if status == StatusNoNeededGlue { + r.verboseLog((depth + 2), "--> Auth resolution of ", ns, " was unsuccessful. No glue to follow", status) + return iterateResult, newTrace, status, err + } else if isStatusAnswer(status) { + r.verboseLog((depth + 1), "--> Auth Resolution of ", ns, " success: ", status) return iterateResult, newTrace, status, err } else if i+1 < len(result.Authorities) { r.verboseLog((depth + 2), "--> Auth resolution of ", ns, " Failed: ", status, ". Will try next authority") @@ -631,6 +633,11 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, // that would normally be cache poison. Because it's "ok" and quite common res, status := checkGlue(server, *result, r.ipVersionMode, r.iterationIPPreference) if status != StatusNoError { + if ok, _ = nameIsBeneath(server, layer); ok { + // The domain we're searching for is beneath us but no glue was returned. We cannot proceed without this Glue. + // Terminating + return "", StatusNoNeededGlue, "", trace + } // Fall through to normal query var q Question q.Name = server @@ -642,7 +649,7 @@ func (r *Resolver) extractAuthority(ctx context.Context, authority interface{}, } res, trace, status, _ = r.iterativeLookup(ctx, q, r.randomRootNameServer(), depth+1, ".", trace) } - if status == StatusIterTimeout { + if status == StatusIterTimeout || status == StatusNoNeededGlue { return "", status, "", trace } if status == StatusNoError { diff --git a/src/zdns/util.go b/src/zdns/util.go index 4b4ae79e..d5571c5d 100644 --- a/src/zdns/util.go +++ b/src/zdns/util.go @@ -92,7 +92,9 @@ func checkGlueHelper(server, ansType string, result SingleQueryResult) (SingleQu if !ok { continue } - if ans.Type == ansType && strings.TrimSuffix(ans.Name, ".") == server { + // sanitize case and trailing dot + // RFC 4343 - states DNS names are case-insensitive + if ans.Type == ansType && strings.EqualFold(strings.TrimSuffix(ans.Name, "."), server) { var retv SingleQueryResult retv.Authorities = make([]interface{}, 0) retv.Answers = make([]interface{}, 0, 1) @@ -150,11 +152,13 @@ func TranslateDNSErrorCode(err int) Status { } // handleStatus is a helper function to deal with a status and error. Error is only returned if the status is an -// Iterative Timeout +// Iterative Timeout or NoNeededGlueRecord func handleStatus(status Status, err error) (Status, error) { switch status { case StatusIterTimeout: return status, err + case StatusNoNeededGlue: + return status, err case StatusNXDomain: return status, nil case StatusServFail: diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index 2de75f8c..650f13d6 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -47,8 +47,8 @@ def test_ipv6_unreachable(self): c = "A --iterative --6=true --4=false" name = "esrg.stanford.edu" cmd, res = self.run_zdns(c, name) - # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, this will fail. - self.assertServFail(res, cmd) + # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, the lookup won't get sufficient glue records to resolve the query. + self.assertEqual(res["status"], "NONEEDEDGLUE", cmd) def test_ipv6_external_lookup_unreachable_nameserver(self): c = "A --6=true --4=false --name-servers=1.1.1.1" From e9b675feaeeac5a4d6d00275961ace1374bcb2d2 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 13 Aug 2024 15:44:42 -0400 Subject: [PATCH 55/59] better err msg if user specifies IPv6 mode on non IPv6 capable machine --- src/cli/worker_manager.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 8e4a373e..1ab84ad3 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -272,10 +272,16 @@ func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Resolv log.Warn("Unable to parse resolvers file. Using ZDNS defaults: ", strings.Join(util.Concat(v4NameServers, v6NameServers), ", ")) } if config.IPVersionMode != zdns.IPv6Only { + if len(v4NameServers) == 0 { + return nil, errors.New("no IPv4 nameservers found. Please specify desired nameservers with --name-servers") + } config.ExternalNameServersV4 = v4NameServers config.RootNameServersV4 = v4NameServers } if config.IPVersionMode != zdns.IPv4Only { + if len(v6NameServers) == 0 { + return nil, errors.New("no IPv6 nameservers found. Please specify desired nameservers with --name-servers") + } config.ExternalNameServersV6 = v6NameServers config.RootNameServersV6 = v6NameServers } From 3a72eddea67ab2a69592c16b49bfc03585d74977 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 14 Aug 2024 12:45:48 -0400 Subject: [PATCH 56/59] infer IP support thru nameservers, use --4/6 as IPvX only --- src/cli/cli.go | 16 ++--- src/cli/config_validation.go | 4 -- src/cli/worker_manager.go | 126 ++++++++++++++++++++++++++++------- 3 files changed, 111 insertions(+), 35 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 13f69566..3cb9befa 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -69,10 +69,10 @@ type CLIConf struct { LookupAllNameServers bool TCPOnly bool UDPOnly bool - IPv4Transport bool - IPv6Transport bool - PreferIPv4Iteration bool // Prefer IPv4/A record lookups during iterative resolution - PreferIPv6Iteration bool // Prefer IPv6/AAAA record lookups during iterative resolution + IPv4TransportOnly bool // IPv4 transport only, incompatible with IPv6 transport only + IPv6TransportOnly bool // IPv6 transport only, incompatible with IPv4 transport only + PreferIPv4Iteration bool // Prefer IPv4/A record lookups during iterative resolution, only used if both IPv4 and IPv6 transport are enabled + PreferIPv6Iteration bool // Prefer IPv6/AAAA record lookups during iterative resolution, only used if both IPv4 and IPv6 transport are enabled RecycleSockets bool LocalAddrSpecified bool LocalAddrs []net.IP @@ -169,10 +169,10 @@ func init() { rootCmd.PersistentFlags().StringVar(&GC.NameServersString, "name-servers", "", "List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53.") rootCmd.PersistentFlags().StringVar(&GC.LocalAddrString, "local-addr", "", "comma-delimited list of local addresses to use, serve as the source IP for outbound queries") rootCmd.PersistentFlags().StringVar(&GC.LocalIfaceString, "local-interface", "", "local interface to use") - rootCmd.PersistentFlags().BoolVar(&GC.IPv4Transport, "4", false, "utilize IPv4 query transport, must have an IPv4 local address") - rootCmd.PersistentFlags().BoolVar(&GC.IPv6Transport, "6", false, "utilize IPv6 query transport, must have an IPv6 local address") - rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", false, "Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both --4 and --6") - rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both --4 and --6") + rootCmd.PersistentFlags().BoolVar(&GC.IPv4TransportOnly, "4", false, "utilize IPv4 query transport only, incompatible with --6") + rootCmd.PersistentFlags().BoolVar(&GC.IPv6TransportOnly, "6", false, "utilize IPv6 query transport only, incompatible with --4") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", false, "Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6") + rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6") rootCmd.PersistentFlags().StringVar(&GC.ConfigFilePath, "conf-file", zdns.DefaultNameServerConfigFile, "config file for DNS servers") rootCmd.PersistentFlags().IntVar(&GC.Timeout, "timeout", 15, "timeout for resolving a individual name, in seconds") diff --git a/src/cli/config_validation.go b/src/cli/config_validation.go index 3d3f06e2..e1964eca 100644 --- a/src/cli/config_validation.go +++ b/src/cli/config_validation.go @@ -31,10 +31,6 @@ func populateNetworkingConfig(gc *CLIConf) error { return errors.New("--local-addr and --local-interface cannot both be specified") } - if gc.IPv4Transport && gc.IPv6Transport && gc.PreferIPv4Iteration && gc.PreferIPv6Iteration { - return errors.New("both prefer IPv4 and prefer IPv6 iteration cannot both be enabled when using both IPv4 and IPv6 transport") - } - if err := parseNameServers(gc); err != nil { return errors.Wrap(err, "name servers could not be parsed") } diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 1ab84ad3..0238119e 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -165,20 +165,6 @@ func populateCLIConfig(gc *CLIConf, flags *pflag.FlagSet) *CLIConf { func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config := zdns.NewResolverConfig() - config.IPVersionMode = zdns.GetIPVersionMode(gc.IPv4Transport, gc.IPv6Transport) - // if we're in IPv4 or IPv6 only mode, set the iteration preference to match - // This is used in extractAuthorities where we need to know whether to request A or AAAA records to continue iteration - if config.IPVersionMode == zdns.IPv4Only { - config.IterationIPPreference = zdns.PreferIPv4 - } else if config.IPVersionMode == zdns.IPv6Only { - config.IterationIPPreference = zdns.PreferIPv6 - } else if config.IPVersionMode == zdns.IPv4OrIPv6 && !gc.PreferIPv4Iteration && !gc.PreferIPv6Iteration { - // need to specify some type of preference, we'll default to IPv4 and inform the user - log.Info("No iteration IP preference specified, defaulting to IPv4 preferred. See --prefer-ipv4-iteration and --prefer-ipv6-iteration for more info") - config.IterationIPPreference = zdns.PreferIPv4 - } else { - config.IterationIPPreference = zdns.GetIterationIPPreference(gc.PreferIPv4Iteration, gc.PreferIPv6Iteration) - } config.TransportMode = zdns.GetTransportMode(gc.UDPOnly, gc.TCPOnly) config.Timeout = time.Second * time.Duration(gc.Timeout) @@ -210,7 +196,27 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { } } // This must occur after setting the DNSConfigFilePath above, so that ZDNS knows where to fetch the DNS Config - config, err := populateNameServers(gc, config) + config, err := populateIPTransportMode(gc, config) + if err != nil { + log.Fatal("could not populate IP transport mode: ", err) + } + // This is used in extractAuthorities where we need to know whether to request A or AAAA records to continue iteration + // Must be set after populating IPTransportMode + if config.IPVersionMode == zdns.IPv4Only { + config.IterationIPPreference = zdns.PreferIPv4 + } else if config.IPVersionMode == zdns.IPv6Only { + config.IterationIPPreference = zdns.PreferIPv6 + } else if config.IPVersionMode == zdns.IPv4OrIPv6 && !gc.PreferIPv4Iteration && !gc.PreferIPv6Iteration { + // need to specify some type of preference, we'll default to IPv4 and inform the user + log.Info("No iteration IP preference specified, defaulting to IPv4 preferred. See --prefer-ipv4-iteration and --prefer-ipv6-iteration for more info") + config.IterationIPPreference = zdns.PreferIPv4 + } else if config.IPVersionMode == zdns.IPv4OrIPv6 && gc.PreferIPv4Iteration && gc.PreferIPv6Iteration { + log.Fatal("Cannot specify both --prefer-ipv4-iteration and --prefer-ipv6-iteration") + } else { + config.IterationIPPreference = zdns.GetIterationIPPreference(gc.PreferIPv4Iteration, gc.PreferIPv6Iteration) + } + // This must occur after setting the DNSConfigFilePath above, so that ZDNS knows where to fetch the DNS Config + config, err = populateNameServers(gc, config) if err != nil { log.Fatal("could not populate name servers: ", err) } @@ -220,6 +226,17 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { config.ExternalNameServersV6 = util.RemoveDuplicates(config.ExternalNameServersV6) config.RootNameServersV6 = util.RemoveDuplicates(config.RootNameServersV6) + if config.IPVersionMode == zdns.IPv4Only { + // Drop any IPv6 nameservers + config.ExternalNameServersV6 = []string{} + config.RootNameServersV6 = []string{} + } + if config.IPVersionMode == zdns.IPv6Only { + // Drop any IPv4 nameservers + config.ExternalNameServersV4 = []string{} + config.RootNameServersV4 = []string{} + } + config, err = populateLocalAddresses(gc, config) if err != nil { log.Fatal("could not populate local addresses: ", err) @@ -227,6 +244,72 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { return config } +// populateIPTransportMode populates the IPTransportMode field of the ResolverConfig +// If user sets --4 (IPv4 Only) or --6 (IPv6 Only), we'll set the IPVersionMode to IPv4Only or IPv6Only, respectively. +// Otherwise, we need to determine the IPVersionMode based on either the OS' default resolver(s) or the user's provided name servers. +// Note: populateNameServers must be called before this function to ensure the nameservers are populated. +func populateIPTransportMode(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.ResolverConfig, error) { + if gc.IPv4TransportOnly { + config.IPVersionMode = zdns.IPv4Only + return config, nil + } + if gc.IPv6TransportOnly { + config.IPVersionMode = zdns.IPv6Only + return config, nil + } + nameServersSupportIPv4 := false + nameServersSupportIPv6 := false + // Check if user provided nameservers + if len(gc.NameServers) != 0 { + // Check that the nameservers have a port and append one if necessary + portValidatedNSs := make([]string, 0, len(gc.NameServers)) + // check that the nameservers have a port and append one if necessary + for _, ns := range gc.NameServers { + portNS, err := util.AddDefaultPortToDNSServerName(ns) + if err != nil { + return nil, fmt.Errorf("could not parse name server: %s. Correct IPv4 format: 1.1.1.1:53 or IPv6 format: [::1]:53", ns) + } + portValidatedNSs = append(portValidatedNSs, portNS) + } + v4NameServers, v6NameServers, err := util.SplitIPv4AndIPv6Addrs(portValidatedNSs) + if err != nil { + return nil, errors.Wrap(err, "could not split IPv4 and IPv6 addresses for nameservers") + } + if len(v4NameServers) != 0 { + nameServersSupportIPv4 = true + } + if len(v6NameServers) != 0 { + nameServersSupportIPv6 = true + } + } else { + // User did not provide nameservers, check the OS' default resolver(s) + v4NameServers, v6NameServers, err := zdns.GetDNSServers(config.DNSConfigFilePath) + if err != nil { + log.Warn("Unable to parse resolvers file to determine if IPv4 or IPv6 is supported. Defaulting to IPv4") + config.IPVersionMode = zdns.IPv4Only + return config, nil + } + if len(v4NameServers) != 0 { + nameServersSupportIPv4 = true + } + if len(v6NameServers) != 0 { + nameServersSupportIPv6 = true + } + } + if nameServersSupportIPv4 && nameServersSupportIPv6 { + config.IPVersionMode = zdns.IPv4OrIPv6 + return config, nil + } else if nameServersSupportIPv4 { + config.IPVersionMode = zdns.IPv4Only + return config, nil + } else if nameServersSupportIPv6 { + config.IPVersionMode = zdns.IPv6Only + return config, nil + } else { + return nil, errors.New("no nameservers found with OS defaults. Please specify desired nameservers with --name-servers") + } +} + func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.ResolverConfig, error) { // Nameservers are populated in this order: // 1. If user provided nameservers, use those @@ -253,14 +336,11 @@ func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Resolv if err != nil { return nil, errors.Wrap(err, "could not split IPv4 and IPv6 addresses for nameservers") } - if config.IPVersionMode != zdns.IPv6Only { - config.ExternalNameServersV4 = v4NameServers - config.RootNameServersV4 = v4NameServers - } - if config.IPVersionMode != zdns.IPv4Only { - config.ExternalNameServersV6 = v6NameServers - config.RootNameServersV6 = v6NameServers - } + // The resolver will ignore IPv6 nameservers if we're doing IPv4 only lookups, and vice versa so this is fine + config.ExternalNameServersV4 = v4NameServers + config.RootNameServersV4 = v4NameServers + config.ExternalNameServersV6 = v6NameServers + config.RootNameServersV6 = v6NameServers return config, nil } // User did not provide nameservers From 433bd3c441ad2456744cb9b0d658cac4c8092fb6 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 14 Aug 2024 13:11:14 -0400 Subject: [PATCH 57/59] disallow both --4 and --6 --- src/cli/worker_manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cli/worker_manager.go b/src/cli/worker_manager.go index 0238119e..51b1bd7f 100644 --- a/src/cli/worker_manager.go +++ b/src/cli/worker_manager.go @@ -249,6 +249,9 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig { // Otherwise, we need to determine the IPVersionMode based on either the OS' default resolver(s) or the user's provided name servers. // Note: populateNameServers must be called before this function to ensure the nameservers are populated. func populateIPTransportMode(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.ResolverConfig, error) { + if gc.IPv4TransportOnly && gc.IPv6TransportOnly { + return nil, errors.New("only one of --4 and --6 allowed") + } if gc.IPv4TransportOnly { config.IPVersionMode = zdns.IPv4Only return config, nil From a71f07a97839174fc0bfdacc41ae25d3db5a2981 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 14 Aug 2024 13:12:36 -0400 Subject: [PATCH 58/59] tests and lints --- src/cli/alookup.go | 1 + src/cli/config_validation_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cli/alookup.go b/src/cli/alookup.go index 0c079840..421d134e 100644 --- a/src/cli/alookup.go +++ b/src/cli/alookup.go @@ -11,6 +11,7 @@ * implied. See the License for the specific language governing * permissions and limitations under the License. */ + package cli import ( diff --git a/src/cli/config_validation_test.go b/src/cli/config_validation_test.go index 53310f01..334a1593 100644 --- a/src/cli/config_validation_test.go +++ b/src/cli/config_validation_test.go @@ -22,17 +22,17 @@ import ( func TestValidateNetworkingConfig(t *testing.T) { t.Run("LocalAddr and LocalInterface both specified", func(t *testing.T) { gc := &CLIConf{ - LocalAddrString: "1.1.1.1", - LocalIfaceString: "eth0", - IPv4Transport: true, + LocalAddrString: "1.1.1.1", + LocalIfaceString: "eth0", + IPv4TransportOnly: true, } err := populateNetworkingConfig(gc) require.NotNil(t, err, "Expected an error but got nil") }) t.Run("Using invalid interface", func(t *testing.T) { gc := &CLIConf{ - LocalIfaceString: "invalid_interface", - IPv4Transport: true, + LocalIfaceString: "invalid_interface", + IPv4TransportOnly: true, } err := populateNetworkingConfig(gc) require.NotNil(t, err, "Expected an error but got nil") @@ -40,7 +40,7 @@ func TestValidateNetworkingConfig(t *testing.T) { t.Run("Using nameserver with port", func(t *testing.T) { gc := &CLIConf{ NameServersString: "127.0.0.1:53", - IPv4Transport: true, + IPv4TransportOnly: true, } err := populateNetworkingConfig(gc) require.Nil(t, err, "Expected no error but got %v", err) From 14969625f9ed59fa53ed0ad30bc87974c9da6516 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Wed, 14 Aug 2024 13:16:23 -0400 Subject: [PATCH 59/59] fixed up ipv6 tests --- testing/ipv6_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/ipv6_tests.py b/testing/ipv6_tests.py index 650f13d6..3205a7b4 100644 --- a/testing/ipv6_tests.py +++ b/testing/ipv6_tests.py @@ -37,14 +37,14 @@ def assertEqualAnswers(self, res, correct, cmd, key="answer"): json.dumps(b, indent=4), json.dumps(a, indent=4)) def test_a_ipv6(self): - c = "A --6=true --name-servers=[2001:4860:4860::8888]:53" + c = "A --name-servers=[2001:4860:4860::8888]:53" name = "zdns-testing.com" cmd, res = self.run_zdns(c, name) self.assertSuccess(res, cmd) self.assertEqualAnswers(res, self.ROOT_A_ANSWERS, cmd) def test_ipv6_unreachable(self): - c = "A --iterative --6=true --4=false" + c = "A --iterative --6" name = "esrg.stanford.edu" cmd, res = self.run_zdns(c, name) # esrg.stanford.edu is hosted on NS's that do not have an IPv6 address. Therefore, the lookup won't get sufficient glue records to resolve the query.