[Ruler][DNS] Don't propagate no such host error if using default resolver

CHANGELOG.md

@@ -11,6 +11,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ## Unreleased
 
+### Fixed
+
+## [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.
+
 ## [v0.16.0](https://github.com/thanos-io/thanos/releases) - Release in progress
 
 ### Fixed

pkg/discovery/dns/provider.go

@@ -37,6 +37,7 @@ type ResolverType string
 const (
 	GolangResolverType   ResolverType = "golang"
 	MiekgdnsResolverType ResolverType = "miekgdns"
+	MockdnsResolverType  ResolverType = "mockdns"
 )
 
 func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
@@ -57,7 +58,7 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
 // If empty resolver type is net.DefaultResolver.
 func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider {
 	p := &Provider{
-		resolver: NewResolver(resolverType.ToResolver(logger)),
+		resolver: NewResolver(resolverType.ToResolver(logger), resolverType),
 		resolved: make(map[string][]string),
 		logger:   logger,
 		resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{

pkg/discovery/dns/resolver.go

@@ -37,12 +37,13 @@ type ipLookupResolver interface {
 }
 
 type dnsSD struct {
-	resolver ipLookupResolver
+	resolver     ipLookupResolver
+	resolverType ResolverType
 }
 
 // NewResolver creates a resolver with given underlying resolver.
-func NewResolver(resolver ipLookupResolver) Resolver {
-	return &dnsSD{resolver: resolver}
+func NewResolver(resolver ipLookupResolver, resolverType ResolverType) Resolver {
+	return &dnsSD{resolver: resolver, resolverType: resolverType}
 }
 
 func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) {
@@ -71,7 +72,10 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		}
 		ips, err := s.resolver.LookupIPAddr(ctx, host)
 		if err != nil {
-			return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			dnsErr, ok := err.(*net.DNSError)
+			if !(s.resolverType == GolangResolverType && ok && dnsErr.IsNotFound) {
+				return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			}
 		}
 		for _, ip := range ips {
 			res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port)))

pkg/discovery/dns/resolver_test.go

@@ -184,7 +184,7 @@ func TestDnsSD_Resolve(t *testing.T) {
 
 func testDnsSd(t *testing.T, tt DNSSDTest) {
 	ctx := context.TODO()
-	dnsSD := dnsSD{tt.resolver}
+	dnsSD := dnsSD{tt.resolver, MockdnsResolverType}
 
 	result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype)
 	if tt.expectedErr != nil {