diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 4ce74da495..374265db16 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -265,6 +265,7 @@ func runQuery( logger, extprom.WrapRegistererWithPrefix("thanos_querier_store_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) for _, store := range strictStores { @@ -277,6 +278,7 @@ func runQuery( logger, extprom.WrapRegistererWithPrefix("thanos_querier_rule_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) var ( diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index f72bbedfd2..17079cb943 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -338,6 +338,7 @@ func runRule( logger, extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) var queryClients []*http_util.Client for _, cfg := range queryCfg { @@ -401,6 +402,7 @@ func runRule( logger, extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg), dns.ResolverType(dnsSDResolver), + false, ) var alertmgrs []*alert.Alertmanager for _, cfg := range alertingCfg.Alertmanagers { diff --git a/pkg/cacheutil/memcached_client.go b/pkg/cacheutil/memcached_client.go index 15751109dc..3becb31e79 100644 --- a/pkg/cacheutil/memcached_client.go +++ b/pkg/cacheutil/memcached_client.go @@ -219,6 +219,7 @@ func newMemcachedClient( logger, extprom.WrapRegistererWithPrefix("thanos_memcached_", reg), dns.GolangResolverType, + true, ) c := &memcachedClient{ diff --git a/pkg/discovery/dns/provider.go b/pkg/discovery/dns/provider.go index 4df7c07f05..11867345ac 100644 --- a/pkg/discovery/dns/provider.go +++ b/pkg/discovery/dns/provider.go @@ -37,7 +37,6 @@ type ResolverType string const ( GolangResolverType ResolverType = "golang" MiekgdnsResolverType ResolverType = "miekgdns" - MockdnsResolverType ResolverType = "mockdns" ) func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver { @@ -56,9 +55,9 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver { // NewProvider returns a new empty provider with a given resolver type. // If empty resolver type is net.DefaultResolver. -func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider { +func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType, returnErrOnNotFound bool) *Provider { p := &Provider{ - resolver: NewResolver(resolverType.ToResolver(logger), resolverType), + resolver: NewResolver(resolverType.ToResolver(logger), logger, returnErrOnNotFound), resolved: make(map[string][]string), logger: logger, resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{ diff --git a/pkg/discovery/dns/provider_test.go b/pkg/discovery/dns/provider_test.go index 668d5a65c8..87277c3d22 100644 --- a/pkg/discovery/dns/provider_test.go +++ b/pkg/discovery/dns/provider_test.go @@ -22,7 +22,7 @@ func TestProvider(t *testing.T) { "127.0.0.5:19095", } - prv := NewProvider(log.NewNopLogger(), nil, "") + prv := NewProvider(log.NewNopLogger(), nil, "", true) prv.resolver = &mockResolver{ res: map[string][]string{ "a": ips[:2], diff --git a/pkg/discovery/dns/resolver.go b/pkg/discovery/dns/resolver.go index d9be897829..9837f8e362 100644 --- a/pkg/discovery/dns/resolver.go +++ b/pkg/discovery/dns/resolver.go @@ -9,6 +9,9 @@ import ( "strconv" "strings" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" ) @@ -37,13 +40,16 @@ type ipLookupResolver interface { } type dnsSD struct { - resolver ipLookupResolver - resolverType ResolverType + resolver ipLookupResolver + logger log.Logger + // https://github.com/thanos-io/thanos/issues/3186 + // This flag is used to prevent components from crashing if hosts are not found. + returnErrOnNotFound bool } // NewResolver creates a resolver with given underlying resolver. -func NewResolver(resolver ipLookupResolver, resolverType ResolverType) Resolver { - return &dnsSD{resolver: resolver, resolverType: resolverType} +func NewResolver(resolver ipLookupResolver, logger log.Logger, returnErrOnNotFound bool) Resolver { + return &dnsSD{resolver: resolver, logger: logger, returnErrOnNotFound: returnErrOnNotFound} } func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) { @@ -73,9 +79,13 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string ips, err := s.resolver.LookupIPAddr(ctx, host) if err != nil { dnsErr, ok := err.(*net.DNSError) - if !(s.resolverType == GolangResolverType && ok && dnsErr.IsNotFound) { + // https://github.com/thanos-io/thanos/issues/3186 + // Default DNS resolver can make thanos components crash if DSN resolutions results in EAI_NONAME. + // the flag returnErrOnNotFound can be used to prevent such crash. + if !(!s.returnErrOnNotFound && ok && dnsErr.IsNotFound) { return nil, errors.Wrapf(err, "lookup IP addresses %q", host) } + level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err) } for _, ip := range ips { res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port))) @@ -110,6 +120,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string return nil, errors.Errorf("invalid lookup scheme %q", qtype) } + // https://github.com/thanos-io/thanos/issues/3186 + // This happens when miekg is used as resolver. When the host cannot be found, nothing is returned. + if res == nil && err == nil { + level.Warn(s.logger).Log("msg", "IP address lookup yielded no results nor errors", "host", host) + } + return res, nil } diff --git a/pkg/discovery/dns/resolver_test.go b/pkg/discovery/dns/resolver_test.go index b6635c8fe3..a12a9be163 100644 --- a/pkg/discovery/dns/resolver_test.go +++ b/pkg/discovery/dns/resolver_test.go @@ -9,6 +9,8 @@ import ( "sort" "testing" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -184,7 +186,7 @@ func TestDnsSD_Resolve(t *testing.T) { func testDnsSd(t *testing.T, tt DNSSDTest) { ctx := context.TODO() - dnsSD := dnsSD{tt.resolver, MockdnsResolverType} + dnsSD := dnsSD{tt.resolver, log.NewNopLogger(), false} result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype) if tt.expectedErr != nil {