Log any FQDN lookup errors and fallback to OS-reported hostname (#34946)

* Log any FQDN lookup errors, fallback to OS hostname, and move on.

* Implement fallback logic

* Bumping up go-sysinfo dependency version

* Update call to host.ReportInfo

* Better log message

* Bumping up version on elastic-agent-system-metrics dependency

* Updating NOTICE.txt

* Log FQDN lookup failure as warning

* Move FQDN initialization to after logging has been configured

* Better log message

* Add metric + test

* Update NOTICE.txt

* Use a single namespace for all add_host_metadata processor instances' monitoring

* Fixing imports

* Create monitoring registry only once

* Check errors

NOTICE.txt

@@ -10626,11 +10626,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.5.0
+Version: v0.5.1-0.20230328163318-a87a0b315c42
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1-0.20230328163318-a87a0b315c42/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -12566,11 +12566,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.9.1-0.20230215152520-f544eca983fb
+Version: v1.9.1-0.20230328042007-6dcfe88b8359
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20230215152520-f544eca983fb/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20230328042007-6dcfe88b8359/LICENSE.txt:
 
 
                                  Apache License
@@ -13684,6 +13684,37 @@ Contents of probable licence file $GOMODCACHE/github.com/fearful-symmetry/gorapl
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/foxcpp/go-mockdns
+Version: v0.0.0-20201212160233-ede2f9158d15
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/foxcpp/[email protected]/LICENSE:
+
+MIT License
+
+Copyright © 2019 Max Mazurov <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/fsevents
 Version: v0.0.0-20181029231046-e1d381a4d270

go.mod

@@ -79,7 +79,7 @@ require (
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
 	github.com/elastic/go-seccomp-bpf v1.3.0
 	github.com/elastic/go-structform v0.0.10
-	github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb
+	github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359
 	github.com/elastic/go-ucfg v0.8.6
 	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.13.0
@@ -196,10 +196,11 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.5.0
 	github.com/elastic/elastic-agent-libs v0.3.3
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
-	github.com/elastic/elastic-agent-system-metrics v0.5.0
+	github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42
 	github.com/elastic/go-elasticsearch/v8 v8.2.0
 	github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
+	github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15
 	github.com/google/cel-go v0.13.0
 	github.com/googleapis/gax-go/v2 v2.6.0
 	github.com/gorilla/handlers v1.5.1

go.sum

@@ -618,8 +618,8 @@ github.com/elastic/elastic-agent-libs v0.3.3 h1:iE8XhqQ0zRBLba+eu6ScZED0DYcVP/r2
 github.com/elastic/elastic-agent-libs v0.3.3/go.mod h1:nRkcK96PSJfK232cJRx17n9+/MVAIOzs5ghZdzXJAMo=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.5.0 h1:1pJHWsq5FRQbUCk4JFTDY3ULa4f4+FA6H6rDhTr051s=
-github.com/elastic/elastic-agent-system-metrics v0.5.0/go.mod h1:v/t/qgYueW3ZOm7SZhYY3ng9GWDddDLu7pmG4Ra3PBs=
+github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42 h1:oJQbTUpFlD/CXKw6N5gWNOjfofwStp+nR49m1wKv6zc=
+github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42/go.mod h1:jBVJTZFtoxQkGmqutGGRSulkRhjgIO2PY059m+HLG4k=
 github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
 github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
@@ -647,8 +647,8 @@ github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZN
 github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
 github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
-github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb h1:dTYv5aVpYzI53KJwDM/Hue68YU/PHkLJD/5jDl1fGSs=
-github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb/go.mod h1:j/gGAinRS+z3loQ/1pO+s9pCyQsna8U13kqhF1Aa0fg=
+github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359 h1:qSCKykLGlUcO/wzq3xH+wtPHM2KCpQnqXOKClSg5K2w=
+github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359/go.mod h1:j/gGAinRS+z3loQ/1pO+s9pCyQsna8U13kqhF1Aa0fg=
 github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
 github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0=
 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
@@ -705,6 +705,7 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNy
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 h1:nLPjjvpUAODOR6vY/7o0hBIk8iTr19Fvmf8aFx/kC7A=
 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=

libbeat/cmd/instance/beat.go

@@ -244,12 +244,6 @@ func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) {
 		return nil, err
 	}
 
-	h, err := sysinfo.Host()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get host information: %w", err)
-	}
-	fqdn := h.Info().FQDN
-
 	fields, err := asset.GetFields(name)
 	if err != nil {
 		return nil, err
@@ -268,7 +262,6 @@ func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) {
 			Version:         v,
 			Name:            hostname,
 			Hostname:        hostname,
-			FQDN:            fqdn,
 			ID:              id,
 			FirstStart:      time.Now(),
 			StartTime:       time.Now(),
@@ -595,7 +588,7 @@ func (b *Beat) RegisterHostname(useFQDN bool) {
 
 	// state.host
 	stateRegistry := monitoring.GetNamespace("state").GetRegistry()
-	monitoring.NewFunc(stateRegistry, "host", host.ReportInfo(useFQDN), monitoring.Report)
+	monitoring.NewFunc(stateRegistry, "host", host.ReportInfo(hostname), monitoring.Report)
 }
 
 // TestConfig check all settings are ok and the beat can be run
@@ -796,6 +789,22 @@ func (b *Beat) configure(settings Settings) error {
 
 	logp.Info("Beat ID: %v", b.Info.ID)
 
+	// Try to get the host's FQDN and set it.
+	h, err := sysinfo.Host()
+	if err != nil {
+		return fmt.Errorf("failed to get host information: %w", err)
+	}
+
+	fqdn, err := h.FQDN()
+	if err != nil {
+		// FQDN lookup is "best effort".  We log the error, fallback to
+		// the OS-reported hostname, and move on.
+		logp.Warn("unable to lookup FQDN: %s, using hostname = %s as FQDN", err.Error(), b.Info.Hostname)
+		b.Info.FQDN = b.Info.Hostname
+	} else {
+		b.Info.FQDN = fqdn
+	}
+
 	// initialize config manager
 	b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID)
 	if err != nil {

libbeat/processors/add_host_metadata/add_host_metadata.go

@@ -22,6 +22,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/elastic/elastic-agent-libs/monitoring"
+
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/features"
 	"github.com/elastic/beats/v7/libbeat/processors"
@@ -34,9 +36,22 @@ import (
 	"github.com/elastic/go-sysinfo"
 )
 
+const processorName = "add_host_metadata"
+const logName = "processor." + processorName
+
+var (
+	reg *monitoring.Registry
+)
+
 func init() {
-	processors.RegisterPlugin("add_host_metadata", New)
+	processors.RegisterPlugin(processorName, New)
 	jsprocessor.RegisterPlugin("AddHostMetadata", New)
+
+	reg = monitoring.Default.NewRegistry(logName, monitoring.DoNotReport)
+}
+
+type metrics struct {
+	FQDNLookupFailed *monitoring.Int
 }
 
 type addHostMetadata struct {
@@ -48,12 +63,9 @@ type addHostMetadata struct {
 	geoData mapstr.M
 	config  Config
 	logger  *logp.Logger
+	metrics metrics
 }
 
-const (
-	processorName = "add_host_metadata"
-)
-
 // New constructs a new add_host_metadata processor.
 func New(cfg *config.C) (processors.Processor, error) {
 	c := defaultConfig()
@@ -64,7 +76,10 @@ func New(cfg *config.C) (processors.Processor, error) {
 	p := &addHostMetadata{
 		config: c,
 		data:   mapstr.NewPointer(nil),
-		logger: logp.NewLogger("add_host_metadata"),
+		logger: logp.NewLogger(logName),
+		metrics: metrics{
+			FQDNLookupFailed: monitoring.NewInt(reg, "fqdn_lookup_failed"),
+		},
 	}
 	if err := p.loadData(); err != nil {
 		return nil, fmt.Errorf("failed to load data: %w", err)
@@ -143,7 +158,25 @@ func (p *addHostMetadata) loadData() error {
 		return err
 	}
 
-	data := host.MapHostInfo(features.FQDN(), h.Info())
+	hostname := h.Info().Hostname
+	if features.FQDN() {
+		fqdn, err := h.FQDN()
+		if err != nil {
+			// FQDN lookup is "best effort". If it fails, we monitor the failure, fallback to
+			// the OS-reported hostname, and move on.
+			p.metrics.FQDNLookupFailed.Inc()
+			p.logger.Debugf(
+				"unable to lookup FQDN (failed attempt counter: %d): %s, using hostname = %s as FQDN",
+				p.metrics.FQDNLookupFailed.Get(),
+				err.Error(),
+				hostname,
+			)
+		} else {
+			hostname = fqdn
+		}
+	}
+
+	data := host.MapHostInfo(h.Info(), hostname)
 	if p.config.NetInfoEnabled {
 		// IP-address and MAC-address
 		var ipList, hwList, err = util.GetNetInfo()

libbeat/processors/add_host_metadata/add_host_metadata_test.go

@@ -19,19 +19,22 @@ package add_host_metadata
 
 import (
 	"fmt"
+	"net"
+	"os"
 	"runtime"
 	"testing"
 	"time"
 
-	"github.com/elastic/beats/v7/libbeat/features"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/features"
 	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/go-sysinfo/types"
+
+	"github.com/foxcpp/go-mockdns"
 )
 
 var (
@@ -492,17 +495,99 @@ func TestExpireCacheOnFQDNReportingChange(t *testing.T) {
 
 	// Toggle the FQDN feature flag; this should cause the cache
 	// to expire.
-	features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{
+	err = features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{
 		"features.fqdn.enabled": true,
 	}))
+	require.NoError(t, err)
+
 	expired = ahmP.expired()
 	require.True(t, expired)
 
 	// Set the FQDN feature flag to the same value; this should NOT
 	// cause the cache to expire.
-	features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{
+	err = features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{
 		"features.fqdn.enabled": true,
 	}))
+	require.NoError(t, err)
+
 	expired = ahmP.expired()
 	require.False(t, expired)
 }
+
+func TestFQDNLookup(t *testing.T) {
+	hostname, err := os.Hostname()
+	require.NoError(t, err)
+
+	tests := map[string]struct {
+		cnameLookupResult             string
+		expectedHostName              string
+		expectedFQDNLookupFailedCount int64
+	}{
+		"lookup_succeeds": {
+			cnameLookupResult:             "foo.bar.baz.",
+			expectedHostName:              "foo.bar.baz",
+			expectedFQDNLookupFailedCount: 0,
+		},
+		"lookup_fails": {
+			cnameLookupResult:             "",
+			expectedHostName:              hostname,
+			expectedFQDNLookupFailedCount: 1,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			// Mock CNAME resolution
+			srv, _ := mockdns.NewServer(map[string]mockdns.Zone{
+				hostname + ".": {
+					CNAME: test.cnameLookupResult,
+				},
+				test.cnameLookupResult: {
+					A: []string{"1.1.1.1"},
+				},
+			}, false)
+			defer srv.Close()
+
+			srv.PatchNet(net.DefaultResolver)
+			defer mockdns.UnpatchNet(net.DefaultResolver)
+
+			// Enable FQDN feature flag
+			err = features.UpdateFromConfig(fqdnFeatureFlagConfig(true))
+			require.NoError(t, err)
+			defer func() {
+				err = features.UpdateFromConfig(fqdnFeatureFlagConfig(true))
+				require.NoError(t, err)
+			}()
+
+			// Create processor and check that FQDN lookup failed
+			testConfig, err := conf.NewConfigFrom(map[string]interface{}{})
+			require.NoError(t, err)
+
+			p, err := New(testConfig)
+			require.NoError(t, err)
+
+			addHostMetadataP, ok := p.(*addHostMetadata)
+			require.True(t, ok)
+			require.Equal(t, test.expectedFQDNLookupFailedCount, addHostMetadataP.metrics.FQDNLookupFailed.Get())
+
+			// Run event through processor and check that hostname reported
+			// by processor is same as OS-reported hostname
+			event := &beat.Event{
+				Fields:    mapstr.M{},
+				Timestamp: time.Now(),
+			}
+			newEvent, err := p.Run(event)
+			require.NoError(t, err)
+
+			v, err := newEvent.GetValue("host.name")
+			require.NoError(t, err)
+			require.Equal(t, test.expectedHostName, v)
+		})
+	}
+}
+
+func fqdnFeatureFlagConfig(fqdnEnabled bool) *conf.C {
+	return conf.MustNewConfigFrom(map[string]interface{}{
+		"features.fqdn.enabled": fqdnEnabled,
+	})
+}