Merge branch 'main' into serverless-aware

.go-version

@@ -1 +1 @@
-1.20.7
+1.20.8

.golangci.yml

@@ -113,7 +113,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -131,19 +131,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
 
   gosec:
     excludes:

CHANGELOG.asciidoc

@@ -3,6 +3,18 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.10.1]]
+=== Beats version 8.10.1
+https://github.com/elastic/beats/compare/v8.10.0\...v8.10.1[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
+- Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
+
+
 [[release-notes-8.10.0]]
 === Beats version 8.10.0
 https://github.com/elastic/beats/compare/v8.9.2\...v8.10.0[View commits]

CHANGELOG.next.asciidoc

@@ -9,10 +9,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 ==== Breaking changes
 
 *Affecting all Beats*
-- Fix status reporting to Elastic-Agent when output configuration is invalid running under Elastic-Agent {pull}35719[35719]
-- Upgrade Go to 1.20.7 {pull}36241[36241]
-- [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
-  Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
 
 *Auditbeat*
 
@@ -63,7 +59,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support build of projects outside of beats directory {pull}36126[36126]
 - Add default cgroup regex for add_process_metadata processor {pull}36484[36484] {issue}32961[32961]
 - Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471]
-- Support fattened `data_stream` object when running under Elastic-Agent {pr}36516[36516]
 
 
 *Auditbeat*
@@ -150,13 +145,16 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
+- Upgrade Go to 1.20.8 {pull}36597[36597]
 - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364]
 - When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183]
 - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
 - Mark `translate_sid` processor is GA. {issue}36279[36279] {pull}36280[36280]
 - dns processor: Add support for forward lookups (`A`, `AAAA`, and `TXT`). {issue}11416[11416] {pull}36394[36394]
 - Mark `syslog` processor as GA, improve docs about how processor handles syslog messages. {issue}36416[36416] {pull}36417[36417]
 - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322]
+- [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
+  Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
 
 *Auditbeat*
 
@@ -302,3 +300,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
+
+
+

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.7
+FROM golang:1.20.8
 
 RUN \
     apt-get update \

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.7
+FROM golang:1.20.8
 
 RUN \
     apt-get update \

heartbeat/hbtest/hbtestutil.go

@@ -40,6 +40,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/ecserr"
 	"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/summarizertesthelper"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil"
 
 	"github.com/elastic/beats/v7/heartbeat/hbtestllext"
 
@@ -49,7 +50,6 @@ import (
 	"github.com/elastic/go-lookslike/isdef"
 	"github.com/elastic/go-lookslike/validator"
 
-	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/libbeat/common/x509util"
 )
 
@@ -172,6 +172,7 @@ func BaseChecks(ip string, status string, typ string) validator.Validator {
 	}
 
 	return lookslike.Compose(
+		hbtestllext.MaybeHasEventType,
 		lookslike.MustCompile(map[string]interface{}{
 			"monitor": map[string]interface{}{
 				"ip":          ipCheck,
@@ -223,8 +224,10 @@ func SimpleURLChecks(t *testing.T, scheme string, host string, port uint16) vali
 
 // URLChecks returns a validator for the given URL's fields
 func URLChecks(t *testing.T, u *url.URL) validator.Validator {
+	t.Helper()
+	require.NotNil(t, u)
 	return lookslike.MustCompile(map[string]interface{}{
-		"url": wrappers.URLFields(u),
+		"url": wraputil.URLFields(u),
 	})
 }
 

heartbeat/hbtestllext/validators.go

@@ -19,6 +19,7 @@ package hbtestllext
 
 import (
 	"github.com/elastic/go-lookslike"
+	"github.com/elastic/go-lookslike/isdef"
 )
 
 // MonitorTimespanValidator is tests for the `next_run` and `next_run_in.us` keys.
@@ -30,3 +31,14 @@ var MonitorTimespanValidator = lookslike.MustCompile(map[string]interface{}{
 		},
 	},
 })
+
+var MaybeHasEventType = lookslike.MustCompile(map[string]interface{}{
+	"event": map[string]interface{}{
+		"type": isdef.Optional(isdef.IsNonEmptyString),
+	},
+	"synthetics.type": isdef.Optional(isdef.IsNonEmptyString),
+})
+
+var MaybeHasDuration = lookslike.MustCompile(map[string]interface{}{
+	"monitor.duration.us": IsInt64,
+})

heartbeat/look/look.go

@@ -31,16 +31,21 @@ import (
 // RTT formats a round-trip-time given as time.Duration into an
 // event field. The duration is stored in `{"us": rtt}`.
 func RTT(rtt time.Duration) mapstr.M {
-	if rtt < 0 {
-		rtt = 0
-	}
-
 	return mapstr.M{
 		// cast to int64 since a go duration is a nano, but we want micros
 		// This makes the types less confusing because other wise the duration
 		// we get back has the wrong unit
-		"us": rtt.Microseconds(),
+		"us": RTTMS(rtt),
+	}
+}
+
+// RTTMS returns the given time.Duration as an int64 in microseconds, with a value of 0
+// if input is negative.
+func RTTMS(rtt time.Duration) int64 {
+	if rtt < 0 {
+		return 0
 	}
+	return rtt.Microseconds()
 }
 
 // Reason formats an error into an error event field.

heartbeat/monitors/active/http/http.go

@@ -23,11 +23,11 @@ import (
 	"net/url"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil"
 	"github.com/elastic/beats/v7/libbeat/version"
 	conf "github.com/elastic/elastic-agent-libs/config"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
-	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
 	"github.com/elastic/elastic-agent-libs/useragent"
@@ -116,7 +116,7 @@ func create(
 
 		// Assign any execution errors to the error field and
 		// assign the url field
-		js[i] = wrappers.WithURLField(u, job)
+		js[i] = wraputil.WithURLField(u, job)
 	}
 
 	return plugin.Plugin{Jobs: js, Endpoints: len(config.Hosts)}, nil

heartbeat/monitors/active/http/http_test.go

@@ -52,6 +52,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil"
 	"github.com/elastic/beats/v7/heartbeat/scheduler/schedule"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common/file"
@@ -110,7 +111,7 @@ func checkServer(t *testing.T, handlerFunc http.HandlerFunc, useUrls bool) (*htt
 func urlChecks(urlStr string) validator.Validator {
 	u, _ := url.Parse(urlStr)
 	return lookslike.MustCompile(map[string]interface{}{
-		"url": wrappers.URLFields(u),
+		"url": wraputil.URLFields(u),
 	})
 }
 

heartbeat/monitors/active/icmp/icmp.go

@@ -23,14 +23,14 @@ import (
 	"net/url"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil"
 	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/heartbeat/look"
 	"github.com/elastic/beats/v7/heartbeat/monitors"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
-	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
@@ -107,7 +107,7 @@ func (jf *jobFactory) makePlugin() (plugin2 plugin.Plugin, err error) {
 			return plugin.Plugin{}, err
 		}
 
-		j = append(j, wrappers.WithURLField(u, job))
+		j = append(j, wraputil.WithURLField(u, job))
 	}
 
 	return plugin.Plugin{Jobs: j, Endpoints: len(jf.config.Hosts)}, nil

heartbeat/monitors/active/tcp/tcp.go

@@ -31,7 +31,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
-	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil"
 	"github.com/elastic/beats/v7/heartbeat/reason"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	conf "github.com/elastic/elastic-agent-libs/config"
@@ -130,7 +130,7 @@ func (jf *jobFactory) makeJobs() ([]jobs.Job, error) {
 			if err != nil {
 				return nil, err
 			}
-			jobs = append(jobs, wrappers.WithURLField(url, endpointJob))
+			jobs = append(jobs, wraputil.WithURLField(url, endpointJob))
 		}
 
 	}
@@ -174,7 +174,7 @@ func (jf *jobFactory) makeDirectEndpointJob(endpointURL *url.URL) (jobs.Job, err
 
 // makeSocksLookupEndpointJob makes jobs that use a Socks5 proxy to perform DNS lookups
 func (jf *jobFactory) makeSocksLookupEndpointJob(endpointURL *url.URL) jobs.Job {
-	return wrappers.WithURLField(endpointURL,
+	return wraputil.WithURLField(endpointURL,
 		jobs.MakeSimpleJob(func(event *beat.Event) error {
 			hostPort := net.JoinHostPort(endpointURL.Hostname(), endpointURL.Port())
 			return jf.dial(event, hostPort, endpointURL)

heartbeat/monitors/logger/logger.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/jobsummary"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
@@ -44,6 +45,7 @@ type MonitorRunInfo struct {
 	Duration  int64  `json:"-"`
 	Steps     *int   `json:"steps,omitempty"`
 	Status    string `json:"status"`
+	Attempt   int    `json:"attempt"`
 }
 
 func (m *MonitorRunInfo) MarshalJSON() ([]byte, error) {
@@ -78,22 +80,33 @@ func extractRunInfo(event *beat.Event) (*MonitorRunInfo, error) {
 	errors := []error{}
 	monitorID, err := event.GetValue("monitor.id")
 	if err != nil {
-		errors = append(errors, err)
+		errors = append(errors, fmt.Errorf("could not extract monitor.id: %w", err))
 	}
 
 	durationUs, err := event.GetValue("monitor.duration.us")
 	if err != nil {
-		errors = append(errors, err)
+		durationUs = int64(0)
 	}
 
 	monType, err := event.GetValue("monitor.type")
 	if err != nil {
-		errors = append(errors, err)
+		errors = append(errors, fmt.Errorf("could not extract monitor.type: %w", err))
 	}
 
 	status, err := event.GetValue("monitor.status")
 	if err != nil {
-		errors = append(errors, err)
+		errors = append(errors, fmt.Errorf("could not extract monitor.status: %w", err))
+	}
+
+	jsIface, err := event.GetValue("summary")
+	var attempt int
+	if err != nil {
+		errors = append(errors, fmt.Errorf("could not extract summary to add attempt info: %w", err))
+	} else {
+		js, ok := jsIface.(*jobsummary.JobSummary)
+		if ok && js != nil {
+			attempt = int(js.Attempt)
+		}
 	}
 
 	if len(errors) > 0 {
@@ -105,6 +118,7 @@ func extractRunInfo(event *beat.Event) (*MonitorRunInfo, error) {
 		Type:      monType.(string),
 		Duration:  durationUs.(int64),
 		Status:    status.(string),
+		Attempt:   attempt,
 	}
 
 	sc, _ := event.Meta.GetValue(META_STEP_COUNT)
@@ -119,7 +133,7 @@ func extractRunInfo(event *beat.Event) (*MonitorRunInfo, error) {
 func LogRun(event *beat.Event) {
 	monitor, err := extractRunInfo(event)
 	if err != nil {
-		getLogger().Errorw("error gathering information to log event: ", err)
+		getLogger().Error(fmt.Errorf("error gathering information to log event: %w", err))
 		return
 	}
 

heartbeat/monitors/logger/logger_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uber.org/zap/zaptest/observer"
 
 	"github.com/elastic/beats/v7/heartbeat/eventext"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/jobsummary"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
@@ -47,6 +48,7 @@ func TestLogRun(t *testing.T) {
 		"monitor.duration.us": durationUs,
 		"monitor.type":        "browser",
 		"monitor.status":      "down",
+		"summary":             jobsummary.NewJobSummary(1, 1, "abc"),
 	}
 
 	event := beat.Event{Fields: fields}
@@ -64,6 +66,7 @@ func TestLogRun(t *testing.T) {
 		Duration:  durationUs,
 		Status:    "down",
 		Steps:     &steps,
+		Attempt:   1,
 	}
 
 	assert.ElementsMatch(t, []zap.Field{