Stop setting observer.id, observer.ephemeral_id

We don't use these fields in the UI anywhere, and
for debugging we can rely on the hostname to
distinguish one APM Server instance from another.

The observer.id and observer.ephemeral_id fields
suggest that we should care about specific instances,
and their continued presence contradicts our
intentions for a more modern "cloud native" approach,
where APM Server instances may come and go.

For now we need to continue persisting a UUID
to disk for stack monitoring, as this is used to
identify instances of APM Server. In the future
we expect to revisit how stack monitoring works,
and at that point we should redesign the documents
to not enshrine a "pets" mentality.

For tail-based sampling we now generate a UUID
(for identifying its documents) at process start.
The field has been renamed from observer.id to
agent.ephemeral_id, the latter of which is an
ECS field (observer.id is not).

apmpackage/apm/changelog.yml

@@ -9,6 +9,9 @@
     - description: Enable synthetic source for metrics data streams
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/9215
+    - description: Remove `observer.id` and `observer.ephemeral_id` fields
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/9412
 - version: "8.5.0"
   changes:
     - description: Add package settings to enable the experimental collection of service metrics

apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml

@@ -5,5 +5,7 @@ processors:
       name: process_ppid
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version

apmpackage/apm/data_stream/app_logs/fields/fields.yml

@@ -26,14 +26,6 @@
   type: keyword
   description: |
     Kubernetes Pod UID
-- name: observer.ephemeral_id
-  type: keyword
-  description: |
-    Ephemeral identifier of the APM Server.
-- name: observer.id
-  type: keyword
-  description: |
-    Unique identifier of the APM Server.
 - name: processor.event
   type: constant_keyword
   description: Processor event.

apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -3,6 +3,8 @@ description: Pipeline for ingesting APM application metrics.
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/app_metrics/fields/fields.yml

@@ -22,14 +22,6 @@
   type: keyword
   description: |
     Network connection type, eg. "wifi", "cell"
-- name: observer.ephemeral_id
-  type: keyword
-  description: |
-    Ephemeral identifier of the APM Server.
-- name: observer.id
-  type: keyword
-  description: |
-    Unique identifier of the APM Server.
 - name: processor.event
   type: constant_keyword
   description: Processor event.

apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml

@@ -3,6 +3,8 @@ description: Pipeline for ingesting APM error events.
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/error_logs/fields/fields.yml

@@ -89,14 +89,6 @@
   type: keyword
   description: |
     Network connection type, eg. "wifi", "cell"
-- name: observer.ephemeral_id
-  type: keyword
-  description: |
-    Ephemeral identifier of the APM Server.
-- name: observer.id
-  type: keyword
-  description: |
-    Unique identifier of the APM Server.
 - name: parent.id
   type: keyword
   description: |

apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -3,6 +3,8 @@ description: Pipeline for ingesting APM internal metrics.
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/internal_metrics/fields/fields.yml

@@ -53,14 +53,6 @@
   type: keyword
   description: |
     Network connection type, eg. "wifi", "cell"
-- name: observer.ephemeral_id
-  type: keyword
-  description: |
-    Ephemeral identifier of the APM Server.
-- name: observer.id
-  type: keyword
-  description: |
-    Unique identifier of the APM Server.
 - name: processor.event
   type: constant_keyword
   description: Processor event.

apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml

@@ -3,6 +3,8 @@ description: Pipeline for ingesting APM RUM trace events.
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/sampled_traces/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,7 @@
+---
+description: Pipeline for ingesting APM tail-sampled trace IDs.
+processors:
+  - rename:
+      field: observer.id
+      target_field: agent.ephemeral_id
+      ignore_missing: true

apmpackage/apm/data_stream/sampled_traces/fields/ecs.yml

@@ -2,3 +2,5 @@
   name: event.ingested
 - external: ecs
   name: trace.id
+- external: ecs
+  name: agent.ephemeral_id

apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/apm_data_stream_migration.yml

@@ -6,6 +6,8 @@ description: |
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml

@@ -3,6 +3,8 @@ description: Pipeline for ingesting APM trace events.
 processors:
   - pipeline:
       name: observer_version
+  - pipeline:
+      name: observer_ids
   - pipeline:
       name: ecs_version
   - pipeline:

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -98,14 +98,6 @@
   type: keyword
   description: |
     Network connection type, eg. "wifi", "cell"
-- name: observer.ephemeral_id
-  type: keyword
-  description: |
-    Ephemeral identifier of the APM Server.
-- name: observer.id
-  type: keyword
-  description: |
-    Unique identifier of the APM Server.
 - name: parent.id
   type: keyword
   description: |

apmpackage/cmd/genpackage/pipelines.go

@@ -37,6 +37,7 @@ import (
 func getCommonPipeline(name string, version *version.V) []map[string]interface{} {
 	commonPipelines := map[string][]map[string]interface{}{
 		"observer_version": getObserverVersionPipeline(version),
+		"observer_ids":     observerIDsPipeline,
 		"ecs_version":      ecsVersionPipeline,
 		"user_agent":       userAgentPipeline,
 		"process_ppid":     processPpidPipeline,
@@ -88,6 +89,18 @@ func getObserverVersionPipeline(version *version.V) []map[string]interface{} {
 	}}
 }
 
+var observerIDsPipeline = []map[string]interface{}{{
+	"remove": map[string]interface{}{
+		// Remove observer.id and observer.ephemeral_id.
+		"field": []string{
+			"observer.id",
+			"observer.ephemeral_id",
+		},
+		"ignore_missing": true,
+		"ignore_failure": true,
+	},
+}}
+
 var ecsVersionPipeline = []map[string]interface{}{{
 	"remove": map[string]interface{}{
 		"field":          "ecs", // remove ecs.version

changelogs/head.asciidoc

@@ -7,6 +7,7 @@ https://github.com/elastic/apm-server/compare/8.5\...main[View commits]
 ==== Breaking Changes
 - `apm-server.decoder.*` stack monitoring metrics are no longer recorded {pull}9210[9210]
 - `ecs.version` is no longer added to document `_source`; it is added as a `constant_keyword` field {pull}9208[9208]
+- `observer.id` and `observer.ephemeral_id` are no longer added to APM documents {pull}9412[9412]
 
 [float]
 ==== Deprecations

cmd/apm-server/main.go

@@ -28,10 +28,8 @@ func main() {
 	rootCmd := beatcmd.NewRootCommand(beatcmd.BeatParams{
 		NewRunner: func(args beatcmd.RunnerParams) (beatcmd.Runner, error) {
 			return beater.NewRunner(beater.RunnerParams{
-				Config:      args.Config,
-				ID:          args.Info.ID,
-				EphemeralID: args.Info.EphemeralID,
-				Logger:      args.Logger,
+				Config: args.Config,
+				Logger: args.Logger,
 			})
 		},
 	})

internal/beater/beater.go

@@ -30,7 +30,6 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/gofrs/uuid"
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	"go.elastic.co/apm/module/apmgrpc/v2"
@@ -75,11 +74,9 @@ import (
 // Runner initialises and runs and orchestrates the APM Server
 // HTTP and gRPC servers, event processing pipeline, and output.
 type Runner struct {
-	wrapServer        WrapServerFunc
-	serverID          uuid.UUID
-	serverEphemeralID uuid.UUID
-	logger            *logp.Logger
-	rawConfig         *agentconfig.C
+	wrapServer WrapServerFunc
+	logger     *logp.Logger
+	rawConfig  *agentconfig.C
 
 	config                    *config.Config
 	fleetConfig               *config.Fleet
@@ -95,18 +92,6 @@ type RunnerParams struct {
 	// and output.* attributes.
 	Config *agentconfig.C
 
-	// ID holds the APM Server `observer.id` value, which persists across
-	// restarts of the process.
-	ID uuid.UUID
-
-	// EphemeralID holds the APM Server `observer.ephemeral_id` value,
-	// which is generated every time the process restarts.
-	//
-	// The value of EphemeralID does NOT change for each invocation of
-	// NewRunner, meaning that `observer.ephemeral_id` will not change
-	// simply because the configuration was updated.
-	EphemeralID uuid.UUID
-
 	// Logger holds a logger to use for logging throughout the APM Server.
 	Logger *logp.Logger
 
@@ -151,11 +136,9 @@ func NewRunner(args RunnerParams) (*Runner, error) {
 		return nil, err
 	}
 	return &Runner{
-		wrapServer:        args.WrapServer,
-		serverID:          args.ID,
-		serverEphemeralID: args.EphemeralID,
-		logger:            logger,
-		rawConfig:         args.Config,
+		wrapServer: args.WrapServer,
+		logger:     logger,
+		rawConfig:  args.Config,
 
 		config:                    cfg,
 		fleetConfig:               unpackedConfig.Fleet,
@@ -347,7 +330,7 @@ func (s *Runner) Run(ctx context.Context) error {
 		// Ensure all events have observer.*, ecs.*, and data_stream.* fields added,
 		// and are counted in metrics. This is done in the final processors to ensure
 		// aggregated metrics are also processed.
-		newObserverBatchProcessor(s.serverID, s.serverEphemeralID),
+		newObserverBatchProcessor(),
 		&modelprocessor.SetDataStream{Namespace: s.config.DataStreams.Namespace},
 		modelprocessor.NewEventCounter(monitoring.Default.GetRegistry("apm-server")),
 
@@ -373,7 +356,6 @@ func (s *Runner) Run(ctx context.Context) error {
 	// Create the runServer function. We start with newBaseRunServer, and then
 	// wrap depending on the configuration in order to inject behaviour.
 	serverParams := ServerParams{
-		UUID:                   s.serverID,
 		Config:                 s.config,
 		Managed:                s.fleetConfig != nil,
 		Namespace:              s.config.DataStreams.Namespace,
@@ -654,8 +636,6 @@ func (s *Runner) newLibbeatFinalBatchProcessor(
 		Version:     version.Version,
 		Hostname:    hostname,
 		Name:        hostname,
-		ID:          s.serverID,
-		EphemeralID: s.serverEphemeralID,
 	}
 
 	stateRegistry := monitoring.GetNamespace("state").GetRegistry()

internal/beater/beatertest/server.go

@@ -27,7 +27,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gofrs/uuid"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
@@ -39,14 +38,6 @@ import (
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
-var (
-	// ServerID is a mock `observer.id` for the server.
-	ServerID = uuid.Must(uuid.FromString("fbba762a-14dd-412c-b7e9-b79f903eb492"))
-
-	// ServerEphemeralID is a mock `observer.ephemeral_id` for the server.
-	ServerEphemeralID = uuid.Must(uuid.FromString("24d4f9c3-cf2e-4b2e-9b0a-2d6a71e82aff"))
-)
-
 // Server runs the core APM Server that, by default, listens on a system-chosen port
 // on the loopback interface, for use in testing package beater.
 type Server struct {
@@ -110,11 +101,9 @@ func NewUnstartedServer(t testing.TB, opts ...option) *Server {
 	require.NoError(t, err)
 
 	runner, err := beater.NewRunner(beater.RunnerParams{
-		Config:      cfg,
-		ID:          ServerID,
-		EphemeralID: ServerEphemeralID,
-		Logger:      logger,
-		WrapServer:  options.wrapServer,
+		Config:     cfg,
+		Logger:     logger,
+		WrapServer: options.wrapServer,
 	})
 	require.NoError(t, err)
 

internal/beater/processors.go

@@ -22,8 +22,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/gofrs/uuid"
-
 	"github.com/elastic/apm-server/internal/beater/auth"
 	"github.com/elastic/apm-server/internal/beater/ratelimit"
 	"github.com/elastic/apm-server/internal/model"
@@ -64,16 +62,12 @@ func rateLimitBatchProcessor(ctx context.Context, batch *model.Batch) error {
 
 // newObserverBatchProcessor returns a model.BatchProcessor that sets
 // observer fields from information about the apm-server process.
-func newObserverBatchProcessor(id, ephemeralID uuid.UUID) model.ProcessBatchFunc {
+func newObserverBatchProcessor() model.ProcessBatchFunc {
 	hostname, _ := os.Hostname()
-	idString := id.String()
-	ephemeralIDString := ephemeralID.String()
 	return func(ctx context.Context, b *model.Batch) error {
 		for i := range *b {
 			observer := &(*b)[i].Observer
-			observer.EphemeralID = ephemeralIDString
 			observer.Hostname = hostname
-			observer.ID = idString
 			observer.Type = "apm-server"
 			observer.Version = version.Version
 		}

internal/beater/server.go

@@ -22,7 +22,6 @@ import (
 	"net"
 	"net/http"
 
-	"github.com/gofrs/uuid"
 	"go.elastic.co/apm/module/apmgorilla/v2"
 	"go.elastic.co/apm/v2"
 	"golang.org/x/sync/errgroup"
@@ -67,9 +66,6 @@ type RunServerFunc func(context.Context, ServerParams) error
 
 // ServerParams holds parameters for running the APM Server.
 type ServerParams struct {
-	// UUID holds a unique ID for the server.
-	UUID uuid.UUID
-
 	// Config is the configuration used for running the APM Server.
 	Config *config.Config