Various integration package enhancements and fixes (#6359)

* apmpackage: various improvements and fixes

- Add client.geo to internal_metrics, so RUM breakdown metrics
  can be indexed. The internal metrics data stream is strictly
  mapped.

- Remove our README generator, update to use the elastic-package
  generator instead. This means we no longer have ECS badges; we
  should enhance the elastic-package generator later if this is
  important. I have also removed the sample documents, as we have
  these in the elastic.co APM documentation.

- Stop generating integration package fields and ingest pipelines.
  Prune unused fields per data stream, and introduce data stream
  specific ingest pipelines.

- Use `external: ecs` for ECS fields, so we get the canonical ECS
  field descriptions and types.

- Use `constant_keyword` for `processor.name` and `processor.event`
  fields where we can.

* systemtest: test geoIP enrichment with integration

* Use built integration package in system tests

Now that we're using `external: ecs` in fields/ecs.yml,
we need to use the output of `elastic-package build`
rather than using the integration package source directly.
This goes for the system tests (i.e. what we mount into
the package-registry container), and also for what we
copy to package-storage.

* testing/docker/package-registry: local first

* apmpackage: make changelog descriptions consistent

Makefile

@@ -112,18 +112,13 @@ docker-compose.override.yml:
 # Rules for updating config files, fields.yml, etc.
 ##############################################################################
 
-update: fields go-generate add-headers copy-docs gen-package notice $(MAGE)
+update: fields go-generate add-headers copy-docs build-package notice $(MAGE)
 	@$(MAGE) update
 
 fields_sources=\
   $(shell find model -name fields.yml) \
   $(shell find x-pack/apm-server/fields -name fields.yml)
 
-.PHONY: gen-package gen-package-only
-gen-package: gen-package-only format-package build-package
-gen-package-only: $(GENPACKAGE)
-	@$(GENPACKAGE)
-
 fields: include/fields.go x-pack/apm-server/include/fields.go
 include/fields.go x-pack/apm-server/include/fields.go: $(MAGE) magefile.go $(fields_sources)
 	@$(MAGE) fields
@@ -232,6 +227,7 @@ check-package: $(ELASTICPACKAGE)
 format-package: $(ELASTICPACKAGE)
 	@(cd apmpackage/apm; $(CURDIR)/$(ELASTICPACKAGE) format)
 build-package: $(ELASTICPACKAGE)
+	@rm -fr ./build/integrations/apm/*
 	@(cd apmpackage/apm; $(CURDIR)/$(ELASTICPACKAGE) build)
 
 .PHONY: check-gofmt check-autopep8 gofmt autopep8
@@ -262,10 +258,6 @@ $(BIN_MAGE): go.mod
 $(MAGE): magefile.go $(BIN_MAGE)
 	$(BIN_MAGE) -compile=$@
 
-.PHONY: $(GENPACKAGE)
-$(GENPACKAGE):
-	@$(GO) build -o $@ github.com/elastic/apm-server/apmpackage/cmd/gen-package
-
 $(GOLINT): go.mod
 	$(GO) build -o $@ golang.org/x/lint/golint
 

apmpackage/README.md

@@ -11,11 +11,11 @@
 
 1. Actual work
     - Make changes in `apmpackage/apm` and/or code as needed
-    - Run `make update`. That will update fields, pipeline and doc files based on apm-server fields and pipeline defs.
+    - Run `make build-package`
 
 2. Run the registry
     - Checkout a fresh master from the registry and run `mage build`
-    - Copy `apmpackage/apm` in apm-server to `build/package-storage/packages/apm/<version>` in the registry
+    - Copy `build/integrations/apm/<version>` in apm-server to `build/package-storage/packages/apm/<version>` in the registry
     - `go run .`
 
 3. Run the stack

apmpackage/apm/_dev/build/build.yml

@@ -0,0 +1,4 @@
+dependencies:
+  ecs:
+    # TODO(axw) make sure this is kept in sync with the ECS version used in libbeat.
+    reference: [email protected]

apmpackage/apm/README.template.md → apmpackage/apm/_dev/build/docs/README.md

@@ -47,61 +47,33 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode,
 ## Traces
 
 Traces are comprised of [spans and transactions](https://www.elastic.co/guide/en/apm/get-started/current/apm-data-model.html).
-Traces are written to `traces-apm.*` indices.
 
-**Exported Fields**
+Traces are written to `traces-apm-*` data streams.
 
-| Field | Description | Type | ECS |
-|---|---|---|:---:|
-{{range .Traces -}}
-| {{- Trim .Name | EscapeMarkdown -}} | {{- Trim .Description | EscapeMarkdown -}} | {{- Trim .Type | EscapeMarkdown -}} | {{if .IsECS}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) {{else}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) {{end}} |
-{{end}}
+{{fields "traces"}}
 
-#### Examples
+## Application Metrics
 
-```json
-{{.TransactionExample}}
-```
+Application metrics are comprised of custom, application-specific metrics, basic system metrics such as CPU and memory usage,
+and runtime metrics such as JVM garbage collection statistics.
 
-```json
-{{.SpanExample}}
-```
+Application metrics are written to service-specific `metrics-apm.app.*-*` data streams.
 
+{{fields "app_metrics"}}
 
-## Metrics
+## Internal Metrics
 
-Metrics include application-based metrics and some basic system metrics.
-Metrics are written to `metrics-apm.app.*`, `metrics-apm.internal.*`, and `metrics-apm.profiling.*` indices.
+Internal metrics comprises metrics produced by Elastic APM agents and Elastic APM server for powering various Kibana charts
+in the APM app, such as "Time spent by span type".
 
-**Exported Fields**
+Internal metrics are written to `metrics-apm.internal-*` data streams.
 
-| Field | Description | Type | ECS |
-|---|---|---|:---:|
-{{range .Metrics -}}
-| {{- Trim .Name | EscapeMarkdown -}} | {{- Trim .Description | EscapeMarkdown -}} | {{- Trim .Type | EscapeMarkdown -}} | {{if .IsECS}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) {{else}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) {{end}} |
-{{end}}
+{{fields "internal_metrics"}}
 
-### Example
+## Application errors
 
-```json
-{{.MetricsExample}}
-```
+Application errors comprises error/exception events occurring in an application.
 
-## Logs
+Application errors are written to `logs-apm.error.*` data stream.
 
-Logs are application error events.
-Logs are written to `logs-apm.error.*` indices.
-
-**Exported Fields**
-
-| Field | Description | Type | ECS |
-|---|---|---|:---:|
-{{range .Logs -}}
-| {{- Trim .Name | EscapeMarkdown -}} | {{- Trim .Description | EscapeMarkdown -}} | {{- Trim .Type | EscapeMarkdown -}} | {{if .IsECS}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) {{else}} ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) {{end}} |
-{{end}}
-
-### Example
-
-```json
-{{.ErrorExample}}
-```
+{{fields "error_logs"}}

apmpackage/apm/changelog.yml

@@ -3,18 +3,30 @@
 # change type can be one of: enhancement, bugfix, breaking-change
 - version: "7.16.0"
   changes:
-    - description: version bump to align package version with stack version
+    - description: updated package version to align with stack version
       type: enhancement
       link: https://github.com/elastic/apm-server/issues/4898
+    - description: added client.geo fields to internal_metrics
+      type: bugfix
+      link: https://github.com/elastic/apm-server/pull/6359
+    - description: removed unused fields
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/6359
+    - description: changed processor.{name,event} to constant_keyword where possible
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/6359
+    - description: changed to data-stream specific ingest pipelines
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/6359
 - version: "0.5.0"
   changes:
-    - description: define index sorting for internal metrics
+    - description: added index sorting for internal metrics
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/6116
-    - description: add histogram dynamic_template to app metrics data stream
+    - description: added histogram dynamic_template to app metrics data stream
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/6043
-    - description: remove warm phase from ILM policies
+    - description: removed warm phase from ILM policies
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/6229
     - description: added privileges to tail-sampled traces data stream
@@ -25,10 +37,10 @@
       link: https://github.com/elastic/apm-server/pull/6139
 - version: "0.4.0"
   changes:
-    - description: add anonymous auth config, replace some RUM config
+    - description: added anonymous auth config, replace some RUM config
       type: breaking-change
       link: https://github.com/elastic/apm-server/pull/5623
-    - description: use new apm-server.auth config
+    - description: updated to use new apm-server.auth config
       type: breaking-change
       link: https://github.com/elastic/apm-server/pull/5691
 - version: "0.3.0"
@@ -44,7 +56,7 @@
     - description: added support for apm-server.rum.allow_service_names
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/5030
-    - description: introduce a configurable default service environment
+    - description: added support a configurable default service environment
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/4861
 - version: "0.1.0"