From 8c8b0b9112f0eb1e6b618f430523bc4319834c1f Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 10 Jan 2024 10:51:53 +0800 Subject: [PATCH] apm-data: fix `@custom` component templates Use `@custom` instead of `apm@custom`. This is an enhancement over what Fleet sets up; it is an additive improvement in the direction of https://github.com/elastic/elasticsearch/issues/97664. The rollup data streams' `@custom` component templates now include the duration, like what Fleet sets up. Add a YAML REST test, and a unit test ensuring consistency across the index templates. --- .../logs-apm.app@template.yaml | 4 +- .../logs-apm.error@template.yaml | 4 +- .../metrics-apm.app@template.yaml | 4 +- .../metrics-apm.internal@template.yaml | 4 +- ...-apm.service_destination.10m@template.yaml | 8 +- ...s-apm.service_destination.1m@template.yaml | 8 +- ...-apm.service_destination.60m@template.yaml | 8 +- ...rics-apm.service_summary.10m@template.yaml | 8 +- ...trics-apm.service_summary.1m@template.yaml | 8 +- ...rics-apm.service_summary.60m@template.yaml | 8 +- ...-apm.service_transaction.10m@template.yaml | 8 +- ...s-apm.service_transaction.1m@template.yaml | 8 +- ...-apm.service_transaction.60m@template.yaml | 8 +- .../metrics-apm.transaction.10m@template.yaml | 8 +- .../metrics-apm.transaction.1m@template.yaml | 8 +- .../metrics-apm.transaction.60m@template.yaml | 8 +- .../traces-apm.rum@template.yaml | 6 +- .../traces-apm.sampled@template.yaml | 6 +- .../index-templates/traces-apm@template.yaml | 4 +- .../APMIndexTemplateRegistryTests.java | 44 +++++++++++ .../test/30_custom_templates.yml | 76 +++++++++++++++++++ 21 files changed, 184 insertions(+), 64 deletions(-) create mode 100644 x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/30_custom_templates.yml diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml index 0ebbb99a1e379..3d9c1490e5a86 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml @@ -11,12 +11,12 @@ composed_of: - apm@mappings - apm@settings - apm-10d@lifecycle -- apm@custom +- logs@custom - logs-apm.app@custom - ecs@mappings ignore_missing_component_templates: +- logs@custom - logs-apm.app@custom -- apm@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml index 831f7cc404415..4adcf125b2df9 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml @@ -13,12 +13,12 @@ composed_of: - apm@settings - apm-10d@lifecycle - logs-apm.error@mappings -- apm@custom +- logs@custom - logs-apm.error@custom - ecs@mappings ignore_missing_component_templates: +- logs@custom - logs-apm.error@custom -- apm@custom template: mappings: properties: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml index bdd1fa363bcf4..c2233469110f8 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml @@ -13,11 +13,11 @@ composed_of: - apm-90d@lifecycle - metrics-apm@mappings - metrics-apm@settings -- apm@custom +- metrics@custom - metrics-apm.app@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom +- metrics@custom - metrics-apm.app@custom template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml index 205784e22e685..3d6d05c58e780 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml @@ -14,11 +14,11 @@ composed_of: - apm-90d@lifecycle - metrics-apm@mappings - metrics-apm@settings -- apm@custom +- metrics@custom - metrics-apm.internal@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom +- metrics@custom - metrics-apm.internal@custom template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml index 6279e044fbfcf..f234b60b1a6ec 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.10m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.10m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml index 10e4ca5b39a52..aa4f212532e56 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml @@ -15,12 +15,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.1m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.1m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml index dbac0d0d17d89..9b1a26486f482 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.60m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_destination@custom +- metrics@custom +- metrics-apm.service_destination.60m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml index af99e419d4a56..c37ec93651d9d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.10m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.10m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml index 29c28953d6b40..3a99bc8472c66 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml @@ -15,12 +15,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.1m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.1m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml index bdbd4900df3bb..d829967f7eddf 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.60m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_summary@custom +- metrics@custom +- metrics-apm.service_summary.60m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml index 8b4e88391a475..bc21b35d4777f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.10m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.10m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml index 811067f8e6f30..87a1e254baea7 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml @@ -15,12 +15,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.1m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.1m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml index db28b7c56aaab..b45ce0ec0fad7 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.60m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.service_transaction@custom +- metrics@custom +- metrics-apm.service_transaction.60m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml index 548f73656fda4..51d3c90cb4af8 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.10m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.10m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml index 6206e7c126c48..8825a93db28dc 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml @@ -15,12 +15,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.1m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.1m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml index 4ad00aecf23a5..e6657fbfe5d28 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml @@ -16,12 +16,12 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.60m@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom -- metrics-apm.transaction@custom +- metrics@custom +- metrics-apm.transaction.60m@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml index 174faf432eb6e..174aec8c5515a 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml @@ -14,14 +14,12 @@ composed_of: - apm-90d@lifecycle - traces-apm@mappings - traces-apm.rum@mappings -- apm@custom -- traces-apm@custom +- traces@custom - traces-apm.rum@custom - ecs@mappings ignore_missing_component_templates: +- traces@custom - traces-apm.rum@custom -- traces-apm@custom -- apm@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml index 8c65c69bc3afa..a39d10897a2ed 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml @@ -11,10 +11,12 @@ composed_of: - traces@mappings - apm@mappings - apm@settings -- apm@custom +- traces@custom +- traces-apm.sampled@custom - ecs@mappings ignore_missing_component_templates: -- apm@custom +- traces@custom +- traces-apm.sampled@custom template: lifecycle: data_retention: 1h diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml index fb6670a7f7143..de9c47dfd3f1b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml @@ -13,12 +13,12 @@ composed_of: - apm@settings - apm-10d@lifecycle - traces-apm@mappings -- apm@custom +- traces@custom - traces-apm@custom - ecs@mappings ignore_missing_component_templates: +- traces@custom - traces-apm@custom -- apm@custom template: settings: index: diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index 7dcd6fdd807e4..c10c3fde45162 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -57,10 +57,12 @@ import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.XPackSettings.APM_DATA_ENABLED; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -249,6 +251,48 @@ public void testIndexTemplates() throws Exception { assertThat(actualInstalledIngestPipelines.get(), equalTo(0)); } + public void testIndexTemplateConventions() throws Exception { + for (Map.Entry entry : apmIndexTemplateRegistry.getComposableTemplateConfigs().entrySet()) { + final String name = entry.getKey(); + final int atIndex = name.lastIndexOf('@'); + assertThat(atIndex, not(equalTo(-1))); + assertThat(name.substring(atIndex + 1), equalTo("template")); + + final String dataStreamType = name.substring(0, name.indexOf('-')); + assertThat(dataStreamType, isIn(List.of("logs", "metrics", "traces"))); + + final ComposableIndexTemplate template = entry.getValue(); + assertThat(template.indexPatterns().size(), equalTo(1)); + + final String namePrefix = name.substring(0, atIndex); + switch (namePrefix) { + case "logs-apm.app", "metrics-apm.app": + // These two data streams have a service-specific dataset. + assertThat(template.indexPatterns().get(0), equalTo(namePrefix + ".*-*")); + break; + default: + assertThat(template.indexPatterns().get(0), equalTo(namePrefix + "-*")); + break; + } + + // Each index template should be composed of the following optional component templates: + // @custom + // -@custom + final List optionalComponentTemplates = template.composedOf() + .stream() + .filter(t -> template.getIgnoreMissingComponentTemplates().contains(t)) + .toList(); + assertThat(optionalComponentTemplates, containsInAnyOrder(namePrefix + "@custom", dataStreamType + "@custom")); + + // There should be no required custom component templates. + final List requiredCustomComponentTemplates = template.getRequiredComponentTemplates() + .stream() + .filter(t -> t.endsWith("@custom")) + .toList(); + assertThat(requiredCustomComponentTemplates, empty()); + } + } + private Map getIndependentComponentTemplateConfigs() { return apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet().stream().filter(template -> { Settings settings = template.getValue().template().settings(); diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/30_custom_templates.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/30_custom_templates.yml new file mode 100644 index 0000000000000..62b36926d01dc --- /dev/null +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/30_custom_templates.yml @@ -0,0 +1,76 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + + - do: + cluster.put_component_template: + name: "metrics@custom" + body: + template: + mappings: + properties: + custom_field1: + type: keyword + meta: + source: metrics@custom + custom_field2: + type: keyword + meta: + source: metrics@custom + + - do: + cluster.put_component_template: + name: "metrics-apm.app@custom" + body: + template: + mappings: + properties: + custom_field2: + type: keyword + meta: + source: metrics-apm.app@custom + custom_field3: + type: keyword + meta: + source: metrics-apm.app@custom + +--- +"Test metrics @custom component templates": + - do: + indices.create_data_stream: + name: metrics-apm.app.svc1-testing + - do: + # Wait for cluster state changes to be applied before + # querying field mappings. + cluster.health: + wait_for_events: languid + - do: + indices.get_field_mapping: + index: metrics-apm.app.svc1-testing + fields: custom_field* + - set: {_arbitrary_key_: index} + - match: + $body.$index.mappings: + custom_field1: + full_name: custom_field1 + mapping: + custom_field1: + type: keyword + meta: + source: metrics@custom + custom_field2: + full_name: custom_field2 + mapping: + custom_field2: + type: keyword + meta: + source: metrics-apm.app@custom + custom_field3: + full_name: custom_field3 + mapping: + custom_field3: + type: keyword + meta: + source: metrics-apm.app@custom