From 980b1b3fcfd6e70b7031207544334126ec75e1c7 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 15 Nov 2022 17:17:07 +0100 Subject: [PATCH] generating routing_path from dimension fields --- .../experimental_datastream_features.test.ts | 79 ++++++++++++++++++- .../experimental_datastream_features.ts | 56 +++++++++++-- 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts index 78cb6642669f2..dddd11dd6ee19 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts @@ -10,7 +10,10 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks import type { NewPackagePolicy, PackagePolicy } from '../../types'; -import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features'; +import { + builRoutingPath, + handleExperimentalDatastreamFeatureOptIn, +} from './experimental_datastream_features'; function getNewTestPackagePolicy({ isSyntheticSourceEnabled, @@ -102,6 +105,12 @@ describe('experimental_datastream_features', () => { // @ts-expect-error mode: 'stored', }, + properties: { + test_dimension: { + type: 'keyword', + time_series_dimension: true, + }, + }, }, }, }, @@ -178,7 +187,9 @@ describe('experimental_datastream_features', () => { expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - settings: expect.objectContaining({ index: { mode: 'time_series' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -280,7 +291,9 @@ describe('experimental_datastream_features', () => { expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - settings: expect.objectContaining({ index: { mode: 'time_series' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -288,4 +301,64 @@ describe('experimental_datastream_features', () => { }); }); }); + it('should build routing path', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['test_dimension']); + }); + + it('should build routing path from nested properties', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['cloud.image.id', 'test_dimension']); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts index b819565849138..3a60733a57eb5 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts @@ -5,6 +5,10 @@ * 2.0. */ +import type { + MappingProperty, + PropertyName, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; @@ -12,6 +16,31 @@ import type { NewPackagePolicy, PackagePolicy } from '../../types'; import { getInstallation } from '../epm/packages'; import { updateDatastreamExperimentalFeatures } from '../epm/packages/update'; +function mapFields(mappingProperties: Record) { + const mappings = Object.keys(mappingProperties).reduce((acc, curr) => { + const property = mappingProperties[curr] as any; + if (property.properties) { + const childMappings = mapFields(property.properties); + Object.keys(childMappings).forEach((key) => { + acc[curr + '.' + key] = childMappings[key]; + }); + } else { + acc[curr] = property; + } + return acc; + }, {} as any); + return mappings; +} + +export function builRoutingPath(properties: Record) { + const mappingsProperties = mapFields(properties); + return Object.keys(mappingsProperties).filter( + (mapping) => + mappingsProperties[mapping].type === 'keyword' && + mappingsProperties[mapping].time_series_dimension + ); +} + export async function handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, @@ -45,14 +74,18 @@ export async function handleExperimentalDatastreamFeatureOptIn({ const isSyntheticSourceOptInChanged = existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source; - if (isSyntheticSourceOptInChanged) { - const componentTemplateName = `${featureMapEntry.data_stream}@package`; - const componentTemplateRes = await esClient.cluster.getComponentTemplate({ - name: componentTemplateName, - }); + const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb; + + if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue; - const componentTemplate = componentTemplateRes.component_templates[0].component_template; + const componentTemplateName = `${featureMapEntry.data_stream}@package`; + const componentTemplateRes = await esClient.cluster.getComponentTemplate({ + name: componentTemplateName, + }); + const componentTemplate = componentTemplateRes.component_templates[0].component_template; + + if (isSyntheticSourceOptInChanged) { const body = { template: { ...componentTemplate.template, @@ -72,9 +105,15 @@ export async function handleExperimentalDatastreamFeatureOptIn({ }); } - const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb; - if (isTSDBOptInChanged && featureMapEntry.features.tsdb) { + const mappingsProperties = componentTemplate.template?.mappings?.properties ?? {}; + + // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path + // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592 + const routingPath = builRoutingPath(mappingsProperties); + + if (routingPath.length === 0) continue; + const indexTemplateRes = await esClient.indices.getIndexTemplate({ name: featureMapEntry.data_stream, }); @@ -88,6 +127,7 @@ export async function handleExperimentalDatastreamFeatureOptIn({ ...(indexTemplate.template?.settings ?? {}), index: { mode: 'time_series', + routing_path: routingPath, }, }, },