diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index e43f622cf4008..716dde9b770b0 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -80,7 +80,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "cb22b422398a785e7e0565a19c6d4d5c7af6f2fd", + "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994", "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 7d5a7966f91c3..e60da12d924cc 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -131,6 +131,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ], @@ -142,6 +143,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ]); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 4ef4a7fdd751b..ca7cc3200da4b 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -352,6 +352,7 @@ export interface RegistryElasticsearch { 'index_template.mappings'?: estypes.MappingTypeMapping; 'ingest_pipeline.name'?: string; source_mode?: 'default' | 'synthetic'; + index_mode?: 'time_series'; } export interface RegistryDataStreamPrivileges { @@ -444,7 +445,7 @@ export type PackageInfo = | Installable>; // TODO - Expand this with other experimental indexing types -export type ExperimentalIndexingFeature = 'synthetic_source'; +export type ExperimentalIndexingFeature = 'synthetic_source' | 'tsdb'; export interface ExperimentalDataStreamFeature { data_stream: string; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 629250c243cea..0a951c0de8fce 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -33,6 +33,7 @@ export interface NewPackagePolicyInputStream { privileges?: { indices?: string[]; }; + index_mode?: string; }; }; release?: RegistryRelease; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 1d0cda70edea4..7bcda09d08d8f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -19,6 +19,7 @@ import { EuiButtonEmpty, } from '@elastic/eui'; +import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; import type { NewPackagePolicy, NewPackagePolicyInput, @@ -125,6 +126,41 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ [packageInputStreams, packagePolicyInput.streams] ); + // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series + let isUpdated = false; + inputStreams.forEach(({ packagePolicyInputStream }) => { + const dataStreamInfo = packageInfo.data_streams?.find( + (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset + ); + + if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') { + if (!packagePolicy.package) return; + if (!packagePolicy.package?.experimental_data_stream_features) + packagePolicy.package!.experimental_data_stream_features = []; + + const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream); + const match = packagePolicy.package!.experimental_data_stream_features.find( + (feat) => feat.data_stream === dsName + ); + if (match) { + if (!match.features.tsdb) { + match.features.tsdb = true; + isUpdated = true; + } + } else { + packagePolicy.package!.experimental_data_stream_features.push({ + data_stream: dsName, + features: { tsdb: true, synthetic_source: false }, + }); + isUpdated = true; + } + } + }); + + if (isUpdated) { + updatePackagePolicy(packagePolicy); + } + return ( <> {/* Header / input-level toggle */} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx index 54b6205da2ed9..2e5f98c8359e7 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState, Fragment, memo, useMemo, useEffect, useRef } from 'react'; +import React, { useState, Fragment, memo, useMemo, useEffect, useRef, useCallback } from 'react'; import ReactMarkdown from 'react-markdown'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -18,13 +18,14 @@ import { EuiSpacer, EuiButtonEmpty, EuiTitle, + EuiToolTip, } from '@elastic/eui'; import { useRouteMatch } from 'react-router-dom'; import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../services/package_prerelease'; import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; - +import type { ExperimentalIndexingFeature } from '../../../../../../../../../common/types/models/epm'; import type { NewPackagePolicy, NewPackagePolicyInputStream, @@ -120,6 +121,62 @@ export const PackagePolicyInputStreamConfig = memo( [advancedVars, inputStreamValidationResults?.vars] ); + const isFeatureEnabled = useCallback( + (feature: ExperimentalIndexingFeature) => + packagePolicy.package?.experimental_data_stream_features?.some( + ({ data_stream: dataStream, features }) => + dataStream === + getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream) && + features[feature] + ) ?? false, + [ + packagePolicy.package?.experimental_data_stream_features, + packagePolicyInputStream.data_stream, + ] + ); + + const newExperimentalIndexingFeature = { + synthetic_source: isFeatureEnabled('synthetic_source'), + tsdb: isFeatureEnabled('tsdb'), + }; + + const onIndexingSettingChange = ( + features: Partial> + ) => { + if (!packagePolicy.package) { + return; + } + + const newExperimentalDataStreamFeatures = [ + ...(packagePolicy.package.experimental_data_stream_features ?? []), + ]; + + const dataStream = getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream); + + const existingSettingRecord = newExperimentalDataStreamFeatures.find( + (x) => x.data_stream === dataStream + ); + + if (existingSettingRecord) { + existingSettingRecord.features = { + ...existingSettingRecord.features, + ...features, + }; + } else { + newExperimentalDataStreamFeatures.push({ + data_stream: dataStream, + features: { ...newExperimentalIndexingFeature, ...features }, + }); + } + + updatePackagePolicy({ + package: { + ...packagePolicy.package, + experimental_data_stream_features: newExperimentalDataStreamFeatures, + }, + }); + }; + return ( <> @@ -311,15 +368,7 @@ export const PackagePolicyInputStreamConfig = memo( - dataStream === - getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ) && features.synthetic_source - ) ?? false - } + checked={isFeatureEnabled('synthetic_source')} label={ ( /> } onChange={(e) => { - if (!packagePolicy.package) { - return; - } - - const newExperimentalDataStreamFeatures = [ - ...(packagePolicy.package.experimental_data_stream_features ?? []), - ]; - - const dataStream = getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ); - - const existingSettingRecord = newExperimentalDataStreamFeatures.find( - (x) => x.data_stream === dataStream - ); - - if (existingSettingRecord) { - existingSettingRecord.features.synthetic_source = e.target.checked; - } else { - newExperimentalDataStreamFeatures.push({ - data_stream: dataStream, - features: { - synthetic_source: e.target.checked, - }, - }); - } - - updatePackagePolicy({ - package: { - ...packagePolicy.package, - experimental_data_stream_features: - newExperimentalDataStreamFeatures, - }, + onIndexingSettingChange({ + synthetic_source: e.target.checked, }); }} /> + + + } + > + + } + onChange={(e) => { + onIndexingSettingChange({ + tsdb: e.target.checked, + }); + }} + /> + + diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9695a83c82bc5..a480675444e50 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -289,6 +289,7 @@ const getSavedObjectTypes = ( type: 'nested', properties: { synthetic_source: { type: 'boolean' }, + tsdb: { type: 'boolean' }, }, }, }, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts index e4ee10c4e270c..32393eebd12e5 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts @@ -67,6 +67,11 @@ describe('parseDataStreamElasticsearchEntry', () => { source_mode: 'synthetic', }); }); + it('Should add index_mode', () => { + expect(parseDataStreamElasticsearchEntry({ index_mode: 'time_series' })).toEqual({ + index_mode: 'time_series', + }); + }); it('Should add index_template mappings and expand dots', () => { expect( parseDataStreamElasticsearchEntry({ diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts index d4ee87dc232f0..1007b1fe35344 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -539,6 +539,10 @@ export function parseDataStreamElasticsearchEntry( ); } + if (expandedElasticsearch?.index_mode) { + parsedElasticsearchEntry.index_mode = expandedElasticsearch.index_mode; + } + return parsedElasticsearchEntry; } diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts index 8c7afa5d30fed..dddd11dd6ee19 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts @@ -10,12 +10,17 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks import type { NewPackagePolicy, PackagePolicy } from '../../types'; -import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features'; +import { + builRoutingPath, + handleExperimentalDatastreamFeatureOptIn, +} from './experimental_datastream_features'; function getNewTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): NewPackagePolicy { const packagePolicy: NewPackagePolicy = { name: 'Test policy', @@ -33,6 +38,7 @@ function getNewTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -44,8 +50,10 @@ function getNewTestPackagePolicy({ function getExistingTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): PackagePolicy { const packagePolicy: PackagePolicy = { id: 'test-policy', @@ -64,6 +72,7 @@ function getExistingTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -83,40 +92,89 @@ describe('experimental_datastream_features', () => { soClient.get.mockClear(); esClient.cluster.getComponentTemplate.mockClear(); esClient.cluster.putComponentTemplate.mockClear(); + + esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ + component_templates: [ + { + name: 'metrics-test.test@package', + component_template: { + template: { + settings: {}, + mappings: { + _source: { + // @ts-expect-error + mode: 'stored', + }, + properties: { + test_dimension: { + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + }, + ], + }); }); const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; describe('when package policy does not exist (create)', () => { - it('updates component template', async () => { - const packagePolicy = getNewTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); + + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -124,12 +182,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -140,12 +200,18 @@ describe('experimental_datastream_features', () => { describe('when package policy exists (update)', () => { describe('when opt in status in unchanged', () => { it('does not update component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: true } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: true, tsdb: false }, + }, ], }, id: 'mocked', @@ -161,34 +227,58 @@ describe('experimental_datastream_features', () => { }); describe('when opt in status is changed', () => { - it('updates component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); + + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -196,12 +286,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -209,4 +301,64 @@ describe('experimental_datastream_features', () => { }); }); }); + it('should build routing path', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['test_dimension']); + }); + + it('should build routing path from nested properties', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['cloud.image.id', 'test_dimension']); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts index 2b8b05aed89c3..3a60733a57eb5 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts @@ -5,6 +5,10 @@ * 2.0. */ +import type { + MappingProperty, + PropertyName, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; @@ -12,6 +16,31 @@ import type { NewPackagePolicy, PackagePolicy } from '../../types'; import { getInstallation } from '../epm/packages'; import { updateDatastreamExperimentalFeatures } from '../epm/packages/update'; +function mapFields(mappingProperties: Record) { + const mappings = Object.keys(mappingProperties).reduce((acc, curr) => { + const property = mappingProperties[curr] as any; + if (property.properties) { + const childMappings = mapFields(property.properties); + Object.keys(childMappings).forEach((key) => { + acc[curr + '.' + key] = childMappings[key]; + }); + } else { + acc[curr] = property; + } + return acc; + }, {} as any); + return mappings; +} + +export function builRoutingPath(properties: Record) { + const mappingsProperties = mapFields(properties); + return Object.keys(mappingsProperties).filter( + (mapping) => + mappingsProperties[mapping].type === 'keyword' && + mappingsProperties[mapping].time_series_dimension + ); +} + export async function handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, @@ -42,13 +71,12 @@ export async function handleExperimentalDatastreamFeatureOptIn({ (optIn) => optIn.data_stream === featureMapEntry.data_stream ); - const isOptInChanged = + const isSyntheticSourceOptInChanged = existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source; - // If the feature opt-in status in unchanged, we don't need to update any component templates - if (!isOptInChanged) { - continue; - } + const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb; + + if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue; const componentTemplateName = `${featureMapEntry.data_stream}@package`; const componentTemplateRes = await esClient.cluster.getComponentTemplate({ @@ -57,23 +85,59 @@ export async function handleExperimentalDatastreamFeatureOptIn({ const componentTemplate = componentTemplateRes.component_templates[0].component_template; - const body = { - template: { - ...componentTemplate.template, - mappings: { - ...componentTemplate.template.mappings, - _source: { - mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + if (isSyntheticSourceOptInChanged) { + const body = { + template: { + ...componentTemplate.template, + mappings: { + ...componentTemplate.template.mappings, + _source: { + mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + }, }, }, - }, - }; + }; - await esClient.cluster.putComponentTemplate({ - name: componentTemplateName, - // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source - body, - }); + await esClient.cluster.putComponentTemplate({ + name: componentTemplateName, + // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source + body, + }); + } + + if (isTSDBOptInChanged && featureMapEntry.features.tsdb) { + const mappingsProperties = componentTemplate.template?.mappings?.properties ?? {}; + + // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path + // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592 + const routingPath = builRoutingPath(mappingsProperties); + + if (routingPath.length === 0) continue; + + const indexTemplateRes = await esClient.indices.getIndexTemplate({ + name: featureMapEntry.data_stream, + }); + const indexTemplate = indexTemplateRes.index_templates[0].index_template; + + const indexTemplateBody = { + ...indexTemplate, + template: { + ...(indexTemplate.template ?? {}), + settings: { + ...(indexTemplate.template?.settings ?? {}), + index: { + mode: 'time_series', + routing_path: routingPath, + }, + }, + }, + }; + + await esClient.indices.putIndexTemplate({ + name: featureMapEntry.data_stream, + body: indexTemplateBody, + }); + } } // Update the installation object to persist the experimental feature map diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 82ff572cd1926..64837501b941b 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -84,6 +84,7 @@ const ExperimentalDataStreamFeatures = schema.arrayOf( data_stream: schema.string(), features: schema.object({ synthetic_source: schema.boolean(), + tsdb: schema.boolean(), }), }) ); @@ -128,7 +129,7 @@ const CreatePackagePolicyProps = { schema.arrayOf( schema.object({ data_stream: schema.string(), - features: schema.object({ synthetic_source: schema.boolean() }), + features: schema.object({ synthetic_source: schema.boolean(), tsdb: schema.boolean() }), }) ) ),