From 0e13d98be8c9fbd3bb1f6224c8399a9a123e00c0 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Thu, 17 Nov 2022 10:13:20 +0100 Subject: [PATCH] Added Time-series indexing (TSDB) to Integrations Experimental Indexing settings (#144974) ## Summary Closes https://github.com/elastic/kibana/issues/144530 Added a new option under integration / data streams / Index settings (experimental) / TSDB - [x] Add a toggle to the "Experimental indexing features" data streams UI for TSDB - Enabling this toggle adds `index.mode: time_series` to the data stream's index template settings. - Note: `index.routing_path` value was not needed to be added in this logic, because there is an existing elasticsearch automation that generates it from dimension fields, see test instructions. - [x] If the current package's manifest contains `index_mode: time_series` for a given data stream, the toggle should be in an "enabled + readonly" state, e.g. it cannot be disabled - Note: currently there is no package that has this setting enabled, I tested locally by manually modifying the package info response. Will need to figure out a way to create a mock package to test this. - [x] Once the toggle is enabled and the policy is saved, the toggle should _not_ be disable-able. Enabling TSDB for a data stream is an irreversible operation - [x] Add a tooltip to make the irreversible nature of this operation clear TSDB setting in System package: image image Tooltip: image ## Test instructions ### Setup local registry - For testing, I created a new version of `System` package, that has a `dimension` field called `test_dimension`. This is needed so that the elasticsearch automation runs to generate the `routing_path` setting. - Download the test data zip: [registry-packages.zip](https://github.com/elastic/kibana/files/10004220/registry-packages.zip) - Start a registry: ``` docker run -p 8080:8080 -v /input-packages/packages:/packages/test-packages -v /input-packages/package_registry_config.yml:/package-registry/config.yml docker.elastic.co/package-registry/package-registry:main ``` - Add this to your kibana config: ``` xpack.fleet.registryUrl: http://localhost:8080 ``` - Open kibana, open Add System integration page (should be version `1.20.5` coming from local registry) - Enable only `Collect metrics from System instances / System cpu metrics` stream, and enable `Time-series indexing (TSDB)` switch under `Advanced options` - Click on save integration. The integration should be added successfully to an agent policy. - Start a fleet server and enroll an agent to the previously created agent policy. Wait for the agent to be healthy, this should trigger system metrics flowing in, this way triggering the creation of the data stream. - Go to `Stack Management / Index Management / Index Templates`, and open the details of `metrics-system.cpu`, check that it has this in `Settings` tab: `index.mode: time_series` image - Go to `Indices` tab and look for `data_stream="metrics-system.cpu-default"` including hidden indices. You should see `routing_path` populated in Settings, and `test_dimension` in Mappings. image image ### Testing scenario of enabling TSDB in package spec by default - Prepared a newer version of `system` package `v1.20.9` that has `index_mode: "time_series"` in `system.cpu` data stream in package manifest. - Download and extract the below zip and restart the local epr container to read from this folder. [registry-packages.zip](https://github.com/elastic/kibana/files/10011702/registry-packages.zip) - Open add system integration page - Check that `tsdb:true` is set in `Preview API request` for `system.cpu` data stream image - Open the system cpu metrics stream on the policy editor, and check that the TSDB switch is turned on. image image ### Update 11/16: - After latest changes, expect `routing_path` with dimension fields to be generated in Index Template when enabling TSDB: image This was added to fix the issue mentioned here https://github.com/elastic/kibana/pull/144974#issuecomment-1315270890 - After adding TSDB, modifying the parent component template should be successful. image ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 4608af41da7dc98403bbdbfba8a3ac02475277e4) --- .../migrations/check_registered_types.test.ts | 2 +- .../simplified_package_policy_helper.test.ts | 2 + .../plugins/fleet/common/types/models/epm.ts | 3 +- .../common/types/models/package_policy.ts | 1 + .../components/package_policy_input_panel.tsx | 36 +++ .../package_policy_input_stream.tsx | 132 ++++++---- .../fleet/server/saved_objects/index.ts | 1 + .../server/services/epm/archive/parse.test.ts | 5 + .../server/services/epm/archive/parse.ts | 4 + .../experimental_datastream_features.test.ts | 226 +++++++++++++++--- .../experimental_datastream_features.ts | 102 ++++++-- .../server/types/models/package_policy.ts | 3 +- 12 files changed, 414 insertions(+), 103 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index e43f622cf4008..716dde9b770b0 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -80,7 +80,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "cb22b422398a785e7e0565a19c6d4d5c7af6f2fd", + "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994", "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 7d5a7966f91c3..e60da12d924cc 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -131,6 +131,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ], @@ -142,6 +143,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ]); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 4ef4a7fdd751b..ca7cc3200da4b 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -352,6 +352,7 @@ export interface RegistryElasticsearch { 'index_template.mappings'?: estypes.MappingTypeMapping; 'ingest_pipeline.name'?: string; source_mode?: 'default' | 'synthetic'; + index_mode?: 'time_series'; } export interface RegistryDataStreamPrivileges { @@ -444,7 +445,7 @@ export type PackageInfo = | Installable>; // TODO - Expand this with other experimental indexing types -export type ExperimentalIndexingFeature = 'synthetic_source'; +export type ExperimentalIndexingFeature = 'synthetic_source' | 'tsdb'; export interface ExperimentalDataStreamFeature { data_stream: string; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 629250c243cea..0a951c0de8fce 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -33,6 +33,7 @@ export interface NewPackagePolicyInputStream { privileges?: { indices?: string[]; }; + index_mode?: string; }; }; release?: RegistryRelease; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 1d0cda70edea4..7bcda09d08d8f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -19,6 +19,7 @@ import { EuiButtonEmpty, } from '@elastic/eui'; +import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; import type { NewPackagePolicy, NewPackagePolicyInput, @@ -125,6 +126,41 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ [packageInputStreams, packagePolicyInput.streams] ); + // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series + let isUpdated = false; + inputStreams.forEach(({ packagePolicyInputStream }) => { + const dataStreamInfo = packageInfo.data_streams?.find( + (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset + ); + + if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') { + if (!packagePolicy.package) return; + if (!packagePolicy.package?.experimental_data_stream_features) + packagePolicy.package!.experimental_data_stream_features = []; + + const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream); + const match = packagePolicy.package!.experimental_data_stream_features.find( + (feat) => feat.data_stream === dsName + ); + if (match) { + if (!match.features.tsdb) { + match.features.tsdb = true; + isUpdated = true; + } + } else { + packagePolicy.package!.experimental_data_stream_features.push({ + data_stream: dsName, + features: { tsdb: true, synthetic_source: false }, + }); + isUpdated = true; + } + } + }); + + if (isUpdated) { + updatePackagePolicy(packagePolicy); + } + return ( <> {/* Header / input-level toggle */} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx index 54b6205da2ed9..2e5f98c8359e7 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState, Fragment, memo, useMemo, useEffect, useRef } from 'react'; +import React, { useState, Fragment, memo, useMemo, useEffect, useRef, useCallback } from 'react'; import ReactMarkdown from 'react-markdown'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -18,13 +18,14 @@ import { EuiSpacer, EuiButtonEmpty, EuiTitle, + EuiToolTip, } from '@elastic/eui'; import { useRouteMatch } from 'react-router-dom'; import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../services/package_prerelease'; import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; - +import type { ExperimentalIndexingFeature } from '../../../../../../../../../common/types/models/epm'; import type { NewPackagePolicy, NewPackagePolicyInputStream, @@ -120,6 +121,62 @@ export const PackagePolicyInputStreamConfig = memo( [advancedVars, inputStreamValidationResults?.vars] ); + const isFeatureEnabled = useCallback( + (feature: ExperimentalIndexingFeature) => + packagePolicy.package?.experimental_data_stream_features?.some( + ({ data_stream: dataStream, features }) => + dataStream === + getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream) && + features[feature] + ) ?? false, + [ + packagePolicy.package?.experimental_data_stream_features, + packagePolicyInputStream.data_stream, + ] + ); + + const newExperimentalIndexingFeature = { + synthetic_source: isFeatureEnabled('synthetic_source'), + tsdb: isFeatureEnabled('tsdb'), + }; + + const onIndexingSettingChange = ( + features: Partial> + ) => { + if (!packagePolicy.package) { + return; + } + + const newExperimentalDataStreamFeatures = [ + ...(packagePolicy.package.experimental_data_stream_features ?? []), + ]; + + const dataStream = getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream); + + const existingSettingRecord = newExperimentalDataStreamFeatures.find( + (x) => x.data_stream === dataStream + ); + + if (existingSettingRecord) { + existingSettingRecord.features = { + ...existingSettingRecord.features, + ...features, + }; + } else { + newExperimentalDataStreamFeatures.push({ + data_stream: dataStream, + features: { ...newExperimentalIndexingFeature, ...features }, + }); + } + + updatePackagePolicy({ + package: { + ...packagePolicy.package, + experimental_data_stream_features: newExperimentalDataStreamFeatures, + }, + }); + }; + return ( <> @@ -311,15 +368,7 @@ export const PackagePolicyInputStreamConfig = memo( - dataStream === - getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ) && features.synthetic_source - ) ?? false - } + checked={isFeatureEnabled('synthetic_source')} label={ ( /> } onChange={(e) => { - if (!packagePolicy.package) { - return; - } - - const newExperimentalDataStreamFeatures = [ - ...(packagePolicy.package.experimental_data_stream_features ?? []), - ]; - - const dataStream = getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ); - - const existingSettingRecord = newExperimentalDataStreamFeatures.find( - (x) => x.data_stream === dataStream - ); - - if (existingSettingRecord) { - existingSettingRecord.features.synthetic_source = e.target.checked; - } else { - newExperimentalDataStreamFeatures.push({ - data_stream: dataStream, - features: { - synthetic_source: e.target.checked, - }, - }); - } - - updatePackagePolicy({ - package: { - ...packagePolicy.package, - experimental_data_stream_features: - newExperimentalDataStreamFeatures, - }, + onIndexingSettingChange({ + synthetic_source: e.target.checked, }); }} /> + + + } + > + + } + onChange={(e) => { + onIndexingSettingChange({ + tsdb: e.target.checked, + }); + }} + /> + + diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9695a83c82bc5..a480675444e50 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -289,6 +289,7 @@ const getSavedObjectTypes = ( type: 'nested', properties: { synthetic_source: { type: 'boolean' }, + tsdb: { type: 'boolean' }, }, }, }, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts index e4ee10c4e270c..32393eebd12e5 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts @@ -67,6 +67,11 @@ describe('parseDataStreamElasticsearchEntry', () => { source_mode: 'synthetic', }); }); + it('Should add index_mode', () => { + expect(parseDataStreamElasticsearchEntry({ index_mode: 'time_series' })).toEqual({ + index_mode: 'time_series', + }); + }); it('Should add index_template mappings and expand dots', () => { expect( parseDataStreamElasticsearchEntry({ diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts index d4ee87dc232f0..1007b1fe35344 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -539,6 +539,10 @@ export function parseDataStreamElasticsearchEntry( ); } + if (expandedElasticsearch?.index_mode) { + parsedElasticsearchEntry.index_mode = expandedElasticsearch.index_mode; + } + return parsedElasticsearchEntry; } diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts index 8c7afa5d30fed..dddd11dd6ee19 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts @@ -10,12 +10,17 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks import type { NewPackagePolicy, PackagePolicy } from '../../types'; -import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features'; +import { + builRoutingPath, + handleExperimentalDatastreamFeatureOptIn, +} from './experimental_datastream_features'; function getNewTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): NewPackagePolicy { const packagePolicy: NewPackagePolicy = { name: 'Test policy', @@ -33,6 +38,7 @@ function getNewTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -44,8 +50,10 @@ function getNewTestPackagePolicy({ function getExistingTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): PackagePolicy { const packagePolicy: PackagePolicy = { id: 'test-policy', @@ -64,6 +72,7 @@ function getExistingTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -83,40 +92,89 @@ describe('experimental_datastream_features', () => { soClient.get.mockClear(); esClient.cluster.getComponentTemplate.mockClear(); esClient.cluster.putComponentTemplate.mockClear(); + + esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ + component_templates: [ + { + name: 'metrics-test.test@package', + component_template: { + template: { + settings: {}, + mappings: { + _source: { + // @ts-expect-error + mode: 'stored', + }, + properties: { + test_dimension: { + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + }, + ], + }); }); const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; describe('when package policy does not exist (create)', () => { - it('updates component template', async () => { - const packagePolicy = getNewTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); + + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -124,12 +182,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -140,12 +200,18 @@ describe('experimental_datastream_features', () => { describe('when package policy exists (update)', () => { describe('when opt in status in unchanged', () => { it('does not update component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: true } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: true, tsdb: false }, + }, ], }, id: 'mocked', @@ -161,34 +227,58 @@ describe('experimental_datastream_features', () => { }); describe('when opt in status is changed', () => { - it('updates component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); + + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -196,12 +286,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -209,4 +301,64 @@ describe('experimental_datastream_features', () => { }); }); }); + it('should build routing path', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['test_dimension']); + }); + + it('should build routing path from nested properties', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['cloud.image.id', 'test_dimension']); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts index 2b8b05aed89c3..3a60733a57eb5 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts @@ -5,6 +5,10 @@ * 2.0. */ +import type { + MappingProperty, + PropertyName, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; @@ -12,6 +16,31 @@ import type { NewPackagePolicy, PackagePolicy } from '../../types'; import { getInstallation } from '../epm/packages'; import { updateDatastreamExperimentalFeatures } from '../epm/packages/update'; +function mapFields(mappingProperties: Record) { + const mappings = Object.keys(mappingProperties).reduce((acc, curr) => { + const property = mappingProperties[curr] as any; + if (property.properties) { + const childMappings = mapFields(property.properties); + Object.keys(childMappings).forEach((key) => { + acc[curr + '.' + key] = childMappings[key]; + }); + } else { + acc[curr] = property; + } + return acc; + }, {} as any); + return mappings; +} + +export function builRoutingPath(properties: Record) { + const mappingsProperties = mapFields(properties); + return Object.keys(mappingsProperties).filter( + (mapping) => + mappingsProperties[mapping].type === 'keyword' && + mappingsProperties[mapping].time_series_dimension + ); +} + export async function handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, @@ -42,13 +71,12 @@ export async function handleExperimentalDatastreamFeatureOptIn({ (optIn) => optIn.data_stream === featureMapEntry.data_stream ); - const isOptInChanged = + const isSyntheticSourceOptInChanged = existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source; - // If the feature opt-in status in unchanged, we don't need to update any component templates - if (!isOptInChanged) { - continue; - } + const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb; + + if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue; const componentTemplateName = `${featureMapEntry.data_stream}@package`; const componentTemplateRes = await esClient.cluster.getComponentTemplate({ @@ -57,23 +85,59 @@ export async function handleExperimentalDatastreamFeatureOptIn({ const componentTemplate = componentTemplateRes.component_templates[0].component_template; - const body = { - template: { - ...componentTemplate.template, - mappings: { - ...componentTemplate.template.mappings, - _source: { - mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + if (isSyntheticSourceOptInChanged) { + const body = { + template: { + ...componentTemplate.template, + mappings: { + ...componentTemplate.template.mappings, + _source: { + mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + }, }, }, - }, - }; + }; - await esClient.cluster.putComponentTemplate({ - name: componentTemplateName, - // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source - body, - }); + await esClient.cluster.putComponentTemplate({ + name: componentTemplateName, + // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source + body, + }); + } + + if (isTSDBOptInChanged && featureMapEntry.features.tsdb) { + const mappingsProperties = componentTemplate.template?.mappings?.properties ?? {}; + + // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path + // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592 + const routingPath = builRoutingPath(mappingsProperties); + + if (routingPath.length === 0) continue; + + const indexTemplateRes = await esClient.indices.getIndexTemplate({ + name: featureMapEntry.data_stream, + }); + const indexTemplate = indexTemplateRes.index_templates[0].index_template; + + const indexTemplateBody = { + ...indexTemplate, + template: { + ...(indexTemplate.template ?? {}), + settings: { + ...(indexTemplate.template?.settings ?? {}), + index: { + mode: 'time_series', + routing_path: routingPath, + }, + }, + }, + }; + + await esClient.indices.putIndexTemplate({ + name: featureMapEntry.data_stream, + body: indexTemplateBody, + }); + } } // Update the installation object to persist the experimental feature map diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 82ff572cd1926..64837501b941b 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -84,6 +84,7 @@ const ExperimentalDataStreamFeatures = schema.arrayOf( data_stream: schema.string(), features: schema.object({ synthetic_source: schema.boolean(), + tsdb: schema.boolean(), }), }) ); @@ -128,7 +129,7 @@ const CreatePackagePolicyProps = { schema.arrayOf( schema.object({ data_stream: schema.string(), - features: schema.object({ synthetic_source: schema.boolean() }), + features: schema.object({ synthetic_source: schema.boolean(), tsdb: schema.boolean() }), }) ) ),