From 4608af41da7dc98403bbdbfba8a3ac02475277e4 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Thu, 17 Nov 2022 10:13:20 +0100 Subject: [PATCH] Added Time-series indexing (TSDB) to Integrations Experimental Indexing settings (#144974) ## Summary Closes https://github.com/elastic/kibana/issues/144530 Added a new option under integration / data streams / Index settings (experimental) / TSDB - [x] Add a toggle to the "Experimental indexing features" data streams UI for TSDB - Enabling this toggle adds `index.mode: time_series` to the data stream's index template settings. - Note: `index.routing_path` value was not needed to be added in this logic, because there is an existing elasticsearch automation that generates it from dimension fields, see test instructions. - [x] If the current package's manifest contains `index_mode: time_series` for a given data stream, the toggle should be in an "enabled + readonly" state, e.g. it cannot be disabled - Note: currently there is no package that has this setting enabled, I tested locally by manually modifying the package info response. Will need to figure out a way to create a mock package to test this. - [x] Once the toggle is enabled and the policy is saved, the toggle should _not_ be disable-able. Enabling TSDB for a data stream is an irreversible operation - [x] Add a tooltip to make the irreversible nature of this operation clear TSDB setting in System package: image image Tooltip: image ## Test instructions ### Setup local registry - For testing, I created a new version of `System` package, that has a `dimension` field called `test_dimension`. This is needed so that the elasticsearch automation runs to generate the `routing_path` setting. - Download the test data zip: [registry-packages.zip](https://github.com/elastic/kibana/files/10004220/registry-packages.zip) - Start a registry: ``` docker run -p 8080:8080 -v /input-packages/packages:/packages/test-packages -v /input-packages/package_registry_config.yml:/package-registry/config.yml docker.elastic.co/package-registry/package-registry:main ``` - Add this to your kibana config: ``` xpack.fleet.registryUrl: http://localhost:8080 ``` - Open kibana, open Add System integration page (should be version `1.20.5` coming from local registry) - Enable only `Collect metrics from System instances / System cpu metrics` stream, and enable `Time-series indexing (TSDB)` switch under `Advanced options` - Click on save integration. The integration should be added successfully to an agent policy. - Start a fleet server and enroll an agent to the previously created agent policy. Wait for the agent to be healthy, this should trigger system metrics flowing in, this way triggering the creation of the data stream. - Go to `Stack Management / Index Management / Index Templates`, and open the details of `metrics-system.cpu`, check that it has this in `Settings` tab: `index.mode: time_series` image - Go to `Indices` tab and look for `data_stream="metrics-system.cpu-default"` including hidden indices. You should see `routing_path` populated in Settings, and `test_dimension` in Mappings. image image ### Testing scenario of enabling TSDB in package spec by default - Prepared a newer version of `system` package `v1.20.9` that has `index_mode: "time_series"` in `system.cpu` data stream in package manifest. - Download and extract the below zip and restart the local epr container to read from this folder. [registry-packages.zip](https://github.com/elastic/kibana/files/10011702/registry-packages.zip) - Open add system integration page - Check that `tsdb:true` is set in `Preview API request` for `system.cpu` data stream image - Open the system cpu metrics stream on the policy editor, and check that the TSDB switch is turned on. image image ### Update 11/16: - After latest changes, expect `routing_path` with dimension fields to be generated in Index Template when enabling TSDB: image This was added to fix the issue mentioned here https://github.com/elastic/kibana/pull/144974#issuecomment-1315270890 - After adding TSDB, modifying the parent component template should be successful. image ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../migrations/check_registered_types.test.ts | 2 +- .../simplified_package_policy_helper.test.ts | 2 + .../plugins/fleet/common/types/models/epm.ts | 3 +- .../common/types/models/package_policy.ts | 1 + .../components/package_policy_input_panel.tsx | 36 +++ .../package_policy_input_stream.tsx | 132 ++++++---- .../fleet/server/saved_objects/index.ts | 1 + .../server/services/epm/archive/parse.test.ts | 5 + .../server/services/epm/archive/parse.ts | 4 + .../experimental_datastream_features.test.ts | 226 +++++++++++++++--- .../experimental_datastream_features.ts | 102 ++++++-- .../server/types/models/package_policy.ts | 3 +- 12 files changed, 414 insertions(+), 103 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index 12f58518952f..33baf258a13d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -80,7 +80,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "cb22b422398a785e7e0565a19c6d4d5c7af6f2fd", + "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994", "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 7d5a7966f91c..e60da12d924c 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -131,6 +131,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ], @@ -142,6 +143,7 @@ describe('toPackagePolicy', () => { data_stream: 'logs-nginx.access', features: { synthetic_source: true, + tsdb: false, }, }, ]); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 4ef4a7fdd751..ca7cc3200da4 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -352,6 +352,7 @@ export interface RegistryElasticsearch { 'index_template.mappings'?: estypes.MappingTypeMapping; 'ingest_pipeline.name'?: string; source_mode?: 'default' | 'synthetic'; + index_mode?: 'time_series'; } export interface RegistryDataStreamPrivileges { @@ -444,7 +445,7 @@ export type PackageInfo = | Installable>; // TODO - Expand this with other experimental indexing types -export type ExperimentalIndexingFeature = 'synthetic_source'; +export type ExperimentalIndexingFeature = 'synthetic_source' | 'tsdb'; export interface ExperimentalDataStreamFeature { data_stream: string; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 629250c243ce..0a951c0de8fc 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -33,6 +33,7 @@ export interface NewPackagePolicyInputStream { privileges?: { indices?: string[]; }; + index_mode?: string; }; }; release?: RegistryRelease; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 1d0cda70edea..7bcda09d08d8 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -19,6 +19,7 @@ import { EuiButtonEmpty, } from '@elastic/eui'; +import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; import type { NewPackagePolicy, NewPackagePolicyInput, @@ -125,6 +126,41 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ [packageInputStreams, packagePolicyInput.streams] ); + // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series + let isUpdated = false; + inputStreams.forEach(({ packagePolicyInputStream }) => { + const dataStreamInfo = packageInfo.data_streams?.find( + (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset + ); + + if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') { + if (!packagePolicy.package) return; + if (!packagePolicy.package?.experimental_data_stream_features) + packagePolicy.package!.experimental_data_stream_features = []; + + const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream); + const match = packagePolicy.package!.experimental_data_stream_features.find( + (feat) => feat.data_stream === dsName + ); + if (match) { + if (!match.features.tsdb) { + match.features.tsdb = true; + isUpdated = true; + } + } else { + packagePolicy.package!.experimental_data_stream_features.push({ + data_stream: dsName, + features: { tsdb: true, synthetic_source: false }, + }); + isUpdated = true; + } + } + }); + + if (isUpdated) { + updatePackagePolicy(packagePolicy); + } + return ( <> {/* Header / input-level toggle */} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx index 54b6205da2ed..2e5f98c8359e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState, Fragment, memo, useMemo, useEffect, useRef } from 'react'; +import React, { useState, Fragment, memo, useMemo, useEffect, useRef, useCallback } from 'react'; import ReactMarkdown from 'react-markdown'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -18,13 +18,14 @@ import { EuiSpacer, EuiButtonEmpty, EuiTitle, + EuiToolTip, } from '@elastic/eui'; import { useRouteMatch } from 'react-router-dom'; import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../services/package_prerelease'; import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services'; - +import type { ExperimentalIndexingFeature } from '../../../../../../../../../common/types/models/epm'; import type { NewPackagePolicy, NewPackagePolicyInputStream, @@ -120,6 +121,62 @@ export const PackagePolicyInputStreamConfig = memo( [advancedVars, inputStreamValidationResults?.vars] ); + const isFeatureEnabled = useCallback( + (feature: ExperimentalIndexingFeature) => + packagePolicy.package?.experimental_data_stream_features?.some( + ({ data_stream: dataStream, features }) => + dataStream === + getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream) && + features[feature] + ) ?? false, + [ + packagePolicy.package?.experimental_data_stream_features, + packagePolicyInputStream.data_stream, + ] + ); + + const newExperimentalIndexingFeature = { + synthetic_source: isFeatureEnabled('synthetic_source'), + tsdb: isFeatureEnabled('tsdb'), + }; + + const onIndexingSettingChange = ( + features: Partial> + ) => { + if (!packagePolicy.package) { + return; + } + + const newExperimentalDataStreamFeatures = [ + ...(packagePolicy.package.experimental_data_stream_features ?? []), + ]; + + const dataStream = getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream); + + const existingSettingRecord = newExperimentalDataStreamFeatures.find( + (x) => x.data_stream === dataStream + ); + + if (existingSettingRecord) { + existingSettingRecord.features = { + ...existingSettingRecord.features, + ...features, + }; + } else { + newExperimentalDataStreamFeatures.push({ + data_stream: dataStream, + features: { ...newExperimentalIndexingFeature, ...features }, + }); + } + + updatePackagePolicy({ + package: { + ...packagePolicy.package, + experimental_data_stream_features: newExperimentalDataStreamFeatures, + }, + }); + }; + return ( <> @@ -311,15 +368,7 @@ export const PackagePolicyInputStreamConfig = memo( - dataStream === - getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ) && features.synthetic_source - ) ?? false - } + checked={isFeatureEnabled('synthetic_source')} label={ ( /> } onChange={(e) => { - if (!packagePolicy.package) { - return; - } - - const newExperimentalDataStreamFeatures = [ - ...(packagePolicy.package.experimental_data_stream_features ?? []), - ]; - - const dataStream = getRegistryDataStreamAssetBaseName( - packagePolicyInputStream.data_stream - ); - - const existingSettingRecord = newExperimentalDataStreamFeatures.find( - (x) => x.data_stream === dataStream - ); - - if (existingSettingRecord) { - existingSettingRecord.features.synthetic_source = e.target.checked; - } else { - newExperimentalDataStreamFeatures.push({ - data_stream: dataStream, - features: { - synthetic_source: e.target.checked, - }, - }); - } - - updatePackagePolicy({ - package: { - ...packagePolicy.package, - experimental_data_stream_features: - newExperimentalDataStreamFeatures, - }, + onIndexingSettingChange({ + synthetic_source: e.target.checked, }); }} /> + + + } + > + + } + onChange={(e) => { + onIndexingSettingChange({ + tsdb: e.target.checked, + }); + }} + /> + + diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index e38e6c966c11..f2ee5b034bf9 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -284,6 +284,7 @@ const getSavedObjectTypes = ( type: 'nested', properties: { synthetic_source: { type: 'boolean' }, + tsdb: { type: 'boolean' }, }, }, }, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts index e4ee10c4e270..32393eebd12e 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts @@ -67,6 +67,11 @@ describe('parseDataStreamElasticsearchEntry', () => { source_mode: 'synthetic', }); }); + it('Should add index_mode', () => { + expect(parseDataStreamElasticsearchEntry({ index_mode: 'time_series' })).toEqual({ + index_mode: 'time_series', + }); + }); it('Should add index_template mappings and expand dots', () => { expect( parseDataStreamElasticsearchEntry({ diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts index d4ee87dc232f..1007b1fe3534 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -539,6 +539,10 @@ export function parseDataStreamElasticsearchEntry( ); } + if (expandedElasticsearch?.index_mode) { + parsedElasticsearchEntry.index_mode = expandedElasticsearch.index_mode; + } + return parsedElasticsearchEntry; } diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts index 8c7afa5d30fe..dddd11dd6ee1 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts @@ -10,12 +10,17 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks import type { NewPackagePolicy, PackagePolicy } from '../../types'; -import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features'; +import { + builRoutingPath, + handleExperimentalDatastreamFeatureOptIn, +} from './experimental_datastream_features'; function getNewTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): NewPackagePolicy { const packagePolicy: NewPackagePolicy = { name: 'Test policy', @@ -33,6 +38,7 @@ function getNewTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -44,8 +50,10 @@ function getNewTestPackagePolicy({ function getExistingTestPackagePolicy({ isSyntheticSourceEnabled, + isTSDBEnabled, }: { isSyntheticSourceEnabled: boolean; + isTSDBEnabled: boolean; }): PackagePolicy { const packagePolicy: PackagePolicy = { id: 'test-policy', @@ -64,6 +72,7 @@ function getExistingTestPackagePolicy({ data_stream: 'metrics-test.test', features: { synthetic_source: isSyntheticSourceEnabled, + tsdb: isTSDBEnabled, }, }, ], @@ -83,40 +92,89 @@ describe('experimental_datastream_features', () => { soClient.get.mockClear(); esClient.cluster.getComponentTemplate.mockClear(); esClient.cluster.putComponentTemplate.mockClear(); + + esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ + component_templates: [ + { + name: 'metrics-test.test@package', + component_template: { + template: { + settings: {}, + mappings: { + _source: { + // @ts-expect-error + mode: 'stored', + }, + properties: { + test_dimension: { + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + }, + ], + }); }); const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; describe('when package policy does not exist (create)', () => { - it('updates component template', async () => { - const packagePolicy = getNewTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getNewTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); + + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -124,12 +182,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -140,12 +200,18 @@ describe('experimental_datastream_features', () => { describe('when package policy exists (update)', () => { describe('when opt in status in unchanged', () => { it('does not update component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: true } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: true, tsdb: false }, + }, ], }, id: 'mocked', @@ -161,34 +227,58 @@ describe('experimental_datastream_features', () => { }); describe('when opt in status is changed', () => { - it('updates component template', async () => { - const packagePolicy = getExistingTestPackagePolicy({ isSyntheticSourceEnabled: true }); - + beforeEach(() => { soClient.get.mockResolvedValueOnce({ attributes: { experimental_data_stream_features: [ - { data_stream: 'metrics-test.test', features: { synthetic_source: false } }, + { + data_stream: 'metrics-test.test', + features: { synthetic_source: false, tsdb: false }, + }, ], }, id: 'mocked', type: 'mocked', references: [], }); + }); + it('updates component template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: true, + isTSDBEnabled: false, + }); + + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); + expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + template: expect.objectContaining({ + mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + }), + }), + }) + ); + }); + + it('should update index template', async () => { + const packagePolicy = getExistingTestPackagePolicy({ + isSyntheticSourceEnabled: false, + isTSDBEnabled: true, + }); - esClient.cluster.getComponentTemplate.mockResolvedValueOnce({ - component_templates: [ + esClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ { - name: 'metrics-test.test@package', - component_template: { + name: 'metrics-test.test', + index_template: { template: { settings: {}, - mappings: { - _source: { - // @ts-expect-error - mode: 'stored', - }, - }, + mappings: {}, }, + composed_of: [], + index_patterns: '', }, }, ], @@ -196,12 +286,14 @@ describe('experimental_datastream_features', () => { await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); - expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled(); - expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith( + expect(esClient.indices.getIndexTemplate).toHaveBeenCalled(); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ body: expect.objectContaining({ template: expect.objectContaining({ - mappings: expect.objectContaining({ _source: { mode: 'synthetic' } }), + settings: expect.objectContaining({ + index: { mode: 'time_series', routing_path: ['test_dimension'] }, + }), }), }), }) @@ -209,4 +301,64 @@ describe('experimental_datastream_features', () => { }); }); }); + it('should build routing path', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['test_dimension']); + }); + + it('should build routing path from nested properties', () => { + const mappingProperties = { + cloud: { + properties: { + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + image: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + time_series_dimension: true, + }, + }, + }, + }, + }, + test_dimension: { + time_series_dimension: true, + type: 'keyword', + }, + '@timestamp': { + type: 'date', + }, + }; + const routingPath = builRoutingPath(mappingProperties as any); + expect(routingPath).toEqual(['cloud.image.id', 'test_dimension']); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts index 2b8b05aed89c..3a60733a57eb 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts @@ -5,6 +5,10 @@ * 2.0. */ +import type { + MappingProperty, + PropertyName, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; @@ -12,6 +16,31 @@ import type { NewPackagePolicy, PackagePolicy } from '../../types'; import { getInstallation } from '../epm/packages'; import { updateDatastreamExperimentalFeatures } from '../epm/packages/update'; +function mapFields(mappingProperties: Record) { + const mappings = Object.keys(mappingProperties).reduce((acc, curr) => { + const property = mappingProperties[curr] as any; + if (property.properties) { + const childMappings = mapFields(property.properties); + Object.keys(childMappings).forEach((key) => { + acc[curr + '.' + key] = childMappings[key]; + }); + } else { + acc[curr] = property; + } + return acc; + }, {} as any); + return mappings; +} + +export function builRoutingPath(properties: Record) { + const mappingsProperties = mapFields(properties); + return Object.keys(mappingsProperties).filter( + (mapping) => + mappingsProperties[mapping].type === 'keyword' && + mappingsProperties[mapping].time_series_dimension + ); +} + export async function handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, @@ -42,13 +71,12 @@ export async function handleExperimentalDatastreamFeatureOptIn({ (optIn) => optIn.data_stream === featureMapEntry.data_stream ); - const isOptInChanged = + const isSyntheticSourceOptInChanged = existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source; - // If the feature opt-in status in unchanged, we don't need to update any component templates - if (!isOptInChanged) { - continue; - } + const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb; + + if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue; const componentTemplateName = `${featureMapEntry.data_stream}@package`; const componentTemplateRes = await esClient.cluster.getComponentTemplate({ @@ -57,23 +85,59 @@ export async function handleExperimentalDatastreamFeatureOptIn({ const componentTemplate = componentTemplateRes.component_templates[0].component_template; - const body = { - template: { - ...componentTemplate.template, - mappings: { - ...componentTemplate.template.mappings, - _source: { - mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + if (isSyntheticSourceOptInChanged) { + const body = { + template: { + ...componentTemplate.template, + mappings: { + ...componentTemplate.template.mappings, + _source: { + mode: featureMapEntry.features.synthetic_source ? 'synthetic' : 'stored', + }, }, }, - }, - }; + }; - await esClient.cluster.putComponentTemplate({ - name: componentTemplateName, - // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source - body, - }); + await esClient.cluster.putComponentTemplate({ + name: componentTemplateName, + // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source + body, + }); + } + + if (isTSDBOptInChanged && featureMapEntry.features.tsdb) { + const mappingsProperties = componentTemplate.template?.mappings?.properties ?? {}; + + // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path + // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592 + const routingPath = builRoutingPath(mappingsProperties); + + if (routingPath.length === 0) continue; + + const indexTemplateRes = await esClient.indices.getIndexTemplate({ + name: featureMapEntry.data_stream, + }); + const indexTemplate = indexTemplateRes.index_templates[0].index_template; + + const indexTemplateBody = { + ...indexTemplate, + template: { + ...(indexTemplate.template ?? {}), + settings: { + ...(indexTemplate.template?.settings ?? {}), + index: { + mode: 'time_series', + routing_path: routingPath, + }, + }, + }, + }; + + await esClient.indices.putIndexTemplate({ + name: featureMapEntry.data_stream, + body: indexTemplateBody, + }); + } } // Update the installation object to persist the experimental feature map diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 82ff572cd192..64837501b941 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -84,6 +84,7 @@ const ExperimentalDataStreamFeatures = schema.arrayOf( data_stream: schema.string(), features: schema.object({ synthetic_source: schema.boolean(), + tsdb: schema.boolean(), }), }) ); @@ -128,7 +129,7 @@ const CreatePackagePolicyProps = { schema.arrayOf( schema.object({ data_stream: schema.string(), - features: schema.object({ synthetic_source: schema.boolean() }), + features: schema.object({ synthetic_source: schema.boolean(), tsdb: schema.boolean() }), }) ) ),