From e9db945670a4bf43408b90222723f4a1096114a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica=20Pais=20da=20Silva?= Date: Tue, 3 Dec 2024 19:23:13 +0100 Subject: [PATCH] [Infra] Exclude frozen/cold data tiers from source queries (#201804) Closes #201568 Adds the exclude data tiers settings to the `/api/metrics/source/hasData` and `/api/metrics/source/{sourceId}` requests. Also applies it to the `getIndexStatus` API call. --------- Co-authored-by: Elastic Machine Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit ac3b60ee907436aeee35fc59df7c30bcb9d66bc8) --- .../server/lib/helpers/tier_filter.ts | 4 +- .../elasticsearch_source_status_adapter.ts | 16 +- .../helpers/get_infra_metrics_client.test.ts | 151 ++++++++++++++++++ .../lib/helpers/get_infra_metrics_client.ts | 27 ++++ .../server/routes/metrics_sources/index.ts | 2 +- .../infra/tsconfig.json | 4 +- .../observability/server/ui_settings.ts | 6 +- 7 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index cad0b03579e3d..31ed11e8d2fa4 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -13,10 +13,10 @@ export function getDataTierFilterCombined({ excludedDataTiers, }: { filter?: QueryDslQueryContainer; - excludedDataTiers: DataTier[]; + excludedDataTiers?: DataTier[]; }): QueryDslQueryContainer | undefined { if (!filter) { - return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined; + return excludedDataTiers?.length ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } return !excludedDataTiers diff --git a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts index 54e2831899482..ff8058011baea 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts @@ -5,11 +5,14 @@ * 2.0. */ +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { InfraPluginRequestHandlerContext } from '../../../types'; import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors'; -import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; -import { InfraDatabaseGetIndicesResponse } from '../framework'; -import { KibanaFramework } from '../framework/kibana_framework_adapter'; +import type { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; +import type { InfraDatabaseGetIndicesResponse } from '../framework'; +import type { KibanaFramework } from '../framework/kibana_framework_adapter'; export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusAdapter { constructor(private readonly framework: KibanaFramework) {} @@ -46,6 +49,12 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA requestContext: InfraPluginRequestHandlerContext, indexNames: string ): Promise { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + + const filter = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : []; + return await this.framework .callWithRequest(requestContext, 'search', { ignore_unavailable: true, @@ -54,6 +63,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA size: 0, terminate_after: 1, track_total_hits: 1, + query: { bool: { filter } }, }) .then( (response) => { diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts new file mode 100644 index 0000000000000..3eb8c47c274d9 --- /dev/null +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { InfraBackendLibs } from '../infra_types'; +import { getInfraMetricsClient } from './get_infra_metrics_client'; +import { InfraPluginRequestHandlerContext } from '../../types'; +import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; + +const withExcludedDataTiers = (tiers: DataTier[]) => ({ + uiSettings: { + client: { + get: () => Promise.resolve(tiers), + }, + }, +}); + +const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) }; + +const infraMetricsTestHarness = + (tiers: DataTier[], input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) => + async () => { + const callWithRequest = jest.fn(); + + const mockedCore = withExcludedDataTiers(tiers); + + const context = { + infra: Promise.resolve(mockedInfra), + core: Promise.resolve(mockedCore), + } as unknown as InfraPluginRequestHandlerContext; + + const client = await getInfraMetricsClient({ + libs: { framework: { callWithRequest } } as unknown as InfraBackendLibs, + context, + request: {} as unknown as KibanaRequest, + }); + + await client.search({ + body: { + query: input, + size: 1, + track_total_hits: false, + }, + }); + + expect(callWithRequest).toBeCalledWith( + context, + 'search', + { + body: { + query: output, + size: 1, + track_total_hits: false, + }, + ignore_unavailable: true, + index: ['*.indices'], + }, + {} + ); + }; + +describe('getInfraMetricsClient', () => { + it( + 'defines an empty must_not query if given no data tiers to filter by', + infraMetricsTestHarness([], undefined, { bool: { must_not: [] } }) + ); + + it( + 'includes excluded data tiers in the request filter by default', + infraMetricsTestHarness(['data_frozen'], undefined, { + bool: { + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + }) + ); + + it( + 'merges provided filters with the excluded data tier filter', + infraMetricsTestHarness( + ['data_frozen'], + { + bool: { + must_not: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + must_not: [ + { + exists: { + field: 'a-field', + }, + }, + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + } + ) + ); + + it( + 'merges other query params with the excluded data tiers filter', + infraMetricsTestHarness( + ['data_frozen'], + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + } + ) + ); +}); diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts index d560ffc358a5c..96ae89b902285 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts @@ -6,6 +6,10 @@ */ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { InfraPluginRequestHandlerContext } from '../../types'; import type { InfraBackendLibs } from '../infra_types'; @@ -29,12 +33,25 @@ export async function getInfraMetricsClient({ }) { const { framework } = libs; const infraContext = await context.infra; + const { uiSettings } = await context.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); const metricsIndices = await infraContext.getMetricsIndices(); + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers)[0].bool!.must_not! + : []; + return { search( searchParams: TParams ): Promise> { + const searchFilter = searchParams.body.query?.bool?.must_not ?? []; + + // This flattens arrays by one level, and non-array values can be added as well, so it all + // results in a nice [QueryDsl, QueryDsl, ...] array. + const mustNot = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery); + return framework.callWithRequest( context, 'search', @@ -42,6 +59,16 @@ export async function getInfraMetricsClient({ ...searchParams, ignore_unavailable: true, index: metricsIndices, + body: { + ...searchParams.body, + query: { + ...searchParams.body.query, + bool: { + ...searchParams.body.query?.bool, + must_not: mustNot, + }, + }, + }, }, request ) as Promise; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts index 20dcbc1b7e20b..77bdee365e3bf 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts @@ -208,7 +208,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => }, async (context, request, response) => { try { - const modules = castArray(request.query.modules); + const modules = request.query.modules ? castArray(request.query.modules) : []; if (modules.length > MAX_MODULES) { throw Boom.badRequest( diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json index 2103350048e4b..c0ffe424066f2 100644 --- a/x-pack/plugins/observability_solution/infra/tsconfig.json +++ b/x-pack/plugins/observability_solution/infra/tsconfig.json @@ -116,7 +116,9 @@ "@kbn/entityManager-plugin", "@kbn/observability-utils", "@kbn/entities-schema", - "@kbn/zod" + "@kbn/zod", + "@kbn/observability-utils-server", + "@kbn/observability-utils-common" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts index dae7e2ad9ab5b..4b007177c5aed 100644 --- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @@ -531,7 +531,7 @@ export const uiSettings: Record = { value: 1.7, description: i18n.translate('xpack.observability.profilingDatacenterPUEUiSettingDescription', { defaultMessage: `Data center power usage effectiveness (PUE) measures how efficiently a data center uses energy. Defaults to 1.7, the average on-premise data center PUE according to the Uptime Institute survey - + You can also use the PUE that corresponds with your cloud provider: '
  • AWS: 1.135
    • @@ -649,8 +649,8 @@ export const uiSettings: Record = { description: i18n.translate( 'xpack.observability.advancedSettings.searchExcludedDataTiersDesc', { - defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. - When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`, + defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. + When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM, Infrastructure`, values: { technicalPreviewLabel: `[${technicalPreviewLabel}]` }, } ),