From 8900cce9bfe4403cc538a0862d784fe0ccff4cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Tue, 26 Nov 2024 15:44:10 +0100 Subject: [PATCH 1/8] [Infra] Exclude frozen/cold data tiers from source queries --- .../server/lib/helpers/tier_filter.ts | 2 +- .../elasticsearch_source_status_adapter.ts | 8 ++++++ .../server/routes/metrics_sources/index.ts | 25 +++++++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index c1f8d5e3fce1f..134bf7556618a 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -19,7 +19,7 @@ export function getDataTierFilterCombined({ return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } - return !excludedDataTiers + return !excludedDataTiers.length ? filter : { bool: { diff --git a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts index 54e2831899482..1d955075c6b7f 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts @@ -5,6 +5,9 @@ * 2.0. */ +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import type { InfraPluginRequestHandlerContext } from '../../../types'; import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors'; import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; @@ -46,6 +49,10 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA requestContext: InfraPluginRequestHandlerContext, indexNames: string ): Promise { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + return await this.framework .callWithRequest(requestContext, 'search', { ignore_unavailable: true, @@ -54,6 +61,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA size: 0, terminate_after: 1, track_total_hits: 1, + query: getDataTierFilterCombined({ excludedDataTiers }), }) .then( (response) => { diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts index 20dcbc1b7e20b..d46aa51f4499e 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts @@ -9,6 +9,9 @@ import { schema } from '@kbn/config-schema'; import Boom from '@hapi/boom'; import { createRouteValidationFunction } from '@kbn/io-ts-utils'; import { termsQuery } from '@kbn/observability-plugin/server'; +import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; import { castArray } from 'lodash'; import { EVENT_MODULE, METRICSET_MODULE } from '../../../common/constants'; import { @@ -209,6 +212,9 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => async (context, request, response) => { try { const modules = castArray(request.query.modules); + const { uiSettings } = await context.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); if (modules.length > MAX_MODULES) { throw Boom.badRequest( @@ -231,15 +237,18 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => size: 0, ...(modules.length > 0 ? { - query: { - bool: { - should: [ - ...termsQuery(EVENT_MODULE, ...modules), - ...termsQuery(METRICSET_MODULE, ...modules), - ], - minimum_should_match: 1, + query: getDataTierFilterCombined({ + filter: { + bool: { + should: [ + ...termsQuery(EVENT_MODULE, ...modules), + ...termsQuery(METRICSET_MODULE, ...modules), + ], + minimum_should_match: 1, + }, }, - }, + excludedDataTiers, + }), } : {}), }, From 7ddf284e46d33fa33161e4e041ad461ebe6e4d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Tue, 26 Nov 2024 17:49:09 +0100 Subject: [PATCH 2/8] Fix false typing of excludedDataTiers --- .../apm_data_access/server/lib/helpers/tier_filter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index 134bf7556618a..174d8380fa038 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -13,13 +13,13 @@ export function getDataTierFilterCombined({ excludedDataTiers, }: { filter?: QueryDslQueryContainer; - excludedDataTiers: DataTier[]; + excludedDataTiers?: DataTier[]; }): QueryDslQueryContainer | undefined { if (!filter) { return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } - return !excludedDataTiers.length + return !excludedDataTiers ? filter : { bool: { From 0183708d27543284f59f5050b040e19bc6f2a9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Tue, 26 Nov 2024 18:51:51 +0100 Subject: [PATCH 3/8] fix bad typing again --- .../apm_data_access/server/lib/helpers/tier_filter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index 174d8380fa038..83171dea7f444 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -16,7 +16,7 @@ export function getDataTierFilterCombined({ excludedDataTiers?: DataTier[]; }): QueryDslQueryContainer | undefined { if (!filter) { - return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined; + return excludedDataTiers?.length ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } return !excludedDataTiers From 0605700f17464f23e8cf65a8e571e74c2f306785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Wed, 27 Nov 2024 15:56:49 +0100 Subject: [PATCH 4/8] Add data tier filtering to infra metrics client, update affect apps label --- .../lib/helpers/get_infra_metrics_client.ts | 13 ++++++++++ .../server/routes/metrics_sources/index.ts | 25 ++++++------------- .../observability/server/ui_settings.ts | 6 ++--- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts index d560ffc358a5c..d8d51cd77b105 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts @@ -6,6 +6,9 @@ */ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import type { InfraPluginRequestHandlerContext } from '../../types'; import type { InfraBackendLibs } from '../infra_types'; @@ -29,6 +32,9 @@ export async function getInfraMetricsClient({ }) { const { framework } = libs; const infraContext = await context.infra; + const { uiSettings } = await context.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); const metricsIndices = await infraContext.getMetricsIndices(); return { @@ -42,6 +48,13 @@ export async function getInfraMetricsClient({ ...searchParams, ignore_unavailable: true, index: metricsIndices, + body: { + ...searchParams.body, + query: getDataTierFilterCombined({ + filter: searchParams.body.query, + excludedDataTiers, + }), + }, }, request ) as Promise; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts index d46aa51f4499e..20dcbc1b7e20b 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts @@ -9,9 +9,6 @@ import { schema } from '@kbn/config-schema'; import Boom from '@hapi/boom'; import { createRouteValidationFunction } from '@kbn/io-ts-utils'; import { termsQuery } from '@kbn/observability-plugin/server'; -import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; -import { DataTier } from '@kbn/observability-shared-plugin/common'; -import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; import { castArray } from 'lodash'; import { EVENT_MODULE, METRICSET_MODULE } from '../../../common/constants'; import { @@ -212,9 +209,6 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => async (context, request, response) => { try { const modules = castArray(request.query.modules); - const { uiSettings } = await context.core; - - const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); if (modules.length > MAX_MODULES) { throw Boom.badRequest( @@ -237,18 +231,15 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => size: 0, ...(modules.length > 0 ? { - query: getDataTierFilterCombined({ - filter: { - bool: { - should: [ - ...termsQuery(EVENT_MODULE, ...modules), - ...termsQuery(METRICSET_MODULE, ...modules), - ], - minimum_should_match: 1, - }, + query: { + bool: { + should: [ + ...termsQuery(EVENT_MODULE, ...modules), + ...termsQuery(METRICSET_MODULE, ...modules), + ], + minimum_should_match: 1, }, - excludedDataTiers, - }), + }, } : {}), }, diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts index 1a387f24fbaed..e5695a3c9034d 100644 --- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @@ -504,7 +504,7 @@ export const uiSettings: Record = { value: 1.7, description: i18n.translate('xpack.observability.profilingDatacenterPUEUiSettingDescription', { defaultMessage: `Data center power usage effectiveness (PUE) measures how efficiently a data center uses energy. Defaults to 1.7, the average on-premise data center PUE according to the Uptime Institute survey - + You can also use the PUE that corresponds with your cloud provider: '
  • AWS: 1.135
    • @@ -622,8 +622,8 @@ export const uiSettings: Record = { description: i18n.translate( 'xpack.observability.advancedSettings.searchExcludedDataTiersDesc', { - defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. - When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`, + defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. + When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM, Infrastructure`, values: { technicalPreviewLabel: `[${technicalPreviewLabel}]` }, } ), From dcd8d23b792b42e9d0eddeea9f3df3a2283674e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Wed, 27 Nov 2024 18:14:42 +0100 Subject: [PATCH 5/8] Avoid using getDataTierFilterCombined for now --- .../elasticsearch_source_status_adapter.ts | 14 ++++++----- .../lib/helpers/get_infra_metrics_client.ts | 24 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts index 1d955075c6b7f..ff8058011baea 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts @@ -5,14 +5,14 @@ * 2.0. */ -import { DataTier } from '@kbn/observability-shared-plugin/common'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; -import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { InfraPluginRequestHandlerContext } from '../../../types'; import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors'; -import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; -import { InfraDatabaseGetIndicesResponse } from '../framework'; -import { KibanaFramework } from '../framework/kibana_framework_adapter'; +import type { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status'; +import type { InfraDatabaseGetIndicesResponse } from '../framework'; +import type { KibanaFramework } from '../framework/kibana_framework_adapter'; export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusAdapter { constructor(private readonly framework: KibanaFramework) {} @@ -53,6 +53,8 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + const filter = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : []; + return await this.framework .callWithRequest(requestContext, 'search', { ignore_unavailable: true, @@ -61,7 +63,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA size: 0, terminate_after: 1, track_total_hits: 1, - query: getDataTierFilterCombined({ excludedDataTiers }), + query: { bool: { filter } }, }) .then( (response) => { diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts index d8d51cd77b105..51e330675e875 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts @@ -7,8 +7,9 @@ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; -import { DataTier } from '@kbn/observability-shared-plugin/common'; -import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { InfraPluginRequestHandlerContext } from '../../types'; import type { InfraBackendLibs } from '../infra_types'; @@ -41,6 +42,14 @@ export async function getInfraMetricsClient({ search( searchParams: TParams ): Promise> { + const searchFilter = searchParams.body.query?.bool?.filter ?? []; + + const excludedQuery = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : []; + + // This flattens arrays by one level, and non-array values can be added as well, so it all + // results in a nice [QueryDsl, QueryDsl, ...] array. + const filter = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery); + return framework.callWithRequest( context, 'search', @@ -50,10 +59,13 @@ export async function getInfraMetricsClient({ index: metricsIndices, body: { ...searchParams.body, - query: getDataTierFilterCombined({ - filter: searchParams.body.query, - excludedDataTiers, - }), + query: { + ...searchParams.body.query, + bool: { + ...searchParams.body.query?.bool, + filter, + }, + }, }, }, request From 4d184744454f48bb2d7dc4c29d32f730da39b10a Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 27 Nov 2024 17:31:13 +0000 Subject: [PATCH 6/8] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/observability_solution/infra/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json index efd8be77b688c..1e130261d5346 100644 --- a/x-pack/plugins/observability_solution/infra/tsconfig.json +++ b/x-pack/plugins/observability_solution/infra/tsconfig.json @@ -116,7 +116,8 @@ "@kbn/entityManager-plugin", "@kbn/entities-schema", "@kbn/zod", - "@kbn/observability-utils-server" + "@kbn/observability-utils-server", + "@kbn/observability-utils-common" ], "exclude": ["target/**/*"] } From a24b7a904b9d00f52ad1a5615306ec1064c45ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Thu, 28 Nov 2024 17:15:39 +0100 Subject: [PATCH 7/8] Add tests for ensuring correct query merging --- .../helpers/get_infra_metrics_client.test.ts | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts new file mode 100644 index 0000000000000..9e930ff31dde7 --- /dev/null +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; +import { InfraBackendLibs } from '../infra_types'; +import { getInfraMetricsClient } from './get_infra_metrics_client'; +import { InfraPluginRequestHandlerContext } from '../../types'; +import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; + +const withExcludedDataTiers = (tiers: DataTier[]) => ({ + uiSettings: { + client: { + get: () => Promise.resolve(tiers), + }, + }, +}); + +const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) }; + +const infraMetricsTestHarness = + (input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) => async () => { + const callWithRequest = jest.fn(); + + const mockedCore = withExcludedDataTiers(['data_frozen']); + + const context = { + infra: Promise.resolve(mockedInfra), + core: Promise.resolve(mockedCore), + } as unknown as InfraPluginRequestHandlerContext; + + const client = await getInfraMetricsClient({ + libs: { framework: { callWithRequest } } as unknown as InfraBackendLibs, + context, + request: {} as unknown as KibanaRequest, + }); + + await client.search({ + body: { + query: input, + size: 1, + track_total_hits: false, + }, + }); + + expect(callWithRequest).toBeCalledWith( + context, + 'search', + { + body: { + query: output, + size: 1, + track_total_hits: false, + }, + ignore_unavailable: true, + index: ['*.indices'], + }, + {} + ); + }; + +describe('getInfraMetricsClient', () => { + it( + 'includes excluded data tiers in the request filter by default', + infraMetricsTestHarness(undefined, { + bool: { + filter: [ + { + bool: { + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + }, + ], + }, + }) + ); + + it( + 'merges provided filters with the excluded data tier filter', + infraMetricsTestHarness( + { + bool: { + filter: [ + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + }, + ], + }, + } + ) + ); + + it( + 'merges other query params with the excluded data tiers filter', + infraMetricsTestHarness( + { + bool: { + must: { + exists: { + field: 'a-field', + }, + }, + }, + }, + { + bool: { + filter: [ + { + bool: { + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], + }, + }, + ], + must: { + exists: { + field: 'a-field', + }, + }, + }, + } + ) + ); +}); From d6cea749b1597260a3aa98c9d43334c80394c817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Rica?= Date: Fri, 29 Nov 2024 12:21:42 +0100 Subject: [PATCH 8/8] Revise query structure merging, fix castArray time bomb --- .../helpers/get_infra_metrics_client.test.ts | 80 +++++++------------ .../lib/helpers/get_infra_metrics_client.ts | 12 +-- .../server/routes/metrics_sources/index.ts | 2 +- 3 files changed, 38 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts index 9e930ff31dde7..3eb8c47c274d9 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts @@ -23,10 +23,11 @@ const withExcludedDataTiers = (tiers: DataTier[]) => ({ const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) }; const infraMetricsTestHarness = - (input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) => async () => { + (tiers: DataTier[], input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) => + async () => { const callWithRequest = jest.fn(); - const mockedCore = withExcludedDataTiers(['data_frozen']); + const mockedCore = withExcludedDataTiers(tiers); const context = { infra: Promise.resolve(mockedInfra), @@ -64,20 +65,19 @@ const infraMetricsTestHarness = }; describe('getInfraMetricsClient', () => { + it( + 'defines an empty must_not query if given no data tiers to filter by', + infraMetricsTestHarness([], undefined, { bool: { must_not: [] } }) + ); + it( 'includes excluded data tiers in the request filter by default', - infraMetricsTestHarness(undefined, { + infraMetricsTestHarness(['data_frozen'], undefined, { bool: { - filter: [ + must_not: [ { - bool: { - must_not: [ - { - terms: { - _tier: ['data_frozen'], - }, - }, - ], + terms: { + _tier: ['data_frozen'], }, }, ], @@ -88,42 +88,27 @@ describe('getInfraMetricsClient', () => { it( 'merges provided filters with the excluded data tier filter', infraMetricsTestHarness( + ['data_frozen'], { bool: { - filter: [ - { - bool: { - must: { - exists: { - field: 'a-field', - }, - }, - }, + must_not: { + exists: { + field: 'a-field', }, - ], + }, }, }, { bool: { - filter: [ + must_not: [ { - bool: { - must: { - exists: { - field: 'a-field', - }, - }, + exists: { + field: 'a-field', }, }, { - bool: { - must_not: [ - { - terms: { - _tier: ['data_frozen'], - }, - }, - ], + terms: { + _tier: ['data_frozen'], }, }, ], @@ -135,6 +120,7 @@ describe('getInfraMetricsClient', () => { it( 'merges other query params with the excluded data tiers filter', infraMetricsTestHarness( + ['data_frozen'], { bool: { must: { @@ -146,24 +132,18 @@ describe('getInfraMetricsClient', () => { }, { bool: { - filter: [ - { - bool: { - must_not: [ - { - terms: { - _tier: ['data_frozen'], - }, - }, - ], - }, - }, - ], must: { exists: { field: 'a-field', }, }, + must_not: [ + { + terms: { + _tier: ['data_frozen'], + }, + }, + ], }, } ) diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts index 51e330675e875..96ae89b902285 100644 --- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts +++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts @@ -38,17 +38,19 @@ export async function getInfraMetricsClient({ const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); const metricsIndices = await infraContext.getMetricsIndices(); + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers)[0].bool!.must_not! + : []; + return { search( searchParams: TParams ): Promise> { - const searchFilter = searchParams.body.query?.bool?.filter ?? []; - - const excludedQuery = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : []; + const searchFilter = searchParams.body.query?.bool?.must_not ?? []; // This flattens arrays by one level, and non-array values can be added as well, so it all // results in a nice [QueryDsl, QueryDsl, ...] array. - const filter = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery); + const mustNot = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery); return framework.callWithRequest( context, @@ -63,7 +65,7 @@ export async function getInfraMetricsClient({ ...searchParams.body.query, bool: { ...searchParams.body.query?.bool, - filter, + must_not: mustNot, }, }, }, diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts index 20dcbc1b7e20b..77bdee365e3bf 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts @@ -208,7 +208,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => }, async (context, request, response) => { try { - const modules = castArray(request.query.modules); + const modules = request.query.modules ? castArray(request.query.modules) : []; if (modules.length > MAX_MODULES) { throw Boom.badRequest(