From f1918769b4717e9a5a40ceb421f09af0f03b72d8 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 9 Sep 2024 15:39:43 +0200 Subject: [PATCH 1/6] Use excluded_data_tiers setting in APM query clients Update config key --- .../apm/common/storage_explorer_types.ts | 20 +++------ .../lib/helpers/get_apm_event_client.ts | 20 ++++++--- .../routes/alerts/alerting_es_client.ts | 22 ++++++++++ .../get_service_group_fields_for_anomaly.ts | 9 +++- .../anomaly/register_anomaly_rule_type.ts | 3 +- .../register_error_count_rule_type.ts | 3 +- ...register_transaction_duration_rule_type.ts | 3 +- ...gister_transaction_error_rate_rule_type.ts | 3 +- .../has_historical_agent_data.ts | 7 ++- .../create_apm_event_client/index.ts | 32 +++++++++++++- .../server/lib/helpers/index.ts | 2 + .../server/lib/helpers/tier_filter.ts | 44 +++++++++++++++++++ .../apm_data_access/server/utils.ts | 1 + .../observability_shared/common/ilm_types.ts | 24 ++++++++++ .../observability_shared/common/index.ts | 5 +++ .../profiling/common/storage_explorer.ts | 20 +++------ 16 files changed, 174 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts create mode 100644 x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts diff --git a/x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts b/x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts index 579417a0f8e03..b6db4dd7405b9 100644 --- a/x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts +++ b/x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts @@ -5,23 +5,13 @@ * 2.0. */ +import { + IndexLifecyclePhaseSelectOption, + indexLifeCyclePhaseToDataTier, +} from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; -export enum IndexLifecyclePhaseSelectOption { - All = 'all', - Hot = 'hot', - Warm = 'warm', - Cold = 'cold', - Frozen = 'frozen', -} - -export const indexLifeCyclePhaseToDataTier = { - [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot', - [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm', - [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold', - [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen', -}; - +export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier }; export const indexLifecyclePhaseRt = t.type({ indexLifecyclePhase: t.union([ t.literal(IndexLifecyclePhaseSelectOption.All), diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts index 8f21bf8f1c691..ab48d13cce237 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts @@ -6,6 +6,8 @@ */ import { UI_SETTINGS } from '@kbn/data-plugin/common'; +import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; import { APMEventClient } from './create_es_client/create_apm_event_client'; import { withApmSpan } from '../../utils/with_apm_span'; import { MinimalAPMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes'; @@ -22,11 +24,18 @@ export async function getApmEventClient({ >): Promise { return withApmSpan('get_apm_event_client', async () => { const coreContext = await context.core; - const [indices, includeFrozen] = await Promise.all([ + const [indices, uiSettings] = await Promise.all([ getApmIndices(), - withApmSpan('get_ui_settings', () => - coreContext.uiSettings.client.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN) - ), + withApmSpan('get_ui_settings', async () => { + const includeFrozen = await coreContext.uiSettings.client.get( + UI_SETTINGS.SEARCH_INCLUDE_FROZEN + ); + const excludedDataTiers = await coreContext.uiSettings.client.get( + searchExcludedDataTiers + ); + + return { includeFrozen, excludedDataTiers }; + }), ]); return new APMEventClient({ @@ -35,7 +44,8 @@ export async function getApmEventClient({ request, indices, options: { - includeFrozen, + includeFrozen: uiSettings.includeFrozen, + excludedDataTiers: uiSettings.excludedDataTiers, inspectableEsQueriesMap, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts index 1a9daf6ad41a6..34e517354989a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts @@ -7,6 +7,11 @@ import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; +import { IUiSettingsClient } from '@kbn/core/server'; +import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import { getExcludedDataTiersFilter } from '@kbn/apm-data-access-plugin/server/utils'; +import { compact } from 'lodash'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; export type APMEventESSearchRequestParams = ESSearchRequest & { body: { size: number; track_total_hits: boolean | number }; @@ -14,13 +19,30 @@ export type APMEventESSearchRequestParams = ESSearchRequest & { export async function alertingEsClient({ scopedClusterClient, + uiSettingsClient, params, }: { scopedClusterClient: RuleExecutorServices['scopedClusterClient']; + uiSettingsClient: IUiSettingsClient; params: TParams; }): Promise> { + const excludedDataTiers = await uiSettingsClient.get( + searchExcludedDataTiers + ); + + const filter = excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined; + const response = await scopedClusterClient.asCurrentUser.search({ ...params, + body: { + ...params.body, + query: { + bool: { + filter, + must: compact([params.body.query]), + }, + }, + }, ignore_unavailable: true, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts index ce8783ad517f9..39de017668454 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { IScopedClusterClient, SavedObjectsClientContract } from '@kbn/core/server'; +import { + IScopedClusterClient, + IUiSettingsClient, + SavedObjectsClientContract, +} from '@kbn/core/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; import { SERVICE_ENVIRONMENT, @@ -23,6 +27,7 @@ export async function getServiceGroupFieldsForAnomaly({ apmIndices, scopedClusterClient, serviceName, + uiSettingsClient, environment, transactionType, timestamp, @@ -31,6 +36,7 @@ export async function getServiceGroupFieldsForAnomaly({ apmIndices: APMIndices; scopedClusterClient: IScopedClusterClient; savedObjectsClient: SavedObjectsClientContract; + uiSettingsClient: IUiSettingsClient; serviceName: string; environment: string; transactionType: string; @@ -70,6 +76,7 @@ export async function getServiceGroupFieldsForAnomaly({ const response = await alertingEsClient({ scopedClusterClient, + uiSettingsClient, params, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts index 4678622a7d122..d9a58d23a5888 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts @@ -129,7 +129,7 @@ export function registerAnomalyRuleType({ } const { params, services, spaceId, startedAt, getTimeRange } = options; - const { alertsClient, savedObjectsClient, scopedClusterClient } = services; + const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services; if (!alertsClient) { throw new AlertsClientError(); } @@ -283,6 +283,7 @@ export function registerAnomalyRuleType({ apmIndices, scopedClusterClient, savedObjectsClient, + uiSettingsClient, serviceName, environment, transactionType, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts index d90aa0e143a14..2539a63ea8575 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts @@ -128,7 +128,7 @@ export function registerErrorCountRuleType({ > ) => { const { params: ruleParams, services, spaceId, startedAt, getTimeRange } = options; - const { alertsClient, savedObjectsClient, scopedClusterClient } = services; + const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services; if (!alertsClient) { throw new AlertsClientError(); } @@ -187,6 +187,7 @@ export function registerErrorCountRuleType({ const response = await alertingEsClient({ scopedClusterClient, + uiSettingsClient, params: searchParams, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 299615e7663ef..96ddbe15c4287 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -140,7 +140,7 @@ export function registerTransactionDurationRuleType({ > ) => { const { params: ruleParams, services, spaceId, getTimeRange } = options; - const { alertsClient, savedObjectsClient, scopedClusterClient } = services; + const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services; if (!alertsClient) { throw new AlertsClientError(); } @@ -221,6 +221,7 @@ export function registerTransactionDurationRuleType({ const response = await alertingEsClient({ scopedClusterClient, + uiSettingsClient, params: searchParams, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts index 81b4612244b1b..cff5a481f9200 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts @@ -138,7 +138,7 @@ export function registerTransactionErrorRateRuleType({ > ) => { const { services, spaceId, params: ruleParams, startedAt, getTimeRange } = options; - const { alertsClient, savedObjectsClient, scopedClusterClient } = services; + const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services; if (!alertsClient) { throw new AlertsClientError(); } @@ -223,6 +223,7 @@ export function registerTransactionErrorRateRuleType({ const response = await alertingEsClient({ scopedClusterClient, + uiSettingsClient, params: searchParams, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts index 1b34aa001dd93..fbb682ce3e05f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts @@ -6,6 +6,7 @@ */ import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export async function hasHistoricalAgentData(apmEventClient: APMEventClient) { @@ -23,8 +24,10 @@ export async function hasHistoricalAgentData(apmEventClient: APMEventClient) { return hasDataUnbounded; } -type DataTier = 'data_hot' | 'data_warm' | 'data_cold' | 'data_frozen'; -async function hasDataRequest(apmEventClient: APMEventClient, dataTiers?: DataTier[]) { +async function hasDataRequest( + apmEventClient: APMEventClient, + dataTiers?: IndexLifeCycleDataTier[] +) { const query = dataTiers ? { terms: { _tier: dataTiers } } : undefined; const params = { diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index 3c195b752c854..d1066207696ef 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -22,6 +22,7 @@ import { compact, omit } from 'lodash'; import { ValuesType } from 'utility-types'; import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; +import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; import { withApmSpan } from '../../../../utils'; import type { ApmDataSource } from '../../../../../common/data_source'; import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort'; @@ -29,6 +30,7 @@ import { callAsyncWithDebug, getDebugBody, getDebugTitle } from '../call_async_w import type { ProcessorEventOfDocumentType } from '../document_type'; import type { APMIndices } from '../../../..'; import { getRequestBase, processorEventsToIndex } from './get_request_base'; +import { getExcludedDataTiersFilter, getIndexFilter } from '../../tier_filter'; export type APMEventESSearchRequest = Omit & { apm: { @@ -88,6 +90,7 @@ export interface APMEventClientConfig { options: { includeFrozen: boolean; inspectableEsQueriesMap?: WeakMap; + excludedDataTiers?: IndexLifeCycleDataTier; }; } @@ -96,7 +99,10 @@ export class APMEventClient { private readonly debug: boolean; private readonly request: KibanaRequest; public readonly indices: APMIndices; + /** @deprecated Use {@link excludedDataTiers} instead. + * See https://www.elastic.co/guide/en/kibana/current/advanced-options.html **/ private readonly includeFrozen: boolean; + private readonly excludedDataTiers?: IndexLifeCycleDataTier; private readonly inspectableEsQueriesMap?: WeakMap; constructor(config: APMEventClientConfig) { @@ -105,6 +111,7 @@ export class APMEventClient { this.request = config.request; this.indices = config.indices; this.includeFrozen = config.options.includeFrozen; + this.excludedDataTiers = config.options.excludedDataTiers; this.inspectableEsQueriesMap = config.options.inspectableEsQueriesMap; } @@ -159,6 +166,10 @@ export class APMEventClient { indices: this.indices, }); + if (this.excludedDataTiers) { + filters.push(getExcludedDataTiersFilter(this.excludedDataTiers)); + } + const searchParams = { ...omit(params, 'apm', 'body'), index, @@ -195,6 +206,10 @@ export class APMEventClient { // Reusing indices configured for errors since both events and errors are stored as logs. const index = processorEventsToIndex([ProcessorEvent.error], this.indices); + const filter = this.excludedDataTiers + ? getExcludedDataTiersFilter(this.excludedDataTiers) + : undefined; + const searchParams = { ...omit(params, 'body'), index, @@ -202,6 +217,7 @@ export class APMEventClient { ...params.body, query: { bool: { + filter, must: compact([params.body.query]), }, }, @@ -234,6 +250,10 @@ export class APMEventClient { indices: this.indices, }); + if (this.excludedDataTiers) { + filters.push(getExcludedDataTiersFilter(this.excludedDataTiers)); + } + const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [ { index, @@ -295,9 +315,13 @@ export class APMEventClient { ): Promise { const index = processorEventsToIndex(params.apm.events, this.indices); - const requestParams = { + const requestParams: Omit & { index: string[] } = { ...omit(params, 'apm'), index, + index_filter: getIndexFilter({ + indexFilter: params.index_filter, + excludedDataTiers: this.excludedDataTiers, + }), }; return this.callAsyncWithDebug({ @@ -314,9 +338,13 @@ export class APMEventClient { ): Promise { const index = processorEventsToIndex(params.apm.events, this.indices); - const requestParams = { + const requestParams: Omit & { index: string } = { ...omit(params, 'apm'), index: index.join(','), + index_filter: getIndexFilter({ + indexFilter: params.index_filter, + excludedDataTiers: this.excludedDataTiers, + }), }; return this.callAsyncWithDebug({ diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts index 30a2ff30d98ee..4cfbbb4d1ae01 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts @@ -21,3 +21,5 @@ export { } from './create_es_client/call_async_with_debug'; export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort'; + +export { getExcludedDataTiersFilter } from './tier_filter'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts new file mode 100644 index 0000000000000..e4f4e881d3465 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; + +export function getExcludedDataTiersFilter( + excludedDataTiers: IndexLifeCycleDataTier +): QueryDslQueryContainer { + return { + bool: { + must_not: [ + { + terms: { + _tier: excludedDataTiers, + }, + }, + ], + }, + }; +} + +export function getIndexFilter({ + indexFilter, + excludedDataTiers, +}: { + indexFilter?: QueryDslQueryContainer; + excludedDataTiers?: IndexLifeCycleDataTier; +}): QueryDslQueryContainer | undefined { + if (!indexFilter) { + return excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined; + } + + return !excludedDataTiers + ? indexFilter + : { + bool: { + must: [indexFilter, getExcludedDataTiersFilter(excludedDataTiers)], + }, + }; +} diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index b1e768edf3733..e7fc88a47eb03 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -11,6 +11,7 @@ export { cancelEsRequestOnAbort, getDebugBody, getDebugTitle, + getExcludedDataTiersFilter, } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts b/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts new file mode 100644 index 0000000000000..eec1b17ebb424 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum IndexLifecyclePhaseSelectOption { + All = 'all', + Hot = 'hot', + Warm = 'warm', + Cold = 'cold', + Frozen = 'frozen', +} + +export const indexLifeCyclePhaseToDataTier = { + [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot', + [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm', + [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold', + [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen', +} as const; + +export type IndexLifeCycleDataTier = + (typeof indexLifeCyclePhaseToDataTier)[keyof typeof indexLifeCyclePhaseToDataTier]; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/index.ts index ab49080f313ba..7e62b01ff6b06 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/index.ts @@ -144,6 +144,11 @@ export { export { type Color, colorTransformer } from './color_palette'; export { ObservabilityTriggerId } from './trigger_ids'; export { getInspectResponse } from './utils/get_inspect_response'; +export { + type IndexLifeCycleDataTier, + indexLifeCyclePhaseToDataTier, + IndexLifecyclePhaseSelectOption, +} from './ilm_types'; export const LOGS_ONBOARDING_FEEDBACK_LINK = 'https://ela.st/logs-onboarding-feedback'; export const LOGS_EXPLORER_FEEDBACK_LINK = 'https://ela.st/explorer-feedback'; diff --git a/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts b/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts index 984619af5ea98..f55f2da1b37a0 100644 --- a/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts +++ b/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts @@ -4,16 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { + IndexLifecyclePhaseSelectOption, + indexLifeCyclePhaseToDataTier, +} from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; -export enum IndexLifecyclePhaseSelectOption { - All = 'all', - Hot = 'hot', - Warm = 'warm', - Cold = 'cold', - Frozen = 'frozen', -} - +export { IndexLifecyclePhaseSelectOption, type indexLifeCyclePhaseToDataTier }; export const indexLifecyclePhaseRt = t.type({ indexLifecyclePhase: t.union([ t.literal(IndexLifecyclePhaseSelectOption.All), @@ -24,13 +21,6 @@ export const indexLifecyclePhaseRt = t.type({ ]), }); -export const indexLifeCyclePhaseToDataTier = { - [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot', - [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm', - [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold', - [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen', -}; - export interface StorageExplorerSummaryAPIResponse { totalProfilingSizeBytes: number; totalSymbolsSizeBytes: number; From 0860b34dcc13a9aef0f5828c92da371c106cfb6d Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 17 Sep 2024 12:19:37 +0200 Subject: [PATCH 2/6] Add tests --- .../lib/helpers/get_apm_event_client.ts | 2 +- .../routes/alerts/alerting_es_client.test.ts | 78 +++++ .../routes/alerts/alerting_es_client.ts | 2 +- .../server/routes/alerts/test_utils/index.ts | 3 + .../create_apm_event_client/index.test.ts | 267 ++++++++++++++---- .../create_apm_event_client/index.ts | 8 +- .../server/lib/helpers/tier_filter.ts | 4 +- .../profiling/common/storage_explorer.ts | 2 +- 8 files changed, 306 insertions(+), 60 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts index ab48d13cce237..7c20db7b1bdff 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts @@ -30,7 +30,7 @@ export async function getApmEventClient({ const includeFrozen = await coreContext.uiSettings.client.get( UI_SETTINGS.SEARCH_INCLUDE_FROZEN ); - const excludedDataTiers = await coreContext.uiSettings.client.get( + const excludedDataTiers = await coreContext.uiSettings.client.get( searchExcludedDataTiers ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts new file mode 100644 index 0000000000000..d655a68445a0e --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APMEventESSearchRequestParams, alertingEsClient } from './alerting_es_client'; +import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; +import { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; +import { ESSearchResponse } from '@kbn/es-types'; + +describe('alertingEsClient', () => { + let scopedClusterClientMock: jest.Mocked<{ + asCurrentUser: jest.Mocked; + }>; + + let uiSettingsClientMock: jest.Mocked; + + beforeEach(() => { + scopedClusterClientMock = { + asCurrentUser: { + search: jest.fn(), + } as unknown as jest.Mocked, + }; + + uiSettingsClientMock = { + get: jest.fn(), + } as unknown as jest.Mocked; + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should call search with filters containing excluded data tiers', async () => { + const excludedDataTiers = ['data_warm', 'data_cold']; + uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers); + + const params = { + body: { + size: 10, + track_total_hits: true, + query: { + match: { field: 'value' }, + }, + }, + }; + + scopedClusterClientMock.asCurrentUser.search.mockResolvedValue({ + hits: { + total: { value: 1, relation: 'eq' }, + hits: [{ _source: {}, _index: '' }], + max_score: 1, + }, + took: 1, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + timed_out: false, + } as unknown as ESSearchResponse); + + await alertingEsClient({ + scopedClusterClient: scopedClusterClientMock as unknown as RuleExecutorServices< + never, + never, + never + >['scopedClusterClient'], + uiSettingsClient: uiSettingsClientMock, + params, + }); + + const searchParams = scopedClusterClientMock.asCurrentUser.search.mock + .calls[0][0] as APMEventESSearchRequestParams; + expect(searchParams?.body?.query?.bool).toEqual({ + filter: { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + must: [{ match: { field: 'value' } }], + }); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts index 34e517354989a..5bf18703ba0f5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts @@ -26,7 +26,7 @@ export async function alertingEsClient> { - const excludedDataTiers = await uiSettingsClient.get( + const excludedDataTiers = await uiSettingsClient.get( searchExcludedDataTiers ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/test_utils/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/test_utils/index.ts index 1f8ddeaff4620..8db29408d4752 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/test_utils/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/test_utils/index.ts @@ -40,6 +40,9 @@ export const createRuleTypeMocks = () => { savedObjectsClient: { get: () => ({ attributes: { consumer: APM_SERVER_FEATURE_ID } }), }, + uiSettingsClient: { + get: jest.fn(), + }, alertFactory: { create: jest.fn(() => ({ scheduleActions, getUuid })), done: {}, diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts index 2239f6d8d8fb0..a349c7c48f687 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts @@ -7,80 +7,245 @@ import { setTimeout as setTimeoutPromise } from 'timers/promises'; import { contextServiceMock, executionContextServiceMock } from '@kbn/core/server/mocks'; import { createHttpService } from '@kbn/core-http-server-mocks'; +import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + TermsEnumRequest, + MsearchMultisearchBody, +} from '@elastic/elasticsearch/lib/api/types'; import supertest from 'supertest'; -import { APMEventClient } from '.'; +import { APMEventClient, type APMEventESSearchRequest, type APMEventFieldCapsRequest } from '.'; +import { APMIndices } from '../../../..'; -describe('APMEventClient', () => { - let server: ReturnType; +import * as cancelEsRequestOnAbortModule from '../cancel_es_request_on_abort'; +import * as observabilityPluginModule from '@kbn/observability-plugin/server'; - beforeEach(() => { - server = createHttpService(); - }); +jest.mock('@kbn/observability-plugin/server', () => ({ + __esModule: true, + ...jest.requireActual('@kbn/observability-plugin/server'), +})); - afterEach(async () => { - await server.stop(); - }); - it('cancels a search when a request is aborted', async () => { - await server.preboot({ - context: contextServiceMock.createPrebootContract(), +describe('APMEventClient', () => { + describe('Abort controller', () => { + let server: ReturnType; + beforeEach(() => { + server = createHttpService(); }); - const { server: innerServer, createRouter } = await server.setup({ - context: contextServiceMock.createSetupContract(), - executionContext: executionContextServiceMock.createInternalSetupContract(), + + afterEach(async () => { + await server.stop(); }); - const router = createRouter('/'); - - let abortSignal: AbortSignal | undefined; - router.get({ path: '/', validate: false }, async (context, request, res) => { - const eventClient = new APMEventClient({ - esClient: { - search: async (params: any, { signal }: { signal: AbortSignal }) => { - abortSignal = signal; - await setTimeoutPromise(3_000, undefined, { - signal: abortSignal, - }); - return {}; + + it('cancels a search when a request is aborted', async () => { + await server.preboot({ + context: contextServiceMock.createPrebootContract(), + }); + const { server: innerServer, createRouter } = await server.setup({ + context: contextServiceMock.createSetupContract(), + executionContext: executionContextServiceMock.createInternalSetupContract(), + }); + const router = createRouter('/'); + + let abortSignal: AbortSignal | undefined; + router.get({ path: '/', validate: false }, async (context, request, res) => { + const eventClient = new APMEventClient({ + esClient: { + search: async (params: any, { signal }: { signal: AbortSignal }) => { + abortSignal = signal; + await setTimeoutPromise(3_000, undefined, { + signal: abortSignal, + }); + return {}; + }, + } as any, + debug: false, + request, + indices: {} as APMIndices, + options: { + includeFrozen: false, }, - } as any, + }); + + await eventClient.search('foo', { + apm: { + events: [], + }, + body: { size: 0, track_total_hits: false }, + }); + + return res.ok({ body: 'ok' }); + }); + + await server.start(); + + expect(abortSignal?.aborted).toBeFalsy(); + + const incomingRequest = supertest(innerServer.listener) + .get('/') + // end required to send request + .end(); + + await new Promise((resolve) => { + setTimeout(() => { + void incomingRequest.on('abort', () => { + setTimeout(() => { + resolve(undefined); + }, 100); + }); + + void incomingRequest.abort(); + }, 200); + }); + + expect(abortSignal?.aborted).toBe(true); + }); + }); + + describe('excludedDataTiers filter', () => { + let esClientMock: jest.Mocked; + let apmEventClient: APMEventClient; + let cancelEsRequestOnAbortSpy: jest.SpyInstance; + let unwrapEsResponseSpy: jest.SpyInstance; + + const esResponse: estypes.SearchResponse = { + hits: { + total: { value: 1, relation: 'eq' }, + hits: [{ _source: {}, _index: '' }], + max_score: 1, + }, + took: 1, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + timed_out: false, + }; + + beforeAll(() => { + jest.resetModules(); + }); + + beforeEach(() => { + cancelEsRequestOnAbortSpy = jest + .spyOn(cancelEsRequestOnAbortModule, 'cancelEsRequestOnAbort') + .mockImplementation(jest.fn()); + + unwrapEsResponseSpy = jest + .spyOn(observabilityPluginModule, 'unwrapEsResponse') + .mockImplementation(jest.fn()); + + esClientMock = { + search: jest.fn(), + msearch: jest.fn(), + eql: { search: jest.fn() }, + fieldCaps: jest.fn(), + termsEnum: jest.fn(), + } as unknown as jest.Mocked; + + apmEventClient = new APMEventClient({ + esClient: esClientMock, debug: false, - request, - indices: {} as any, + request: {} as KibanaRequest, + indices: {} as APMIndices, options: { includeFrozen: false, + excludedDataTiers: ['data_warm', 'data_cold'], }, }); + }); + + afterAll(() => { + cancelEsRequestOnAbortSpy.mockReset(); + unwrapEsResponseSpy.mockReset(); + }); - await eventClient.search('foo', { - apm: { - events: [], + it('includes excludedDataTiers filter in search params', async () => { + esClientMock.search.mockResolvedValue(esResponse); + + await apmEventClient.search('testOperation', { + apm: { events: [] }, + body: { + size: 0, + track_total_hits: false, + query: { bool: { filter: [{ match_all: {} }] } }, }, - body: { size: 0, track_total_hits: false }, }); - return res.ok({ body: 'ok' }); + const searchParams = esClientMock.search.mock.calls[0][0] as APMEventESSearchRequest; + + expect(searchParams.body.query?.bool).toEqual({ + filter: [ + { terms: { 'processor.event': [] } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], + must: [{ bool: { filter: [{ match_all: {} }] } }], + }); }); - await server.start(); + it('includes excludedDataTiers filter in msearch params', async () => { + esClientMock.msearch.mockResolvedValue({ responses: [esResponse], took: 1 }); - expect(abortSignal?.aborted).toBeFalsy(); + await apmEventClient.msearch('testOperation', { + apm: { events: [] }, + body: { + size: 0, + track_total_hits: false, + query: { bool: { filter: [{ match_all: {} }] } }, + }, + }); - const incomingRequest = supertest(innerServer.listener) - .get('/') - // end required to send request - .end(); + const msearchParams = esClientMock.msearch.mock.calls[0][0] as { + searches: MsearchMultisearchBody[]; + }; - await new Promise((resolve) => { - setTimeout(() => { - void incomingRequest.on('abort', () => { - setTimeout(() => { - resolve(undefined); - }, 100); - }); + expect(msearchParams.searches[1].query?.bool).toEqual({ + filter: [ + { bool: { filter: [{ match_all: {} }] } }, + { terms: { 'processor.event': [] } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], + }); + }); + + it('includes excludedDataTiers filter in fieldCaps params', async () => { + esClientMock.fieldCaps.mockResolvedValue({ + fields: {}, + indices: '', + }); - void incomingRequest.abort(); - }, 200); + await apmEventClient.fieldCaps('testOperation', { + apm: { events: [] }, + fields: ['field1'], + index_filter: { bool: { filter: [{ match_all: {} }] } }, + }); + + const fieldCapsParams = esClientMock.fieldCaps.mock.calls[0][0] as APMEventFieldCapsRequest; + expect(fieldCapsParams?.index_filter?.bool).toEqual({ + must: [ + { bool: { filter: [{ match_all: {} }] } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], + }); }); - expect(abortSignal?.aborted).toBe(true); + it('includes excludedDataTiers filter in termsEnum params', async () => { + esClientMock.termsEnum.mockResolvedValue({ + terms: [''], + _shards: { total: 1, successful: 1, failed: 0 }, + complete: true, + }); + + await apmEventClient.termsEnum('testOperation', { + apm: { events: [] }, + field: 'field1', + index_filter: { bool: { filter: [{ match_all: {} }] } }, + }); + + const termsEnumParams = esClientMock.termsEnum.mock.calls[0][0] as TermsEnumRequest; + + expect(termsEnumParams.index_filter?.bool).toEqual({ + must: [ + { bool: { filter: [{ match_all: {} }] } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], + }); + }); }); }); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index d1066207696ef..3d359041df369 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -53,9 +53,9 @@ type APMEventWrapper = Omit & { apm: { events: ProcessorEvent[] }; }; -type APMEventTermsEnumRequest = APMEventWrapper; +export type APMEventTermsEnumRequest = APMEventWrapper; type APMEventEqlSearchRequest = APMEventWrapper; -type APMEventFieldCapsRequest = APMEventWrapper; +export type APMEventFieldCapsRequest = APMEventWrapper; type TypeOfProcessorEvent = { [ProcessorEvent.error]: APMError; @@ -90,7 +90,7 @@ export interface APMEventClientConfig { options: { includeFrozen: boolean; inspectableEsQueriesMap?: WeakMap; - excludedDataTiers?: IndexLifeCycleDataTier; + excludedDataTiers?: IndexLifeCycleDataTier[]; }; } @@ -102,7 +102,7 @@ export class APMEventClient { /** @deprecated Use {@link excludedDataTiers} instead. * See https://www.elastic.co/guide/en/kibana/current/advanced-options.html **/ private readonly includeFrozen: boolean; - private readonly excludedDataTiers?: IndexLifeCycleDataTier; + private readonly excludedDataTiers?: IndexLifeCycleDataTier[]; private readonly inspectableEsQueriesMap?: WeakMap; constructor(config: APMEventClientConfig) { diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index e4f4e881d3465..a57fdd2f504ad 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -8,7 +8,7 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/type import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; export function getExcludedDataTiersFilter( - excludedDataTiers: IndexLifeCycleDataTier + excludedDataTiers: IndexLifeCycleDataTier[] ): QueryDslQueryContainer { return { bool: { @@ -28,7 +28,7 @@ export function getIndexFilter({ excludedDataTiers, }: { indexFilter?: QueryDslQueryContainer; - excludedDataTiers?: IndexLifeCycleDataTier; + excludedDataTiers?: IndexLifeCycleDataTier[]; }): QueryDslQueryContainer | undefined { if (!indexFilter) { return excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined; diff --git a/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts b/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts index f55f2da1b37a0..7705988274c41 100644 --- a/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts +++ b/x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts @@ -10,7 +10,7 @@ import { } from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; -export { IndexLifecyclePhaseSelectOption, type indexLifeCyclePhaseToDataTier }; +export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier }; export const indexLifecyclePhaseRt = t.type({ indexLifecyclePhase: t.union([ t.literal(IndexLifecyclePhaseSelectOption.All), From 291f5dcdcbf0a2c3bb0400c162e07585b614bc46 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 18 Sep 2024 12:24:40 +0200 Subject: [PATCH 3/6] More tests --- .../lib/helpers/get_apm_alerts_client.test.ts | 95 +++++++++++++++++++ .../lib/helpers/get_apm_alerts_client.ts | 28 ++++-- .../routes/alerts/alerting_es_client.test.ts | 87 ++++++++++------- .../routes/alerts/alerting_es_client.ts | 15 +-- .../create_apm_event_client/index.ts | 10 +- .../server/lib/helpers/index.ts | 2 +- .../server/lib/helpers/tier_filter.ts | 12 +-- .../apm_data_access/server/utils.ts | 1 + 8 files changed, 186 insertions(+), 64 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.test.ts diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.test.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.test.ts new file mode 100644 index 0000000000000..2479cad9f213b --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type ApmAlertsRequiredParams, getApmAlertsClient } from './get_apm_alerts_client'; +import type { + IScopedClusterClient, + IUiSettingsClient, + KibanaRequest, + SavedObjectsClientContract, +} from '@kbn/core/server'; +import { AlertsClient, RuleRegistryPluginStartContract } from '@kbn/rule-registry-plugin/server'; + +describe('get_apm_alerts_client', () => { + let ruleRegistryMock: jest.Mocked; + let alertClient: jest.Mocked; + let uiSettingsClientMock: jest.Mocked; + + const params: ApmAlertsRequiredParams = { + size: 10, + track_total_hits: true, + query: { + match: { field: 'value' }, + }, + }; + + beforeEach(async () => { + uiSettingsClientMock = { + get: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + + alertClient = { + find: jest.fn().mockResolvedValue({}), + getAuthorizedAlertsIndices: jest.fn().mockResolvedValue(['apm']), + } as unknown as jest.Mocked; + + ruleRegistryMock = { + getRacClientWithRequest: jest.fn().mockResolvedValue(alertClient), + alerting: jest.fn(), + } as unknown as jest.Mocked; + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + // Helper function to create the APM alerts client + const createApmAlertsClient = async () => { + return await getApmAlertsClient({ + context: { + core: Promise.resolve({ + uiSettings: { client: uiSettingsClientMock }, + elasticsearch: { client: {} as IScopedClusterClient }, + savedObjects: { client: {} as SavedObjectsClientContract }, + }), + } as any, + plugins: { + ruleRegistry: { + start: jest.fn().mockResolvedValue(ruleRegistryMock), + setup: {} as any, + }, + } as any, + request: {} as KibanaRequest, + }); + }; + + it('should call search', async () => { + const apmAlertsClient = await createApmAlertsClient(); + + await apmAlertsClient.search(params); + + const searchParams = alertClient.find.mock.calls[0][0] as ApmAlertsRequiredParams; + expect(searchParams.query).toEqual({ match: { field: 'value' } }); + }); + + it('should call search with filters containing excluded data tiers', async () => { + const excludedDataTiers = ['data_warm', 'data_cold']; + uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers); + + const apmAlertsClient = await createApmAlertsClient(); + + await apmAlertsClient.search(params); + + const searchParams = alertClient.find.mock.calls[0][0] as ApmAlertsRequiredParams; + expect(searchParams.query?.bool).toEqual({ + must: [ + { match: { field: 'value' } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], + }); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts index 3c885eef658d5..a0fea97c4b950 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts @@ -8,14 +8,27 @@ import { isEmpty } from 'lodash'; import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; +import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import { estypes } from '@elastic/elasticsearch'; +import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import type { MinimalAPMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes'; export type ApmAlertsClient = Awaited>; +export type ApmAlertsRequiredParams = ESSearchRequest & { + size: number; + track_total_hits: boolean | number; + query?: estypes.QueryDslQueryContainer; +}; + export async function getApmAlertsClient({ + context, plugins, request, -}: Pick) { +}: Pick) { + const coreContext = await context.core; + const ruleRegistryPluginStart = await plugins.ruleRegistry.start(); const alertsClient = await ruleRegistryPluginStart.getRacClientWithRequest(request); const apmAlertsIndices = await alertsClient.getAuthorizedAlertsIndices(['apm']); @@ -24,17 +37,20 @@ export async function getApmAlertsClient({ throw Error('No alert indices exist for "apm"'); } - type RequiredParams = ESSearchRequest & { - size: number; - track_total_hits: boolean | number; - }; + const excludedDataTiers = await coreContext.uiSettings.client.get( + searchExcludedDataTiers + ); return { - search( + search( searchParams: TParams ): Promise> { return alertsClient.find({ ...searchParams, + query: getDataTierFilterCombined({ + filter: searchParams.query, + excludedDataTiers, + }), index: apmAlertsIndices.join(','), }) as Promise; }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts index d655a68445a0e..757b199940547 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { APMEventESSearchRequestParams, alertingEsClient } from './alerting_es_client'; -import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; -import { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; -import { ESSearchResponse } from '@kbn/es-types'; +import { type APMEventESSearchRequestParams, alertingEsClient } from './alerting_es_client'; +import type { RuleExecutorServices } from '@kbn/alerting-plugin/server'; +import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; +import type { ESSearchResponse } from '@kbn/es-types'; describe('alertingEsClient', () => { let scopedClusterClientMock: jest.Mocked<{ @@ -17,15 +17,36 @@ describe('alertingEsClient', () => { let uiSettingsClientMock: jest.Mocked; + const params = { + body: { + size: 10, + track_total_hits: true, + query: { + match: { field: 'value' }, + }, + }, + }; + + const mockSearchResponse = { + hits: { + total: { value: 1, relation: 'eq' }, + hits: [{ _source: {}, _index: '' }], + max_score: 1, + }, + took: 1, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + timed_out: false, + } as unknown as ESSearchResponse; + beforeEach(() => { scopedClusterClientMock = { asCurrentUser: { - search: jest.fn(), + search: jest.fn().mockResolvedValue(mockSearchResponse), } as unknown as jest.Mocked, }; uiSettingsClientMock = { - get: jest.fn(), + get: jest.fn().mockResolvedValue(undefined), } as unknown as jest.Mocked; }); @@ -33,46 +54,40 @@ describe('alertingEsClient', () => { jest.resetAllMocks(); }); - it('should call search with filters containing excluded data tiers', async () => { - const excludedDataTiers = ['data_warm', 'data_cold']; - uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers); - - const params = { - body: { - size: 10, - track_total_hits: true, - query: { - match: { field: 'value' }, - }, - }, - }; - - scopedClusterClientMock.asCurrentUser.search.mockResolvedValue({ - hits: { - total: { value: 1, relation: 'eq' }, - hits: [{ _source: {}, _index: '' }], - max_score: 1, - }, - took: 1, - _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, - timed_out: false, - } as unknown as ESSearchResponse); - - await alertingEsClient({ + // Helper function to perform the search + const performSearch = async (searchParams: APMEventESSearchRequestParams) => { + return await alertingEsClient({ scopedClusterClient: scopedClusterClientMock as unknown as RuleExecutorServices< never, never, never >['scopedClusterClient'], uiSettingsClient: uiSettingsClientMock, - params, + params: searchParams, }); + }; + + it('should call search with default params', async () => { + await performSearch(params); + + const searchParams = scopedClusterClientMock.asCurrentUser.search.mock + .calls[0][0] as APMEventESSearchRequestParams; + expect(searchParams.body?.query).toEqual({ match: { field: 'value' } }); + }); + + it('should call search with filters containing excluded data tiers', async () => { + const excludedDataTiers = ['data_warm', 'data_cold']; + uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers); + + await performSearch(params); const searchParams = scopedClusterClientMock.asCurrentUser.search.mock .calls[0][0] as APMEventESSearchRequestParams; - expect(searchParams?.body?.query?.bool).toEqual({ - filter: { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, - must: [{ match: { field: 'value' } }], + expect(searchParams.body?.query?.bool).toEqual({ + must: [ + { match: { field: 'value' } }, + { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } }, + ], }); }); }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts index 5bf18703ba0f5..981efb13e821e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts @@ -9,8 +9,7 @@ import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; import { IUiSettingsClient } from '@kbn/core/server'; import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; -import { getExcludedDataTiersFilter } from '@kbn/apm-data-access-plugin/server/utils'; -import { compact } from 'lodash'; +import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; export type APMEventESSearchRequestParams = ESSearchRequest & { @@ -30,18 +29,14 @@ export async function alertingEsClient & { apm: { @@ -318,8 +318,8 @@ export class APMEventClient { const requestParams: Omit & { index: string[] } = { ...omit(params, 'apm'), index, - index_filter: getIndexFilter({ - indexFilter: params.index_filter, + index_filter: getDataTierFilterCombined({ + filter: params.index_filter, excludedDataTiers: this.excludedDataTiers, }), }; @@ -341,8 +341,8 @@ export class APMEventClient { const requestParams: Omit & { index: string } = { ...omit(params, 'apm'), index: index.join(','), - index_filter: getIndexFilter({ - indexFilter: params.index_filter, + index_filter: getDataTierFilterCombined({ + filter: params.index_filter, excludedDataTiers: this.excludedDataTiers, }), }; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts index 4cfbbb4d1ae01..a4a9a53fa348d 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts @@ -22,4 +22,4 @@ export { export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort'; -export { getExcludedDataTiersFilter } from './tier_filter'; +export { getExcludedDataTiersFilter, getDataTierFilterCombined } from './tier_filter'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index a57fdd2f504ad..0ab042848e9d1 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -23,22 +23,22 @@ export function getExcludedDataTiersFilter( }; } -export function getIndexFilter({ - indexFilter, +export function getDataTierFilterCombined({ + filter, excludedDataTiers, }: { - indexFilter?: QueryDslQueryContainer; + filter?: QueryDslQueryContainer; excludedDataTiers?: IndexLifeCycleDataTier[]; }): QueryDslQueryContainer | undefined { - if (!indexFilter) { + if (!filter) { return excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined; } return !excludedDataTiers - ? indexFilter + ? filter : { bool: { - must: [indexFilter, getExcludedDataTiersFilter(excludedDataTiers)], + must: [filter, getExcludedDataTiersFilter(excludedDataTiers)], }, }; } diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index e7fc88a47eb03..9c1cf318743da 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -12,6 +12,7 @@ export { getDebugBody, getDebugTitle, getExcludedDataTiersFilter, + getDataTierFilterCombined, } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; From 630a2136ec22c3df2c00ded25ddf15c2cedfc8e7 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 20 Sep 2024 12:09:53 +0200 Subject: [PATCH 4/6] CR changes --- .../apm/server/routes/alerts/alerting_es_client.ts | 2 +- .../rule_types/anomaly/get_service_group_fields_for_anomaly.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts index 981efb13e821e..ba4d8b65acfa2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts @@ -8,7 +8,7 @@ import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; import { IUiSettingsClient } from '@kbn/core/server'; -import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts index 39de017668454..c617bfd74dc22 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { +import type { IScopedClusterClient, IUiSettingsClient, SavedObjectsClientContract, From 28db3d8be6ff63fbf300a39dd9af8a2c4e9371bc Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 20 Sep 2024 14:22:42 +0200 Subject: [PATCH 5/6] move and rename exclude_tiers_query --- .../es/queries/exclude_frozen_query.ts | 15 ++--------- .../es/queries/exclude_tiers_query.ts | 26 +++++++++++++++++++ .../lib/helpers/get_apm_alerts_client.ts | 4 +-- .../lib/helpers/get_apm_event_client.ts | 4 +-- .../routes/alerts/alerting_es_client.ts | 6 ++--- .../has_historical_agent_data.ts | 7 ++--- .../create_apm_event_client/index.ts | 17 ++++++------ .../server/lib/helpers/index.ts | 2 +- .../server/lib/helpers/tier_filter.ts | 25 ++++-------------- .../apm_data_access/server/utils.ts | 1 - .../apm_data_access/tsconfig.json | 3 ++- .../observability_shared/common/ilm_types.ts | 2 +- .../observability_shared/common/index.ts | 2 +- 13 files changed, 54 insertions(+), 60 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts b/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts index f348d925c41ca..1dd0bb736c644 100644 --- a/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts +++ b/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts @@ -5,19 +5,8 @@ * 2.0. */ import type { estypes } from '@elastic/elasticsearch'; +import { excludeTiersQuery } from './exclude_tiers_query'; export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] { - return [ - { - bool: { - must_not: [ - { - term: { - _tier: 'data_frozen', - }, - }, - ], - }, - }, - ]; + return excludeTiersQuery(['data_frozen']); } diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts b/x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts new file mode 100644 index 0000000000000..16bb9e24f505a --- /dev/null +++ b/x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +export function excludeTiersQuery( + excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'> +): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + terms: { + _tier: excludedDataTiers, + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts index a0fea97c4b950..b0e601fd4c0db 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts @@ -8,7 +8,7 @@ import { isEmpty } from 'lodash'; import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; -import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; import { estypes } from '@elastic/elasticsearch'; import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; @@ -37,7 +37,7 @@ export async function getApmAlertsClient({ throw Error('No alert indices exist for "apm"'); } - const excludedDataTiers = await coreContext.uiSettings.client.get( + const excludedDataTiers = await coreContext.uiSettings.client.get( searchExcludedDataTiers ); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts index 7c20db7b1bdff..8d2f61a20500d 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts @@ -6,7 +6,7 @@ */ import { UI_SETTINGS } from '@kbn/data-plugin/common'; -import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import { DataTier } from '@kbn/observability-shared-plugin/common'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; import { APMEventClient } from './create_es_client/create_apm_event_client'; import { withApmSpan } from '../../utils/with_apm_span'; @@ -30,7 +30,7 @@ export async function getApmEventClient({ const includeFrozen = await coreContext.uiSettings.client.get( UI_SETTINGS.SEARCH_INCLUDE_FROZEN ); - const excludedDataTiers = await coreContext.uiSettings.client.get( + const excludedDataTiers = await coreContext.uiSettings.client.get( searchExcludedDataTiers ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts index ba4d8b65acfa2..5638acd293538 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts @@ -8,7 +8,7 @@ import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; import { IUiSettingsClient } from '@kbn/core/server'; -import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils'; import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; @@ -25,9 +25,7 @@ export async function alertingEsClient> { - const excludedDataTiers = await uiSettingsClient.get( - searchExcludedDataTiers - ); + const excludedDataTiers = await uiSettingsClient.get(searchExcludedDataTiers); const response = await scopedClusterClient.asCurrentUser.search({ ...params, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts index fbb682ce3e05f..f1741db20026f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts @@ -6,7 +6,7 @@ */ import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export async function hasHistoricalAgentData(apmEventClient: APMEventClient) { @@ -24,10 +24,7 @@ export async function hasHistoricalAgentData(apmEventClient: APMEventClient) { return hasDataUnbounded; } -async function hasDataRequest( - apmEventClient: APMEventClient, - dataTiers?: IndexLifeCycleDataTier[] -) { +async function hasDataRequest(apmEventClient: APMEventClient, dataTiers?: DataTier[]) { const query = dataTiers ? { terms: { _tier: dataTiers } } : undefined; const params = { diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index 3575cdc63b6c7..c6c68830ae10c 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -22,7 +22,8 @@ import { compact, omit } from 'lodash'; import { ValuesType } from 'utility-types'; import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; -import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; import { withApmSpan } from '../../../../utils'; import type { ApmDataSource } from '../../../../../common/data_source'; import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort'; @@ -30,7 +31,7 @@ import { callAsyncWithDebug, getDebugBody, getDebugTitle } from '../call_async_w import type { ProcessorEventOfDocumentType } from '../document_type'; import type { APMIndices } from '../../../..'; import { getRequestBase, processorEventsToIndex } from './get_request_base'; -import { getExcludedDataTiersFilter, getDataTierFilterCombined } from '../../tier_filter'; +import { getDataTierFilterCombined } from '../../tier_filter'; export type APMEventESSearchRequest = Omit & { apm: { @@ -90,7 +91,7 @@ export interface APMEventClientConfig { options: { includeFrozen: boolean; inspectableEsQueriesMap?: WeakMap; - excludedDataTiers?: IndexLifeCycleDataTier[]; + excludedDataTiers?: DataTier[]; }; } @@ -102,7 +103,7 @@ export class APMEventClient { /** @deprecated Use {@link excludedDataTiers} instead. * See https://www.elastic.co/guide/en/kibana/current/advanced-options.html **/ private readonly includeFrozen: boolean; - private readonly excludedDataTiers?: IndexLifeCycleDataTier[]; + private readonly excludedDataTiers?: DataTier[]; private readonly inspectableEsQueriesMap?: WeakMap; constructor(config: APMEventClientConfig) { @@ -167,7 +168,7 @@ export class APMEventClient { }); if (this.excludedDataTiers) { - filters.push(getExcludedDataTiersFilter(this.excludedDataTiers)); + filters.push(...excludeTiersQuery(this.excludedDataTiers)); } const searchParams = { @@ -206,9 +207,7 @@ export class APMEventClient { // Reusing indices configured for errors since both events and errors are stored as logs. const index = processorEventsToIndex([ProcessorEvent.error], this.indices); - const filter = this.excludedDataTiers - ? getExcludedDataTiersFilter(this.excludedDataTiers) - : undefined; + const filter = this.excludedDataTiers ? excludeTiersQuery(this.excludedDataTiers) : undefined; const searchParams = { ...omit(params, 'body'), @@ -251,7 +250,7 @@ export class APMEventClient { }); if (this.excludedDataTiers) { - filters.push(getExcludedDataTiersFilter(this.excludedDataTiers)); + filters.push(...excludeTiersQuery(this.excludedDataTiers)); } const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [ diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts index a4a9a53fa348d..a912b2a1d60bb 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts @@ -22,4 +22,4 @@ export { export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort'; -export { getExcludedDataTiersFilter, getDataTierFilterCombined } from './tier_filter'; +export { getDataTierFilterCombined } from './tier_filter'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index 0ab042848e9d1..ae29575c044c6 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -5,40 +5,25 @@ * 2.0. */ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common'; - -export function getExcludedDataTiersFilter( - excludedDataTiers: IndexLifeCycleDataTier[] -): QueryDslQueryContainer { - return { - bool: { - must_not: [ - { - terms: { - _tier: excludedDataTiers, - }, - }, - ], - }, - }; -} +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; export function getDataTierFilterCombined({ filter, excludedDataTiers, }: { filter?: QueryDslQueryContainer; - excludedDataTiers?: IndexLifeCycleDataTier[]; + excludedDataTiers?: DataTier[]; }): QueryDslQueryContainer | undefined { if (!filter) { - return excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined; + return excludedDataTiers ? excludeTiersQuery(excludedDataTiers)[0] : undefined; } return !excludedDataTiers ? filter : { bool: { - must: [filter, getExcludedDataTiersFilter(excludedDataTiers)], + must: [filter, ...excludeTiersQuery(excludedDataTiers)], }, }; } diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index 9c1cf318743da..2fac072a8cdb5 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -11,7 +11,6 @@ export { cancelEsRequestOnAbort, getDebugBody, getDebugTitle, - getExcludedDataTiersFilter, getDataTierFilterCombined, } from './lib/helpers'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index 589d08ba56b4e..ea3ebf77b25be 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -19,6 +19,7 @@ "@kbn/core-http-server-mocks", "@kbn/apm-utils", "@kbn/core-http-server", - "@kbn/security-plugin-types-server" + "@kbn/security-plugin-types-server", + "@kbn/observability-utils" ] } diff --git a/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts b/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts index eec1b17ebb424..9a96f8c39c459 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts @@ -20,5 +20,5 @@ export const indexLifeCyclePhaseToDataTier = { [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen', } as const; -export type IndexLifeCycleDataTier = +export type DataTier = (typeof indexLifeCyclePhaseToDataTier)[keyof typeof indexLifeCyclePhaseToDataTier]; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/index.ts index 9fcca6dac070d..d845ea1d398fd 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/index.ts @@ -145,7 +145,7 @@ export { type Color, colorTransformer } from './color_palette'; export { ObservabilityTriggerId } from './trigger_ids'; export { getInspectResponse } from './utils/get_inspect_response'; export { - type IndexLifeCycleDataTier, + type DataTier, indexLifeCyclePhaseToDataTier, IndexLifecyclePhaseSelectOption, } from './ilm_types'; From b764250de24bfec6af7431c671c5ec22db898fcc Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 23 Sep 2024 09:42:40 +0200 Subject: [PATCH 6/6] Mark observability:searchExcludedDataTiers as tech preview --- .../server/routes/historical_data/has_historical_agent_data.ts | 2 ++ .../observability_solution/observability/server/ui_settings.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts index f1741db20026f..5489d893f86f1 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts @@ -25,6 +25,8 @@ export async function hasHistoricalAgentData(apmEventClient: APMEventClient) { } async function hasDataRequest(apmEventClient: APMEventClient, dataTiers?: DataTier[]) { + // the `observability:searchExcludedDataTiers` setting will also be considered + // in the `search` function to exclude data tiers from the search const query = dataTiers ? { terms: { _tier: dataTiers } } : undefined; const params = { diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts index 81c0596722106..cabd851703dbd 100644 --- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @@ -649,8 +649,9 @@ export const uiSettings: Record = { description: i18n.translate( 'xpack.observability.advancedSettings.searchExcludedDataTiersDesc', { - defaultMessage: `Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. + defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`, + values: { technicalPreviewLabel: `[${technicalPreviewLabel}]` }, } ), value: [],