Use excluded_data_tiers setting in APM query clients

Update config key

x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts

@@ -5,23 +5,13 @@
  * 2.0.
  */
 
+import {
+  IndexLifecyclePhaseSelectOption,
+  indexLifeCyclePhaseToDataTier,
+} from '@kbn/observability-shared-plugin/common';
 import * as t from 'io-ts';
 
-export enum IndexLifecyclePhaseSelectOption {
-  All = 'all',
-  Hot = 'hot',
-  Warm = 'warm',
-  Cold = 'cold',
-  Frozen = 'frozen',
-}
-
-export const indexLifeCyclePhaseToDataTier = {
-  [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot',
-  [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm',
-  [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold',
-  [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen',
-};
-
+export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier };
 export const indexLifecyclePhaseRt = t.type({
   indexLifecyclePhase: t.union([
     t.literal(IndexLifecyclePhaseSelectOption.All),

x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts

@@ -6,6 +6,8 @@
  */
 
 import { UI_SETTINGS } from '@kbn/data-plugin/common';
+import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
 import { APMEventClient } from './create_es_client/create_apm_event_client';
 import { withApmSpan } from '../../utils/with_apm_span';
 import { MinimalAPMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes';
@@ -22,11 +24,18 @@ export async function getApmEventClient({
 >): Promise<APMEventClient> {
   return withApmSpan('get_apm_event_client', async () => {
     const coreContext = await context.core;
-    const [indices, includeFrozen] = await Promise.all([
+    const [indices, uiSettings] = await Promise.all([
       getApmIndices(),
-      withApmSpan('get_ui_settings', () =>
-        coreContext.uiSettings.client.get<boolean>(UI_SETTINGS.SEARCH_INCLUDE_FROZEN)
-      ),
+      withApmSpan('get_ui_settings', async () => {
+        const includeFrozen = await coreContext.uiSettings.client.get<boolean>(
+          UI_SETTINGS.SEARCH_INCLUDE_FROZEN
+        );
+        const excludedDataTiers = await coreContext.uiSettings.client.get<IndexLifeCycleDataTier>(
+          searchExcludedDataTiers
+        );
+
+        return { includeFrozen, excludedDataTiers };
+      }),
     ]);
 
     return new APMEventClient({
@@ -35,7 +44,8 @@ export async function getApmEventClient({
       request,
       indices,
       options: {
-        includeFrozen,
+        includeFrozen: uiSettings.includeFrozen,
+        excludedDataTiers: uiSettings.excludedDataTiers,
         inspectableEsQueriesMap,
       },
     });

x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts

@@ -7,20 +7,42 @@
 
 import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types';
 import { RuleExecutorServices } from '@kbn/alerting-plugin/server';
+import { IUiSettingsClient } from '@kbn/core/server';
+import { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common';
+import { getExcludedDataTiersFilter } from '@kbn/apm-data-access-plugin/server/utils';
+import { compact } from 'lodash';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
 
 export type APMEventESSearchRequestParams = ESSearchRequest & {
   body: { size: number; track_total_hits: boolean | number };
 };
 
 export async function alertingEsClient<TParams extends APMEventESSearchRequestParams>({
   scopedClusterClient,
+  uiSettingsClient,
   params,
 }: {
   scopedClusterClient: RuleExecutorServices<never, never, never>['scopedClusterClient'];
+  uiSettingsClient: IUiSettingsClient;
   params: TParams;
 }): Promise<ESSearchResponse<unknown, TParams>> {
+  const excludedDataTiers = await uiSettingsClient.get<IndexLifeCycleDataTier>(
+    searchExcludedDataTiers
+  );
+
+  const filter = excludedDataTiers ? getExcludedDataTiersFilter(excludedDataTiers) : undefined;
+
   const response = await scopedClusterClient.asCurrentUser.search({
     ...params,
+    body: {
+      ...params.body,
+      query: {
+        bool: {
+          filter,
+          must: compact([params.body.query]),
+        },
+      },
+    },
     ignore_unavailable: true,
   });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { IScopedClusterClient, SavedObjectsClientContract } from '@kbn/core/server';
+import {
+  IScopedClusterClient,
+  IUiSettingsClient,
+  SavedObjectsClientContract,
+} from '@kbn/core/server';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import {
   SERVICE_ENVIRONMENT,
@@ -23,6 +27,7 @@ export async function getServiceGroupFieldsForAnomaly({
   apmIndices,
   scopedClusterClient,
   serviceName,
+  uiSettingsClient,
   environment,
   transactionType,
   timestamp,
@@ -31,6 +36,7 @@ export async function getServiceGroupFieldsForAnomaly({
   apmIndices: APMIndices;
   scopedClusterClient: IScopedClusterClient;
   savedObjectsClient: SavedObjectsClientContract;
+  uiSettingsClient: IUiSettingsClient;
   serviceName: string;
   environment: string;
   transactionType: string;
@@ -70,6 +76,7 @@ export async function getServiceGroupFieldsForAnomaly({
 
   const response = await alertingEsClient({
     scopedClusterClient,
+    uiSettingsClient,
     params,
   });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts

@@ -129,7 +129,7 @@ export function registerAnomalyRuleType({
       }
 
       const { params, services, spaceId, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -283,6 +283,7 @@ export function registerAnomalyRuleType({
           apmIndices,
           scopedClusterClient,
           savedObjectsClient,
+          uiSettingsClient,
           serviceName,
           environment,
           transactionType,

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts

@@ -128,7 +128,7 @@ export function registerErrorCountRuleType({
       >
     ) => {
       const { params: ruleParams, services, spaceId, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -187,6 +187,7 @@ export function registerErrorCountRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts

@@ -140,7 +140,7 @@ export function registerTransactionDurationRuleType({
       >
     ) => {
       const { params: ruleParams, services, spaceId, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -221,6 +221,7 @@ export function registerTransactionDurationRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts

@@ -138,7 +138,7 @@ export function registerTransactionErrorRateRuleType({
       >
     ) => {
       const { services, spaceId, params: ruleParams, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -223,6 +223,7 @@ export function registerTransactionErrorRateRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts

@@ -6,6 +6,7 @@
  */
 
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
 export async function hasHistoricalAgentData(apmEventClient: APMEventClient) {
@@ -23,8 +24,10 @@ export async function hasHistoricalAgentData(apmEventClient: APMEventClient) {
   return hasDataUnbounded;
 }
 
-type DataTier = 'data_hot' | 'data_warm' | 'data_cold' | 'data_frozen';
-async function hasDataRequest(apmEventClient: APMEventClient, dataTiers?: DataTier[]) {
+async function hasDataRequest(
+  apmEventClient: APMEventClient,
+  dataTiers?: IndexLifeCycleDataTier[]
+) {
   const query = dataTiers ? { terms: { _tier: dataTiers } } : undefined;
 
   const params = {

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -22,13 +22,15 @@ import { compact, omit } from 'lodash';
 import { ValuesType } from 'utility-types';
 import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui';
 import type { InspectResponse } from '@kbn/observability-plugin/typings/common';
+import type { IndexLifeCycleDataTier } from '@kbn/observability-shared-plugin/common';
 import { withApmSpan } from '../../../../utils';
 import type { ApmDataSource } from '../../../../../common/data_source';
 import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort';
 import { callAsyncWithDebug, getDebugBody, getDebugTitle } from '../call_async_with_debug';
 import type { ProcessorEventOfDocumentType } from '../document_type';
 import type { APMIndices } from '../../../..';
 import { getRequestBase, processorEventsToIndex } from './get_request_base';
+import { getExcludedDataTiersFilter, getIndexFilter } from '../../tier_filter';
 
 export type APMEventESSearchRequest = Omit<ESSearchRequest, 'index'> & {
   apm: {
@@ -88,6 +90,7 @@ export interface APMEventClientConfig {
   options: {
     includeFrozen: boolean;
     inspectableEsQueriesMap?: WeakMap<KibanaRequest, InspectResponse>;
+    excludedDataTiers?: IndexLifeCycleDataTier;
   };
 }
 
@@ -96,7 +99,10 @@ export class APMEventClient {
   private readonly debug: boolean;
   private readonly request: KibanaRequest;
   public readonly indices: APMIndices;
+  /** @deprecated Use {@link excludedDataTiers} instead.
+   * See https://www.elastic.co/guide/en/kibana/current/advanced-options.html **/
   private readonly includeFrozen: boolean;
+  private readonly excludedDataTiers?: IndexLifeCycleDataTier;
   private readonly inspectableEsQueriesMap?: WeakMap<KibanaRequest, InspectResponse>;
 
   constructor(config: APMEventClientConfig) {
@@ -105,6 +111,7 @@ export class APMEventClient {
     this.request = config.request;
     this.indices = config.indices;
     this.includeFrozen = config.options.includeFrozen;
+    this.excludedDataTiers = config.options.excludedDataTiers;
     this.inspectableEsQueriesMap = config.options.inspectableEsQueriesMap;
   }
 
@@ -159,6 +166,10 @@ export class APMEventClient {
       indices: this.indices,
     });
 
+    if (this.excludedDataTiers) {
+      filters.push(getExcludedDataTiersFilter(this.excludedDataTiers));
+    }
+
     const searchParams = {
       ...omit(params, 'apm', 'body'),
       index,
@@ -195,13 +206,18 @@ export class APMEventClient {
     // Reusing indices configured for errors since both events and errors are stored as logs.
     const index = processorEventsToIndex([ProcessorEvent.error], this.indices);
 
+    const filter = this.excludedDataTiers
+      ? getExcludedDataTiersFilter(this.excludedDataTiers)
+      : undefined;
+
     const searchParams = {
       ...omit(params, 'body'),
       index,
       body: {
         ...params.body,
         query: {
           bool: {
+            filter,
             must: compact([params.body.query]),
           },
         },
@@ -234,6 +250,10 @@ export class APMEventClient {
           indices: this.indices,
         });
 
+        if (this.excludedDataTiers) {
+          filters.push(getExcludedDataTiersFilter(this.excludedDataTiers));
+        }
+
         const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [
           {
             index,
@@ -295,9 +315,13 @@ export class APMEventClient {
   ): Promise<FieldCapsResponse> {
     const index = processorEventsToIndex(params.apm.events, this.indices);
 
-    const requestParams = {
+    const requestParams: Omit<APMEventFieldCapsRequest, 'apm'> & { index: string[] } = {
       ...omit(params, 'apm'),
       index,
+      index_filter: getIndexFilter({
+        indexFilter: params.index_filter,
+        excludedDataTiers: this.excludedDataTiers,
+      }),
     };
 
     return this.callAsyncWithDebug({
@@ -314,9 +338,13 @@ export class APMEventClient {
   ): Promise<TermsEnumResponse> {
     const index = processorEventsToIndex(params.apm.events, this.indices);
 
-    const requestParams = {
+    const requestParams: Omit<APMEventTermsEnumRequest, 'apm'> & { index: string } = {
       ...omit(params, 'apm'),
       index: index.join(','),
+      index_filter: getIndexFilter({
+        indexFilter: params.index_filter,
+        excludedDataTiers: this.excludedDataTiers,
+      }),
     };
 
     return this.callAsyncWithDebug({

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts

@@ -21,3 +21,5 @@ export {
 } from './create_es_client/call_async_with_debug';
 
 export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort';
+
+export { getExcludedDataTiersFilter } from './tier_filter';