[APM] Use excluded data tiers setting (elastic#192373)

closes [elastic#190559](elastic#190559)

## Summary

This PR updates the ES clients in APM to respect the excluded tier
configuration. When this config is set, the ES clients will
automatically add a filter to exclude the specified tiers from queries.

<img width="600" alt="image"
src="https://github.com/user-attachments/assets/9b0de76d-242c-4343-bc30-d5c787316f59">

All queries in APM should have the `_tier` filter (via
`get_apm_events_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/c525602f-f239-4be8-99c4-65d617962656">

This change also affects alerting (via `alerting_es_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/750df4d7-5b49-4de5-9294-7afedf11d7e5">

And it impacts the alerts column (via `get_apm_alert_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/44bd9129-1e72-4a3a-af32-d42a9cd9164d">

### What won't automatically add a filter for `_tier`

- Embeddables
- ML queries

### How to test
- Set the config in Advanced Settings to exclude `data_frozen` and
`data_cold` (optional)
- Navigate to APM and check the query `Inspect` to see if the filter is
present.
- Click through APM to confirm things still work.
- Create one of each type of APM alerts
- Without the config set, queries should not include the `_tier` filter`

---------

Co-authored-by: Elastic Machine <[email protected]>
(cherry picked from commit ee5ef81)

# Conflicts:
#	x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts
#	x-pack/plugins/cloud_integrations/cloud_experiments/public/launch_darkly_client/index.ts
#	x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts
#	x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts
#	x-pack/plugins/observability_solution/apm/server/lib/helpers/tier_filter.ts
#	x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts
#	x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts
#	x-pack/plugins/observability_solution/apm_data_access/tsconfig.json
#	x-pack/plugins/observability_solution/observability/server/ui_settings.ts

x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+
+export function excludeTiersQuery(
+  excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'>
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: excludedDataTiers,
+            },
+          },
+        ],
+      },
+    },
+  ];
+}

x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts

@@ -5,23 +5,13 @@
  * 2.0.
  */
 
+import {
+  IndexLifecyclePhaseSelectOption,
+  indexLifeCyclePhaseToDataTier,
+} from '@kbn/observability-shared-plugin/common';
 import * as t from 'io-ts';
 
-export enum IndexLifecyclePhaseSelectOption {
-  All = 'all',
-  Hot = 'hot',
-  Warm = 'warm',
-  Cold = 'cold',
-  Frozen = 'frozen',
-}
-
-export const indexLifeCyclePhaseToDataTier = {
-  [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot',
-  [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm',
-  [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold',
-  [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen',
-};
-
+export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier };
 export const indexLifecyclePhaseRt = t.type({
   indexLifecyclePhase: t.union([
     t.literal(IndexLifecyclePhaseSelectOption.All),

x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts

@@ -7,77 +7,245 @@
 import { setTimeout as setTimeoutPromise } from 'timers/promises';
 import { contextServiceMock, executionContextServiceMock } from '@kbn/core/server/mocks';
 import { createHttpService } from '@kbn/core-http-server-mocks';
+import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  TermsEnumRequest,
+  MsearchMultisearchBody,
+} from '@elastic/elasticsearch/lib/api/types';
 import supertest from 'supertest';
-import { APMEventClient } from '.';
+import { APMEventClient, type APMEventESSearchRequest, type APMEventFieldCapsRequest } from '.';
+import { APMIndices } from '../../../..';
 
-describe('APMEventClient', () => {
-  let server: ReturnType<typeof createHttpService>;
+import * as cancelEsRequestOnAbortModule from '../cancel_es_request_on_abort';
+import * as observabilityPluginModule from '@kbn/observability-plugin/server';
 
-  beforeEach(() => {
-    server = createHttpService();
-  });
+jest.mock('@kbn/observability-plugin/server', () => ({
+  __esModule: true,
+  ...jest.requireActual('@kbn/observability-plugin/server'),
+}));
 
-  afterEach(async () => {
-    await server.stop();
-  });
-  it('cancels a search when a request is aborted', async () => {
-    await server.preboot({
-      context: contextServiceMock.createPrebootContract(),
+describe('APMEventClient', () => {
+  describe('Abort controller', () => {
+    let server: ReturnType<typeof createHttpService>;
+    beforeEach(() => {
+      server = createHttpService();
     });
-    const { server: innerServer, createRouter } = await server.setup({
-      context: contextServiceMock.createSetupContract(),
-      executionContext: executionContextServiceMock.createInternalSetupContract(),
+
+    afterEach(async () => {
+      await server.stop();
     });
-    const router = createRouter('/');
-
-    let abortSignal: AbortSignal | undefined;
-    router.get({ path: '/', validate: false }, async (context, request, res) => {
-      const eventClient = new APMEventClient({
-        esClient: {
-          search: async (params: any, { signal }: { signal: AbortSignal }) => {
-            abortSignal = signal;
-            await setTimeoutPromise(3_000);
-            return {};
+
+    it('cancels a search when a request is aborted', async () => {
+      await server.preboot({
+        context: contextServiceMock.createPrebootContract(),
+      });
+      const { server: innerServer, createRouter } = await server.setup({
+        context: contextServiceMock.createSetupContract(),
+        executionContext: executionContextServiceMock.createInternalSetupContract(),
+      });
+      const router = createRouter('/');
+
+      let abortSignal: AbortSignal | undefined;
+      router.get({ path: '/', validate: false }, async (context, request, res) => {
+        const eventClient = new APMEventClient({
+          esClient: {
+            search: async (params: any, { signal }: { signal: AbortSignal }) => {
+              abortSignal = signal;
+              await setTimeoutPromise(3_000, undefined, {
+                signal: abortSignal,
+              });
+              return {};
+            },
+          } as any,
+          debug: false,
+          request,
+          indices: {} as APMIndices,
+          options: {
+            includeFrozen: false,
           },
-        } as any,
+        });
+
+        await eventClient.search('foo', {
+          apm: {
+            events: [],
+          },
+          body: { size: 0, track_total_hits: false },
+        });
+
+        return res.ok({ body: 'ok' });
+      });
+
+      await server.start();
+
+      expect(abortSignal?.aborted).toBeFalsy();
+
+      const incomingRequest = supertest(innerServer.listener)
+        .get('/')
+        // end required to send request
+        .end();
+
+      await new Promise((resolve) => {
+        setTimeout(() => {
+          void incomingRequest.on('abort', () => {
+            setTimeout(() => {
+              resolve(undefined);
+            }, 100);
+          });
+
+          void incomingRequest.abort();
+        }, 200);
+      });
+
+      expect(abortSignal?.aborted).toBe(true);
+    });
+  });
+
+  describe('excludedDataTiers filter', () => {
+    let esClientMock: jest.Mocked<ElasticsearchClient>;
+    let apmEventClient: APMEventClient;
+    let cancelEsRequestOnAbortSpy: jest.SpyInstance;
+    let unwrapEsResponseSpy: jest.SpyInstance;
+
+    const esResponse: estypes.SearchResponse = {
+      hits: {
+        total: { value: 1, relation: 'eq' },
+        hits: [{ _source: {}, _index: '' }],
+        max_score: 1,
+      },
+      took: 1,
+      _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+      timed_out: false,
+    };
+
+    beforeAll(() => {
+      jest.resetModules();
+    });
+
+    beforeEach(() => {
+      cancelEsRequestOnAbortSpy = jest
+        .spyOn(cancelEsRequestOnAbortModule, 'cancelEsRequestOnAbort')
+        .mockImplementation(jest.fn());
+
+      unwrapEsResponseSpy = jest
+        .spyOn(observabilityPluginModule, 'unwrapEsResponse')
+        .mockImplementation(jest.fn());
+
+      esClientMock = {
+        search: jest.fn(),
+        msearch: jest.fn(),
+        eql: { search: jest.fn() },
+        fieldCaps: jest.fn(),
+        termsEnum: jest.fn(),
+      } as unknown as jest.Mocked<ElasticsearchClient>;
+
+      apmEventClient = new APMEventClient({
+        esClient: esClientMock,
         debug: false,
-        request,
-        indices: {} as any,
+        request: {} as KibanaRequest,
+        indices: {} as APMIndices,
         options: {
           includeFrozen: false,
+          excludedDataTiers: ['data_warm', 'data_cold'],
         },
       });
+    });
+
+    afterAll(() => {
+      cancelEsRequestOnAbortSpy.mockReset();
+      unwrapEsResponseSpy.mockReset();
+    });
 
-      await eventClient.search('foo', {
-        apm: {
-          events: [],
+    it('includes excludedDataTiers filter in search params', async () => {
+      esClientMock.search.mockResolvedValue(esResponse);
+
+      await apmEventClient.search('testOperation', {
+        apm: { events: [] },
+        body: {
+          size: 0,
+          track_total_hits: false,
+          query: { bool: { filter: [{ match_all: {} }] } },
         },
-        body: { size: 0, track_total_hits: false },
       });
 
-      return res.ok({ body: 'ok' });
+      const searchParams = esClientMock.search.mock.calls[0][0] as APMEventESSearchRequest;
+
+      expect(searchParams.body.query?.bool).toEqual({
+        filter: [
+          { terms: { 'processor.event': [] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+        must: [{ bool: { filter: [{ match_all: {} }] } }],
+      });
     });
 
-    await server.start();
+    it('includes excludedDataTiers filter in msearch params', async () => {
+      esClientMock.msearch.mockResolvedValue({ responses: [esResponse], took: 1 });
 
-    expect(abortSignal?.aborted).toBeFalsy();
+      await apmEventClient.msearch('testOperation', {
+        apm: { events: [] },
+        body: {
+          size: 0,
+          track_total_hits: false,
+          query: { bool: { filter: [{ match_all: {} }] } },
+        },
+      });
 
-    const incomingRequest = supertest(innerServer.listener)
-      .get('/')
-      // end required to send request
-      .end();
+      const msearchParams = esClientMock.msearch.mock.calls[0][0] as {
+        searches: MsearchMultisearchBody[];
+      };
 
-    await new Promise((resolve) => {
-      setTimeout(() => {
-        void incomingRequest.on('abort', () => {
-          setTimeout(() => {
-            resolve(undefined);
-          }, 100);
-        });
-        void incomingRequest.abort();
-      }, 100);
+      expect(msearchParams.searches[1].query?.bool).toEqual({
+        filter: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { terms: { 'processor.event': [] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
     });
 
-    expect(abortSignal?.aborted).toBe(true);
+    it('includes excludedDataTiers filter in fieldCaps params', async () => {
+      esClientMock.fieldCaps.mockResolvedValue({
+        fields: {},
+        indices: '',
+      });
+
+      await apmEventClient.fieldCaps('testOperation', {
+        apm: { events: [] },
+        fields: ['field1'],
+        index_filter: { bool: { filter: [{ match_all: {} }] } },
+      });
+
+      const fieldCapsParams = esClientMock.fieldCaps.mock.calls[0][0] as APMEventFieldCapsRequest;
+      expect(fieldCapsParams?.index_filter?.bool).toEqual({
+        must: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
+    });
+
+    it('includes excludedDataTiers filter in termsEnum params', async () => {
+      esClientMock.termsEnum.mockResolvedValue({
+        terms: [''],
+        _shards: { total: 1, successful: 1, failed: 0 },
+        complete: true,
+      });
+
+      await apmEventClient.termsEnum('testOperation', {
+        apm: { events: [] },
+        field: 'field1',
+        index_filter: { bool: { filter: [{ match_all: {} }] } },
+      });
+
+      const termsEnumParams = esClientMock.termsEnum.mock.calls[0][0] as TermsEnumRequest;
+
+      expect(termsEnumParams.index_filter?.bool).toEqual({
+        must: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
+    });
   });
 });