[Infra] Exclude frozen/cold data tiers from source queries (elastic#2…

…01804) Closes elastic#201568 Adds the exclude data tiers settings to the `/api/metrics/source/hasData` and `/api/metrics/source/{sourceId}` requests. Also applies it to the `getIndexStatus` API call. --------- Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: kibanamachine <[email protected]> (cherry picked from commit ac3b60e)
Bluefinger · Dec 4, 2024 · e9db945 · e9db945
1 parent e0516f3
commit e9db945
Show file tree

Hide file tree

Showing 7 changed files with 200 additions and 10 deletions.
diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts
@@ -13,10 +13,10 @@ export function getDataTierFilterCombined({
   excludedDataTiers,
 }: {
   filter?: QueryDslQueryContainer;
-  excludedDataTiers: DataTier[];
+  excludedDataTiers?: DataTier[];
 }): QueryDslQueryContainer | undefined {
   if (!filter) {
-    return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined;
+    return excludedDataTiers?.length ? excludeTiersQuery(excludedDataTiers)[0] : undefined;
   }
 
   return !excludedDataTiers

diff --git a/...y_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/...y_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts
@@ -5,11 +5,14 @@
  * 2.0.
  */
 
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
 import type { InfraPluginRequestHandlerContext } from '../../../types';
 import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors';
-import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status';
-import { InfraDatabaseGetIndicesResponse } from '../framework';
-import { KibanaFramework } from '../framework/kibana_framework_adapter';
+import type { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status';
+import type { InfraDatabaseGetIndicesResponse } from '../framework';
+import type { KibanaFramework } from '../framework/kibana_framework_adapter';
 
 export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusAdapter {
   constructor(private readonly framework: KibanaFramework) {}
@@ -46,6 +49,12 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA
     requestContext: InfraPluginRequestHandlerContext,
     indexNames: string
   ): Promise<SourceIndexStatus> {
+    const { uiSettings } = await requestContext.core;
+
+    const excludedDataTiers = await uiSettings.client.get<DataTier[]>(searchExcludedDataTiers);
+
+    const filter = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : [];
+
     return await this.framework
       .callWithRequest(requestContext, 'search', {
         ignore_unavailable: true,
@@ -54,6 +63,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA
         size: 0,
         terminate_after: 1,
         track_total_hits: 1,
+        query: { bool: { filter } },
       })
       .then(
         (response) => {

diff --git a/.../plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts b/.../plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaRequest } from '@kbn/core-http-server';
+import { DataTier } from '@kbn/observability-shared-plugin/common';
+import { InfraBackendLibs } from '../infra_types';
+import { getInfraMetricsClient } from './get_infra_metrics_client';
+import { InfraPluginRequestHandlerContext } from '../../types';
+import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
+
+const withExcludedDataTiers = (tiers: DataTier[]) => ({
+  uiSettings: {
+    client: {
+      get: () => Promise.resolve(tiers),
+    },
+  },
+});
+
+const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) };
+
+const infraMetricsTestHarness =
+  (tiers: DataTier[], input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) =>
+  async () => {
+    const callWithRequest = jest.fn();
+
+    const mockedCore = withExcludedDataTiers(tiers);
+
+    const context = {
+      infra: Promise.resolve(mockedInfra),
+      core: Promise.resolve(mockedCore),
+    } as unknown as InfraPluginRequestHandlerContext;
+
+    const client = await getInfraMetricsClient({
+      libs: { framework: { callWithRequest } } as unknown as InfraBackendLibs,
+      context,
+      request: {} as unknown as KibanaRequest,
+    });
+
+    await client.search({
+      body: {
+        query: input,
+        size: 1,
+        track_total_hits: false,
+      },
+    });
+
+    expect(callWithRequest).toBeCalledWith(
+      context,
+      'search',
+      {
+        body: {
+          query: output,
+          size: 1,
+          track_total_hits: false,
+        },
+        ignore_unavailable: true,
+        index: ['*.indices'],
+      },
+      {}
+    );
+  };
+
+describe('getInfraMetricsClient', () => {
+  it(
+    'defines an empty must_not query if given no data tiers to filter by',
+    infraMetricsTestHarness([], undefined, { bool: { must_not: [] } })
+  );
+
+  it(
+    'includes excluded data tiers in the request filter by default',
+    infraMetricsTestHarness(['data_frozen'], undefined, {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: ['data_frozen'],
+            },
+          },
+        ],
+      },
+    })
+  );
+
+  it(
+    'merges provided filters with the excluded data tier filter',
+    infraMetricsTestHarness(
+      ['data_frozen'],
+      {
+        bool: {
+          must_not: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+        },
+      },
+      {
+        bool: {
+          must_not: [
+            {
+              exists: {
+                field: 'a-field',
+              },
+            },
+            {
+              terms: {
+                _tier: ['data_frozen'],
+              },
+            },
+          ],
+        },
+      }
+    )
+  );
+
+  it(
+    'merges other query params with the excluded data tiers filter',
+    infraMetricsTestHarness(
+      ['data_frozen'],
+      {
+        bool: {
+          must: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+        },
+      },
+      {
+        bool: {
+          must: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+          must_not: [
+            {
+              terms: {
+                _tier: ['data_frozen'],
+              },
+            },
+          ],
+        },
+      }
+    )
+  );
+});
diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts
@@ -6,6 +6,10 @@
  */
 import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import type { InfraPluginRequestHandlerContext } from '../../types';
 import type { InfraBackendLibs } from '../infra_types';
 
@@ -29,19 +33,42 @@ export async function getInfraMetricsClient({
 }) {
   const { framework } = libs;
   const infraContext = await context.infra;
+  const { uiSettings } = await context.core;
+
+  const excludedDataTiers = await uiSettings.client.get<DataTier[]>(searchExcludedDataTiers);
   const metricsIndices = await infraContext.getMetricsIndices();
 
+  const excludedQuery = excludedDataTiers.length
+    ? excludeTiersQuery(excludedDataTiers)[0].bool!.must_not!
+    : [];
+
   return {
     search<TDocument, TParams extends RequiredParams>(
       searchParams: TParams
     ): Promise<InferSearchResponseOf<TDocument, TParams>> {
+      const searchFilter = searchParams.body.query?.bool?.must_not ?? [];
+
+      // This flattens arrays by one level, and non-array values can be added as well, so it all
+      // results in a nice [QueryDsl, QueryDsl, ...] array.
+      const mustNot = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery);
+
       return framework.callWithRequest(
         context,
         'search',
         {
           ...searchParams,
           ignore_unavailable: true,
           index: metricsIndices,
+          body: {
+            ...searchParams.body,
+            query: {
+              ...searchParams.body.query,
+              bool: {
+                ...searchParams.body.query?.bool,
+                must_not: mustNot,
+              },
+            },
+          },
         },
         request
       ) as Promise<any>;

diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts
@@ -208,7 +208,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) =>
     },
     async (context, request, response) => {
       try {
-        const modules = castArray(request.query.modules);
+        const modules = request.query.modules ? castArray(request.query.modules) : [];
 
         if (modules.length > MAX_MODULES) {
           throw Boom.badRequest(

diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json
@@ -116,7 +116,9 @@
     "@kbn/entityManager-plugin",
     "@kbn/observability-utils",
     "@kbn/entities-schema",
-    "@kbn/zod"
+    "@kbn/zod",
+    "@kbn/observability-utils-server",
+    "@kbn/observability-utils-common"
   ],
   "exclude": ["target/**/*"]
 }
diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts
@@ -531,7 +531,7 @@ export const uiSettings: Record<string, UiSettings> = {
     value: 1.7,
     description: i18n.translate('xpack.observability.profilingDatacenterPUEUiSettingDescription', {
       defaultMessage: `Data center power usage effectiveness (PUE) measures how efficiently a data center uses energy. Defaults to 1.7, the average on-premise data center PUE according to the <a>Uptime Institute</a> survey
-      
+
       You can also use the PUE that corresponds with your cloud provider:
       '<ul style="list-style-type: none;margin-left: 4px;">
         <li><strong>AWS:</strong> 1.135</li>
@@ -649,8 +649,8 @@ export const uiSettings: Record<string, UiSettings> = {
     description: i18n.translate(
       'xpack.observability.advancedSettings.searchExcludedDataTiersDesc',
       {
-        defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. 
-        When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`,
+        defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen.
+        When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM, Infrastructure`,
         values: { technicalPreviewLabel: `<em>[${technicalPreviewLabel}]</em>` },
       }
     ),