[8.x] [Dataset quality] Adding size column and estimated size to serv…

…erless (#193998) (#194520)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Dataset quality] Adding size column and estimated size to serverless
(#193998)](#193998)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Yngrid
Coello","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-01T08:01:22Z","message":"[Dataset
quality] Adding size column and estimated size to serverless
(#193998)\n\nCloses
https://github.com/elastic/logs-dev/issues/179.\r\n\r\nThis PR aims to
enable:\r\nThe estimated size is displayed in Serverless in the
following areas:\r\n- Estimated and size column in main page\r\n- Size
in Overview section of Dataset details\r\n\r\n###
Demo\r\n\r\nhttps://github.com/user-attachments/assets/b0ef03fb-061d-44e5-8e1a-c47ece58de37","sha":"ea9850683a2b24ede2b792717eb6c2ed9271777d","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","ci:project-deploy-observability"],"title":"[Dataset
quality] Adding size column and estimated size to
serverless","number":193998,"url":"https://github.com/elastic/kibana/pull/193998","mergeCommit":{"message":"[Dataset
quality] Adding size column and estimated size to serverless
(#193998)\n\nCloses
https://github.com/elastic/logs-dev/issues/179.\r\n\r\nThis PR aims to
enable:\r\nThe estimated size is displayed in Serverless in the
following areas:\r\n- Estimated and size column in main page\r\n- Size
in Overview section of Dataset details\r\n\r\n###
Demo\r\n\r\nhttps://github.com/user-attachments/assets/b0ef03fb-061d-44e5-8e1a-c47ece58de37","sha":"ea9850683a2b24ede2b792717eb6c2ed9271777d"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193998","number":193998,"mergeCommit":{"message":"[Dataset
quality] Adding size column and estimated size to serverless
(#193998)\n\nCloses
https://github.com/elastic/logs-dev/issues/179.\r\n\r\nThis PR aims to
enable:\r\nThe estimated size is displayed in Serverless in the
following areas:\r\n- Estimated and size column in main page\r\n- Size
in Overview section of Dataset details\r\n\r\n###
Demo\r\n\r\nhttps://github.com/user-attachments/assets/b0ef03fb-061d-44e5-8e1a-c47ece58de37","sha":"ea9850683a2b24ede2b792717eb6c2ed9271777d"}}]}]
BACKPORT-->

Co-authored-by: Yngrid Coello <[email protected]>

x-pack/plugins/observability_solution/dataset_quality/README.md

@@ -92,34 +92,34 @@ unset FLEET_PACKAGE_REGISTRY_PORT
 
 ### Functional Tests
 
-### Stateful
-#### FTR Server
+#### Stateful
+##### FTR Server
 ```
 yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts
 ```
 
-#### FTR Runner
+##### FTR Runner
 ```
 yarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/index.ts
 ```
 
-#### Running Individual Tests
+##### Running Individual Tests
 ```
 yarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/$1
 ```
 
-### Serverless
+#### Serverless
 
-#### Server
+##### Server
 ```
 yarn test:ftr:server --config ./x-pack/test_serverless/functional/test_suites/observability/config.ts
 ```
 
-#### Runner
+##### Runner
 ```
 yarn test:ftr:runner --config ./x-pack/test_serverless/functional/test_suites/observability/config.ts --include ./x-pack/test_serverless/functional/test_suites/observability/dataset_quality/index.ts
 ```
-#### Running Individual Tests
+##### Running Individual Tests
 ```
 yarn test:ftr:runner --config ./x-pack/test_serverless/functional/test_suites/observability/config.ts --include ./x-pack/test_serverless/functional/test_suites/observability/dataset_quality/$1
 ```

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -31,7 +31,7 @@ export const dataStreamStatRt = rt.intersection([
     sizeBytes: rt.number,
     lastActivity: rt.number,
     integration: rt.string,
-    totalDocs: rt.union([rt.null, rt.number]), // rt.null is only needed for https://github.com/elastic/kibana/issues/178954
+    totalDocs: rt.number,
   }),
 ]);
 
@@ -132,7 +132,7 @@ export const dataStreamDetailsRt = rt.partial({
   lastActivity: rt.number,
   degradedDocsCount: rt.number,
   docsCount: rt.number,
-  sizeBytes: rt.union([rt.null, rt.number]), // rt.null is only needed for https://github.com/elastic/kibana/issues/178954
+  sizeBytes: rt.number,
   services: rt.record(rt.string, rt.array(rt.string)),
   hosts: rt.record(rt.string, rt.array(rt.string)),
   userPrivileges: userPrivilegesRt,
@@ -158,7 +158,7 @@ export const getDataStreamsSettingsResponseRt = rt.exact(dataStreamSettingsRt);
 export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt);
 
 export const dataStreamsEstimatedDataInBytesRT = rt.type({
-  estimatedDataInBytes: rt.union([rt.number, rt.null]), // Null in serverless: https://github.com/elastic/kibana/issues/178954
+  estimatedDataInBytes: rt.number,
 });
 
 export const getDataStreamsEstimatedDataInBytesResponseRt = rt.exact(

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/summary_panel.tsx

@@ -8,25 +8,22 @@
 import React from 'react';
 
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
-import { useSummaryPanelContext } from '../../../hooks';
 import { DatasetsQualityIndicators } from './datasets_quality_indicators';
 import { DatasetsActivity } from './datasets_activity';
 import { EstimatedData } from './estimated_data';
 
 // Allow for lazy loading
 // eslint-disable-next-line import/no-default-export
 export default function SummaryPanel() {
-  const { isEstimatedDataDisabled } = useSummaryPanelContext();
   return (
     <EuiFlexGroup gutterSize="m">
       <EuiFlexItem>
         <DatasetsQualityIndicators />
       </EuiFlexItem>
-
       <EuiFlexItem>
         <EuiFlexGroup gutterSize="m">
           <DatasetsActivity />
-          {!isEstimatedDataDisabled && <EstimatedData />}
+          <EstimatedData />
         </EuiFlexGroup>
       </EuiFlexItem>
     </EuiFlexGroup>

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx

@@ -161,7 +161,6 @@ export const getDatasetQualityTableColumns = ({
   loadingDataStreamStats,
   loadingDegradedStats,
   showFullDatasetNames,
-  isSizeStatsAvailable,
   isActiveDataset,
   timeRange,
   urlService,
@@ -172,7 +171,6 @@ export const getDatasetQualityTableColumns = ({
   loadingDataStreamStats: boolean;
   loadingDegradedStats: boolean;
   showFullDatasetNames: boolean;
-  isSizeStatsAvailable: boolean;
   isActiveDataset: (lastActivity: number) => boolean;
   timeRange: TimeRangeConfig;
   urlService: BrowserUrlService;
@@ -226,7 +224,7 @@ export const getDatasetQualityTableColumns = ({
       ),
       width: '160px',
     },
-    ...(isSizeStatsAvailable && canUserMonitorDataset && canUserMonitorAnyDataStream
+    ...(canUserMonitorDataset && canUserMonitorAnyDataStream
       ? [
           {
             name: (

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx

@@ -26,7 +26,6 @@ export default function Summary() {
   const {
     isSummaryPanelLoading,
     totalDocsCount,
-    sizeInBytesAvailable,
     sizeInBytes,
     isUserAllowedToSeeSizeInBytes,
     totalServicesCount,
@@ -41,14 +40,12 @@ export default function Summary() {
           value={totalDocsCount}
           isLoading={isSummaryPanelLoading}
         />
-        {sizeInBytesAvailable && (
-          <PanelIndicator
-            label={overviewPanelDocumentsIndicatorSize}
-            value={sizeInBytes}
-            isLoading={isSummaryPanelLoading}
-            userHasPrivilege={isUserAllowedToSeeSizeInBytes}
-          />
-        )}
+        <PanelIndicator
+          label={overviewPanelDocumentsIndicatorSize}
+          value={sizeInBytes}
+          isLoading={isSummaryPanelLoading}
+          userHasPrivilege={isUserAllowedToSeeSizeInBytes}
+        />
       </Panel>
       <Panel title={overviewPanelTitleResources}>
         <PanelIndicator

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx

@@ -39,7 +39,6 @@ export const useDatasetQualityTable = () => {
 
   const { page, rowsPerPage, sort } = useSelector(service, (state) => state.context.table);
 
-  const isSizeStatsAvailable = useSelector(service, (state) => state.context.isSizeStatsAvailable);
   const canUserMonitorDataset = useSelector(
     service,
     (state) => state.context.datasetUserPrivileges.canMonitor
@@ -102,7 +101,6 @@ export const useDatasetQualityTable = () => {
         loadingDataStreamStats,
         loadingDegradedStats,
         showFullDatasetNames,
-        isSizeStatsAvailable,
         isActiveDataset: isActive,
         timeRange,
         urlService: url,
@@ -114,7 +112,6 @@ export const useDatasetQualityTable = () => {
       loadingDataStreamStats,
       loadingDegradedStats,
       showFullDatasetNames,
-      isSizeStatsAvailable,
       isActive,
       timeRange,
       url,
@@ -211,6 +208,5 @@ export const useDatasetQualityTable = () => {
     canUserMonitorAnyDataStream,
     toggleInactiveDatasets,
     toggleFullDatasetNames,
-    isSizeStatsAvailable,
   };
 };

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts

@@ -26,10 +26,9 @@ export const useOverviewSummaryPanel = () => {
     .map((key: string) => services[key].length)
     .reduce((a, b) => a + b, 0);
 
-  const totalDocsCount = formatNumber(dataStreamDetails?.docsCount ?? 0, NUMBER_FORMAT);
+  const totalDocsCount = formatNumber(dataStreamDetails.docsCount, NUMBER_FORMAT);
 
-  const sizeInBytesAvailable = dataStreamDetails?.sizeBytes !== null;
-  const sizeInBytes = formatNumber(dataStreamDetails?.sizeBytes ?? 0, BYTE_NUMBER_FORMAT);
+  const sizeInBytes = formatNumber(dataStreamDetails.sizeBytes, BYTE_NUMBER_FORMAT);
   const isUserAllowedToSeeSizeInBytes = dataStreamDetails?.userPrivileges?.canMonitor ?? true;
 
   const hosts = dataStreamDetails?.hosts ?? {};
@@ -57,7 +56,6 @@ export const useOverviewSummaryPanel = () => {
 
   return {
     totalDocsCount,
-    sizeInBytesAvailable,
     sizeInBytes,
     isUserAllowedToSeeSizeInBytes,
     totalServicesCount,

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts

@@ -14,13 +14,8 @@ import { filterInactiveDatasets } from '../utils';
 
 const useSummaryPanel = () => {
   const { service } = useDatasetQualityContext();
-  const {
-    filteredItems,
-    isSizeStatsAvailable,
-    canUserMonitorDataset,
-    canUserMonitorAnyDataStream,
-    loading,
-  } = useDatasetQualityTable();
+  const { filteredItems, canUserMonitorDataset, canUserMonitorAnyDataStream, loading } =
+    useDatasetQualityTable();
 
   const { timeRange } = useSelector(service, (state) => state.context.filters);
 
@@ -84,7 +79,6 @@ const useSummaryPanel = () => {
 
     isEstimatedDataLoading,
     estimatedData,
-    isEstimatedDataDisabled: !isSizeStatsAvailable,
 
     isDatasetsActivityLoading,
     datasetsActivity,

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts

@@ -55,6 +55,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
     types: [DEFAULT_DATASET_TYPE],
   },
   datasets: [],
-  isSizeStatsAvailable: true,
   nonAggregatableDatasets: [],
 };

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts

@@ -323,13 +323,8 @@ export const createPureDatasetQualityControllerStateMachine = (
             const dataStreamStats = event.data.dataStreamsStats as DataStreamStat[];
             const datasetUserPrivileges = event.data.datasetUserPrivileges;
 
-            // Check if any DataStreamStat has null; to check for serverless
-            const isSizeStatsAvailable =
-              !dataStreamStats.length || dataStreamStats.some((stat) => stat.totalDocs !== null);
-
             return {
               dataStreamStats,
-              isSizeStatsAvailable,
               datasetUserPrivileges,
             };
           }

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts

@@ -60,7 +60,6 @@ export interface WithNonAggregatableDatasets {
 
 export interface WithDatasets {
   datasets: DataStreamStat[];
-  isSizeStatsAvailable: boolean;
 }
 
 export interface WithIntegrations {

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts

@@ -6,7 +6,7 @@
  */
 
 import { badRequest } from '@hapi/boom';
-import type { ElasticsearchClient } from '@kbn/core/server';
+import type { ElasticsearchClient, IScopedClusterClient } from '@kbn/core/server';
 import {
   findInventoryFields,
   InventoryItemType,
@@ -20,6 +20,7 @@ import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_ty
 import { createDatasetQualityESClient } from '../../../utils';
 import { dataStreamService, datasetQualityPrivileges } from '../../../services';
 import { getDataStreams } from '../get_data_streams';
+import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats';
 
 export async function getDataStreamSettings({
   esClient,
@@ -51,41 +52,50 @@ export async function getDataStreamDetails({
   end,
   isServerless,
 }: {
-  esClient: ElasticsearchClient;
+  esClient: IScopedClusterClient;
   dataStream: string;
   start: number;
   end: number;
   isServerless: boolean;
 }): Promise<DataStreamDetails> {
   throwIfInvalidDataStreamParams(dataStream);
 
+  // Query datastreams as the current user as the Kibana internal user may not have all the required permissions
+  const esClientAsCurrentUser = esClient.asCurrentUser;
+  const esClientAsSecondaryAuthUser = esClient.asSecondaryAuthUser;
+
   const hasAccessToDataStream = (
-    await datasetQualityPrivileges.getHasIndexPrivileges(esClient, [dataStream], ['monitor'])
+    await datasetQualityPrivileges.getHasIndexPrivileges(
+      esClientAsCurrentUser,
+      [dataStream],
+      ['monitor']
+    )
   )[dataStream];
 
   const esDataStream = hasAccessToDataStream
     ? (
         await getDataStreams({
-          esClient,
+          esClient: esClientAsCurrentUser,
           datasetQuery: dataStream,
         })
       ).dataStreams[0]
     : undefined;
 
   try {
     const dataStreamSummaryStats = await getDataStreamSummaryStats(
-      esClient,
+      esClientAsCurrentUser,
       dataStream,
       start,
       end
     );
 
-    const whenSizeStatsNotAvailable = NaN; // This will indicate size cannot be calculated
-    const avgDocSizeInBytes = isServerless
-      ? whenSizeStatsNotAvailable
-      : hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
-      ? await getAvgDocSizeInBytes(esClient, dataStream)
-      : 0;
+    const avgDocSizeInBytes =
+      hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
+        ? isServerless
+          ? await getMeteringAvgDocSizeInBytes(esClientAsSecondaryAuthUser, dataStream)
+          : await getAvgDocSizeInBytes(esClientAsCurrentUser, dataStream)
+        : 0;
+
     const sizeBytes = Math.ceil(avgDocSizeInBytes * dataStreamSummaryStats.docsCount);
 
     return {
@@ -172,6 +182,18 @@ async function getDataStreamSummaryStats(
   };
 }
 
+async function getMeteringAvgDocSizeInBytes(esClient: ElasticsearchClient, index: string) {
+  const meteringStats = await getDataStreamsMeteringStats({
+    esClient,
+    dataStreams: [index],
+  });
+
+  const docCount = meteringStats[index].totalDocs ?? 0;
+  const sizeInBytes = meteringStats[index].sizeBytes ?? 0;
+
+  return docCount ? sizeInBytes / docCount : 0;
+}
+
 async function getAvgDocSizeInBytes(esClient: ElasticsearchClient, index: string) {
   const indexStats = await esClient.indices.stats({ index });
   const docCount = indexStats._all.total?.docs?.count ?? 0;