[Dataset quality] Integration non-default dataset is now marked as part of the integration

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts

@@ -63,20 +63,20 @@ export class DataStreamStat {
 
   public static fromDegradedDocStat({
     degradedDocStat,
-    integrationMap,
+    datasetIntegrationMap,
   }: {
     degradedDocStat: DegradedDocsStat;
-    integrationMap: Record<string, { integration: Integration; title: string }>;
+    datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
   }) {
     const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);
 
     const dataStreamStatProps = {
       rawName: degradedDocStat.dataset,
       type,
       name: dataset,
-      title: integrationMap[dataset]?.title || dataset,
+      title: datasetIntegrationMap[dataset]?.title || dataset,
       namespace,
-      integration: integrationMap[dataset]?.integration,
+      integration: datasetIntegrationMap[dataset]?.integration,
       degradedDocs: {
         percentage: degradedDocStat.percentage,
         count: degradedDocStat.count,

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts

@@ -6,15 +6,14 @@
  */
 
 import { APIClientRequestParamsOf, APIReturnType } from '../rest';
-import { DataStreamStat } from './data_stream_stat';
 
 export type GetDataStreamsStatsParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params'];
 export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query'];
 export type GetDataStreamsStatsResponse =
   APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>;
 export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0];
-export type DataStreamStatServiceResponse = DataStreamStat[];
+export type DataStreamStatServiceResponse = DataStreamStatType[];
 
 export type GetIntegrationsParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params'];

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx

@@ -76,10 +76,18 @@ export const useDatasetQualityFilters = () => {
   );
 
   const integrationItems: IntegrationItem[] = useMemo(() => {
+    const integrationsMap = datasets.reduce(
+      (acc, dataset) => ({
+        ...acc,
+        ...(dataset.integration && !acc[dataset.integration.name]
+          ? { [dataset.integration.name]: dataset.integration }
+          : {}),
+      }),
+      {} as { [key: string]: Integration }
+    );
+
     const integrations = [
-      ...datasets
-        .map((dataset) => dataset.integration)
-        .filter((integration): integration is Integration => !!integration),
+      ...Object.values(integrationsMap),
       ...(datasets.some((dataset) => !dataset.integration)
         ? [Integration.create({ name: 'none', title: 'None' })]
         : []),

x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts

@@ -27,7 +27,6 @@ import {
   GetIntegrationsParams,
   IntegrationsResponse,
 } from '../../../common/data_streams_stats';
-import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat';
 import { IDataStreamsStatsClient } from './types';
 
 export class DataStreamsStatsClient implements IDataStreamsStatsClient {
@@ -50,7 +49,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
         new GetDataStreamsStatsError(`Failed to decode data streams stats response: ${message}`)
     )(response);
 
-    return dataStreamsStats.map(DataStreamStat.create);
+    return dataStreamsStats;
   }
 
   public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) {

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts

@@ -8,13 +8,13 @@
 import { IToasts } from '@kbn/core/public';
 import { getDateISORange } from '@kbn/timerange';
 import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
+import { DataStreamStat } from '../../../../common/api_types';
 import { Integration } from '../../../../common/data_streams_stats/integration';
 import { IDataStreamDetailsClient } from '../../../services/data_stream_details';
 import {
   DashboardType,
   DataStreamSettings,
   DataStreamDetails,
-  DataStreamStat,
   GetDataStreamsStatsQuery,
   GetIntegrationsParams,
 } from '../../../../common/data_streams_stats';

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts

@@ -18,6 +18,7 @@ import {
   DataStreamStatServiceResponse,
   IntegrationsResponse,
   DataStreamStat,
+  DataStreamStatType,
 } from '../../../../common/data_streams_stats';
 
 export type FlyoutDataset = Omit<
@@ -66,7 +67,7 @@ export interface WithFilters {
 }
 
 export interface WithDataStreamStats {
-  dataStreamStats: DataStreamStat[];
+  dataStreamStats: DataStreamStatType[];
 }
 
 export interface WithDegradedDocs {

x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts

@@ -6,9 +6,9 @@
  */
 
 import { indexNameToDataStreamParts } from '../../common/utils';
-import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
 import { Integration } from '../../common/data_streams_stats/integration';
 import { generateDatasets } from './generate_datasets';
+import { DataStreamStatType } from '../../common/data_streams_stats/types';
 
 describe('generateDatasets', () => {
   const integrations: Integration[] = [
@@ -34,34 +34,19 @@ describe('generateDatasets', () => {
     },
   ];
 
-  const dataStreamStats: DataStreamStat[] = [
+  const dataStreamStats: DataStreamStatType[] = [
     {
-      name: 'system.application',
-      title: 'system.application',
-      type: 'logs',
-      namespace: 'default',
+      name: 'logs-system.application-default',
       lastActivity: 1712911241117,
       size: '82.1kb',
       sizeBytes: 84160,
-      rawName: 'logs-system.application-default',
-      degradedDocs: {
-        percentage: 0,
-        count: 0,
-      },
+      integration: 'system',
     },
     {
-      name: 'synth',
-      title: 'synth',
-      type: 'logs',
-      namespace: 'default',
+      name: 'logs-synth-default',
       lastActivity: 1712911241117,
-      rawName: 'logs-synth-default',
       size: '62.5kb',
       sizeBytes: 64066,
-      degradedDocs: {
-        percentage: 0,
-        count: 0,
-      },
     },
   ];
 
@@ -84,11 +69,29 @@ describe('generateDatasets', () => {
     expect(datasets).toEqual([
       {
         ...dataStreamStats[0],
-        title: integrations[0].datasets[dataStreamStats[0].name],
+        name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset,
+        namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace,
+        title:
+          integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset],
+        type: indexNameToDataStreamParts(dataStreamStats[0].name).type,
+        rawName: dataStreamStats[0].name,
         integration: integrations[0],
+        degradedDocs: {
+          count: 0,
+          percentage: 0,
+        },
       },
       {
         ...dataStreamStats[1],
+        name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
+        namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace,
+        title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
+        type: indexNameToDataStreamParts(dataStreamStats[1].name).type,
+        rawName: dataStreamStats[1].name,
+        degradedDocs: {
+          count: 0,
+          percentage: 0,
+        },
       },
     ]);
   });
@@ -137,7 +140,12 @@ describe('generateDatasets', () => {
     expect(datasets).toEqual([
       {
         ...dataStreamStats[0],
-        title: integrations[0].datasets[dataStreamStats[0].name],
+        name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset,
+        namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace,
+        title:
+          integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset],
+        type: indexNameToDataStreamParts(dataStreamStats[0].name).type,
+        rawName: dataStreamStats[0].name,
         integration: integrations[0],
         degradedDocs: {
           percentage: degradedDocs[0].percentage,
@@ -146,6 +154,11 @@ describe('generateDatasets', () => {
       },
       {
         ...dataStreamStats[1],
+        name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
+        namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace,
+        title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
+        type: indexNameToDataStreamParts(dataStreamStats[1].name).type,
+        rawName: dataStreamStats[1].name,
         degradedDocs: {
           percentage: degradedDocs[1].percentage,
           count: degradedDocs[1].count,
@@ -154,6 +167,36 @@ describe('generateDatasets', () => {
     ]);
   });
 
+  it('merges integration information with dataStreamStats when dataset is not an integration default one', () => {
+    const dataset = 'logs-system.custom-default';
+
+    const nonDefaultDataset = {
+      name: dataset,
+      lastActivity: 1712911241117,
+      size: '82.1kb',
+      sizeBytes: 84160,
+      integration: 'system',
+    };
+
+    const datasets = generateDatasets([nonDefaultDataset], undefined, integrations);
+
+    expect(datasets).toEqual([
+      {
+        ...nonDefaultDataset,
+        title: indexNameToDataStreamParts(dataset).dataset,
+        name: indexNameToDataStreamParts(dataset).dataset,
+        namespace: indexNameToDataStreamParts(dataset).namespace,
+        type: indexNameToDataStreamParts(dataset).type,
+        rawName: nonDefaultDataset.name,
+        integration: integrations[0],
+        degradedDocs: {
+          count: 0,
+          percentage: 0,
+        },
+      },
+    ]);
+  });
+
   it('returns an empty array if no valid object is provided', () => {
     expect(generateDatasets(undefined, undefined, integrations)).toEqual([]);
   });

x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts

@@ -5,20 +5,21 @@
  * 2.0.
  */
 
+import { DataStreamStatType } from '../../common/data_streams_stats/types';
 import { Integration } from '../../common/data_streams_stats/integration';
 import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
 import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat';
 
 export function generateDatasets(
-  dataStreamStats: DataStreamStat[] = [],
+  dataStreamStats: DataStreamStatType[] = [],
   degradedDocStats: DegradedDocsStat[] = [],
   integrations: Integration[]
-) {
+): DataStreamStat[] {
   if (!dataStreamStats.length && !integrations.length) {
     return [];
   }
 
-  const integrationMap: Record<string, { integration: Integration; title: string }> =
+  const datasetIntegrationMap: Record<string, { integration: Integration; title: string }> =
     integrations.reduce((integrationMapAcc, integration) => {
       return {
         ...integrationMapAcc,
@@ -37,10 +38,20 @@ export function generateDatasets(
 
   if (!dataStreamStats.length) {
     return degradedDocStats.map((degradedDocStat) =>
-      DataStreamStat.fromDegradedDocStat({ degradedDocStat, integrationMap })
+      DataStreamStat.fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap })
     );
   }
 
+  const integrationsMap: Record<string, Integration> = integrations.reduce(
+    (integrationMapAcc, integration) => {
+      return {
+        ...integrationMapAcc,
+        [integration.name]: integration,
+      };
+    },
+    {}
+  );
+
   const degradedMap: Record<
     DegradedDocsStat['dataset'],
     {
@@ -53,10 +64,16 @@ export function generateDatasets(
     {}
   );
 
-  return dataStreamStats?.map((dataStream) => ({
-    ...dataStream,
-    title: integrationMap[dataStream.name]?.title || dataStream.title,
-    integration: integrationMap[dataStream.name]?.integration,
-    degradedDocs: degradedMap[dataStream.rawName] || dataStream.degradedDocs,
-  }));
+  return dataStreamStats?.map((dataStream) => {
+    const dataset = DataStreamStat.create(dataStream);
+
+    return {
+      ...dataset,
+      title: datasetIntegrationMap[dataset.name]?.title || dataset.title,
+      integration:
+        datasetIntegrationMap[dataset.name]?.integration ??
+        integrationsMap[dataStream.integration ?? ''],
+      degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs,
+    };
+  });
 }