diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index f3b68e2c1c2a0..0cf0a40b1b45b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -65,10 +65,10 @@ export class DataStreamStat { public static fromDegradedDocStat({ degradedDocStat, - integrationMap, + datasetIntegrationMap, }: { degradedDocStat: DegradedDocsStat; - integrationMap: Record; + datasetIntegrationMap: Record; }) { const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); @@ -76,9 +76,9 @@ export class DataStreamStat { rawName: degradedDocStat.dataset, type, name: dataset, - title: integrationMap[dataset]?.title || dataset, + title: datasetIntegrationMap[dataset]?.title || dataset, namespace, - integration: integrationMap[dataset]?.integration, + integration: datasetIntegrationMap[dataset]?.integration, degradedDocs: { percentage: degradedDocStat.percentage, count: degradedDocStat.count, diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 66fbffd452dc6..0c9456c2d7257 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -6,7 +6,6 @@ */ import { APIClientRequestParamsOf, APIReturnType } from '../rest'; -import { DataStreamStat } from './data_stream_stat'; export type GetDataStreamsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params']; @@ -14,7 +13,7 @@ export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query']; export type GetDataStreamsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>; export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0]; -export type DataStreamStatServiceResponse = DataStreamStat[]; +export type DataStreamStatServiceResponse = DataStreamStatType[]; export type GetIntegrationsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params']; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx index d6eccbbe545f2..a4d7be4fca46e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx @@ -36,26 +36,22 @@ export const useDatasetQualityFilters = () => { interface Filters { namespaces: string[]; qualities: QualityIndicators[]; - integrations: Integration[]; - hasNoneIntegration: boolean; + filteredIntegrations: string[]; } const datasets = useSelector(service, (state) => state.context.datasets); - const { namespaces, qualities, integrations } = useMemo( + const integrations = useSelector(service, (state) => state.context.integrations); + const { namespaces, qualities, filteredIntegrations } = useMemo( () => datasets.reduce( - (acc: Filters, dataset) => { - acc.namespaces.push(dataset.namespace); - acc.qualities.push(dataset.degradedDocs.quality); - if (dataset.integration) { - acc.integrations.push(dataset.integration); - } else if (!acc.hasNoneIntegration) { - acc.integrations.push(Integration.create({ name: 'none', title: 'None' })); - acc.hasNoneIntegration = true; - } - return acc; - }, - { namespaces: [], qualities: [], integrations: [], hasNoneIntegration: false } + (acc: Filters, dataset) => ({ + namespaces: [...new Set([...acc.namespaces, dataset.namespace])], + qualities: [...new Set([...acc.qualities, dataset.degradedDocs.quality])], + filteredIntegrations: [ + ...new Set([...acc.filteredIntegrations, dataset.integration?.name ?? 'none']), + ], + }), + { namespaces: [], qualities: [], filteredIntegrations: [] } ), [datasets] ); @@ -100,15 +96,24 @@ export const useDatasetQualityFilters = () => { [service, timeRange] ); - const integrationItems: IntegrationItem[] = useMemo( - () => - integrations.map((integration) => ({ - ...integration, - label: integration.title, - checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined, - })), - [integrations, selectedIntegrations] - ); + const integrationItems: IntegrationItem[] = useMemo(() => { + const integrationsMap = + integrations?.reduce( + (acc, integration) => ({ + ...acc, + [integration.name]: integration, + }), + {} as { [key: string]: Integration } + ) ?? {}; + + integrationsMap.none = Integration.create({ name: 'none', title: 'None' }); + + return filteredIntegrations.map((name) => ({ + ...integrationsMap[name], + label: integrationsMap[name]?.title, + checked: selectedIntegrations.includes(name) ? 'on' : undefined, + })); + }, [integrations, filteredIntegrations, selectedIntegrations]); const onIntegrationsChange = useCallback( (newIntegrationItems: IntegrationItem[]) => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 2fc2b388f3a8d..d908c8665c1f4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -27,7 +27,6 @@ import { GetIntegrationsParams, IntegrationsResponse, } from '../../../common/data_streams_stats'; -import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; import { IDataStreamsStatsClient } from './types'; export class DataStreamsStatsClient implements IDataStreamsStatsClient { @@ -50,7 +49,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { new GetDataStreamsStatsError(`Failed to decode data streams stats response: ${message}`) )(response); - return dataStreamsStats.map(DataStreamStat.create); + return dataStreamsStats; } public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index bc60bad709a7b..7d54e268bb5df 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,13 +8,13 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; +import { DataStreamStat } from '../../../../common/api_types'; import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { DashboardType, DataStreamSettings, DataStreamDetails, - DataStreamStat, GetDataStreamsStatsQuery, GetIntegrationsParams, } from '../../../../common/data_streams_stats'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index d4c1236329f2b..10c08f8fa0bfc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -19,6 +19,7 @@ import { DataStreamStatServiceResponse, IntegrationsResponse, DataStreamStat, + DataStreamStatType, } from '../../../../common/data_streams_stats'; export type FlyoutDataset = Omit< @@ -68,7 +69,7 @@ export interface WithFilters { } export interface WithDataStreamStats { - dataStreamStats: DataStreamStat[]; + dataStreamStats: DataStreamStatType[]; } export interface WithDegradedDocs { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index 0db27c57dfc1f..9b9a007f0c54b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -6,9 +6,9 @@ */ import { indexNameToDataStreamParts } from '../../common/utils'; -import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { Integration } from '../../common/data_streams_stats/integration'; import { generateDatasets } from './generate_datasets'; +import { DataStreamStatType } from '../../common/data_streams_stats/types'; describe('generateDatasets', () => { const integrations: Integration[] = [ @@ -34,36 +34,19 @@ describe('generateDatasets', () => { }, ]; - const dataStreamStats: DataStreamStat[] = [ + const dataStreamStats: DataStreamStatType[] = [ { - name: 'system.application', - title: 'system.application', - type: 'logs', - namespace: 'default', + name: 'logs-system.application-default', lastActivity: 1712911241117, size: '82.1kb', sizeBytes: 84160, - rawName: 'logs-system.application-default', - degradedDocs: { - percentage: 0, - count: 0, - quality: 'good', - }, + integration: 'system', }, { - name: 'synth', - title: 'synth', - type: 'logs', - namespace: 'default', + name: 'logs-synth-default', lastActivity: 1712911241117, - rawName: 'logs-synth-default', size: '62.5kb', sizeBytes: 64066, - degradedDocs: { - percentage: 0, - count: 0, - quality: 'good', - }, }, ]; @@ -88,16 +71,31 @@ describe('generateDatasets', () => { expect(datasets).toEqual([ { ...dataStreamStats[0], - title: integrations[0].datasets[dataStreamStats[0].name], + name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, + title: + integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], + type: indexNameToDataStreamParts(dataStreamStats[0].name).type, + rawName: dataStreamStats[0].name, integration: integrations[0], degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { ...dataStreamStats[1], + name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, + title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + type: indexNameToDataStreamParts(dataStreamStats[1].name).type, + rawName: dataStreamStats[1].name, + degradedDocs: { + count: 0, + percentage: 0, + quality: 'good', + }, }, ]); }); @@ -120,7 +118,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { @@ -136,7 +134,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[1].percentage, count: degradedDocs[1].count, - quality: 'poor', + quality: degradedDocs[1].quality, }, }, ]); @@ -148,20 +146,61 @@ describe('generateDatasets', () => { expect(datasets).toEqual([ { ...dataStreamStats[0], - title: integrations[0].datasets[dataStreamStats[0].name], + name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, + title: + integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], + type: indexNameToDataStreamParts(dataStreamStats[0].name).type, + rawName: dataStreamStats[0].name, integration: integrations[0], degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { ...dataStreamStats[1], + name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, + title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + type: indexNameToDataStreamParts(dataStreamStats[1].name).type, + rawName: dataStreamStats[1].name, degradedDocs: { percentage: degradedDocs[1].percentage, count: degradedDocs[1].count, - quality: 'poor', + quality: degradedDocs[1].quality, + }, + }, + ]); + }); + + it('merges integration information with dataStreamStats when dataset is not an integration default one', () => { + const dataset = 'logs-system.custom-default'; + + const nonDefaultDataset = { + name: dataset, + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, + integration: 'system', + }; + + const datasets = generateDatasets([nonDefaultDataset], undefined, integrations); + + expect(datasets).toEqual([ + { + ...nonDefaultDataset, + title: indexNameToDataStreamParts(dataset).dataset, + name: indexNameToDataStreamParts(dataset).dataset, + namespace: indexNameToDataStreamParts(dataset).namespace, + type: indexNameToDataStreamParts(dataset).type, + rawName: nonDefaultDataset.name, + integration: integrations[0], + degradedDocs: { + count: 0, + percentage: 0, + quality: 'good', }, }, ]); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 3d592794413b0..e2f85d8846c99 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -5,40 +5,52 @@ * 2.0. */ +import { DataStreamStatType } from '../../common/data_streams_stats/types'; import { mapPercentageToQuality } from '../../common/utils'; import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; export function generateDatasets( - dataStreamStats: DataStreamStat[] = [], + dataStreamStats: DataStreamStatType[] = [], degradedDocStats: DegradedDocsStat[] = [], integrations: Integration[] -) { +): DataStreamStat[] { if (!dataStreamStats.length && !integrations.length) { return []; } - const integrationMap: Record = - integrations.reduce((integrationMapAcc, integration) => { + const { + datasetIntegrationMap, + integrationsMap, + }: { + datasetIntegrationMap: Record; + integrationsMap: Record; + } = integrations.reduce( + (acc, integration) => { return { - ...integrationMapAcc, - ...Object.keys(integration.datasets).reduce( - (datasetsAcc, dataset) => - Object.assign(datasetsAcc, { - [dataset]: { - integration, - title: integration.datasets[dataset], - }, - }), - {} - ), + datasetIntegrationMap: { + ...acc.datasetIntegrationMap, + ...Object.keys(integration.datasets).reduce( + (datasetsAcc, dataset) => + Object.assign(datasetsAcc, { + [dataset]: { + integration, + title: integration.datasets[dataset], + }, + }), + {} + ), + }, + integrationsMap: { ...acc.integrationsMap, [integration.name]: integration }, }; - }, {}); + }, + { datasetIntegrationMap: {}, integrationsMap: {} } + ); if (!dataStreamStats.length) { return degradedDocStats.map((degradedDocStat) => - DataStreamStat.fromDegradedDocStat({ degradedDocStat, integrationMap }) + DataStreamStat.fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap }) ); } @@ -61,10 +73,16 @@ export function generateDatasets( {} ); - return dataStreamStats?.map((dataStream) => ({ - ...dataStream, - title: integrationMap[dataStream.name]?.title || dataStream.title, - integration: integrationMap[dataStream.name]?.integration, - degradedDocs: degradedMap[dataStream.rawName] || dataStream.degradedDocs, - })); + return dataStreamStats?.map((dataStream) => { + const dataset = DataStreamStat.create(dataStream); + + return { + ...dataset, + title: datasetIntegrationMap[dataset.name]?.title || dataset.title, + integration: + datasetIntegrationMap[dataset.name]?.integration ?? + integrationsMap[dataStream.integration ?? ''], + degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, + }; + }); }