From de89bb2c61bc730a86eddc5e76cf62063fca9bd2 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Fri, 26 Apr 2024 14:41:55 +0200 Subject: [PATCH 1/3] Using integration coming from DataStreamStats when non-default dataset --- .../data_streams_stats/data_stream_stat.ts | 8 +- .../common/data_streams_stats/types.ts | 3 +- .../data_streams_stats_client.ts | 3 +- .../src/state_machine.ts | 2 +- .../dataset_quality_controller/src/types.ts | 3 +- .../public/utils/generate_datasets.test.ts | 87 ++++++++++++++----- .../public/utils/generate_datasets.ts | 37 +++++--- 7 files changed, 101 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 7b1a171356a61..5bbb166da0a1e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -63,10 +63,10 @@ export class DataStreamStat { public static fromDegradedDocStat({ degradedDocStat, - integrationMap, + datasetIntegrationMap, }: { degradedDocStat: DegradedDocsStat; - integrationMap: Record; + datasetIntegrationMap: Record; }) { const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); @@ -74,9 +74,9 @@ export class DataStreamStat { rawName: degradedDocStat.dataset, type, name: dataset, - title: integrationMap[dataset]?.title || dataset, + title: datasetIntegrationMap[dataset]?.title || dataset, namespace, - integration: integrationMap[dataset]?.integration, + integration: datasetIntegrationMap[dataset]?.integration, degradedDocs: { percentage: degradedDocStat.percentage, count: degradedDocStat.count, diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 66fbffd452dc6..0c9456c2d7257 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -6,7 +6,6 @@ */ import { APIClientRequestParamsOf, APIReturnType } from '../rest'; -import { DataStreamStat } from './data_stream_stat'; export type GetDataStreamsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params']; @@ -14,7 +13,7 @@ export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query']; export type GetDataStreamsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>; export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0]; -export type DataStreamStatServiceResponse = DataStreamStat[]; +export type DataStreamStatServiceResponse = DataStreamStatType[]; export type GetIntegrationsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params']; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 2fc2b388f3a8d..d908c8665c1f4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -27,7 +27,6 @@ import { GetIntegrationsParams, IntegrationsResponse, } from '../../../common/data_streams_stats'; -import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; import { IDataStreamsStatsClient } from './types'; export class DataStreamsStatsClient implements IDataStreamsStatsClient { @@ -50,7 +49,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { new GetDataStreamsStatsError(`Failed to decode data streams stats response: ${message}`) )(response); - return dataStreamsStats.map(DataStreamStat.create); + return dataStreamsStats; } public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index f13e2c7340851..c0b71987111cc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,13 +8,13 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; +import { DataStreamStat } from '../../../../common/api_types'; import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { DashboardType, DataStreamSettings, DataStreamDetails, - DataStreamStat, GetDataStreamsStatsQuery, GetIntegrationsParams, } from '../../../../common/data_streams_stats'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 3cfb3e9c5a548..e50ebb6eed4e3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -18,6 +18,7 @@ import { DataStreamStatServiceResponse, IntegrationsResponse, DataStreamStat, + DataStreamStatType, } from '../../../../common/data_streams_stats'; export type FlyoutDataset = Omit< @@ -66,7 +67,7 @@ export interface WithFilters { } export interface WithDataStreamStats { - dataStreamStats: DataStreamStat[]; + dataStreamStats: DataStreamStatType[]; } export interface WithDegradedDocs { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index cc48eb1dfdcb5..63f98e60d6b10 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -6,9 +6,9 @@ */ import { indexNameToDataStreamParts } from '../../common/utils'; -import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { Integration } from '../../common/data_streams_stats/integration'; import { generateDatasets } from './generate_datasets'; +import { DataStreamStatType } from '../../common/data_streams_stats/types'; describe('generateDatasets', () => { const integrations: Integration[] = [ @@ -34,34 +34,19 @@ describe('generateDatasets', () => { }, ]; - const dataStreamStats: DataStreamStat[] = [ + const dataStreamStats: DataStreamStatType[] = [ { - name: 'system.application', - title: 'system.application', - type: 'logs', - namespace: 'default', + name: 'logs-system.application-default', lastActivity: 1712911241117, size: '82.1kb', sizeBytes: 84160, - rawName: 'logs-system.application-default', - degradedDocs: { - percentage: 0, - count: 0, - }, + integration: 'system', }, { - name: 'synth', - title: 'synth', - type: 'logs', - namespace: 'default', + name: 'logs-synth-default', lastActivity: 1712911241117, - rawName: 'logs-synth-default', size: '62.5kb', sizeBytes: 64066, - degradedDocs: { - percentage: 0, - count: 0, - }, }, ]; @@ -84,11 +69,29 @@ describe('generateDatasets', () => { expect(datasets).toEqual([ { ...dataStreamStats[0], - title: integrations[0].datasets[dataStreamStats[0].name], + name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, + title: + integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], + type: indexNameToDataStreamParts(dataStreamStats[0].name).type, + rawName: dataStreamStats[0].name, integration: integrations[0], + degradedDocs: { + count: 0, + percentage: 0, + }, }, { ...dataStreamStats[1], + name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, + title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + type: indexNameToDataStreamParts(dataStreamStats[1].name).type, + rawName: dataStreamStats[1].name, + degradedDocs: { + count: 0, + percentage: 0, + }, }, ]); }); @@ -137,7 +140,12 @@ describe('generateDatasets', () => { expect(datasets).toEqual([ { ...dataStreamStats[0], - title: integrations[0].datasets[dataStreamStats[0].name], + name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, + title: + integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], + type: indexNameToDataStreamParts(dataStreamStats[0].name).type, + rawName: dataStreamStats[0].name, integration: integrations[0], degradedDocs: { percentage: degradedDocs[0].percentage, @@ -146,6 +154,11 @@ describe('generateDatasets', () => { }, { ...dataStreamStats[1], + name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, + title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, + type: indexNameToDataStreamParts(dataStreamStats[1].name).type, + rawName: dataStreamStats[1].name, degradedDocs: { percentage: degradedDocs[1].percentage, count: degradedDocs[1].count, @@ -154,6 +167,36 @@ describe('generateDatasets', () => { ]); }); + it('merges integration information with dataStreamStats when dataset is not an integration default one', () => { + const dataset = 'logs-system.custom-default'; + + const nonDefaultDataset = { + name: dataset, + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, + integration: 'system', + }; + + const datasets = generateDatasets([nonDefaultDataset], undefined, integrations); + + expect(datasets).toEqual([ + { + ...nonDefaultDataset, + title: indexNameToDataStreamParts(dataset).dataset, + name: indexNameToDataStreamParts(dataset).dataset, + namespace: indexNameToDataStreamParts(dataset).namespace, + type: indexNameToDataStreamParts(dataset).type, + rawName: nonDefaultDataset.name, + integration: integrations[0], + degradedDocs: { + count: 0, + percentage: 0, + }, + }, + ]); + }); + it('returns an empty array if no valid object is provided', () => { expect(generateDatasets(undefined, undefined, integrations)).toEqual([]); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 951ba9ccafc8b..2ee6841afaee7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -5,20 +5,21 @@ * 2.0. */ +import { DataStreamStatType } from '../../common/data_streams_stats/types'; import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; export function generateDatasets( - dataStreamStats: DataStreamStat[] = [], + dataStreamStats: DataStreamStatType[] = [], degradedDocStats: DegradedDocsStat[] = [], integrations: Integration[] -) { +): DataStreamStat[] { if (!dataStreamStats.length && !integrations.length) { return []; } - const integrationMap: Record = + const datasetIntegrationMap: Record = integrations.reduce((integrationMapAcc, integration) => { return { ...integrationMapAcc, @@ -37,10 +38,20 @@ export function generateDatasets( if (!dataStreamStats.length) { return degradedDocStats.map((degradedDocStat) => - DataStreamStat.fromDegradedDocStat({ degradedDocStat, integrationMap }) + DataStreamStat.fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap }) ); } + const integrationsMap: Record = integrations.reduce( + (integrationMapAcc, integration) => { + return { + ...integrationMapAcc, + [integration.name]: integration, + }; + }, + {} + ); + const degradedMap: Record< DegradedDocsStat['dataset'], { @@ -53,10 +64,16 @@ export function generateDatasets( {} ); - return dataStreamStats?.map((dataStream) => ({ - ...dataStream, - title: integrationMap[dataStream.name]?.title || dataStream.title, - integration: integrationMap[dataStream.name]?.integration, - degradedDocs: degradedMap[dataStream.rawName] || dataStream.degradedDocs, - })); + return dataStreamStats?.map((dataStream) => { + const dataset = DataStreamStat.create(dataStream); + + return { + ...dataset, + title: datasetIntegrationMap[dataset.name]?.title || dataset.title, + integration: + datasetIntegrationMap[dataset.name]?.integration ?? + integrationsMap[dataStream.integration ?? ''], + degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, + }; + }); } From 7c143478d395fe2d14596314e2901b1f1f46d900 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Fri, 26 Apr 2024 14:42:23 +0200 Subject: [PATCH 2/3] Fixing duplicated integrations entries --- .../public/hooks/use_dataset_quality_filters.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx index f6dc0920a8813..56739c15c2616 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx @@ -76,10 +76,18 @@ export const useDatasetQualityFilters = () => { ); const integrationItems: IntegrationItem[] = useMemo(() => { + const integrationsMap = datasets.reduce( + (acc, dataset) => ({ + ...acc, + ...(dataset.integration && !acc[dataset.integration.name] + ? { [dataset.integration.name]: dataset.integration } + : {}), + }), + {} as { [key: string]: Integration } + ); + const integrations = [ - ...datasets - .map((dataset) => dataset.integration) - .filter((integration): integration is Integration => !!integration), + ...Object.values(integrationsMap), ...(datasets.some((dataset) => !dataset.integration) ? [Integration.create({ name: 'none', title: 'None' })] : []), From 1e35bc4832384d8daa43f4b04fd08593a412916f Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 29 Apr 2024 14:41:53 +0200 Subject: [PATCH 3/3] fixing PR comments --- .../hooks/use_dataset_quality_filters.tsx | 75 +++++++------------ .../public/utils/generate_datasets.test.ts | 12 +-- .../public/utils/generate_datasets.ts | 49 ++++++------ 3 files changed, 60 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx index 27a228ce1afcf..a4d7be4fca46e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx @@ -36,26 +36,22 @@ export const useDatasetQualityFilters = () => { interface Filters { namespaces: string[]; qualities: QualityIndicators[]; - integrations: Integration[]; - hasNoneIntegration: boolean; + filteredIntegrations: string[]; } const datasets = useSelector(service, (state) => state.context.datasets); - const { namespaces, qualities, integrations } = useMemo( + const integrations = useSelector(service, (state) => state.context.integrations); + const { namespaces, qualities, filteredIntegrations } = useMemo( () => datasets.reduce( - (acc: Filters, dataset) => { - acc.namespaces.push(dataset.namespace); - acc.qualities.push(dataset.degradedDocs.quality); - if (dataset.integration) { - acc.integrations.push(dataset.integration); - } else if (!acc.hasNoneIntegration) { - acc.integrations.push(Integration.create({ name: 'none', title: 'None' })); - acc.hasNoneIntegration = true; - } - return acc; - }, - { namespaces: [], qualities: [], integrations: [], hasNoneIntegration: false } + (acc: Filters, dataset) => ({ + namespaces: [...new Set([...acc.namespaces, dataset.namespace])], + qualities: [...new Set([...acc.qualities, dataset.degradedDocs.quality])], + filteredIntegrations: [ + ...new Set([...acc.filteredIntegrations, dataset.integration?.name ?? 'none']), + ], + }), + { namespaces: [], qualities: [], filteredIntegrations: [] } ), [datasets] ); @@ -100,39 +96,24 @@ export const useDatasetQualityFilters = () => { [service, timeRange] ); -/* const integrationItems: IntegrationItem[] = useMemo(() => { - const integrationsMap = datasets.reduce( - (acc, dataset) => ({ - ...acc, - ...(dataset.integration && !acc[dataset.integration.name] - ? { [dataset.integration.name]: dataset.integration } - : {}), - }), - {} as { [key: string]: Integration } - ); - - const integrations = [ - ...Object.values(integrationsMap), - ...(datasets.some((dataset) => !dataset.integration) - ? [Integration.create({ name: 'none', title: 'None' })] - : []), - ]; - - return integrations.map((integration) => ({ - ...integration, - label: integration.title, - checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined, + const integrationItems: IntegrationItem[] = useMemo(() => { + const integrationsMap = + integrations?.reduce( + (acc, integration) => ({ + ...acc, + [integration.name]: integration, + }), + {} as { [key: string]: Integration } + ) ?? {}; + + integrationsMap.none = Integration.create({ name: 'none', title: 'None' }); + + return filteredIntegrations.map((name) => ({ + ...integrationsMap[name], + label: integrationsMap[name]?.title, + checked: selectedIntegrations.includes(name) ? 'on' : undefined, })); - }, [datasets, selectedIntegrations]); */ - const integrationItems: IntegrationItem[] = useMemo( - () => - integrations.map((integration) => ({ - ...integration, - label: integration.title, - checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined, - })), - [integrations, selectedIntegrations] - ); + }, [integrations, filteredIntegrations, selectedIntegrations]); const onIntegrationsChange = useCallback( (newIntegrationItems: IntegrationItem[]) => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index cb9d3aaf7847b..9b9a007f0c54b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -81,7 +81,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { @@ -94,6 +94,7 @@ describe('generateDatasets', () => { degradedDocs: { count: 0, percentage: 0, + quality: 'good', }, }, ]); @@ -117,7 +118,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { @@ -133,7 +134,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[1].percentage, count: degradedDocs[1].count, - quality: 'poor', + quality: degradedDocs[1].quality, }, }, ]); @@ -155,7 +156,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[0].percentage, count: degradedDocs[0].count, - quality: 'good', + quality: degradedDocs[0].quality, }, }, { @@ -168,7 +169,7 @@ describe('generateDatasets', () => { degradedDocs: { percentage: degradedDocs[1].percentage, count: degradedDocs[1].count, - quality: 'poor', + quality: degradedDocs[1].quality, }, }, ]); @@ -199,6 +200,7 @@ describe('generateDatasets', () => { degradedDocs: { count: 0, percentage: 0, + quality: 'good', }, }, ]); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 9480a34b40fa3..e2f85d8846c99 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -20,22 +20,33 @@ export function generateDatasets( return []; } - const datasetIntegrationMap: Record = - integrations.reduce((integrationMapAcc, integration) => { + const { + datasetIntegrationMap, + integrationsMap, + }: { + datasetIntegrationMap: Record; + integrationsMap: Record; + } = integrations.reduce( + (acc, integration) => { return { - ...integrationMapAcc, - ...Object.keys(integration.datasets).reduce( - (datasetsAcc, dataset) => - Object.assign(datasetsAcc, { - [dataset]: { - integration, - title: integration.datasets[dataset], - }, - }), - {} - ), + datasetIntegrationMap: { + ...acc.datasetIntegrationMap, + ...Object.keys(integration.datasets).reduce( + (datasetsAcc, dataset) => + Object.assign(datasetsAcc, { + [dataset]: { + integration, + title: integration.datasets[dataset], + }, + }), + {} + ), + }, + integrationsMap: { ...acc.integrationsMap, [integration.name]: integration }, }; - }, {}); + }, + { datasetIntegrationMap: {}, integrationsMap: {} } + ); if (!dataStreamStats.length) { return degradedDocStats.map((degradedDocStat) => @@ -43,16 +54,6 @@ export function generateDatasets( ); } - const integrationsMap: Record = integrations.reduce( - (integrationMapAcc, integration) => { - return { - ...integrationMapAcc, - [integration.name]: integration, - }; - }, - {} - ); - const degradedMap: Record< DegradedDocsStat['dataset'], {