Skip to content

Commit

Permalink
[Dataset quality] Integration non-default dataset is now marked as pa…
Browse files Browse the repository at this point in the history 
…rt of the integration (elastic#181836)

Closes elastic#181542

## 📝  Summary

This PR uses`IndicesDataStream` metadata info to determine whether a
non-default dataset belongs to an integration or not

## 🎥 Demo

### Before changes


https://github.com/elastic/kibana/assets/1313018/23fbbb9d-ae52-4e30-b525-8176f89d6462



### After changes


https://github.com/elastic/kibana/assets/1313018/5423dba2-fbcf-47b4-87b7-2987f5a1f6d2

---------

Co-authored-by: mohamedhamed-ahmed <[email protected]>
  • Loading branch information
@yngrdyn @mohamedhamed-ahmed
yngrdyn and mohamedhamed-ahmed authored Apr 29, 2024
1 parent 0eb7453 commit 4ad5d46
Show file tree
Hide file tree
Showing 8 changed files with 147 additions and 86 deletions.
8 changes: 4 additions & 4 deletions ...gins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,20 +65,20 @@ export class DataStreamStat {

public static fromDegradedDocStat({
degradedDocStat,
integrationMap,
datasetIntegrationMap,
}: {
degradedDocStat: DegradedDocsStat;
integrationMap: Record<string, { integration: Integration; title: string }>;
datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
}) {
const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);

const dataStreamStatProps = {
rawName: degradedDocStat.dataset,
type,
name: dataset,
title: integrationMap[dataset]?.title || dataset,
title: datasetIntegrationMap[dataset]?.title || dataset,
namespace,
integration: integrationMap[dataset]?.integration,
integration: datasetIntegrationMap[dataset]?.integration,
degradedDocs: {
percentage: degradedDocStat.percentage,
count: degradedDocStat.count,
Expand Down
3 changes: 1 addition & 2 deletions x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@
*/

import { APIClientRequestParamsOf, APIReturnType } from '../rest';
import { DataStreamStat } from './data_stream_stat';

export type GetDataStreamsStatsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params'];
export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query'];
export type GetDataStreamsStatsResponse =
APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>;
export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0];
export type DataStreamStatServiceResponse = DataStreamStat[];
export type DataStreamStatServiceResponse = DataStreamStatType[];

export type GetIntegrationsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params'];
Expand Down
53 changes: 29 additions & 24 deletions ...ugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,26 +36,22 @@ export const useDatasetQualityFilters = () => {
interface Filters {
namespaces: string[];
qualities: QualityIndicators[];
integrations: Integration[];
hasNoneIntegration: boolean;
filteredIntegrations: string[];
}

const datasets = useSelector(service, (state) => state.context.datasets);
const { namespaces, qualities, integrations } = useMemo(
const integrations = useSelector(service, (state) => state.context.integrations);
const { namespaces, qualities, filteredIntegrations } = useMemo(
() =>
datasets.reduce(
(acc: Filters, dataset) => {
acc.namespaces.push(dataset.namespace);
acc.qualities.push(dataset.degradedDocs.quality);
if (dataset.integration) {
acc.integrations.push(dataset.integration);
} else if (!acc.hasNoneIntegration) {
acc.integrations.push(Integration.create({ name: 'none', title: 'None' }));
acc.hasNoneIntegration = true;
}
return acc;
},
{ namespaces: [], qualities: [], integrations: [], hasNoneIntegration: false }
(acc: Filters, dataset) => ({
namespaces: [...new Set([...acc.namespaces, dataset.namespace])],
qualities: [...new Set([...acc.qualities, dataset.degradedDocs.quality])],
filteredIntegrations: [
...new Set([...acc.filteredIntegrations, dataset.integration?.name ?? 'none']),
],
}),
{ namespaces: [], qualities: [], filteredIntegrations: [] }
),
[datasets]
);
Expand Down Expand Up @@ -100,15 +96,24 @@ export const useDatasetQualityFilters = () => {
[service, timeRange]
);

const integrationItems: IntegrationItem[] = useMemo(
() =>
integrations.map((integration) => ({
...integration,
label: integration.title,
checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined,
})),
[integrations, selectedIntegrations]
);
const integrationItems: IntegrationItem[] = useMemo(() => {
const integrationsMap =
integrations?.reduce(
(acc, integration) => ({
...acc,
[integration.name]: integration,
}),
{} as { [key: string]: Integration }
) ?? {};

integrationsMap.none = Integration.create({ name: 'none', title: 'None' });

return filteredIntegrations.map((name) => ({
...integrationsMap[name],
label: integrationsMap[name]?.title,
checked: selectedIntegrations.includes(name) ? 'on' : undefined,
}));
}, [integrations, filteredIntegrations, selectedIntegrations]);

const onIntegrationsChange = useCallback(
(newIntegrationItems: IntegrationItem[]) => {
Expand Down
3 changes: 1 addition & 2 deletions ..._solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import {
GetIntegrationsParams,
IntegrationsResponse,
} from '../../../common/data_streams_stats';
import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat';
import { IDataStreamsStatsClient } from './types';

export class DataStreamsStatsClient implements IDataStreamsStatsClient {
Expand All @@ -50,7 +49,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
new GetDataStreamsStatsError(`Failed to decode data streams stats response: ${message}`)
)(response);

return dataStreamsStats.map(DataStreamStat.create);
return dataStreamsStats;
}

public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) {
Expand Down
2 changes: 1 addition & 1 deletion ...ion/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@
import { IToasts } from '@kbn/core/public';
import { getDateISORange } from '@kbn/timerange';
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
import { DataStreamStat } from '../../../../common/api_types';
import { Integration } from '../../../../common/data_streams_stats/integration';
import { IDataStreamDetailsClient } from '../../../services/data_stream_details';
import {
DashboardType,
DataStreamSettings,
DataStreamDetails,
DataStreamStat,
GetDataStreamsStatsQuery,
GetIntegrationsParams,
} from '../../../../common/data_streams_stats';
Expand Down
3 changes: 2 additions & 1 deletion ...ty_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
DataStreamStatServiceResponse,
IntegrationsResponse,
DataStreamStat,
DataStreamStatType,
} from '../../../../common/data_streams_stats';

export type FlyoutDataset = Omit<
Expand Down Expand Up @@ -68,7 +69,7 @@ export interface WithFilters {
}

export interface WithDataStreamStats {
dataStreamStats: DataStreamStat[];
dataStreamStats: DataStreamStatType[];
}

export interface WithDegradedDocs {
Expand Down
97 changes: 68 additions & 29 deletions x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
*/

import { indexNameToDataStreamParts } from '../../common/utils';
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
import { Integration } from '../../common/data_streams_stats/integration';
import { generateDatasets } from './generate_datasets';
import { DataStreamStatType } from '../../common/data_streams_stats/types';

describe('generateDatasets', () => {
const integrations: Integration[] = [
Expand All @@ -34,36 +34,19 @@ describe('generateDatasets', () => {
},
];

const dataStreamStats: DataStreamStat[] = [
const dataStreamStats: DataStreamStatType[] = [
{
name: 'system.application',
title: 'system.application',
type: 'logs',
namespace: 'default',
name: 'logs-system.application-default',
lastActivity: 1712911241117,
size: '82.1kb',
sizeBytes: 84160,
rawName: 'logs-system.application-default',
degradedDocs: {
percentage: 0,
count: 0,
quality: 'good',
},
integration: 'system',
},
{
name: 'synth',
title: 'synth',
type: 'logs',
namespace: 'default',
name: 'logs-synth-default',
lastActivity: 1712911241117,
rawName: 'logs-synth-default',
size: '62.5kb',
sizeBytes: 64066,
degradedDocs: {
percentage: 0,
count: 0,
quality: 'good',
},
},
];

Expand All @@ -88,16 +71,31 @@ describe('generateDatasets', () => {
expect(datasets).toEqual([
{
...dataStreamStats[0],
title: integrations[0].datasets[dataStreamStats[0].name],
name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset,
namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace,
title:
integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset],
type: indexNameToDataStreamParts(dataStreamStats[0].name).type,
rawName: dataStreamStats[0].name,
integration: integrations[0],
degradedDocs: {
percentage: degradedDocs[0].percentage,
count: degradedDocs[0].count,
quality: 'good',
quality: degradedDocs[0].quality,
},
},
{
...dataStreamStats[1],
name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace,
title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
type: indexNameToDataStreamParts(dataStreamStats[1].name).type,
rawName: dataStreamStats[1].name,
degradedDocs: {
count: 0,
percentage: 0,
quality: 'good',
},
},
]);
});
Expand All @@ -120,7 +118,7 @@ describe('generateDatasets', () => {
degradedDocs: {
percentage: degradedDocs[0].percentage,
count: degradedDocs[0].count,
quality: 'good',
quality: degradedDocs[0].quality,
},
},
{
Expand All @@ -136,7 +134,7 @@ describe('generateDatasets', () => {
degradedDocs: {
percentage: degradedDocs[1].percentage,
count: degradedDocs[1].count,
quality: 'poor',
quality: degradedDocs[1].quality,
},
},
]);
Expand All @@ -148,20 +146,61 @@ describe('generateDatasets', () => {
expect(datasets).toEqual([
{
...dataStreamStats[0],
title: integrations[0].datasets[dataStreamStats[0].name],
name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset,
namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace,
title:
integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset],
type: indexNameToDataStreamParts(dataStreamStats[0].name).type,
rawName: dataStreamStats[0].name,
integration: integrations[0],
degradedDocs: {
percentage: degradedDocs[0].percentage,
count: degradedDocs[0].count,
quality: 'good',
quality: degradedDocs[0].quality,
},
},
{
...dataStreamStats[1],
name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace,
title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset,
type: indexNameToDataStreamParts(dataStreamStats[1].name).type,
rawName: dataStreamStats[1].name,
degradedDocs: {
percentage: degradedDocs[1].percentage,
count: degradedDocs[1].count,
quality: 'poor',
quality: degradedDocs[1].quality,
},
},
]);
});

it('merges integration information with dataStreamStats when dataset is not an integration default one', () => {
const dataset = 'logs-system.custom-default';

const nonDefaultDataset = {
name: dataset,
lastActivity: 1712911241117,
size: '82.1kb',
sizeBytes: 84160,
integration: 'system',
};

const datasets = generateDatasets([nonDefaultDataset], undefined, integrations);

expect(datasets).toEqual([
{
...nonDefaultDataset,
title: indexNameToDataStreamParts(dataset).dataset,
name: indexNameToDataStreamParts(dataset).dataset,
namespace: indexNameToDataStreamParts(dataset).namespace,
type: indexNameToDataStreamParts(dataset).type,
rawName: nonDefaultDataset.name,
integration: integrations[0],
degradedDocs: {
count: 0,
percentage: 0,
quality: 'good',
},
},
]);
Expand Down
Loading

0 comments on commit 4ad5d46

Please sign in to comment.