From 7ff00882a0a58f1bdac2c899ca40a7e7f5715ca4 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Wed, 17 Apr 2024 09:52:41 +0200 Subject: [PATCH] [Dataset quality] Split integration request from dataStreamStats (#180560) Relates to https://github.com/elastic/kibana/issues/179638. ## :memo: Summary This PR is all about decoupling `integrations` from `DataStreamStats` request. This change is needed in order to render dataset quality table from only `DegradedDocsStats` or `DataStreamStats`, this will allow us to show the users the information as soon as it arrives, also will help us to introduce soonish states according to user privileges. ### Changes - New internal endpoint `GET /internal/dataset_quality/integrations` that will return all the installed integrations that are of a specific type, e.g. `logs`. - Generating datasets when integrations request has finished, so we render the integration information correctly and show the information available: dataStreamStats and/or degradedDocs. ### App statechart image ### Demos #### dataStreamStats taking longer to resolve https://github.com/elastic/kibana/assets/1313018/c1127ec2-2cfe-4796-a331-47a3ef718e98 #### degradedDocs taking longer to resolve https://github.com/elastic/kibana/assets/1313018/b6f9954f-8e2b-445f-89a5-b6d213abe4b1 #### dataStreamStats and degradedDocs loading https://github.com/elastic/kibana/assets/1313018/e7987657-41cd-4cfc-b24e-6ad47aed0df1 #### Integration request failed but we still show information related to datasets https://github.com/elastic/kibana/assets/1313018/965558f3-4660-47e9-a7a1-068491e08a8a --- .../dataset_quality/common/api_types.ts | 17 +- .../data_streams_stats/data_stream_stat.ts | 31 +++- .../common/data_streams_stats/integration.ts | 3 + .../common/data_streams_stats/types.ts | 17 +- .../dataset_quality/table/columns.tsx | 39 +++-- .../hooks/use_dataset_quality_filters.tsx | 36 ++-- .../hooks/use_dataset_quality_table.tsx | 13 +- .../data_streams_stats_client.ts | 44 +++-- .../services/data_streams_stats/types.ts | 3 + .../src/defaults.ts | 1 - .../src/notifications.ts | 9 + .../src/state_machine.ts | 130 +++++++++----- .../dataset_quality_controller/src/types.ts | 8 +- .../summary_panel/src/state_machine.ts | 2 +- .../public/utils/generate_datasets.test.ts | 160 ++++++++++++++++++ .../public/utils/generate_datasets.ts | 62 +++++++ .../dataset_quality/public/utils/index.ts | 2 +- .../merge_degraded_docs_into_datastreams.ts | 31 ---- .../server/routes/data_streams/routes.ts | 41 +---- .../dataset_quality/server/routes/index.ts | 2 + .../get_integration_dashboards.ts} | 56 +----- .../routes/integrations/get_integrations.ts | 96 +++++++++++ .../server/routes/integrations/routes.ts | 70 ++++++++ .../dataset_quality/src/url_schema_v1.ts | 29 +++- .../integration_dashboards.spec.ts | 23 +-- .../tests/integrations/integrations.spec.ts | 131 ++++++++++++++ .../tests/integrations/package_utils.ts | 65 +++++++ .../dataset_quality/dataset_quality_table.ts | 2 + .../page_objects/dataset_quality.ts | 7 +- .../dataset_quality/dataset_quality_table.ts | 2 + 30 files changed, 877 insertions(+), 255 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts delete mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/utils/merge_degraded_docs_into_datastreams.ts rename x-pack/plugins/observability_solution/dataset_quality/server/routes/{data_streams/get_integrations.ts => integrations/get_integration_dashboards.ts} (57%) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts rename x-pack/test/dataset_quality_api_integration/tests/{data_streams => integrations}/integration_dashboards.spec.ts (86%) create mode 100644 x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts create mode 100644 x-pack/test/dataset_quality_api_integration/tests/integrations/package_utils.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index c12269c6e7060..ae098c08c8ec8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -62,6 +62,12 @@ export const integrationRt = rt.intersection([ export type Integration = rt.TypeOf; +export const getIntegrationsResponseRt = rt.exact( + rt.type({ + integrations: rt.array(integrationRt), + }) +); + export const degradedDocsRt = rt.type({ dataset: rt.string, percentage: rt.number, @@ -78,14 +84,9 @@ export const dataStreamDetailsRt = rt.partial({ export type DataStreamDetails = rt.TypeOf; export const getDataStreamsStatsResponseRt = rt.exact( - rt.intersection([ - rt.type({ - dataStreamsStats: rt.array(dataStreamStatRt), - }), - rt.type({ - integrations: rt.array(integrationRt), - }), - ]) + rt.type({ + dataStreamsStats: rt.array(dataStreamStatRt), + }) ); export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact( diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 6e88662ac8cf2..7b1a171356a61 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -9,6 +9,7 @@ import { DEFAULT_DEGRADED_DOCS } from '../constants'; import { DataStreamType } from '../types'; import { indexNameToDataStreamParts } from '../utils'; import { Integration } from './integration'; +import { DegradedDocsStat } from './malformed_docs_stat'; import { DataStreamStatType } from './types'; export class DataStreamStat { @@ -49,17 +50,39 @@ export class DataStreamStat { rawName: dataStreamStat.name, type, name: dataset, - title: dataStreamStat.integration?.datasets?.[dataset] ?? dataset, + title: dataset, namespace, size: dataStreamStat.size, sizeBytes: dataStreamStat.sizeBytes, lastActivity: dataStreamStat.lastActivity, - integration: dataStreamStat.integration - ? Integration.create(dataStreamStat.integration) - : undefined, degradedDocs: DEFAULT_DEGRADED_DOCS, }; return new DataStreamStat(dataStreamStatProps); } + + public static fromDegradedDocStat({ + degradedDocStat, + integrationMap, + }: { + degradedDocStat: DegradedDocsStat; + integrationMap: Record; + }) { + const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); + + const dataStreamStatProps = { + rawName: degradedDocStat.dataset, + type, + name: dataset, + title: integrationMap[dataset]?.title || dataset, + namespace, + integration: integrationMap[dataset]?.integration, + degradedDocs: { + percentage: degradedDocStat.percentage, + count: degradedDocStat.count, + }, + }; + + return new DataStreamStat(dataStreamStatProps); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/integration.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/integration.ts index 97d0a5f001d69..ad8fecd91c7e0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/integration.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/integration.ts @@ -11,6 +11,7 @@ export class Integration { name: IntegrationType['name']; title: string; version: string; + datasets: Record; icons?: IntegrationType['icons']; dashboards?: DashboardType[]; @@ -20,6 +21,7 @@ export class Integration { this.version = integration.version || '1.0.0'; this.icons = integration.icons; this.dashboards = integration.dashboards || []; + this.datasets = integration.datasets || {}; } public static create(integration: IntegrationType) { @@ -28,6 +30,7 @@ export class Integration { title: integration.title || integration.name, version: integration.version || '1.0.0', dashboards: integration.dashboards || [], + datasets: integration.datasets || {}, }; return new Integration(integrationProps); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index a2fc5f42fed4a..7fbbf4d8f3e90 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -7,21 +7,20 @@ import { APIClientRequestParamsOf, APIReturnType } from '../rest'; import { DataStreamStat } from './data_stream_stat'; -import { Integration } from './integration'; export type GetDataStreamsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params']; export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query']; export type GetDataStreamsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>; -export interface DataStreamStatServiceResponse { - dataStreamStats: DataStreamStat[]; - integrations: Integration[]; -} -export type IntegrationType = GetDataStreamsStatsResponse['integrations'][0]; -export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0] & { - integration?: IntegrationType; -}; +export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0]; +export type DataStreamStatServiceResponse = DataStreamStat[]; + +export type GetIntegrationsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params']; +export type GetIntegrationsResponse = APIReturnType<`GET /internal/dataset_quality/integrations`>; +export type IntegrationType = GetIntegrationsResponse['integrations'][0]; +export type IntegrationsResponse = IntegrationType[]; export type GetDataStreamsDegradedDocsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params']; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index dd85160f17b80..1fb229e1b7155 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -16,6 +16,7 @@ import { EuiToolTip, EuiButtonIcon, EuiText, + EuiSkeletonRectangle, } from '@elastic/eui'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; @@ -126,12 +127,14 @@ export const getDatasetQualityTableColumns = ({ fieldFormats, selectedDataset, openFlyout, + loadingDataStreamStats, loadingDegradedStats, showFullDatasetNames, isActiveDataset, }: { fieldFormats: FieldFormatsStart; selectedDataset?: FlyoutDataset; + loadingDataStreamStats: boolean; loadingDegradedStats: boolean; showFullDatasetNames: boolean; openFlyout: (selectedDataset: FlyoutDataset) => void; @@ -197,7 +200,16 @@ export const getDatasetQualityTableColumns = ({ name: sizeColumnName, field: 'sizeBytes', sortable: true, - render: (_, dataStreamStat: DataStreamStat) => formatBytes(dataStreamStat.sizeBytes || 0), + render: (_, dataStreamStat: DataStreamStat) => ( + + {formatBytes(dataStreamStat.sizeBytes || 0)} + + ), width: '100px', }, { @@ -222,22 +234,27 @@ export const getDatasetQualityTableColumns = ({ { name: lastActivityColumnName, field: 'lastActivity', - render: (timestamp: number) => { - if (!isActiveDataset(timestamp)) { - return ( + render: (timestamp: number) => ( + + {!isActiveDataset(timestamp) ? ( {inactiveDatasetActivityColumnDescription} - ); - } - - return fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) - .convert(timestamp); - }, + ) : ( + fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) + .convert(timestamp) + )} + + ), width: '300px', sortable: true, }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx index 899a837b01a45..f6dc0920a8813 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.tsx @@ -8,6 +8,7 @@ import { OnRefreshChangeProps } from '@elastic/eui'; import { useSelector } from '@xstate/react'; import { useCallback, useMemo } from 'react'; +import { Integration } from '../../common/data_streams_stats/integration'; import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { IntegrationItem } from '../components/dataset_quality/filters/integrations_selector'; import { NamespaceItem } from '../components/dataset_quality/filters/namespaces_selector'; @@ -15,14 +16,20 @@ import { NamespaceItem } from '../components/dataset_quality/filters/namespaces_ export const useDatasetQualityFilters = () => { const { service } = useDatasetQualityContext(); - const isLoading = useSelector(service, (state) => state.matches('datasets.fetching')); + const isLoading = useSelector( + service, + (state) => + state.matches('integrations.fetching') && + (state.matches('datasets.fetching') || state.matches('degradedDocs.fetching')) + ); + const { timeRange, integrations: selectedIntegrations, namespaces: selectedNamespaces, query: selectedQuery, } = useSelector(service, (state) => state.context.filters); - const integrations = useSelector(service, (state) => state.context.integrations); + const datasets = useSelector(service, (state) => state.context.datasets); const namespaces = useSelector(service, (state) => state.context.datasets).map( (dataset) => dataset.namespace @@ -68,15 +75,22 @@ export const useDatasetQualityFilters = () => { [service, timeRange] ); - const integrationItems: IntegrationItem[] = useMemo( - () => - integrations.map((integration) => ({ - ...integration, - label: integration.title, - checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined, - })), - [integrations, selectedIntegrations] - ); + const integrationItems: IntegrationItem[] = useMemo(() => { + const integrations = [ + ...datasets + .map((dataset) => dataset.integration) + .filter((integration): integration is Integration => !!integration), + ...(datasets.some((dataset) => !dataset.integration) + ? [Integration.create({ name: 'none', title: 'None' })] + : []), + ]; + + return integrations.map((integration) => ({ + ...integration, + label: integration.title, + checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined, + })); + }, [datasets, selectedIntegrations]); const onIntegrationsChange = useCallback( (newIntegrationItems: IntegrationItem[]) => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 2b2c48d43998d..116e15bd92c93 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -45,7 +45,16 @@ export const useDatasetQualityTable = () => { const flyout = useSelector(service, (state) => state.context.flyout); - const loading = useSelector(service, (state) => state.matches('datasets.fetching')); + const loading = useSelector( + service, + (state) => + state.matches('datasets.fetching') || + state.matches('integrations.fetching') || + state.matches('degradedDocs.fetching') + ); + const loadingDataStreamStats = useSelector(service, (state) => + state.matches('datasets.fetching') + ); const loadingDegradedStats = useSelector(service, (state) => state.matches('degradedDocs.fetching') ); @@ -104,6 +113,7 @@ export const useDatasetQualityTable = () => { fieldFormats, selectedDataset: flyout?.dataset, openFlyout, + loadingDataStreamStats, loadingDegradedStats, showFullDatasetNames, isActiveDataset: isActive, @@ -112,6 +122,7 @@ export const useDatasetQualityTable = () => { fieldFormats, flyout?.dataset, openFlyout, + loadingDataStreamStats, loadingDegradedStats, showFullDatasetNames, isActive, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 69e8b2364006e..2fc2b388f3a8d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -7,14 +7,14 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; -import { find, merge } from 'lodash'; import { Integration } from '../../../common/data_streams_stats/integration'; import { getDataStreamsDegradedDocsStatsResponseRt, getDataStreamsStatsResponseRt, getDataStreamsEstimatedDataInBytesResponseRt, + getIntegrationsResponseRt, } from '../../../common/api_types'; -import { DEFAULT_DATASET_TYPE, NONE } from '../../../common/constants'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, @@ -24,6 +24,8 @@ import { GetDataStreamsStatsResponse, GetDataStreamsEstimatedDataInBytesParams, GetDataStreamsEstimatedDataInBytesResponse, + GetIntegrationsParams, + IntegrationsResponse, } from '../../../common/data_streams_stats'; import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; import { IDataStreamsStatsClient } from './types'; @@ -42,27 +44,13 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { throw new GetDataStreamsStatsError(`Failed to fetch data streams stats: ${error}`); }); - const { dataStreamsStats, integrations } = decodeOrThrow( + const { dataStreamsStats } = decodeOrThrow( getDataStreamsStatsResponseRt, (message: string) => new GetDataStreamsStatsError(`Failed to decode data streams stats response: ${message}`) )(response); - const mergedDataStreamsStats = dataStreamsStats.map((statsItem) => { - const integration = find(integrations, { name: statsItem.integration }); - - return merge({}, statsItem, { integration }); - }); - - const uncategorizedDatasets = dataStreamsStats.some((dataStream) => !dataStream.integration); - - return { - dataStreamStats: mergedDataStreamsStats.map(DataStreamStat.create), - integrations: (uncategorizedDatasets - ? [...integrations, { name: NONE, title: 'None' }] - : integrations - ).map(Integration.create), - }; + return dataStreamsStats.map(DataStreamStat.create); } public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) { @@ -117,4 +105,24 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return dataStreamsEstimatedDataInBytes; } + + public async getIntegrations( + params: GetIntegrationsParams['query'] = { type: DEFAULT_DATASET_TYPE } + ): Promise { + const response = await this.http + .get('/internal/dataset_quality/integrations', { + query: params, + }) + .catch((error) => { + throw new GetDataStreamsStatsError(`Failed to fetch integrations: ${error}`); + }); + + const { integrations } = decodeOrThrow( + getIntegrationsResponseRt, + (message: string) => + new GetDataStreamsStatsError(`Failed to decode integrations response: ${message}`) + )(response); + + return integrations.map(Integration.create); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index ed454233d36eb..bfd36db4e8375 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -13,6 +13,8 @@ import { GetDataStreamsStatsQuery, GetDataStreamsEstimatedDataInBytesParams, GetDataStreamsEstimatedDataInBytesResponse, + GetIntegrationsParams, + IntegrationsResponse, } from '../../../common/data_streams_stats'; export type DataStreamsStatsServiceSetup = void; @@ -33,4 +35,5 @@ export interface IDataStreamsStatsClient { getDataStreamsEstimatedDataInBytes( params: GetDataStreamsEstimatedDataInBytesParams ): Promise; + getIntegrations(params: GetIntegrationsParams['query']): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 6a5ab9ccc8340..b2a839e4f1dfb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -40,5 +40,4 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { }, flyout: {}, datasets: [], - integrations: [], }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index ca249e3fa6022..4d399552ec5e8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -44,6 +44,15 @@ export const fetchIntegrationDashboardsFailedNotifier = (toasts: IToasts, error: }); }; +export const fetchIntegrationsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchIntegrationsFailed', { + defaultMessage: "We couldn't get your integrations.", + }), + text: error.message, + }); +}; + export const noDatasetSelected = i18n.translate( 'xpack.datasetQuality.fetchDatasetDetailsFailed.noDatasetSelected', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index b996bc3171237..39d630a7aa770 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,24 +8,27 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; +import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { DashboardType, DataStreamDetails, - DataStreamStatServiceResponse, + DataStreamStat, GetDataStreamsStatsQuery, + GetIntegrationsParams, } from '../../../../common/data_streams_stats'; import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; import { DataStreamType } from '../../../../common/types'; import { dataStreamPartsToIndexName } from '../../../../common/utils'; import { IDataStreamsStatsClient } from '../../../services/data_streams_stats'; -import { mergeDegradedStatsIntoDataStreams } from '../../../utils'; +import { generateDatasets } from '../../../utils'; import { DEFAULT_CONTEXT } from './defaults'; import { fetchDatasetDetailsFailedNotifier, fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, fetchIntegrationDashboardsFailedNotifier, + fetchIntegrationsFailedNotifier, noDatasetSelected, } from './notifications'; import { @@ -38,7 +41,7 @@ import { export const createPureDatasetQualityControllerStateMachine = ( initialContext: DatasetQualityControllerContext ) => - /** @xstate-layout N4IgpgJg5mDOIC5QBECGAXVszoIoFdUAbAS3QE8BhAewDt0AnaoosBgOggyx1gGIAqgAVkAQQAqAUQD64gJIBZGQCVRAOQDikgNoAGALqJQAB2qwyJOkZAAPRAE4A7ABoQ5RACYAjLoAs7AFYPAA4PR2CAgN1HewBmXwBfBNc0TGw8QlIKGnomFjZObnT+ZUkAMVKAZQAJaTFxUT1DJBBTc3RLWms7BCdXdwQPXQD-R10wxwA2X2jfey8klKKcAmIyKjpGZlYOLjTeQREJGTk1KQ1VeQB5NUqm6zaLKxaevrdEL0dfR3Zg+xHdMFJhFdMDgosQKkeBk1tlNnkdoV9uh+MJ6jI1KIlJUhKJKJI7gYHmYnl0Xg4XO8EF5gr5YuxProfLEvJNYvZJqyIVD0qsshtctsCntoaijlJpLgBJJlABNe4tR4dZ6gHqTDz0rwBP6g0LTLUefofbX0jxOL5gnwBSbc5Yw-k5Lb5XZ22DsABmOAAxgALEi0KB8CB0MDsf0AN2oAGtQzyVpl1o6EcLXR7vX6AwgI9QvRhOk0FSYScqyarEAF7PZ2LE-h5JpMgqzHI5DVSvPYgdXZlEZvFAbbkXzE-ChS7kW7Pehff7A2wmBxjEQMO7qAwALbsOP24eC51I0VpqcZqBZ2iR3MlgtExXFzrdcuV6u1+uNybN1sDbzDBn-AKORt0t4sQDtCQ5wruiIisU7BENQqAQJAhzorIogAEIADIyJQyhyFIOGNNeRbtHe5IIH8ATsICHjalM9bhLEkxGoMTJVqCvi+ECXzBBE3wgbyCbgU6kGprB8GIeIVwaBomHSKceLyAAajI9SiJUkjiISzREaS95kc2DK+NMsQeHWQI9kxNJOAy-66ICsQsvYgF8fGsICkJKbjjBcEIRAfASVJMllAI6HoXUEiqep0iYtihatLeKq2IgQLBOwYzWtqXjePW1EWcETLsPYtkmeMMzNokySQnaYFucmuxgFADBiRAyA5mKyHyEo0iqJoOiEXFxEJT0LZMRqMTVr4QweHMZq6JW5VLIOAk1aOnD1Y1PktV6JTlFUtQqbFSokWWCDDVSGpmoErJBGEEQxI4znboJtWrQ1TWbRO6YzkGIZhme0axlVS1JitCGvRtrWHtOmbZhe+YGAd8WlolCAjBR0zan4JlhBWI0zF41aVsZ7H2QxXwPdVwN7qD62QO9kPHnwc6ruwi7LquG5bhTI5U2tb0Q5OUMnjDeZ0FeWn9TppGo+w6N5RNJn-vYI2xLNDLUdxc3BO2bLk0D3OIu6RDkNQ+DoHwamYZQ4hRZIADqYUNGp4gIwNSNDR+iDsv4oQsvEYS6MZWu665lMG0bJtm5Q6FXGp0hlOhspXAIzt9Ydg2IKdn50jLfg0ir8yTfYwcOvrBSG8bpu-RYawAF5fcGtChrAmDoADi0h6XHDlxHVcdLXM4u5Lx2OZqRUB3+oRml4Fk0vjPF0uyLbxFMxc7u5Xfh5X-rV6QdcBvulSMGAqBrsgOCoCQRAfUe9c-dmMaboDHcQWXm-oL3JD9-vIqHwwx+n+fS+19BannPCLWgYtiSu10r4dsT57Bmi+IZWabILKzR+FqXwQQA6shKvdCqnM9Yvw3hXd+28+67xnAfI+J8z6YCAfTL6TMFxLnQCudcj924l2IR6N+H8v5QGoX-WhgCr6MOhn9WGot4ap0RjAuBNYEFOHYpMFBsQ0F+F+FrQEJodGxHwQtUCRD168NIfwyh+9-St1eiWNAsAfQACM4IMAgMAhmDdQz3zbkY5+Jju5b1oDvEge9BFWN5rYrAjjnGuPEULSR4DIE3mgaRdiFEhj52CF8WaZkAgWXZPjEqLINT-G4oZVeT0Vr+LIYEihwSqFhJsZ0OxUTUAuLcUwhg84WasPYRzJ+3C-F8PIZ-CxoT6DhKaZEpxrSYkC2PKAnMCSZHizTm7RAqT2DpOyVkjsDZcltjNClGymSYg+EcF4eI5Tlp7iqV5JqfAo4xxkPHROydB4ll0ggpiS9KJOAQRqTkcwyYEP6WvZ6tzRI+SQscWStw5AaGqBpWQigVDqC0O8o6yNMqaJutaEEnIvCZTQaEVK3xbLtjpNEYYVzQ6vzMV6WC2BfJXCEJINQccE5JxTisuRpEGJVkUSrJkxkWRErOgEfGwITQ1n0eMWywEIS0GoAheALRCG+NqlAoeyMAC0WCmLAgKiaaaIwaQhHBCCrhYKQaui1R8lJSs2z1l0IEFscxPgvm1EXS1PiBnPSgrwWJdrMU9Hshg7i4RgQNn+BxfZn5AQpX+CrOsXx5grx9fxDVNrPKQsgMG9OvQGwFXGGm9GPhCUjUJQK9kYQOIzBKjSzuL0abNVavmtZgxfAjRMnPEtjhk0ciBI2nh1M+ZbSDUk7VPQzQUT+A5GsWo8qckpJ+Yy+NjKIIXpMeYNoM0uT9SDXm4Nx25ogO23SWt8YxC1uaoYhlYhxs8GG1KzqOw1g1L2YdgzSHnr5RZa0myb31k+HWJRX7wVDJqSMupAZf3HQlUxYIKtNkPtA6o2adJvWGMzQem5kGgkhKEf-OhF8r5weRpWJiabAgZWBAXeI+jwOVPw7UwjP8aEAPoWIuZM5yM9C8N8NBnxKLhAuYVTBWsDGVStRUvDZjhkCKIyIrjboPF8cQNul15zATNkiFg2kj7qQBw8DLQqgIdH9u8B4Jjcme4KdGb9axjUIn2OmW09TvRHUDEyoS9gE0YgjDylWyINmw7yag4phpznJmueie02Dk77XHQEyuj4LIXWORiOlpNsDd3Yf3da2zASCP1PGY0ugzS3MxLU4lkNGcANzCBN8QL-5OR5LrDRjsLY4jnMvaFulPdT0ecJRxAqYn1RVvmFqGeSHKJZQ5BNbWCr8uPWuWFnuDKzB5tqwWusJnMqk3ZDxBBqXBj1j8xyKYAQWS6Z8LupIQA */ + /** @xstate-layout N4IgpgJg5mDOIC5QBECGAXVszoIoFdUAbAS3QE8BhAewDt0AnaoosBgOggyx1gGIAqgAVkAQQAqAUQD64gJIBZGQCVRAOQDikgNoAGALqJQAB2qwyJOkZAAPRAFZ77AEwAWAGwAOXfc+f39u7uAIwBADQg5IieAMwAnOwA7PbOujHucTHpzs72AL55EWiY2HiEpBQ09EwsbJzcpfzKkgBizQDKABLSYuKieoZIIKbm6Ja01nYIzu4xLv4ZMcGJwZ7OiXERUQieronsurtxibquufYrMQVFDTgExGRUdIzMrBxcJbzsAGY4AMYACxItCgfAgdDA7GBADdqABrSHFHhlB6VZ41N71T7oWA-f5AkEIGHUP4YcYDAbWEYWKxDKbpJwxNJZHIxHK6DKJLaIYI+fauTzBeyuOLOOKuGKxK6FEBI0r3CpPaqvOofZG437oQHA0FsJgcYxEDDfagMAC27Dld3KjyqL1q71uOLxWoJUCJtFhpLGdApBipZhpEzpiD2nhccVWx2CwVch1W9m5Owy7HciT89klIr2rnyMqtKMVdoxqrAUAYqAgkGQJP4wl6MnkSmkqk0On9Q2pPuDoCm3nc7F26aynLOIqTsQSyVS6Uy2Tc11lToVtvRKveZYrVYgNb+TVaHW6vX6HZMge7kx5umWB3WYaynkSMWFE-czlTrj2s2C6yf9mCi4FiuaLKg6nCbpW1a1i62ogmCEJQp68KIsuNogfamJVuWkE7tBmqwe6xLeuSBiUp257jJeCAxiE7AeOmJzPokzF7EmvIXHRgrCqK4qSlkgGoaiSoYaW2HbruGr4jqfB6qa7CGsapoWkBaHCSWG5iVBe4wW6HpemSvqkaewwUbSvY8ss+zXnEsShB49iRlykSIHEhzzMxKQ-s4ni+HmNzYsBanroh6AQd2dYiBIjaKCo6haGRZ6jJRIYIMKqarMsbIhB5QqJs5CCfq4g4xs+nkhDEEruAJAWqcWwXAqF2HhXwzRtJIXQ9BIJ6DIlQZUTkRWZisiQij5qzuBySbOEK7AXCNwTHLxqQpNVyKBXVYENWF4wRQ20hyGoUgaKo8gAPJqO0CUmUlZm2A4HGxJ+Sz+CsDl5dsoRON4mRxvYPgcuKq3yrVa6bfQ210LtUXSGoohKO0QiiJQ7VXV2yXmalngJEyMTDXEKQCksU1pi4cZY2moRzq4QPWkJG2YltTU7YIkVSNIuACJIygAJqo6ZPZ3dRqxFRkszJFluPLGxGzsDZv7pCscbrH5S41XToMM+DTOQzp0ngrQkLEgilqCUWGt1IzFbhbrhJEQZtB+j1119Slua6IOzE+VmPhvm4Sa44yey6AxyzTb4NOFquoGa41Vs7TbuoMPq8lGugJrmibatm9HFta3HOv4bpdvdo7AY3QL9K8rNcTuAK3ixgKyRJoE7t7AHkZxKKDlVfmptRyJHCW-buJENQOEs3tfQAEIADIyJQyhyFIi-dWXLsY2m4a8Y+-7LJvSYTYNWQ1xkFyPjMEfrebg958P7Cj+P4inRoGhz-tsOUPIABqMjHu0kjiEusZNGt0+wzHYBlUUWNj4XFcEmCU+w2Q-lnP4T8mRL4gxzjfWOd8H7bj4E-F+b8WgCBnjPTqfR-7iBhnDFGwD+ZUUFPsea14BS6EyFGf26ROKhw5FjDwAoVYqXVlgn4RByDUHwOgPg-856fxhpIAA6hQ0QVC+blyoimdhvgTiZmcFkBBbEKpFSfN5ZIwQ2S42pr3LO-d1JiIkVIvglAZ6nX-tIFoM9uanQEOIdR69Bb-icJ+RwARYysifEY0IsssZ+A2KcCqmYMEiIHg4yR6BEIWAeAALz1ghWAmBQqZzWpg1J3xxHpMyWMHJOp-EXhSryGacZEj6O8p3XGxw2L4zmMkGywpprDXWMk7OZSKlSKqSQGpIIsSoHaIwMAqAzTIBwKgEgRBJKujyQbRCsJjbCJGfY8pjiMnAiyaQXJ0y1RzIYAspZKy1kbIInpEk9tS7kQ0a7fRqYVhYwWv+XMViuluHYLjYx6xBTsk8MMuxwUjmVNOdU85OoZnXNucszADyE4ySTnJBSaclLFOBikw5YyTm0DOSQC5UAUXzMWei1Z6yE7POIoZfQdT0aCzOMEJI2jryxiCaKJyH10xTl4r4fwPhhSJGhehElxyJlTOpUPbsaBYAAgAEZjwYBAR5bp4LbKNihWxsrYWkoVUi6Zyrxiqo1VqnVTLi4kTZfQj5GM0EuCWN5NY1k1h+3yjkRwEDQ4BAmiOR8MqgpgTheMhFkyLVKtviqrAtrUDat1dJWSBpU7p2Un3E1UazWxsVSFCGtAbWatTfawuOpmWvKMk7EBFdQwig9T+PwqQFq+rgf60I74rEXF0OsTIpwI30zqNGjJeDIDONce4zx3jfHstAYgEaCQT6jWWjmZwbE9hODFKHY4iwghQpsSU4lpr5VTogBPaGB12hyA0J0QBsgYotjiu2BtDCGmdoOEybR4TDjMTYrA1MQRUjBxrrkLwo7r5pPGX8Ue2Br2nSEJINQHivE+L8S6gJUxwUQIqsxNMMxEi13SDumuBx-D40fKcE4J6ZS0GoFWeAQx9kwodGvepGMAC0rhuWSlWCRyUA1djvUQDxre00cwZE-LoYO-gYOiLVI0LjHKpj7DcF4HwfgQ0hHCPlacIKFhiiyJKR81j-JnoOcFFTXxq0gjU8uoW4Z5O7DSIKSUswBRJnFAOLIvhPwzElhkJTqS7POivU5ptqVwFCksSJiU-Hgi+Z8rNGYf0mR-XWOkML9isJbi0qx3q3HBbJHDEOKUo43CbEMykYzYG5zmZGnl2zEFxJ4Sko595uHoiRlmvxxwrkuInBS-lY4744n+FCN4Ea6ZWtgQKzhCS98x7bmi1RCxaW1h6Nco4CqaQ2LXjmMRxw15DhsmFAtmOpbivO1K1MAcIQmFZUskE8TCB0x0QlJkMUPgDFLGu7nHB1sHNQA2w08USRYi128Bmc+7gppQ8Ysgi434fJCLzZGm72taAjzW5ACHG8zgQN8KESUwdgtCocDkA4AQRTsLo14OIQOOATqJ2Vun9FZwnDTHNtiOQEhvk-N4JkHheTOFZ3BslFKqUc6mIGjk3POQcg8mxEITgOQVUFIzrMlnVbWY45iCd5rKXIqubSu5GL1ny6vHMJYsxIyEfxtNRH+VliDrol63TrkYz4ylybot8aaU3LpfcxlYPbfUX-CC8nTu9gu9CGxXYRVTiOVmFxbykvT1EpswW+VQezeXO4KisP1vcT6zAFH0+ECctCkjOcIIyeHLfN5LsGM00KrZ6s7no347C3ksRUXhNIPrXJorWmqPvI5hK7TDz1XTd-XB01+w1YLT4isIDwP2XyKrV0HLXa9N3WSvqZ5DH2fT4Vd84+wMkxPFO4zjTD3HvtM8-G+30PqlJbccH8rRXiEUepG7sYGmYFiwoSCMQU0C0rcsYrkiwZwawW+l6BOEAU+wc3KHCjEaOYoQG7ugQCQ-gHawW-GsQ3eBuve+a7+8qCGZghOPWD290A44oLSg66+2YkB7uEo742iTIywvgg6K0BQeQQAA */ createMachine< DatasetQualityControllerContext, DatasetQualityControllerEvent, @@ -66,22 +69,7 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, - loaded: { - on: { - UPDATE_TABLE_CRITERIA: { - target: 'loaded', - actions: ['storeTableOptions'], - }, - TOGGLE_INACTIVE_DATASETS: { - target: 'loaded', - actions: ['storeInactiveDatasetsVisibility', 'resetPage'], - }, - TOGGLE_FULL_DATASET_NAMES: { - target: 'loaded', - actions: ['storeFullDatasetNamesVisibility'], - }, - }, - }, + loaded: {}, }, on: { UPDATE_TIME_RANGE: { @@ -91,17 +79,6 @@ export const createPureDatasetQualityControllerStateMachine = ( REFRESH_DATA: { target: 'datasets.fetching', }, - UPDATE_INTEGRATIONS: { - target: 'datasets.loaded', - actions: ['storeIntegrations'], - }, - UPDATE_NAMESPACES: { - target: 'datasets.loaded', - actions: ['storeNamespaces'], - }, - UPDATE_QUERY: { - actions: ['storeQuery'], - }, }, }, degradedDocs: { @@ -132,6 +109,64 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, + integrations: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadIntegrations', + onDone: { + target: 'loaded', + actions: ['storeIntegrations', 'storeDatasets'], + }, + onError: { + target: 'loaded', + actions: [ + 'notifyFetchIntegrationsFailed', + 'storeEmptyIntegrations', + 'storeDatasets', + ], + }, + }, + }, + loaded: { + on: { + UPDATE_TABLE_CRITERIA: { + target: 'loaded', + actions: ['storeTableOptions'], + }, + TOGGLE_INACTIVE_DATASETS: { + target: 'loaded', + actions: ['storeInactiveDatasetsVisibility', 'resetPage'], + }, + TOGGLE_FULL_DATASET_NAMES: { + target: 'loaded', + actions: ['storeFullDatasetNamesVisibility'], + }, + }, + }, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'integrations.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'integrations.fetching', + }, + UPDATE_INTEGRATIONS: { + target: 'integrations.loaded', + actions: ['storeIntegrationsFilter'], + }, + UPDATE_NAMESPACES: { + target: 'integrations.loaded', + actions: ['storeNamespaces'], + }, + UPDATE_QUERY: { + actions: ['storeQuery'], + }, + }, + }, flyout: { initial: 'closed', states: { @@ -262,7 +297,7 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), - storeIntegrations: assign((context, event) => { + storeIntegrationsFilter: assign((context, event) => { return 'integrations' in event ? { filters: { @@ -320,8 +355,7 @@ export const createPureDatasetQualityControllerStateMachine = ( storeDataStreamStats: assign((_context, event) => { return 'data' in event ? { - dataStreamStats: (event.data as DataStreamStatServiceResponse).dataStreamStats, - integrations: (event.data as DataStreamStatServiceResponse).integrations, + dataStreamStats: event.data as DataStreamStat[], } : {}; }), @@ -342,6 +376,18 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeIntegrations: assign((_context, event) => { + return 'data' in event + ? { + integrations: event.data as Integration[], + } + : {}; + }), + storeEmptyIntegrations: assign((_context) => { + return { + integrations: [], + }; + }), storeIntegrationDashboards: assign((context, event) => { return 'data' in event && 'dashboards' in event.data ? { @@ -359,16 +405,15 @@ export const createPureDatasetQualityControllerStateMachine = ( : {}; }), storeDatasets: assign((context, _event) => { - return context.dataStreamStats && context.degradedDocStats + return context.integrations && (context.dataStreamStats || context.degradedDocStats) ? { - datasets: mergeDegradedStatsIntoDataStreams( + datasets: generateDatasets( context.dataStreamStats, - context.degradedDocStats + context.degradedDocStats, + context.integrations ), } - : context.dataStreamStats - ? { datasets: context.dataStreamStats } - : { datasets: [] }; + : {}; }), }, } @@ -397,6 +442,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchDatasetDetailsFailedNotifier(toasts, event.data), notifyFetchIntegrationDashboardsFailed: (_context, event: DoneInvokeEvent) => fetchIntegrationDashboardsFailedNotifier(toasts, event.data), + notifyFetchIntegrationsFailed: (_context, event: DoneInvokeEvent) => + fetchIntegrationsFailedNotifier(toasts, event.data), }, services: { loadDataStreamStats: (context) => @@ -414,6 +461,11 @@ export const createDatasetQualityControllerStateMachine = ({ end, }); }, + loadIntegrations: (context) => { + return dataStreamStatsClient.getIntegrations({ + type: context.type as GetIntegrationsParams['query']['type'], + }); + }, loadDataStreamDetails: (context) => { if (!context.flyout.dataset) { fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected)); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 3009ba6e5a985..5af18adf8313d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -15,6 +15,7 @@ import { DataStreamDegradedDocsStatServiceResponse, DataStreamDetails, DataStreamStatServiceResponse, + IntegrationsResponse, } from '../../../../common/data_streams_stats'; import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; @@ -84,7 +85,7 @@ export type DefaultDatasetQualityControllerState = { type: string } & WithTableO WithFlyoutOptions & WithDatasets & WithFilters & - WithIntegrations; + Partial; type DefaultDatasetQualityStateContext = DefaultDatasetQualityControllerState & Partial; @@ -118,6 +119,10 @@ export type DatasetQualityControllerTypeState = value: 'datasets.loaded'; context: DefaultDatasetQualityStateContext; } + | { + value: 'integrations.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'flyout.initializing.dataStreamDetails.fetching'; context: DefaultDatasetQualityStateContext; @@ -181,4 +186,5 @@ export type DatasetQualityControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/summary_panel/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/summary_panel/src/state_machine.ts index 075c8fe268d96..55daf83af13bf 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/summary_panel/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/summary_panel/src/state_machine.ts @@ -227,7 +227,7 @@ export const createDatasetsSummaryPanelStateMachine = ({ return { percentages }; }, loadDatasetsActivity: async (_context) => { - const { dataStreamStats } = await dataStreamStatsClient.getDataStreamsStats(); + const dataStreamStats = await dataStreamStatsClient.getDataStreamsStats(); const activeDataStreams = filterInactiveDatasets({ datasets: dataStreamStats }); return { total: dataStreamStats.length, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts new file mode 100644 index 0000000000000..cc48eb1dfdcb5 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { indexNameToDataStreamParts } from '../../common/utils'; +import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; +import { Integration } from '../../common/data_streams_stats/integration'; +import { generateDatasets } from './generate_datasets'; + +describe('generateDatasets', () => { + const integrations: Integration[] = [ + { + name: 'system', + title: 'System', + version: '1.54.0', + datasets: { + 'system.application': 'Windows Application Events', + 'system.auth': 'System auth logs', + 'system.security': 'Security logs', + 'system.syslog': 'System syslog logs', + 'system.system': 'Windows System Events', + }, + }, + { + name: 'custom', + title: 'Custom', + version: '1.0.0', + datasets: { + custom: 'Custom', + }, + }, + ]; + + const dataStreamStats: DataStreamStat[] = [ + { + name: 'system.application', + title: 'system.application', + type: 'logs', + namespace: 'default', + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, + rawName: 'logs-system.application-default', + degradedDocs: { + percentage: 0, + count: 0, + }, + }, + { + name: 'synth', + title: 'synth', + type: 'logs', + namespace: 'default', + lastActivity: 1712911241117, + rawName: 'logs-synth-default', + size: '62.5kb', + sizeBytes: 64066, + degradedDocs: { + percentage: 0, + count: 0, + }, + }, + ]; + + const degradedDocs = [ + { + dataset: 'logs-system.application-default', + percentage: 0, + count: 0, + }, + { + dataset: 'logs-synth-default', + percentage: 11.320754716981131, + count: 6, + }, + ]; + + it('merges integrations information with dataStreamStats', () => { + const datasets = generateDatasets(dataStreamStats, undefined, integrations); + + expect(datasets).toEqual([ + { + ...dataStreamStats[0], + title: integrations[0].datasets[dataStreamStats[0].name], + integration: integrations[0], + }, + { + ...dataStreamStats[1], + }, + ]); + }); + + it('merges integrations information with degradedDocs', () => { + const datasets = generateDatasets(undefined, degradedDocs, integrations); + + expect(datasets).toEqual([ + { + rawName: degradedDocs[0].dataset, + name: indexNameToDataStreamParts(degradedDocs[0].dataset).dataset, + type: indexNameToDataStreamParts(degradedDocs[0].dataset).type, + lastActivity: undefined, + size: undefined, + sizeBytes: undefined, + namespace: indexNameToDataStreamParts(degradedDocs[0].dataset).namespace, + title: + integrations[0].datasets[indexNameToDataStreamParts(degradedDocs[0].dataset).dataset], + integration: integrations[0], + degradedDocs: { + percentage: degradedDocs[0].percentage, + count: degradedDocs[0].count, + }, + }, + { + rawName: degradedDocs[1].dataset, + name: indexNameToDataStreamParts(degradedDocs[1].dataset).dataset, + type: indexNameToDataStreamParts(degradedDocs[1].dataset).type, + lastActivity: undefined, + size: undefined, + sizeBytes: undefined, + namespace: indexNameToDataStreamParts(degradedDocs[1].dataset).namespace, + title: indexNameToDataStreamParts(degradedDocs[1].dataset).dataset, + integration: undefined, + degradedDocs: { + percentage: degradedDocs[1].percentage, + count: degradedDocs[1].count, + }, + }, + ]); + }); + + it('merges integrations information with dataStreamStats and degradedDocs', () => { + const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations); + + expect(datasets).toEqual([ + { + ...dataStreamStats[0], + title: integrations[0].datasets[dataStreamStats[0].name], + integration: integrations[0], + degradedDocs: { + percentage: degradedDocs[0].percentage, + count: degradedDocs[0].count, + }, + }, + { + ...dataStreamStats[1], + degradedDocs: { + percentage: degradedDocs[1].percentage, + count: degradedDocs[1].count, + }, + }, + ]); + }); + + it('returns an empty array if no valid object is provided', () => { + expect(generateDatasets(undefined, undefined, integrations)).toEqual([]); + }); +}); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts new file mode 100644 index 0000000000000..951ba9ccafc8b --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Integration } from '../../common/data_streams_stats/integration'; +import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; +import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; + +export function generateDatasets( + dataStreamStats: DataStreamStat[] = [], + degradedDocStats: DegradedDocsStat[] = [], + integrations: Integration[] +) { + if (!dataStreamStats.length && !integrations.length) { + return []; + } + + const integrationMap: Record = + integrations.reduce((integrationMapAcc, integration) => { + return { + ...integrationMapAcc, + ...Object.keys(integration.datasets).reduce( + (datasetsAcc, dataset) => + Object.assign(datasetsAcc, { + [dataset]: { + integration, + title: integration.datasets[dataset], + }, + }), + {} + ), + }; + }, {}); + + if (!dataStreamStats.length) { + return degradedDocStats.map((degradedDocStat) => + DataStreamStat.fromDegradedDocStat({ degradedDocStat, integrationMap }) + ); + } + + const degradedMap: Record< + DegradedDocsStat['dataset'], + { + percentage: DegradedDocsStat['percentage']; + count: DegradedDocsStat['count']; + } + > = degradedDocStats.reduce( + (degradedMapAcc, { dataset, percentage, count }) => + Object.assign(degradedMapAcc, { [dataset]: { percentage, count } }), + {} + ); + + return dataStreamStats?.map((dataStream) => ({ + ...dataStream, + title: integrationMap[dataStream.name]?.title || dataStream.title, + integration: integrationMap[dataStream.name]?.integration, + degradedDocs: degradedMap[dataStream.rawName] || dataStream.degradedDocs, + })); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts index a6b7c09d0fd57..3185367e39aca 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts @@ -6,5 +6,5 @@ */ export * from './filter_inactive_datasets'; -export * from './merge_degraded_docs_into_datastreams'; +export * from './generate_datasets'; export * from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/merge_degraded_docs_into_datastreams.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/merge_degraded_docs_into_datastreams.ts deleted file mode 100644 index 5039bfc31761d..0000000000000 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/merge_degraded_docs_into_datastreams.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DataStreamStat } from '../../common/data_streams_stats'; -import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; - -export function mergeDegradedStatsIntoDataStreams( - dataStreamStats: DataStreamStat[], - degradedDocStats: DegradedDocsStat[] -) { - const degradedMap: Record< - DegradedDocsStat['dataset'], - { - percentage: DegradedDocsStat['percentage']; - count: DegradedDocsStat['count']; - } - > = degradedDocStats.reduce( - (degradedMapAcc, { dataset, percentage, count }) => - Object.assign(degradedMapAcc, { [dataset]: { percentage, count } }), - {} - ); - - return dataStreamStats?.map((dataStream) => ({ - ...dataStream, - degradedDocs: degradedMap[dataStream.rawName] || dataStream.degradedDocs, - })); -} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 80cea5c9de877..a8475ed8cc8ef 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -13,8 +13,6 @@ import { DataStreamsEstimatedDataInBytes, DataStreamStat, DegradedDocs, - Integration, - IntegrationDashboards, } from '../../../common/api_types'; import { rangeRt, typeRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; @@ -22,7 +20,6 @@ import { getDataStreamDetails } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; -import { getIntegrationDashboards, getIntegrations } from './get_integrations'; import { getEstimatedDataInBytes } from './get_estimated_data_in_bytes'; const statsRoute = createDatasetQualityServerRoute({ @@ -40,17 +37,13 @@ const statsRoute = createDatasetQualityServerRoute({ }, async handler(resources): Promise<{ dataStreamsStats: DataStreamStat[]; - integrations: Integration[]; }> { - const { context, params, plugins } = resources; + const { context, params } = resources; const coreContext = await context.core; // Query datastreams as the current user as the Kibana internal user may not have all the required permissions const esClient = coreContext.elasticsearch.client.asCurrentUser; - const fleetPluginStart = await plugins.fleet.start(); - const packageClient = fleetPluginStart.packageService.asInternalUser; - const [dataStreams, dataStreamsStats] = await Promise.all([ getDataStreams({ esClient, @@ -64,7 +57,6 @@ const statsRoute = createDatasetQualityServerRoute({ dataStreamsStats: values( merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name')) ), - integrations: await getIntegrations({ packageClient, dataStreams: dataStreams.items }), }; }, }); @@ -170,40 +162,9 @@ const estimatedDataInBytesRoute = createDatasetQualityServerRoute({ }, }); -const integrationDashboardsRoute = createDatasetQualityServerRoute({ - endpoint: 'GET /internal/dataset_quality/integrations/{integration}/dashboards', - params: t.type({ - path: t.type({ - integration: t.string, - }), - }), - options: { - tags: [], - }, - async handler(resources): Promise { - const { context, params, plugins } = resources; - const { integration } = params.path; - const { savedObjects } = await context.core; - - const fleetPluginStart = await plugins.fleet.start(); - const packageClient = fleetPluginStart.packageService.asInternalUser; - - const integrationDashboards = await getIntegrationDashboards( - packageClient, - savedObjects.client, - integration - ); - - return { - dashboards: integrationDashboards, - }; - }, -}); - export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, ...dataStreamDetailsRoute, ...estimatedDataInBytesRoute, - ...integrationDashboardsRoute, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/index.ts index bb242e0db8721..3a02e7e99b2dc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/index.ts @@ -6,10 +6,12 @@ */ import type { EndpointOf, ServerRouteRepository } from '@kbn/server-route-repository'; import { dataStreamsRouteRepository } from './data_streams/routes'; +import { integrationsRouteRepository } from './integrations/routes'; function getTypedDatasetQualityServerRouteRepository() { const repository = { ...dataStreamsRouteRepository, + ...integrationsRouteRepository, }; return repository; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_integrations.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integration_dashboards.ts similarity index 57% rename from x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_integrations.ts rename to x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integration_dashboards.ts index f13e3661942c0..4bea7e34a8140 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_integrations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integration_dashboards.ts @@ -8,8 +8,7 @@ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { DASHBOARD_SAVED_OBJECT_TYPE } from '@kbn/deeplinks-analytics/constants'; import { PackageClient } from '@kbn/fleet-plugin/server'; -import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors'; -import { Dashboard, DataStreamStat, Integration } from '../../../common/api_types'; +import { Dashboard } from '../../../common/api_types'; export async function getIntegrationDashboards( packageClient: PackageClient, @@ -55,56 +54,3 @@ export async function getIntegrationDashboards( return packageDashboards; } - -export async function getIntegrations(options: { - packageClient: PackageClient; - dataStreams: DataStreamStat[]; -}): Promise { - const { packageClient, dataStreams } = options; - - const packages = await packageClient.getPackages(); - const installedPackages = dataStreams.map((item) => item.integration); - - return Promise.all( - packages - .filter((pkg) => installedPackages.includes(pkg.name)) - .map(async (p) => ({ - name: p.name, - title: p.title, - version: p.version, - icons: p.icons, - datasets: await getDatasets({ - packageClient, - name: p.name, - version: p.version, - }), - })) - ); -} - -const getDatasets = async (options: { - packageClient: PackageClient; - name: string; - version: string; -}) => { - try { - const { packageClient, name, version } = options; - - const pkg = await packageClient.getPackage(name, version); - - return pkg.packageInfo.data_streams?.reduce( - (acc, curr) => ({ - ...acc, - [curr.dataset]: curr.title, - }), - {} - ); - } catch (error) { - // Custom integration - if (error instanceof PackageNotFoundError) { - return {}; - } - - throw error; - } -}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts new file mode 100644 index 0000000000000..70aa86f08efc3 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/core/server'; +import { PackageClient } from '@kbn/fleet-plugin/server'; +import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors'; +import { PackageListItem, RegistryDataStream } from '@kbn/fleet-plugin/common'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; +import { DataStreamType } from '../../../common/types'; +import { Integration } from '../../../common/api_types'; + +export async function getIntegrations(options: { + packageClient: PackageClient; + logger: Logger; + type?: DataStreamType; +}): Promise { + const { packageClient, logger, type = DEFAULT_DATASET_TYPE } = options; + + const packages = await packageClient.getPackages(); + const installedPackages = packages.filter((p) => p.status === 'installed'); + + const integrations = await Promise.all( + installedPackages.map(async (p) => ({ + name: p.name, + title: p.title, + version: p.version, + icons: p.icons, + datasets: await getDatasets({ packageClient, logger, pkg: p, type }), + })) + ); + + return integrations.filter((integration) => Object.keys(integration.datasets).length > 0); +} + +const getDatasets = async (options: { + packageClient: PackageClient; + logger: Logger; + pkg: PackageListItem; + type: DataStreamType; +}) => { + const { packageClient, logger, pkg, type } = options; + + return ( + (await fetchDatasets({ + packageClient, + logger, + name: pkg.name, + version: pkg.version, + type, + })) ?? getDatasetsReadableName(pkg.data_streams ?? []) + ); +}; + +const fetchDatasets = async (options: { + packageClient: PackageClient; + logger: Logger; + name: string; + version: string; + type: DataStreamType; +}) => { + try { + const { packageClient, name, version, type } = options; + + const pkg = await packageClient.getPackage(name, version); + + return getDatasetsReadableName( + (pkg.packageInfo.data_streams ?? []).filter((ds) => ds.type === type) + ); + } catch (error) { + // Custom integration + if (error instanceof PackageNotFoundError) { + return null; + } + + const { name, version, logger } = options; + logger.error( + `There was an error when trying to fetch information about package ${name} version ${version}: ${error}` + ); + + return {}; + } +}; + +const getDatasetsReadableName = (dataStreams: RegistryDataStream[]) => { + return dataStreams.reduce( + (acc, curr) => ({ + ...acc, + [curr.dataset]: curr.title, + }), + {} + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts new file mode 100644 index 0000000000000..dfc0b8a2824e3 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { Integration, IntegrationDashboards } from '../../../common/api_types'; +import { typeRt } from '../../types/default_api_types'; +import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; +import { getIntegrations } from './get_integrations'; +import { getIntegrationDashboards } from './get_integration_dashboards'; + +const integrationsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/integrations', + params: t.type({ + query: typeRt, + }), + options: { + tags: [], + }, + async handler(resources): Promise<{ + integrations: Integration[]; + }> { + const { params, plugins, logger } = resources; + + const fleetPluginStart = await plugins.fleet.start(); + const packageClient = fleetPluginStart.packageService.asInternalUser; + + const integrations = await getIntegrations({ packageClient, logger, ...params.query }); + + return { integrations }; + }, +}); + +const integrationDashboardsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/integrations/{integration}/dashboards', + params: t.type({ + path: t.type({ + integration: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params, plugins } = resources; + const { integration } = params.path; + const { savedObjects } = await context.core; + + const fleetPluginStart = await plugins.fleet.start(); + const packageClient = fleetPluginStart.packageService.asInternalUser; + + const integrationDashboards = await getIntegrationDashboards( + packageClient, + savedObjects.client, + integration + ); + + return { + dashboards: integrationDashboards, + }; + }, +}); + +export const integrationsRouteRepository = { + ...integrationsRoute, + ...integrationDashboardsRoute, +}; diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/state_machines/dataset_quality/src/url_schema_v1.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/public/state_machines/dataset_quality/src/url_schema_v1.ts index 512219c9b0099..d49ab302e1908 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/state_machines/dataset_quality/src/url_schema_v1.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/state_machines/dataset_quality/src/url_schema_v1.ts @@ -5,7 +5,10 @@ * 2.0. */ -import { DatasetQualityPublicStateUpdate } from '@kbn/dataset-quality-plugin/public/controller'; +import { + DatasetQualityFlyoutOptions, + DatasetQualityPublicStateUpdate, +} from '@kbn/dataset-quality-plugin/public/controller'; import * as rt from 'io-ts'; import { datasetQualityUrlSchemaV1, deepCompactObject } from '../../../../common'; @@ -14,7 +17,7 @@ export const getStateFromUrlValue = ( ): DatasetQualityPublicStateUpdate => deepCompactObject({ table: urlValue.table, - flyout: urlValue.flyout, + flyout: getFlyoutFromUrlValue(urlValue.flyout), filters: urlValue.filters, }); @@ -41,3 +44,25 @@ const stateFromUrlSchemaRT = new rt.Type< export const stateFromUntrustedUrlRT = datasetQualityUrlSchemaV1.urlSchemaRT.pipe(stateFromUrlSchemaRT); + +const getFlyoutFromUrlValue = ( + flyout?: datasetQualityUrlSchemaV1.UrlSchema['flyout'] +): DatasetQualityFlyoutOptions => + deepCompactObject({ + ...(flyout + ? { + ...flyout, + dataset: flyout.dataset + ? { + ...flyout.dataset, + integration: flyout.dataset.integration + ? { + ...flyout.dataset.integration, + datasets: {}, + } + : undefined, + } + : undefined, + } + : {}), + }); diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/integration_dashboards.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/integrations/integration_dashboards.spec.ts similarity index 86% rename from x-pack/test/dataset_quality_api_integration/tests/data_streams/integration_dashboards.spec.ts rename to x-pack/test/dataset_quality_api_integration/tests/integrations/integration_dashboards.spec.ts index e29b934223a31..4481e0120c8b4 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/integration_dashboards.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/integrations/integration_dashboards.spec.ts @@ -8,11 +8,7 @@ import expect from '@kbn/expect'; import { DatasetQualityApiClientKey } from '../../common/config'; import { FtrProviderContext } from '../../common/ftr_provider_context'; - -interface IntegrationPackage { - name: string; - version: string; -} +import { installPackage, IntegrationPackage, uninstallPackage } from './package_utils'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); @@ -32,17 +28,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, ]; - async function installPackage({ name, version }: IntegrationPackage) { - return supertest - .post(`/api/fleet/epm/packages/${name}/${version}`) - .set('kbn-xsrf', 'xxxx') - .send({ force: true }); - } - - async function uninstallPackage({ name, version }: IntegrationPackage) { - return supertest.delete(`/api/fleet/epm/packages/${name}/${version}`).set('kbn-xsrf', 'xxxx'); - } - async function callApiAs(integration: string) { const user = 'datasetQualityLogsUser' as DatasetQualityApiClientKey; return await datasetQualityApiClient[user]({ @@ -59,7 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('gets the installed integration dashboards', () => { before(async () => { await Promise.all( - integrationPackages.map((pkg: IntegrationPackage) => installPackage(pkg)) + integrationPackages.map((pkg: IntegrationPackage) => installPackage({ supertest, pkg })) ); }); @@ -108,7 +93,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { after( async () => await Promise.all( - integrationPackages.map((pkg: IntegrationPackage) => uninstallPackage(pkg)) + integrationPackages.map((pkg: IntegrationPackage) => + uninstallPackage({ supertest, pkg }) + ) ) ); }); diff --git a/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts new file mode 100644 index 0000000000000..ee8c392e75317 --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + CustomIntegration, + installCustomIntegration, + installPackage, + IntegrationPackage, + uninstallPackage, +} from './package_utils'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const supertest = getService('supertest'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + + const integrationPackages: IntegrationPackage[] = [ + { + // logs based integration + name: 'system', + version: '1.0.0', + }, + { + // logs based integration + name: 'apm', + version: '8.0.0', + }, + { + // non-logs based integration + name: 'synthetics', + version: '1.0.0', + }, + ]; + + const customIntegrations: CustomIntegration[] = [ + { + integrationName: 'my.custom.integration', + datasets: [ + { + name: 'my.custom.integration', + type: 'logs', + }, + ], + }, + ]; + + async function callApiAs() { + const user = 'datasetQualityLogsUser' as DatasetQualityApiClientKey; + + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/integrations', + params: { + query: { + type: 'logs', + }, + }, + }); + } + + registry.when('Integration', { config: 'basic' }, () => { + describe('gets the installed integrations', () => { + before(async () => { + await Promise.all( + integrationPackages.map((pkg: IntegrationPackage) => installPackage({ supertest, pkg })) + ); + }); + + it('returns only log based integrations and its datasets map', async () => { + const resp = await callApiAs(); + + expect(resp.body.integrations.map((integration) => integration.name)).to.eql([ + 'apm', + 'system', + ]); + + expect(resp.body.integrations[0].datasets).not.empty(); + expect(resp.body.integrations[1].datasets).not.empty(); + }); + + after( + async () => + await Promise.all( + integrationPackages.map((pkg: IntegrationPackage) => + uninstallPackage({ supertest, pkg }) + ) + ) + ); + }); + + describe('gets the custom installed integrations', () => { + before(async () => { + await Promise.all( + customIntegrations.map((customIntegration: CustomIntegration) => + installCustomIntegration({ supertest, customIntegration }) + ) + ); + }); + + it('returns custom integrations and its datasets map', async () => { + const resp = await callApiAs(); + + expect(resp.body.integrations.map((integration) => integration.name)).to.eql([ + 'my.custom.integration', + ]); + + expect(resp.body.integrations[0].datasets).to.eql({ + 'my.custom.integration': 'My.custom.integration', + }); + }); + + after( + async () => + await Promise.all( + customIntegrations.map((customIntegration: CustomIntegration) => + uninstallPackage({ + supertest, + pkg: { name: customIntegration.integrationName, version: '1.0.0' }, + }) + ) + ) + ); + }); + }); +} diff --git a/x-pack/test/dataset_quality_api_integration/tests/integrations/package_utils.ts b/x-pack/test/dataset_quality_api_integration/tests/integrations/package_utils.ts new file mode 100644 index 0000000000000..4aa403ee6d62a --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/integrations/package_utils.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SuperTest, Test } from 'supertest'; + +export interface IntegrationPackage { + name: string; + version: string; +} + +export interface CustomIntegration { + integrationName: string; + datasets: IntegrationDataset[]; +} + +export interface IntegrationDataset { + name: string; + type: 'logs' | 'metrics' | 'synthetics'; +} + +export async function installCustomIntegration({ + supertest, + customIntegration, +}: { + supertest: SuperTest; + customIntegration: CustomIntegration; +}) { + const { integrationName, datasets } = customIntegration; + + return supertest + .post(`/api/fleet/epm/custom_integrations`) + .set('kbn-xsrf', 'xxxx') + .send({ integrationName, datasets }); +} + +export async function installPackage({ + supertest, + pkg, +}: { + supertest: SuperTest; + pkg: IntegrationPackage; +}) { + const { name, version } = pkg; + + return supertest + .post(`/api/fleet/epm/packages/${name}/${version}`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }); +} + +export async function uninstallPackage({ + supertest, + pkg, +}: { + supertest: SuperTest; + pkg: IntegrationPackage; +}) { + const { name, version } = pkg; + + return supertest.delete(`/api/fleet/epm/packages/${name}/${version}`).set('kbn-xsrf', 'xxxx'); +} diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index ceaeaaf576ed6..f3bbadc0f89f8 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -222,6 +222,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); it('hides inactive datasets', async () => { + await PageObjects.datasetQuality.waitUntilTableLoaded(); + // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); const lastActivityCol = cols['Last Activity']; diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 0a9ba23f007b4..cd59faf3c9053 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -40,7 +40,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv const euiSelectable = getService('selectable'); const retry = getService('retry'); const find = getService('find'); - const browser = getService('browser'); const selectors = { datasetQualityTable: '[data-test-subj="datasetQualityTable"]', @@ -49,6 +48,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv datasetSearchInput: '[placeholder="Filter datasets"]', showFullDatasetNamesSwitch: 'button[aria-label="Show full dataset names"]', showInactiveDatasetsNamesSwitch: 'button[aria-label="Show inactive datasets"]', + superDatePickerApplyButton: '.euiQuickSelect__applyButton', }; const testSubjectSelectors = { @@ -300,10 +300,9 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv await timeUnitSelect.focus(); await timeUnitSelect.type(unit); - await browser.pressKeys(browser.keys.ENTER); + (await datePickerQuickMenu.findByCssSelector(selectors.superDatePickerApplyButton)).click(); - // Close the date picker quick menu - return testSubjects.click(testSubjectSelectors.superDatePickerToggleQuickMenuButton); + return testSubjects.missingOrFail(testSubjectSelectors.superDatePickerQuickMenu); }, /** diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts index 61db9caeb05fc..475984c4873ea 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts @@ -225,6 +225,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('hides inactive datasets', async () => { + await PageObjects.datasetQuality.waitUntilTableLoaded(); + // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); const lastActivityCol = cols['Last Activity'];