From 2441e9ac8f38a561c5572a08162d0822f24074d8 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 27 Aug 2024 17:28:53 +0200 Subject: [PATCH] [Dataset quality] Enable page for metrics and traces (#190043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/elastic/observability-dev/issues/3454. This PR enables Dataset quality for being used for `traces` and `metrics`. ### Changes - Added `KNOWN_TYPES` array containing the types that we allow in dataset quality page. - `datasets`, `degradedDocs` and `nonAggregatableDatasets` are now wrapped in an state. This allow us to retrigger the calls whenever we need to do it (e.g. when changing timeframe), more importantly now whenever we select a new type of dataset. - The `invoke` `degradedDocs` is created dynamically depending on the types present in `KNOWN_TYPES`. - `GET /internal/dataset_quality/data_streams/stats` and `GET /internal/dataset_quality/data_streams/non_aggregatable` now accept an array of `DataStreamType`. This allow us to query the information for multiple types of dataStreams at the same time. - degradedDocs are stored now locally as a nested structure. This nested structure allow us to update just the needed portion instead of updating all datasets whenever a change occurs (e.g. a type is deselected). - redirectLink now takes into account the datastream type, it only redirects to logs explorer if it's enabled and the type of the datastream is logs. ### 🎥 Demo https://github.com/user-attachments/assets/082bd4e9-a8f8-4af9-a425-267adc3b30df --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../dataset_quality_url_schema_v1.ts | 1 + x-pack/plugins/data_quality/server/plugin.ts | 14 + .../dataset_quality/common/constants.ts | 6 +- .../common/data_stream_details/types.ts | 3 - .../common/data_streams_stats/types.ts | 3 - .../dataset_quality/filters/filters.tsx | 31 ++ .../dataset_quality/filters/selector.tsx | 102 ++++++ .../components/dataset_quality/header.tsx | 13 +- .../dataset_quality/table/columns.tsx | 13 + .../document_trends/degraded_docs/index.tsx | 6 +- .../hooks/use_dataset_quality_filters.ts | 28 +- .../hooks/use_dataset_quality_table.tsx | 11 +- .../hooks/use_dataset_quality_warnings.ts | 7 +- .../public/hooks/use_empty_state.ts | 4 +- .../public/hooks/use_redirect_link.ts | 7 +- .../public/hooks/use_summary_panel.ts | 13 +- .../data_stream_details_client.ts | 6 +- .../data_streams_stats_client.ts | 23 +- .../services/data_streams_stats/types.ts | 3 +- .../src/defaults.ts | 11 +- .../src/state_machine.ts | 305 ++++++++++-------- .../dataset_quality_controller/src/types.ts | 36 ++- .../state_machine.ts | 4 +- .../public/utils/flatten_stats.ts | 16 + .../public/utils/generate_datasets.test.ts | 108 ++++--- .../public/utils/generate_datasets.ts | 12 +- .../get_data_streams/get_data_streams.test.ts | 14 +- .../data_streams/get_data_streams/index.ts | 22 +- .../get_non_aggregatable_data_streams.ts | 9 +- .../server/routes/data_streams/routes.ts | 33 +- .../routes/integrations/get_integrations.ts | 18 +- .../server/routes/integrations/routes.ts | 8 +- .../server/types/default_api_types.ts | 16 +- .../dataset_quality/tsconfig.json | 3 +- .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../tests/data_streams/stats.spec.ts | 3 +- .../tests/integrations/integrations.spec.ts | 8 +- 39 files changed, 610 insertions(+), 313 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/filters/selector.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/utils/flatten_stats.ts diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_url_schema_v1.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_url_schema_v1.ts index 08313ddc4860..5121f1b7f64d 100644 --- a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_url_schema_v1.ts +++ b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_url_schema_v1.ts @@ -13,6 +13,7 @@ export const filtersRT = rt.exact( inactive: rt.boolean, fullNames: rt.boolean, timeRange: timeRangeRT, + types: rt.array(rt.string), integrations: rt.array(rt.string), namespaces: rt.array(rt.string), qualities: rt.array(rt.union([rt.literal('poor'), rt.literal('degraded'), rt.literal('good')])), diff --git a/x-pack/plugins/data_quality/server/plugin.ts b/x-pack/plugins/data_quality/server/plugin.ts index 4977e6f09a5c..088905b8975d 100644 --- a/x-pack/plugins/data_quality/server/plugin.ts +++ b/x-pack/plugins/data_quality/server/plugin.ts @@ -25,6 +25,20 @@ export class DataQualityPlugin implements Plugin { ['logs-*-*']: ['read'], }, }, + { + ui: [], + requiredClusterPrivileges: [], + requiredIndexPrivileges: { + ['traces-*-*']: ['read'], + }, + }, + { + ui: [], + requiredClusterPrivileges: [], + requiredIndexPrivileges: { + ['metrics-*-*']: ['read'], + }, + }, ], }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index a6c6754befb8..895381703742 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { QualityIndicators } from './types'; +import { DataStreamType, QualityIndicators } from './types'; export const DATASET_QUALITY_APP_ID = 'dataset_quality'; -export const DEFAULT_DATASET_TYPE = 'logs'; +export const DEFAULT_DATASET_TYPE: DataStreamType = 'logs'; export const DEFAULT_LOGS_DATA_VIEW = 'logs-*-*'; export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3; @@ -41,3 +41,5 @@ export const MAX_DEGRADED_FIELDS = 1000; export const MASKED_FIELD_PLACEHOLDER = ''; export const UNKOWN_FIELD_PLACEHOLDER = ''; + +export const KNOWN_TYPES: DataStreamType[] = ['logs', 'metrics', 'traces']; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index 74f065339644..82d6d2651be5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -5,9 +5,6 @@ * 2.0. */ -import { DataStreamType } from '../types'; - export interface GetDataStreamIntegrationParams { - type: DataStreamType; integrationName: string; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 1963c73d263e..79fdbfbd7dd9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -15,9 +15,6 @@ export type GetDataStreamsStatsResponse = export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0]; export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse; -export type GetIntegrationsParams = - APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params']; - export type GetDataStreamsDegradedDocsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params']; export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query']; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/filters/filters.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/filters/filters.tsx index 8c247104415c..4c63b8480d3b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/filters/filters.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/filters/filters.tsx @@ -10,12 +10,33 @@ import { EuiSuperDatePicker } from '@elastic/eui'; import { UI_SETTINGS } from '@kbn/data-service'; import { TimePickerQuickRange } from '@kbn/observability-shared-plugin/public/hooks/use_quick_time_ranges'; import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; import { useDatasetQualityFilters } from '../../../hooks/use_dataset_quality_filters'; import { useKibanaContextForPlugin } from '../../../utils/use_kibana'; import { FilterBar } from './filter_bar'; import { IntegrationsSelector } from './integrations_selector'; import { NamespacesSelector } from './namespaces_selector'; import { QualitiesSelector } from './qualities_selector'; +import { Selector } from './selector'; + +const typesLabel = i18n.translate('xpack.datasetQuality.types.label', { + defaultMessage: 'Types', +}); + +const typesSearchPlaceholder = i18n.translate( + 'xpack.datasetQuality.selector.types.search.placeholder', + { + defaultMessage: 'Filter types', + } +); + +const typesNoneMatching = i18n.translate('xpack.datasetQuality.selector.types.noneMatching', { + defaultMessage: 'No types found', +}); + +const typesNoneAvailable = i18n.translate('xpack.datasetQuality.selector.types.noneAvailable', { + defaultMessage: 'No types available', +}); // Allow for lazy loading // eslint-disable-next-line import/no-default-export @@ -29,9 +50,11 @@ export default function Filters() { integrations, namespaces, qualities, + types, onIntegrationsChange, onNamespacesChange, onQualitiesChange, + onTypesChange, selectedQuery, onQueryChange, } = useDatasetQualityFilters(); @@ -65,6 +88,14 @@ export default function Filters() { integrations={integrations} onIntegrationsChange={onIntegrationsChange} /> + void; +} + +export interface Item { + label: string; + checked?: EuiSelectableOptionCheckedType; +} + +export function Selector({ + isLoading, + options, + loadingMessage, + label, + searchPlaceholder, + noneAvailableMessage, + noneMatchingMessage, + onOptionsChange, +}: SelectorProps) { + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + + const onButtonClick = () => { + setIsPopoverOpen(!isPopoverOpen); + }; + + const closePopover = () => { + setIsPopoverOpen(false); + }; + + const renderOption = (option: Item) => {option.label}; + + const button = ( + item.checked === 'on')} + numActiveFilters={options.filter((item) => item.checked === 'on').length} + > + {label} + + ); + + return ( + + renderOption(option)} + > + {(list, search) => ( +
+ {search} + {list} +
+ )} + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/header.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/header.tsx index 4ec200b1b98c..66e2cba5ed87 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/header.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/header.tsx @@ -10,7 +10,7 @@ import { EuiBetaBadge, EuiLink, EuiPageHeader, EuiCode } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; -import { DEFAULT_LOGS_DATA_VIEW } from '../../../common/constants'; +import { KNOWN_TYPES } from '../../../common/constants'; import { datasetQualityAppTitle } from '../../../common/translations'; // Allow for lazy loading @@ -33,9 +33,16 @@ export default function Header() { description={ {DEFAULT_LOGS_DATA_VIEW}, + types: KNOWN_TYPES.map((type, index) => { + return ( + <> + {index > 0 && ', '} + {type} + + ); + }), dsNamingSchemeLink: ( ( + {dataStreamStat.type} + ), + width: '160px', + }, ...(isSizeStatsAvailable && canUserMonitorDataset && canUserMonitorAnyDataStream ? [ { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx index 7c3aea15a6cf..a0875f136770 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx @@ -6,6 +6,7 @@ */ import React, { useCallback, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, @@ -133,7 +134,10 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe { const { service } = useDatasetQualityContext(); @@ -22,13 +24,14 @@ export const useDatasetQualityFilters = () => { service, (state) => state.matches('integrations.fetching') && - (state.matches('datasets.fetching') || state.matches('degradedDocs.fetching')) + (state.matches('stats.datasets.fetching') || state.matches('stats.degradedDocs.fetching')) ); const { timeRange, integrations: selectedIntegrations, namespaces: selectedNamespaces, + types: selectedTypes, qualities: selectedQualities, query: selectedQuery, } = useSelector(service, (state) => state.context.filters); @@ -169,6 +172,25 @@ export const useDatasetQualityFilters = () => { [service] ); + const typeItems: Item[] = useMemo(() => { + return KNOWN_TYPES.map((type) => ({ + label: type, + checked: selectedTypes.includes(type) ? 'on' : undefined, + })); + }, [selectedTypes]); + + const onTypesChange = useCallback( + (newTypeItems: Item[]) => { + service.send({ + type: 'UPDATE_TYPES', + types: newTypeItems + .filter((quality) => quality.checked === 'on') + .map((type) => type.label as DataStreamType), + }); + }, + [service] + ); + const onQueryChange = useCallback( (query: string) => { service.send({ @@ -187,9 +209,11 @@ export const useDatasetQualityFilters = () => { integrations: integrationItems, namespaces: namespaceItems, qualities: qualityItems, + types: typeItems, onIntegrationsChange, onNamespacesChange, onQualitiesChange, + onTypesChange, isLoading, selectedQuery, onQueryChange, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 3a3475eabeed..77764acae1e1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -48,8 +48,7 @@ export const useDatasetQualityTable = () => { service, (state) => !state.context.dataStreamStats || - !state.context.dataStreamStats.length || - state.context.dataStreamStats.some((s) => s.userPrivileges.canMonitor) + state.context.datasets.some((s) => s.userPrivileges?.canMonitor) ); const { @@ -66,15 +65,15 @@ export const useDatasetQualityTable = () => { const loading = useSelector( service, (state) => - state.matches('datasets.fetching') || + state.matches('stats.datasets.fetching') || state.matches('integrations.fetching') || - state.matches('degradedDocs.fetching') + state.matches('stats.degradedDocs.fetching') ); const loadingDataStreamStats = useSelector(service, (state) => - state.matches('datasets.fetching') + state.matches('stats.datasets.fetching') ); const loadingDegradedStats = useSelector(service, (state) => - state.matches('degradedDocs.fetching') + state.matches('stats.degradedDocs.fetching') ); const datasets = useSelector(service, (state) => state.context.datasets); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts index ddb116dc4830..f5ba4d01f898 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts @@ -16,8 +16,11 @@ export function useDatasetQualityWarnings() { ); const isNonAggregatableDatasetsLoading = useSelector(service, (state) => - state.matches('nonAggregatableDatasets.fetching') + state.matches('stats.nonAggregatableDatasets.fetching') ); - return { loading: isNonAggregatableDatasetsLoading, nonAggregatableDatasets }; + return { + loading: isNonAggregatableDatasetsLoading, + nonAggregatableDatasets, + }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_empty_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_empty_state.ts index a7de315a3589..f60a7914275b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_empty_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_empty_state.ts @@ -18,9 +18,9 @@ export function useEmptyState() { const isDatasetEmpty = useSelector( service, (state) => - !state.matches('datasets.fetching') && + !state.matches('stats.datasets.fetching') && !state.matches('integrations.fetching') && - !state.matches('degradedDocs.fetching') && + !state.matches('stats.degradedDocs.fetching') && (state.context.datasets?.length ?? 0) === 0 ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_redirect_link.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_redirect_link.ts index d83ad8299e26..b015d2335d62 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_redirect_link.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_redirect_link.ts @@ -47,7 +47,8 @@ export const useRedirectLink = ({ navigate: () => void; isLogsExplorerAvailable: boolean; }>(() => { - const config = logsExplorerLocator + const isLogsExplorerAvailable = !!logsExplorerLocator && dataStreamStat.type === 'logs'; + const config = isLogsExplorerAvailable ? buildLogsExplorerConfig({ locator: logsExplorerLocator, dataStreamStat, @@ -83,7 +84,7 @@ export const useRedirectLink = ({ onClick: onClickWithTelemetry, }, navigate: navigateWithTelemetry, - isLogsExplorerAvailable: !!logsExplorerLocator, + isLogsExplorerAvailable, }; }, [ breakdownField, @@ -181,7 +182,7 @@ const buildDiscoverConfig = ({ dataViewId, dataViewSpec: { id: dataViewId, - title: dataViewTitle, + title: dataViewId, }, query, breakdownField, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts index 622f495fc477..ce8acd58b750 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts @@ -19,6 +19,7 @@ const useSummaryPanel = () => { isSizeStatsAvailable, canUserMonitorDataset, canUserMonitorAnyDataStream, + loading, } = useDatasetQualityTable(); const { timeRange } = useSelector(service, (state) => state.context.filters); @@ -32,7 +33,7 @@ const useSummaryPanel = () => { }; const isDatasetsQualityLoading = useSelector(service, (state) => - state.matches('degradedDocs.fetching') + state.matches('stats.degradedDocs.fetching') ); /* @@ -42,8 +43,9 @@ const useSummaryPanel = () => { (item) => item.userPrivileges?.canMonitor ?? true ); - const isUserAuthorizedForDataset = - canUserMonitorDataset && canUserMonitorAnyDataStream && canUserMonitorAllFilteredDataStreams; + const isUserAuthorizedForDataset = !loading + ? canUserMonitorDataset && canUserMonitorAnyDataStream && canUserMonitorAllFilteredDataStreams + : true; /* Datasets Activity @@ -57,7 +59,7 @@ const useSummaryPanel = () => { }; const isDatasetsActivityLoading = useSelector(service, (state) => - state.matches('datasets.fetching') + state.matches('stats.datasets.fetching') ); /* @@ -70,7 +72,8 @@ const useSummaryPanel = () => { const isEstimatedDataLoading = useSelector( service, - (state) => state.matches('datasets.fetching') || state.matches('degradedDocs.fetching') + (state) => + state.matches('stats.datasets.fetching') || state.matches('stats.degradedDocs.fetching') ); return { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 55d21286d7f6..5f944bd3fcde 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -123,11 +123,9 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { public async getDataStreamIntegration( params: GetDataStreamIntegrationParams ): Promise { - const { type, integrationName } = params; + const { integrationName } = params; const response = await this.http - .get('/internal/dataset_quality/integrations', { - query: { type }, - }) + .get('/internal/dataset_quality/integrations') .catch((error) => { throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 35cd67a1724f..1e75088853d3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -7,6 +7,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; +import rison from '@kbn/rison'; +import { KNOWN_TYPES } from '../../../common/constants'; import { getDataStreamsDegradedDocsStatsResponseRt, getDataStreamsStatsResponseRt, @@ -15,14 +17,12 @@ import { IntegrationResponse, NonAggregatableDatasets, } from '../../../common/api_types'; -import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, GetDataStreamsDegradedDocsStatsResponse, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, - GetIntegrationsParams, GetNonAggregatableDataStreamsParams, } from '../../../common/data_streams_stats'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -33,11 +33,15 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { constructor(private readonly http: HttpStart) {} public async getDataStreamsStats( - params: GetDataStreamsStatsQuery = { type: DEFAULT_DATASET_TYPE } + params: GetDataStreamsStatsQuery ): Promise { + const types = params.types.length === 0 ? KNOWN_TYPES : params.types; const response = await this.http .get('/internal/dataset_quality/data_streams/stats', { - query: params, + query: { + ...params, + types: rison.encodeArray(types), + }, }) .catch((error) => { throw new DatasetQualityError(`Failed to fetch data streams stats: ${error}`, error); @@ -59,7 +63,6 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { { query: { ...params, - type: DEFAULT_DATASET_TYPE, }, } ) @@ -86,7 +89,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { .get('/internal/dataset_quality/data_streams/non_aggregatable', { query: { ...params, - type: DEFAULT_DATASET_TYPE, + types: rison.encodeArray(params.types), }, }) .catch((error) => { @@ -102,13 +105,9 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return nonAggregatableDatasets; } - public async getIntegrations( - params: GetIntegrationsParams['query'] = { type: DEFAULT_DATASET_TYPE } - ): Promise { + public async getIntegrations(): Promise { const response = await this.http - .get('/internal/dataset_quality/integrations', { - query: params, - }) + .get('/internal/dataset_quality/integrations') .catch((error) => { throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index 785cdb4a88cc..ae1c72c23df0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -11,7 +11,6 @@ import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, GetDataStreamsStatsQuery, - GetIntegrationsParams, GetNonAggregatableDataStreamsParams, } from '../../../common/data_streams_stats'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -32,7 +31,7 @@ export interface IDataStreamsStatsClient { getDataStreamsDegradedStats( params?: GetDataStreamsDegradedDocsStatsQuery ): Promise; - getIntegrations(params: GetIntegrationsParams['query']): Promise; + getIntegrations(): Promise; getNonAggregatableDatasets( params: GetNonAggregatableDataStreamsParams ): Promise; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 87a6a398df8f..d0b2b9978080 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -14,8 +14,15 @@ import { DefaultDatasetQualityControllerState } from './types'; const ONE_MINUTE_IN_MS = 60000; +export const DEFAULT_DICTIONARY_TYPE = { + logs: [], + metrics: [], + traces: [], + synthetics: [], + profiling: [], +}; + export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { - type: DEFAULT_DATASET_TYPE, table: { page: 0, rowsPerPage: 10, @@ -30,6 +37,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { canViewIntegrations: true, }, dataStreamStats: [], + degradedDocStats: DEFAULT_DICTIONARY_TYPE, filters: { inactive: true, fullNames: false, @@ -44,6 +52,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { integrations: [], namespaces: [], qualities: [], + types: [DEFAULT_DATASET_TYPE], }, datasets: [], isSizeStatsAvailable: true, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 7ce3c5b5d8f6..127e56a755c8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -9,34 +9,43 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; import { DataStreamStat, NonAggregatableDatasets } from '../../../../common/api_types'; -import { Integration } from '../../../../common/data_streams_stats/integration'; +import { KNOWN_TYPES } from '../../../../common/constants'; import { - GetDataStreamsStatsQuery, - GetIntegrationsParams, - GetNonAggregatableDataStreamsParams, + DataStreamDegradedDocsStatServiceResponse, DataStreamStatServiceResponse, } from '../../../../common/data_streams_stats'; -import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; +import { Integration } from '../../../../common/data_streams_stats/integration'; +import { DataStreamType } from '../../../../common/types'; import { IDataStreamsStatsClient } from '../../../services/data_streams_stats'; import { generateDatasets } from '../../../utils'; +import { fetchNonAggregatableDatasetsFailedNotifier } from '../../common/notifications'; import { DEFAULT_CONTEXT } from './defaults'; import { fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, fetchIntegrationsFailedNotifier, } from './notifications'; -import { fetchNonAggregatableDatasetsFailedNotifier } from '../../common/notifications'; import { DatasetQualityControllerContext, DatasetQualityControllerEvent, DatasetQualityControllerTypeState, - DefaultDatasetQualityControllerState, } from './types'; +const generateInvokePerType = ({ src }: { src: string }) => ({ + invoke: KNOWN_TYPES.map((type) => ({ + id: `${type}`, + src, + data: { type }, + })), +}); + +const isTypeSelected = (type: DataStreamType, context: DatasetQualityControllerContext) => + context.filters.types.length === 0 || context.filters.types.includes(type); + export const createPureDatasetQualityControllerStateMachine = ( initialContext: DatasetQualityControllerContext ) => - /** @xstate-layout N4IgpgJg5mDOIC5QBECGAXVszoIoFdUAbAS3QE8BhAewDt0AnaoosBgOggyx1gGIAqgAVkAQQAqAUQD64gJIBZGQCVRAOQDikgNoAGALqJQAB2qwyJOkZAAPRAFZ77AEwAWAGwAOXfc+f39u7uAIwBADQg5IieAMwAnOwA7PbOujHucTHpzs72AL55EWiY2HiEpBQ09EwsbJzcpfzKkgBizQDKABLSYuKieoZIIKbm6Ja01nYIzu4xLv4ZMcGJwZ7OiXERUQieronsurtxibquufYrMQVFDTgExGRUdIzMrBxcJbzsAGY4AMYACxItCgfAgdDA7GBADdqABrSHFHhlB6VZ41N71T7oWA-f5AkEIGHUP4YcYDAbWEYWKxDKbpJwxNJZHIxHK6DKJLaIYI+fauTzBeyuOLOOKuGKxK6FEBI0r3CpPaqvOofZG437oQHA0FsJgcYxEDDfagMAC27Dld3KjyqL1q71uOLxWoJUCJtFhpLGdApBipZhpEzpiD2nhccVWx2CwVch1W9m5Owy7HciT89klIr2rnyMqtKMVdoxqrAUAYqAgkGQJP4wl6MnkSmkqk0On9Q2pPuDoCm3nc7F26aynLOIqTsQSyVS6Uy2Tc11lToVtvRKveZYrVYgNb+TVaHW6vX6HZMge7kx5umWB3WYaynkSMWFE-czlTrj2s2C6yf9mCi4FiuaLKg6nCbpW1a1i62ogmCEJQp68KIsuNogfamJVuWkE7tBmqwe6xLeuSBiUp257jJeCAxiE7AeOmJzPokzF7EmvIXHRgrCqK4qSlkgGoaiSoYaW2HbruGr4jqfB6qa7CGsapoWkBaHCSWG5iVBe4wW6HpemSvqkaewwUbSvY8ss+zXnEsShB49iRlykSIHEhzzMxKQ-s4ni+HmNzYsBanroh6AQd2dYiBIjaKCo6haGRZ6jJRIYIMKqarMsbIhB5QqJs5CCfq4g4xs+nkhDEEruAJAWqcWwXAqF2HhXwzRtJIXQ9BIJ6DIlQZUTkRWZisiQij5qzuBySbOEK7AXCNwTHLxqQpNVyKBXVYENWF4wRQ20hyGoUgaKo8gAPJqO0CUmUlZm2A4HGxJ+Sz+CsDl5dsoRON4mRxvYPgcuKq3yrVa6bfQ210LtUXSGoohKO0QiiJQ7VXV2yXmalngJEyMTDXEKQCksU1pi4cZY2moRzq4QPWkJG2YltTU7YIkVSNIuACJIygAJqo6ZPZ3dRqxFRkszJFluPLGxGzsDZv7pCscbrH5S41XToMM+DTOQzp0ngrQkLEgilqCUWGt1IzFbhbrhJEQZtB+j1119Slua6IOzE+VmPhvm4Sa44yey6AxyzTb4NOFquoGa41Vs7TbuoMPq8lGugJrmibatm9HFta3HOv4bpdvdo7AY3QL9K8rNcTuAK3ixgKyRJoE7t7AHkZxKKDlVfmptRyJHCW-buJENQOEs3tfQAEIADIyJQyhyFIi-dWXLsY2m4a8Y+-7LJvSYTYNWQ1xkFyPjMEfrebg958P7Cj+P4inRoGhz-tsOUPIABqMjHu0kjiEusZNGt0+wzHYBlUUWNj4XFcEmCU+w2Q-lnP4T8mRL4gxzjfWOd8H7bj4E-F+b8WgCBnjPTqfR-7iBhnDFGwD+ZUUFPsea14BS6EyFGf26ROKhw5FjDwAoVYqXVlgn4RByDUHwOgPg-856fxhpIAA6hQ0QVC+blyoimdhvgTiZmcFkBBbEKpFSfN5ZIwQ2S42pr3LO-d1JiIkVIvglAZ6nX-tIFoM9uanQEOIdR69Bb-icJ+RwARYysifEY0IsssZ+A2KcCqmYMEiIHg4yR6BEIWAeAALz1ghWAmBQqZzWpg1J3xxHpMyWMHJOp-EXhSryGacZEj6O8p3XGxw2L4zmMkGywpprDXWMk7OZSKlSKqSQGpIIsSoHaIwMAqAzTIBwKgEgRBJKujyQbRCsJjbCJGfY8pjiMnAiyaQXJ0y1RzIYAspZKy1kbIInpEk9tS7kQ0a7fRqYVhYwWv+XMViuluHYLjYx6xBTsk8MMuxwUjmVNOdU85OoZnXNucszADyE4ySTnJBSaclLFOBikw5YyTm0DOSQC5UAUXzMWei1Z6yE7POIoZfQdT0aCzOMEJI2jryxiCaKJyH10xTl4r4fwPhhSJGhehElxyJlTOpUPbsaBYAAgAEZjwYBAR5bp4LbKNihWxsrYWkoVUi6Zyrxiqo1VqnVTLi4kTZfQj5GM0EuCWN5NY1k1h+3yjkRwEDQ4BAmiOR8MqgpgTheMhFkyLVKtviqrAtrUDat1dJWSBpU7p2Un3E1UazWxsVSFCGtAbWatTfawuOpmWvKMk7EBFdQwig9T+PwqQFq+rgf60I74rEXF0OsTIpwI30zqNGjJeDIDONce4zx3jfHstAYgEaCQT6jWWjmZwbE9hODFKHY4iwghQpsSU4lpr5VTogBPaGB12hyA0J0QBsgYotjiu2BtDCGmdoOEybR4TDjMTYrA1MQRUjBxrrkLwo7r5pPGX8Ue2Br2nSEJINQHivE+L8S6gJUxwUQIqsxNMMxEi13SDumuBx-D40fKcE4J6ZS0GoFWeAQx9kwodGvepGMAC0rhuWSlWCRyUA1djvUQDxre00cwZE-LoYO-gYOiLVI0LjHKpj7DcF4HwfgQ0hHCPlacIKFhiiyJKR81j-JnoOcFFTXxq0gjU8uoW4Z5O7DSIKSUswBRJnFAOLIvhPwzElhkJTqS7POivU5ptqVwFCksSJiU-Hgi+Z8rNGYf0mR-XWOkML9isJbi0qx3q3HBbJHDEOKUo43CbEMykYzYG5zmZGnl2zEFxJ4Sko595uHoiRlmvxxwrkuInBS-lY4744n+FCN4Ea6ZWtgQKzhCS98x7bmi1RCxaW1h6Nco4CqaQ2LXjmMRxw15DhsmFAtmOpbivO1K1MAcIQmFZUskE8TCB0x0QlJkMUPgDFLGu7nHB1sHNQA2w08USRYi128Bmc+7gppQ8Ysgi434fJCLzZGm72taAjzW5ACHG8zgQN8KESUwdgtCocDkA4AQRTsLo14OIQOOATqJ2Vun9FZwnDTHNtiOQEhvk-N4JkHheTOFZ3BslFKqUc6mIGjk3POQcg8mxEITgOQVUFIzrMlnVbWY45iCd5rKXIqubSu5GL1ny6vHMJYsxIyEfxtNRH+VliDrol63TrkYz4ylybot8aaU3LpfcxlYPbfUX-CC8nTu9gu9CGxXYRVTiOVmFxbykvT1EpswW+VQezeXO4KisP1vcT6zAFH0+ECctCkjOcIIyeHLfN5LsGM00KrZ6s7no347C3ksRUXhNIPrXJorWmqPvI5hK7TDz1XTd-XB01+w1YLT4isIDwP2XyKrV0HLXa9N3WSvqZ5DH2fT4Vd84+wMkxPFO4zjTD3HvtM8-G+30PqlJbccH8rRXiEUepG7sYGmYFiwoSCMQU0C0rcsYrkiwZwawW+l6BOEAU+wc3KHCjEaOYoQG7ugQCQ-gHawW-GsQ3eBuve+a7+8qCGZghOPWD290A44oLSg66+2YkB7uEo742iTIywvgg6K0BQeQQAA */ + /** @xstate-layout N4IgpgJg5mDOIC5QBECGAXVszoIoFdUAbAS3QE8BhAewDt0AnaoosBgOlk3VnYgyw5YAYgDKAQQBqAUQD6ycQBVxoxQCVp4gLKzVS0QG0ADAF1EoAA7VYZEnXMgAHogCMANgDMAdnYAONwBMbgAsAKxh3kYAnL4ANCDkiMHBHsHsRgG+vh4xIb6hLgC+hfFomNh4hKQUNPRMLGyc3Lz85ULCAHIA8ooAkgBiAJrySirqmjp6iqKy-eK9ADLSyMZmSCBWNuh2tA7OCO7efoEh4aGRMfGJCCkevunBLsmBOQEeHsWlAhUExGRUdEYzFYHC4GBa33aAFUAAoKRRyPpaORqcQdADi0lWDk2tns632hx8-iCYQiXmicQSSQCwQC6XCRlCQRcHjcLlCnxAZUElT+NUB9RBTXBfEhPGEGn6GlEAAkRspsetcdt8aBCZ5iScyecKZdqTcXFEXOwvC5zXcjMEvMFfEUStzxb9qgC6sDGmCeHwwFAGKgIJBkNQAMYiCQyeTSdGo5DLeRdSgzKaGUw46x43YE1ya46ks4XKnXYJRUJRdieKJ0sJRY0FLk8n5Vf61IENUHNb2+-2BkMibp9IaR6PiWPIeOJ3TKaazeZLFap5Xp1WZ9XZo4k07kylXJJuPfsKJeLLsm1eUJW+tOpsCt1tkVegNdgMQIOhzo9AbDWPD0fjpNTmY5kWZYDBcNZLCXHY9jXLU8y3fUi38Dxy2ZIwPGiLwPHOD4HQbHBnWbQV3XbUVHz9Z9XxEWF4URXpkVkVEMSxBcIK2KCswOHMNx1AsdxuLw3B8Pdgk1VIvFrS82j5F0WyFD0OzI7sX17SVpGlaQ5QVcQlVYjNoM49dtXzPVCySM0fA8AIXC8AIAiPay7Uk3kCJvVthU9XhaDocQoF9H0BAAI1YPCJXDORug6WRxHRaMo1GAAhJYtNEaRph0jZILVJwYNzTddW3A0cn8dgglCUJskPDIjScxt+VdNz5NFLzaB8vyoEC4LxT7D9BwiqKYo0dEEqS+EVFSwDZxAliMrYrKNUMuD8oQxBvByErizcNDfCiTbMhq-Dr3quSSK9ZrWoYfzMCCsAQu6gdhj66LYqG5RErkUaUunIC51A8CZr0jiiVyniTL4u4oiMEr7Ih1IRKMIwvH26TCNvdyOzO3yLvaq7Oqkqi4SUWj6MYzF0pVdjVwM2C8t4wqKoPLDvErHaUn8JGXKO4j7087zMcu1Brtu1T1M00aycyldsqp4HjIK65vG29hS1pEIonQ9DOVwq86tkrmSHoH0-WXfGaNkJEUTRUnpvJubEEPPiRJs9gIjso1fAyNwonZw7dbvfX0ENjAdhEKUZXlMXrYl-T7YNETbPYc00NCG0nlZNxvZ1oi-YNrtjeEajCdkXoOgRYc+i6DoUz+m3Jf2GOiyNE0Nb3EI2RtAIM5krPhX9wO84LhFZA6bQNJhcRKA08XZtru2vAdzIfACNWjE2le3nTrWpI532e5zo3g-zgnB9wKFxAWXo+knyPp+jufY7uZDMitc43F8cTxM7lGGo4Xvc4PgfESDBhFfauUdAYuDtM7ZIh5X7WXcA7SyyFHi0m2mhJeWRP6uWOuwX++86Am0LifaQahBhTwBpTKI8dKzRAyDtSyKcHYZFCMcKqm0vA2gKB3Tezkfbd0aLgoO+D2AADMcDBgABb6ygMICAdAwA4NoAAN2oAAa3kSFbefCf570EbQXgoj0ASKkQgfWyjgy6NWGQ5c+lQivz8FkCB8MV55DvkWZkbhTQp3POeBGGDuG1S7qjfhOjjYiLEZI2g0i2BMA4BYIgGBhHUAYAAW3YBo3hQTtEBz-kIgxRjIkmKUSGCxpgrEUylrY+4WQ7Tu3hsJN+DsQgeOSC4IwRoFbMi9v4g6mdMkKOyXgvR7AiDUCUofU2r0kqUDUBfYhvRtLX3IVLfwZZbRGF8LaM0ENHhRAduVNIZU1ZL09hkFImDObZwGbo3gIyxmKC6DFJKxdx59AjB9caZTbYIHyCacIFJrSVjPN4FwDsRImm2lhcS1pU6ay+FvDJ39+l92DsM0Zz5hD3MeXIfoUIFgLGSqlIeI8q5phvhxDZyFWnMzXm-CBrikhsmYRDdk+QNnJ0POcnejRhFEHINQfA6AxDSCWJQRQQ9pAAHUCWKE+TPBANkwYligTWVkNY7R2Q3nCnhvTEU8r5QK4QlAFhdBSrMBYgwuhQhlYs6xHEFUGlpMhN+ORwiPAVncWFjp4U6uwXq-l6AFG2D+AALykTIuR951Ha0Cbq3l-rA3bBDVI2V+lWS0gPBA4IGRLLlTcOVPitI0KmnXiWV+JZHLdORlgrmfqBUJpIEmyJ7AMZtQ6jdcUYTDEROkbI2g8jTGqKjd6mNvq411v1kG0goam0tqxm2kKnb8lQEKWYkpJgU0cQ2ukM0WQSwFCwpkPiTxKwlVsUedeAkbI4S1QEr+o79UBonYmqdUjm281bTjdtUlF3duENEpJ7A4kJKSak9JPqa1jsfbQSdJBp1QDfS1Pm2MBa415D+4xA7zHLksTa8p+wt0IwgdtMqrJmSmQOOVReYQjR0nEoeIInKtEiMg-Wxt8HZ380Fh23tYBxmF2LqIXo6JZTTnNgxS2zFQFkspsyZhZ5X6+CXmmtW9LOLuxKu8Ta7JKF2XeIxvptaoMwbgwh86nHUMVD4HIoVIqxXvOtVJpZ+xPBpCtBDCkR4UiljcEeoqJVPZ0prGeSs9ob09JHRBh9rGX1NtaKgUQjAwCoGSaIHA2xIn6PCWGnjCjlFqLSdGu9kX41PobTF+DcWEsXWS6l9A6WoCZa7RhopWGdg4cc7aymdJ7gcmqW07wbDggFreCaMIngl42jQp5-TsaoulbY2KTAVWkspbS1IxrS6-0MBiYB+J6BEkpIK8Oord5DPRdg6+yriWatrYy+hgpmG10bq65kBO5Usj9YEgjIbDrLJpGSNaR4eaggf0rZogzLH5vlcW-F67q26vrfO8GsAvQQkooEcuZAOBUAkCIBt39OWB35bAxF07kPoPPou7FgQy2bsI7u1D2DKO0dCIxzsLHmBcf4+a6u7DpTcNfLIweEITi0KCSyKEAttL2DvBsoJR4GzFczfvSVinZWqcVZp3D2r9XeCM+R6jq5oS2d0A5zjvH92onbYA0B-bIGjvatJ8KM7+vLta+q-D3XSPmdG-Ryz2gZuueW5XcUvn66BdyqFyzUXbcJcFvYfcVIEuyor0zcr4r461cLaux7nXiP9c++Raz-3aBYDiICqMhgEBueRPDX23Lg6He3urWTubWfoc55W3nhn7emeG6L0Mk3AesDl8r9X4Pj2w-PYqVmhOGzX6e2LLkVTHh3AePYW8MqyCKQr3T631Xxm3dLe17dhr3v+85MHyXkfFfUBV5r1bnbtuDugcKy3535PD-U+P7n0-eve8G7+7-6+6m435j4P4h6tZ0DtakpOaICMhz4iQrJL7+Ar4BClimibQHoRDmiIxg4Ioq6Z5f6a4-5d5-7n5AFIqX6l6j537j55K-r-qxJ7Yv5N7hYnYf5t7EEw606e754AGF6X7AED40G3734T4tZPZgSwGdYz6QyZoL7MzL5gwIzIRHg5BebwxZoIx76cEH6U4mad7JaB4W4MHZYRpE5DqO4cHcqf4GFH6w4e4mEQGT5tb84dZ4YMppBsgQzZABArwryVgeBHpGjMKKZLxvAtKKbRC6G2FcH2Hf6OErbOGW5bZP4sH24k42EcAu4AEOG8EpFmEPaSFT4R76QpDeEnJ+EBGL7BEGiJz0jZDUYchLyoShZerWHv5xH6Hq6GHu7JHY5B48Z8aDwCZCYiYzBiYkySYyGeE3CpAy5VGWQ1FBFHp1J+DQrWQwwQxcJhZVoXJ6FEEJEkFJHGGDEW7DHxQaDiAADSyAXQEqkU-QvQwqY4lAsoEm0++wJISs9G9C7Ir85wIRRa2QnCKQvhiCsRORdhvR+RcOWOT4kA-QJAYARA9BWWtehORSxOb+Bx3RRxsJiRBRgcz4yJqJ6JTWxRvObh4eHhgu0QwupY1CWQ7CLg6BR6MQJoLcea0QVkIQZy+B4G++BJ2e-RZxiJEAZJaJD+aRNuGRh2WRXR0J8RhJJxxJEpUpFJS6kBT2ZRHE54ZYO0TJWaLJZo7JDq32KEIkAKgQqeUJzGKpoppB4p5ESJKJ0pVmfaIxcgYxwmomdEFsTEXx8B7wCcERbSdIh4O0qmRGzCgkEMtiJGiZeBex4Os2PRTppxySCJrpkp7p4+wxACQ4MYcYzxrxkx4gb0sg0ysyMyCydJcq2QkMX2rSym6ynsHJDJ7CycqhlYSB16HRzeeJypGZ0OWA2ADA6A8U1WKisiAA7rQJqb0LANIKGKkViXllYUOVyiOSKWObABOVOTOfOYufmcuauS4SUTScGTcIJP5naJhGVJkAUGeGDH4X4DulaO4AUHWIKU7viUZscewOOWwEeUlrOdQAuUuSuWuUUY-nKcBgqbiTuQ6aORrsBQeaBdOeBSedBReRIdSdAe4bMV8o7PedZFhGRi+ZLoVJhGWFkO8DplRe0YqcOahXWrcuikaiatiuapag5iRXKpQnxDZP9qtAjC8BwimYOewUqexQGsGCMtgBAMIF0MAk8XxVajeT4QePDGaSDqgjRdcOgSaKWlZDuucBAqkMUA6F5AGPAOsKxTuYJfpAALS2J8R2IljsLeC2Tz7hC+D2keQuWAzhDgpGTwTkZYSeBKyEY7TOoAmepOVMYeQw4VAOW6SyGEjlT0URVLTkaPBWinoTbBblSMpBUKRdSW4hWUxGhHglSKaJnuzWi0hGVJBGhlh3DQr0auosXIUpWVV4yopKQ1VSx1X3CZDoF5rNWiVtXzF2K6gg7PBqxhAVWkQkk9ihijXZWKzcSyzLQ3DJz3CL6ljwzZCvxrUPgbXKSwUYlQDbWIC2RtKmhvzxUcjwzGiqbWjZDlgnDuBZriS7EyX7EoWpWKQUS9jDXPgPUIBPUmhHjiSwI+KfUOwxD3AwLlShHJzsiXUtDXWUTsD4C0CoACriJJJM4QAw1w0vWI12jI3WSNL7gtFNIPmMy42mZIbzpdQw29a5WLS0zyw8mMnoFsgIxLzs0cbIZcZDVwVU3oGGmHLeJHjnABBvmhnmitKCTuCslJX9V9KpWS1c1DWcWQBy0YF7olhMjK2WSMIZD+bRXqwZCRES3vpzqfq3SE3E2k3k3I6U2LjSZSy2Tm2K1W35A220V3DOyRFPBMgQrWj2lD4ZX-RZVJC7KxytJOrWibSVToSewJ2UGy3+1wEICy7lgli2RL6eCr4-ZFh5o+BPAbSZDbQ1hA3JV9KJ1Q2m1F0p0IDUJKzlTnDwxnAQqNL5BKyOxvCr65ocj2mGYw2uXkYcjNK80CTuwNGt163pl7ka4w1Zp8TZCmWr5aEpB5oBWz0wkLaG3u3igw01i+atJ+AWiMxhV8nn2OnQ5X0oZfpoaF2ZVzFPAxl0iUpXraYxBGlv1oUmaf3S1oY8Yw3ZCJ5vw2SULuw2TsJHrMhNwvxZpNIFB7gQPb19HOl8EZbwPkZWQpAHgxDWh6gnKarA1pmEGAWqk8En705n6-3J3-3WjDaUM7QAm+XeCbIEPMOZm8Hd5n4F4X6DJJ01yppsgFpYHOz0J3CezoQljBAiNI5wm-7sP-7GaCEyNUGDLOE80bKKMQx+DGjwyZCBBmgz1-nZHyXaNElsNe5SOUFD6FF3VU3h3GXiSQysqKYJ5tKKZaOu6uO6PuMCHSPXLGO6IpFwPd1zGKbIRWiFr0ab5S4rxKwWhPCK2sgDlt1b2iMd5ikkOSMxOePX5l5iHV5mM12PW5Ay5TXiTvC+IiThN5GRNkF6MUEgFX4DOiHgHVXJNfJvCq2FTRBIJPBvxWgCTgOONyW5HcFGEVP6MGGGNxNeNgF0EtByI81nUy5hBWSZAljoS0hgytkHjWlMKLUlhdOrPlMSMbO9FbPG41O0HiFE0k3oBk0MAU2HPrLHOYNnOD2XOFToE+AVFNItzoSYSPNAVrMvOelgA82BBgwKxR1o2YSnhmiIssNrOmNjNCVp3XDGidWoLsjWRO1hNLNsUrNIvlPeOUn3Ukupo8P1EezFpnia3JArwbIEtiPwnnH7N9ow2YQeI1g1i2KMUpAM31Fsk+DrL+GWS7pbSaP0soWMuEvMvXWamyNgIUJkuuDiQTX+GA13A0YcpatMY6vCtOH6v5kP482cvXBGhMj93eXWg9lGj0PFNMMuNqkisanOuou32QLGgpDvx03z4FptLNlV1tICRkjSUBsZ6lPoUgWTnYWoAQVQVnkwWGsB2EhGhgzA5R2cLux+FYRCv7mHm5v5unnknnm3Wsvosr5nj0g5CWRBDry2R9XHbLMX31tYXHmQXNtomttitovst2ohCmizPUuoRRWr0fnRmshMKr5pub2Bsm1+1-1fJPBrR0jYSK6UI5CqZvCHjpDCQjatKYSDudEMssaKXWBd2Htyp9vljWmNzIOWQliKP13+EkjGhuz8M2WFBAA */ createMachine< DatasetQualityControllerContext, DatasetQualityControllerEvent, @@ -48,66 +57,105 @@ export const createPureDatasetQualityControllerStateMachine = ( id: 'DatasetQualityController', type: 'parallel', states: { - datasets: { - initial: 'fetching', + stats: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDataStreamStats', - onDone: { - target: 'loaded', - actions: ['storeDataStreamStats', 'storeDatasets'], + datasets: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDataStreamStats', + onDone: { + target: 'loaded', + actions: ['storeDataStreamStats', 'storeDatasets'], + }, + onError: { + target: 'loaded', + actions: ['notifyFetchDatasetStatsFailed'], + }, + }, }, - onError: { - target: 'loaded', - actions: ['notifyFetchDatasetStatsFailed'], + loaded: {}, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'datasets.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'datasets.fetching', }, }, }, - loaded: {}, - }, - on: { - UPDATE_TIME_RANGE: { - target: 'datasets.fetching', - actions: ['storeTimeRange'], - }, - REFRESH_DATA: { - target: 'datasets.fetching', - }, - }, - }, - degradedDocs: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadDegradedDocs', - onDone: { - target: 'loaded', + degradedDocs: { + initial: 'fetching', + states: { + fetching: { + ...generateInvokePerType({ + src: 'loadDegradedDocs', + }), + }, + loaded: {}, + unauthorized: { type: 'final' }, + }, + on: { + SAVE_DEGRADED_DOCS_STATS: { + target: 'degradedDocs.loaded', actions: ['storeDegradedDocStats', 'storeDatasets'], }, - onError: [ + NOTIFY_DEGRADED_DOCS_STATS_FAILED: [ { - target: 'unauthorized', + target: 'degradedDocs.unauthorized', cond: 'checkIfActionForbidden', }, { - target: 'loaded', + target: 'degradedDocs.loaded', actions: ['notifyFetchDegradedStatsFailed'], }, ], + UPDATE_TIME_RANGE: { + target: 'degradedDocs.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'degradedDocs.fetching', + }, }, }, - loaded: {}, - unauthorized: { type: 'final' }, - }, - on: { - UPDATE_TIME_RANGE: { - target: 'degradedDocs.fetching', - actions: ['storeTimeRange'], - }, - REFRESH_DATA: { - target: 'degradedDocs.fetching', + nonAggregatableDatasets: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadNonAggregatableDatasets', + onDone: { + target: 'loaded', + actions: ['storeNonAggregatableDatasets'], + }, + onError: [ + { + target: 'unauthorized', + cond: 'checkIfActionForbidden', + }, + { + target: 'loaded', + actions: ['notifyFetchNonAggregatableDatasetsFailed'], + }, + ], + }, + }, + loaded: {}, + unauthorized: { type: 'final' }, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'nonAggregatableDatasets.fetching', + }, + REFRESH_DATA: { + target: 'nonAggregatableDatasets.fetching', + }, + }, }, }, }, @@ -168,45 +216,15 @@ export const createPureDatasetQualityControllerStateMachine = ( target: 'integrations.loaded', actions: ['storeQualities'], }, + UPDATE_TYPES: { + target: '#DatasetQualityController.stats', + actions: ['storeTypes'], + }, UPDATE_QUERY: { actions: ['storeQuery'], }, }, }, - nonAggregatableDatasets: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadNonAggregatableDatasets', - onDone: { - target: 'loaded', - actions: ['storeNonAggregatableDatasets'], - }, - onError: [ - { - target: 'unauthorized', - cond: 'checkIfActionForbidden', - }, - { - target: 'loaded', - actions: ['notifyFetchNonAggregatableDatasetsFailed'], - }, - ], - }, - }, - loaded: {}, - unauthorized: { type: 'final' }, - }, - on: { - UPDATE_TIME_RANGE: { - target: 'nonAggregatableDatasets.fetching', - }, - REFRESH_DATA: { - target: 'nonAggregatableDatasets.fetching', - }, - }, - }, }, }, { @@ -280,6 +298,16 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeTypes: assign((context, event) => { + return 'types' in event + ? { + filters: { + ...context.filters, + types: event.types, + }, + } + : {}; + }), storeQuery: assign((context, event) => { return 'query' in event ? { @@ -292,42 +320,37 @@ export const createPureDatasetQualityControllerStateMachine = ( }), storeDataStreamStats: assign( (_context, event: DoneInvokeEvent) => { - if ('data' in event && 'dataStreamsStats' in event.data) { - const dataStreamStats = event.data.dataStreamsStats as DataStreamStat[]; - const datasetUserPrivileges = event.data.datasetUserPrivileges; + const dataStreamStats = event.data.dataStreamsStats as DataStreamStat[]; + const datasetUserPrivileges = event.data.datasetUserPrivileges; - // Check if any DataStreamStat has null; to check for serverless - const isSizeStatsAvailable = - !dataStreamStats.length || dataStreamStats.some((stat) => stat.totalDocs !== null); + // Check if any DataStreamStat has null; to check for serverless + const isSizeStatsAvailable = + !dataStreamStats.length || dataStreamStats.some((stat) => stat.totalDocs !== null); - return { - dataStreamStats, - isSizeStatsAvailable, - datasetUserPrivileges, - }; - } - return {}; + return { + dataStreamStats, + isSizeStatsAvailable, + datasetUserPrivileges, + }; } ), - storeDegradedDocStats: assign((_context, event) => { - return 'data' in event - ? { - degradedDocStats: event.data as DegradedDocsStat[], - } - : {}; - }), - storeNonAggregatableDatasets: assign( - ( - _context: DefaultDatasetQualityControllerState, - event: DoneInvokeEvent - ) => { - return 'data' in event - ? { - nonAggregatableDatasets: event.data.datasets, - } - : {}; + storeDegradedDocStats: assign( + (context, event: DoneInvokeEvent, meta) => { + const type = meta._event.origin as DataStreamType; + + return { + degradedDocStats: { + ...context.degradedDocStats, + [type]: event.data, + }, + }; } ), + storeNonAggregatableDatasets: assign( + (_context, event: DoneInvokeEvent) => ({ + nonAggregatableDatasets: event.data.datasets, + }) + ), storeIntegrations: assign((_context, event) => { return 'data' in event ? { @@ -341,7 +364,7 @@ export const createPureDatasetQualityControllerStateMachine = ( }; }), storeDatasets: assign((context, _event) => { - return context.integrations && (context.dataStreamStats || context.degradedDocStats) + return context.integrations ? { datasets: generateDatasets( context.dataStreamStats, @@ -388,35 +411,49 @@ export const createDatasetQualityControllerStateMachine = ({ fetchIntegrationsFailedNotifier(toasts, event.data), }, services: { - loadDataStreamStats: (context) => + loadDataStreamStats: (context, _event) => dataStreamStatsClient.getDataStreamsStats({ - type: context.type as GetDataStreamsStatsQuery['type'], + types: context.filters.types as DataStreamType[], datasetQuery: context.filters.query, }), - loadDegradedDocs: (context) => { - const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); + loadDegradedDocs: + (context, _event, { data: { type } }) => + async (send) => { + try { + const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); - return dataStreamStatsClient.getDataStreamsDegradedStats({ - type: context.type as GetDataStreamsStatsQuery['type'], - datasetQuery: context.filters.query, - start, - end, - }); - }, - loadIntegrations: (context) => { - return dataStreamStatsClient.getIntegrations({ - type: context.type as GetIntegrationsParams['query']['type'], - }); - }, + const degradedDocsStats = await (isTypeSelected(type, context) + ? dataStreamStatsClient.getDataStreamsDegradedStats({ + type, + datasetQuery: context.filters.query, + start, + end, + }) + : Promise.resolve([])); + + send({ + type: 'SAVE_DEGRADED_DOCS_STATS', + data: degradedDocsStats, + }); + } catch (e) { + send({ + type: 'NOTIFY_DEGRADED_DOCS_STATS_FAILED', + data: e, + }); + } + }, loadNonAggregatableDatasets: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); return dataStreamStatsClient.getNonAggregatableDatasets({ - type: context.type as GetNonAggregatableDataStreamsParams['type'], + types: context.filters.types as DataStreamType[], start, end, }); }, + loadIntegrations: () => { + return dataStreamStatsClient.getIntegrations(); + }, }, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index a03bddcc9e93..624fef066afc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -6,18 +6,23 @@ */ import { DoneInvokeEvent } from 'xstate'; -import { QualityIndicators, TableCriteria, TimeRangeConfig } from '../../../../common/types'; import { DatasetUserPrivileges, NonAggregatableDatasets } from '../../../../common/api_types'; -import { Integration } from '../../../../common/data_streams_stats/integration'; -import { DatasetTableSortField } from '../../../hooks'; -import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; import { DataStreamDegradedDocsStatServiceResponse, DataStreamDetails, - DataStreamStatServiceResponse, DataStreamStat, + DataStreamStatServiceResponse, DataStreamStatType, } from '../../../../common/data_streams_stats'; +import { Integration } from '../../../../common/data_streams_stats/integration'; +import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; +import { + DataStreamType, + QualityIndicators, + TableCriteria, + TimeRangeConfig, +} from '../../../../common/types'; +import { DatasetTableSortField } from '../../../hooks'; interface FiltersCriteria { inactive: boolean; @@ -26,6 +31,7 @@ interface FiltersCriteria { integrations: string[]; namespaces: string[]; qualities: QualityIndicators[]; + types: string[]; query?: string; } @@ -37,13 +43,15 @@ export interface WithFilters { filters: FiltersCriteria; } +export type DictionaryType = Record; + export interface WithDataStreamStats { datasetUserPrivileges: DatasetUserPrivileges; dataStreamStats: DataStreamStatType[]; } export interface WithDegradedDocs { - degradedDocStats: DegradedDocsStat[]; + degradedDocStats: DictionaryType; } export interface WithNonAggregatableDatasets { @@ -59,9 +67,9 @@ export interface WithIntegrations { integrations: Integration[]; } -export type DefaultDatasetQualityControllerState = { type: string } & WithTableOptions & +export type DefaultDatasetQualityControllerState = WithTableOptions & WithDataStreamStats & - Partial & + WithDegradedDocs & WithDatasets & WithFilters & WithNonAggregatableDatasets & @@ -71,19 +79,19 @@ type DefaultDatasetQualityStateContext = DefaultDatasetQualityControllerState; export type DatasetQualityControllerTypeState = | { - value: 'datasets.fetching'; + value: 'stats.datasets.fetching'; context: DefaultDatasetQualityStateContext; } | { - value: 'datasets.loaded'; + value: 'stats.datasets.loaded'; context: DefaultDatasetQualityStateContext; } | { - value: 'datasets.loaded.idle'; + value: 'stats.degradedDocs.fetching'; context: DefaultDatasetQualityStateContext; } | { - value: 'degradedDocs.fetching'; + value: 'stats.nonAggregatableDatasets.fetching'; context: DefaultDatasetQualityStateContext; } | { @@ -135,6 +143,10 @@ export type DatasetQualityControllerEvent = type: 'UPDATE_QUERY'; query: string; } + | { + type: 'UPDATE_TYPES'; + types: DataStreamType[]; + } | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 74dca9499019..c18f3d479ee9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -427,7 +427,7 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ const { startDate: start, endDate: end } = getDateISORange(context.timeRange); return dataStreamStatsClient.getNonAggregatableDatasets({ - type, + types: [type], start, end, dataStream: context.dataStream, @@ -479,9 +479,7 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ }, loadDataStreamIntegration: (context) => { if ('dataStreamSettings' in context && context.dataStreamSettings?.integration) { - const { type } = indexNameToDataStreamParts(context.dataStream); return dataStreamDetailsClient.getDataStreamIntegration({ - type, integrationName: context.dataStreamSettings.integration, }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/flatten_stats.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/flatten_stats.ts new file mode 100644 index 000000000000..70a861191ba9 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/flatten_stats.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStreamType } from '../../common/types'; + +export function flattenStats( + stats: Record +): Array { + return Object.entries(stats).flatMap(([type, dataStreams]) => + dataStreams.map((dataStream) => ({ ...dataStream, type: type as DataStreamType })) + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index b6ce00628dd5..6f2e46baacf8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -8,7 +8,8 @@ import { indexNameToDataStreamParts } from '../../common/utils'; import { Integration } from '../../common/data_streams_stats/integration'; import { generateDatasets } from './generate_datasets'; -import { DataStreamStatType } from '../../common/data_streams_stats/types'; +import { DataStreamStatType } from '../../common/data_streams_stats'; +import { DEFAULT_DICTIONARY_TYPE } from '../state_machines/dataset_quality_controller'; describe('generateDatasets', () => { const integrations: Integration[] = [ @@ -56,25 +57,28 @@ describe('generateDatasets', () => { }, ]; - const degradedDocs = [ - { - dataset: 'logs-system.application-default', - percentage: 0, - count: 0, - docsCount: 0, - quality: 'good' as const, - }, - { - dataset: 'logs-synth-default', - percentage: 11.320754716981131, - count: 6, - docsCount: 0, - quality: 'poor' as const, - }, - ]; + const degradedDocs = { + ...DEFAULT_DICTIONARY_TYPE, + logs: [ + { + dataset: 'logs-system.application-default', + percentage: 0, + count: 0, + docsCount: 0, + quality: 'good' as const, + }, + { + dataset: 'logs-synth-default', + percentage: 11.320754716981131, + count: 6, + docsCount: 0, + quality: 'poor' as const, + }, + ], + }; it('merges integrations information with dataStreamStats', () => { - const datasets = generateDatasets(dataStreamStats, undefined, integrations); + const datasets = generateDatasets(dataStreamStats, DEFAULT_DICTIONARY_TYPE, integrations); expect(datasets).toEqual([ { @@ -87,10 +91,10 @@ describe('generateDatasets', () => { rawName: dataStreamStats[0].name, integration: integrations[0], degradedDocs: { - percentage: degradedDocs[0].percentage, - count: degradedDocs[0].count, - docsCount: degradedDocs[0].docsCount, - quality: degradedDocs[0].quality, + percentage: degradedDocs.logs[0].percentage, + count: degradedDocs.logs[0].count, + docsCount: degradedDocs.logs[0].docsCount, + quality: degradedDocs.logs[0].quality, }, }, { @@ -115,40 +119,42 @@ describe('generateDatasets', () => { expect(datasets).toEqual([ { - rawName: degradedDocs[0].dataset, - name: indexNameToDataStreamParts(degradedDocs[0].dataset).dataset, - type: indexNameToDataStreamParts(degradedDocs[0].dataset).type, + rawName: degradedDocs.logs[0].dataset, + name: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).dataset, + type: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).type, lastActivity: undefined, size: undefined, sizeBytes: undefined, userPrivileges: undefined, - namespace: indexNameToDataStreamParts(degradedDocs[0].dataset).namespace, + namespace: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).namespace, title: - integrations[0].datasets[indexNameToDataStreamParts(degradedDocs[0].dataset).dataset], + integrations[0].datasets[ + indexNameToDataStreamParts(degradedDocs.logs[0].dataset).dataset + ], integration: integrations[0], degradedDocs: { - percentage: degradedDocs[0].percentage, - count: degradedDocs[0].count, - docsCount: degradedDocs[0].docsCount, - quality: degradedDocs[0].quality, + percentage: degradedDocs.logs[0].percentage, + count: degradedDocs.logs[0].count, + docsCount: degradedDocs.logs[0].docsCount, + quality: degradedDocs.logs[0].quality, }, }, { - rawName: degradedDocs[1].dataset, - name: indexNameToDataStreamParts(degradedDocs[1].dataset).dataset, - type: indexNameToDataStreamParts(degradedDocs[1].dataset).type, + rawName: degradedDocs.logs[1].dataset, + name: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).dataset, + type: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).type, lastActivity: undefined, size: undefined, sizeBytes: undefined, userPrivileges: undefined, - namespace: indexNameToDataStreamParts(degradedDocs[1].dataset).namespace, - title: indexNameToDataStreamParts(degradedDocs[1].dataset).dataset, + namespace: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).namespace, + title: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).dataset, integration: undefined, degradedDocs: { - percentage: degradedDocs[1].percentage, - count: degradedDocs[1].count, - docsCount: degradedDocs[1].docsCount, - quality: degradedDocs[1].quality, + percentage: degradedDocs.logs[1].percentage, + count: degradedDocs.logs[1].count, + docsCount: degradedDocs.logs[1].docsCount, + quality: degradedDocs.logs[1].quality, }, }, ]); @@ -168,10 +174,10 @@ describe('generateDatasets', () => { rawName: dataStreamStats[0].name, integration: integrations[0], degradedDocs: { - percentage: degradedDocs[0].percentage, - count: degradedDocs[0].count, - docsCount: degradedDocs[0].docsCount, - quality: degradedDocs[0].quality, + percentage: degradedDocs.logs[0].percentage, + count: degradedDocs.logs[0].count, + docsCount: degradedDocs.logs[0].docsCount, + quality: degradedDocs.logs[0].quality, }, }, { @@ -182,10 +188,10 @@ describe('generateDatasets', () => { type: indexNameToDataStreamParts(dataStreamStats[1].name).type, rawName: dataStreamStats[1].name, degradedDocs: { - percentage: degradedDocs[1].percentage, - count: degradedDocs[1].count, - docsCount: degradedDocs[1].docsCount, - quality: degradedDocs[1].quality, + percentage: degradedDocs.logs[1].percentage, + count: degradedDocs.logs[1].count, + docsCount: degradedDocs.logs[1].docsCount, + quality: degradedDocs.logs[1].quality, }, }, ]); @@ -205,7 +211,7 @@ describe('generateDatasets', () => { }, }; - const datasets = generateDatasets([nonDefaultDataset], undefined, integrations); + const datasets = generateDatasets([nonDefaultDataset], DEFAULT_DICTIONARY_TYPE, integrations); expect(datasets).toEqual([ { @@ -225,8 +231,4 @@ describe('generateDatasets', () => { }, ]); }); - - it('returns an empty array if no valid object is provided', () => { - expect(generateDatasets(undefined, undefined, integrations)).toEqual([]); - }); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index ca6dc793d35e..fb479198bbac 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -10,10 +10,12 @@ import { mapPercentageToQuality } from '../../common/utils'; import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; +import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types'; +import { flattenStats } from './flatten_stats'; export function generateDatasets( dataStreamStats: DataStreamStatType[] = [], - degradedDocStats: DegradedDocsStat[] = [], + degradedDocStats: DictionaryType, integrations: Integration[] ): DataStreamStat[] { if (!dataStreamStats.length && !integrations.length) { @@ -48,8 +50,10 @@ export function generateDatasets( { datasetIntegrationMap: {}, integrationsMap: {} } ); + const degradedDocs = flattenStats(degradedDocStats); + if (!dataStreamStats.length) { - return degradedDocStats.map((degradedDocStat) => + return degradedDocs.map((degradedDocStat) => DataStreamStat.fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap }) ); } @@ -62,8 +66,8 @@ export function generateDatasets( docsCount: DegradedDocsStat['docsCount']; quality: DegradedDocsStat['quality']; } - > = degradedDocStats.reduce( - (degradedMapAcc, { dataset, percentage, count, docsCount, quality }) => + > = degradedDocs.reduce( + (degradedMapAcc, { dataset, percentage, count, docsCount }) => Object.assign(degradedMapAcc, { [dataset]: { percentage, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts index ae28384e7a16..907c0c0fdc05 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts @@ -40,13 +40,13 @@ describe('getDataStreams', () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); const result = await getDataStreams({ esClient: esClientMock, - type: 'logs', - datasetQuery: 'nginx', + types: ['logs'], + datasetQuery: 'nginx-*', uncategorisedOnly: true, }); expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith( expect.anything(), - 'logs-*nginx*' + 'logs-nginx-*' ); expect(result.datasetUserPrivileges.canMonitor).toBe(true); @@ -57,8 +57,8 @@ describe('getDataStreams', () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); const results = await getDataStreams({ esClient: esClientMock, - type: 'logs', - datasetQuery: 'nginx', + types: ['logs'], + datasetQuery: 'nginx-*', uncategorisedOnly: true, }); expect(results.items.length).toBe(1); @@ -67,8 +67,8 @@ describe('getDataStreams', () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); const results = await getDataStreams({ esClient: esClientMock, - type: 'logs', - datasetQuery: 'nginx', + types: ['logs'], + datasetQuery: 'nginx-*', uncategorisedOnly: false, }); expect(results.items.length).toBe(5); diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index 346946d8bed5..853a0cf208b4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -6,27 +6,28 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { DEFAULT_DATASET_TYPE } from '../../../../common/constants'; import { streamPartsToIndexPattern } from '../../../../common/utils'; import { DataStreamType } from '../../../../common/types'; import { dataStreamService, datasetQualityPrivileges } from '../../../services'; export async function getDataStreams(options: { esClient: ElasticsearchClient; - type?: DataStreamType; + types: DataStreamType[]; datasetQuery?: string; uncategorisedOnly: boolean; }) { - const { esClient, type = DEFAULT_DATASET_TYPE, datasetQuery, uncategorisedOnly } = options; + const { esClient, types, datasetQuery, uncategorisedOnly } = options; - const datasetName = streamPartsToIndexPattern({ - typePattern: type, - datasetPattern: datasetQuery ? `*${datasetQuery}*` : '*-*', - }); + const datasetNames = types.map((type) => + streamPartsToIndexPattern({ + typePattern: type, + datasetPattern: datasetQuery ? `${datasetQuery}` : '*-*', + }) + ); const datasetUserPrivileges = await datasetQualityPrivileges.getDatasetPrivileges( esClient, - datasetName + datasetNames.join(',') ); if (!datasetUserPrivileges.canMonitor) { @@ -36,7 +37,10 @@ export async function getDataStreams(options: { }; } - const allDataStreams = await dataStreamService.getMatchingDataStreams(esClient, datasetName); + const allDataStreams = await dataStreamService.getMatchingDataStreams( + esClient, + datasetNames.join(',') + ); const filteredDataStreams = uncategorisedOnly ? allDataStreams.filter((stream) => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts index b59999d0c3c2..6137bc5426f8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts @@ -8,28 +8,29 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { rangeQuery } from '@kbn/observability-plugin/server/utils/queries'; import { extractIndexNameFromBackingIndex } from '../../../common/utils'; -import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; import { _IGNORED } from '../../../common/es_fields'; import { DataStreamType } from '../../../common/types'; import { createDatasetQualityESClient } from '../../utils'; export async function getNonAggregatableDataStreams({ esClient, - type = DEFAULT_DATASET_TYPE, + types, start, end, dataStream, }: { esClient: ElasticsearchClient; - type?: DataStreamType; + types: DataStreamType[]; start: number; end: number; dataStream?: string; }) { const datasetQualityESClient = createDatasetQualityESClient(esClient); + const dataStreamTypes = types.map((type) => `${type}-*-*`).join(','); + const response = await datasetQualityESClient.fieldCaps({ - index: dataStream ?? `${type}-*-*`, + index: dataStream ?? dataStreamTypes, fields: [_IGNORED], index_filter: { ...rangeQuery(start, end)[0], diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 9862b11cf16e..54a229cb790e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -16,7 +16,7 @@ import { DegradedFieldResponse, DatasetUserPrivileges, } from '../../../common/api_types'; -import { rangeRt, typeRt } from '../../types/default_api_types'; +import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; import { datasetQualityPrivileges } from '../../services'; import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details'; @@ -30,7 +30,7 @@ const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', params: t.type({ query: t.intersection([ - typeRt, + t.type({ types: typesRt }), t.partial({ datasetQuery: t.string, }), @@ -59,6 +59,7 @@ const statsRoute = createDatasetQualityServerRoute({ const privilegedDataStreams = items.filter((stream) => { return stream.userPrivileges.canMonitor; }); + const dataStreamsStats = await getDataStreamsStats({ esClient, dataStreams: privilegedDataStreams.map((stream) => stream.name), @@ -116,7 +117,7 @@ const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({ params: t.type({ query: t.intersection([ rangeRt, - typeRt, + t.type({ types: typesRt }), t.partial({ dataStream: t.string, }), @@ -131,11 +132,36 @@ const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({ const esClient = coreContext.elasticsearch.client.asCurrentUser; + return await getNonAggregatableDataStreams({ + esClient, + ...params.query, + }); + }, +}); + +const nonAggregatableDatasetRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/non_aggregatable', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + query: t.intersection([rangeRt, typeRt]), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + await datasetQualityPrivileges.throwIfCannotReadDataset(esClient, params.query.type); return await getNonAggregatableDataStreams({ esClient, ...params.query, + types: [params.query.type], }); }, }); @@ -230,6 +256,7 @@ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, ...nonAggregatableDatasetsRoute, + ...nonAggregatableDatasetRoute, ...degradedFieldsRoute, ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts index 208ed7e70ab3..10e89db800a2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts @@ -9,16 +9,13 @@ import { Logger } from '@kbn/core/server'; import { PackageClient } from '@kbn/fleet-plugin/server'; import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors'; import { PackageListItem, RegistryDataStream } from '@kbn/fleet-plugin/common'; -import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; -import { DataStreamType } from '../../../common/types'; import { IntegrationType } from '../../../common/api_types'; export async function getIntegrations(options: { packageClient: PackageClient; logger: Logger; - type?: DataStreamType; }): Promise { - const { packageClient, logger, type = DEFAULT_DATASET_TYPE } = options; + const { packageClient, logger } = options; const packages = await packageClient.getPackages(); const installedPackages = packages.filter((p) => p.status === 'installed'); @@ -29,7 +26,7 @@ export async function getIntegrations(options: { title: p.title, version: p.version, icons: p.icons, - datasets: await getDatasets({ packageClient, logger, pkg: p, type }), + datasets: await getDatasets({ packageClient, logger, pkg: p }), })) ); @@ -40,9 +37,8 @@ const getDatasets = async (options: { packageClient: PackageClient; logger: Logger; pkg: PackageListItem; - type: DataStreamType; }) => { - const { packageClient, logger, pkg, type } = options; + const { packageClient, logger, pkg } = options; return ( (await fetchDatasets({ @@ -50,7 +46,6 @@ const getDatasets = async (options: { logger, name: pkg.name, version: pkg.version, - type, })) ?? getDatasetsReadableName(pkg.data_streams ?? []) ); }; @@ -60,16 +55,13 @@ const fetchDatasets = async (options: { logger: Logger; name: string; version: string; - type: DataStreamType; }) => { try { - const { packageClient, name, version, type } = options; + const { packageClient, name, version } = options; const pkg = await packageClient.getPackage(name, version); - return getDatasetsReadableName( - (pkg.packageInfo.data_streams ?? []).filter((ds) => ds.type === type) - ); + return getDatasetsReadableName(pkg.packageInfo.data_streams ?? []); } catch (error) { // Custom integration if (error instanceof PackageNotFoundError) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts index 1a21d7e1a14c..dfeb2f3329ba 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/routes.ts @@ -7,28 +7,24 @@ import * as t from 'io-ts'; import { IntegrationType, IntegrationDashboardsResponse } from '../../../common/api_types'; -import { typeRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; import { getIntegrations } from './get_integrations'; import { getIntegrationDashboards } from './get_integration_dashboards'; const integrationsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/integrations', - params: t.type({ - query: typeRt, - }), options: { tags: [], }, async handler(resources): Promise<{ integrations: IntegrationType[]; }> { - const { params, plugins, logger } = resources; + const { plugins, logger } = resources; const fleetPluginStart = await plugins.fleet.start(); const packageClient = fleetPluginStart.packageService.asInternalUser; - const integrations = await getIntegrations({ packageClient, logger, ...params.query }); + const integrations = await getIntegrations({ packageClient, logger }); return { integrations }; }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/types/default_api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/server/types/default_api_types.ts index 91eb8698dfca..c2b746e65528 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/types/default_api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/types/default_api_types.ts @@ -7,12 +7,24 @@ import { isoToEpochRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; -import { dataStreamTypesRt } from '../../common/types'; +import { DataStreamType, dataStreamTypesRt } from '../../common/types'; -export const typeRt = t.partial({ +export const typeRt = t.type({ type: dataStreamTypesRt, }); +export const typesRt = new t.Type( + 'typesRt', + (input: unknown): input is DataStreamType[] => + (typeof input === 'string' && input.split(',').every((value) => dataStreamTypesRt.is(value))) || + (Array.isArray(input) && input.every((value) => dataStreamTypesRt.is(value))), + (input, context) => + typeof input === 'string' && input.split(',').every((value) => dataStreamTypesRt.is(value)) + ? t.success(input.split(',') as DataStreamType[]) + : t.failure(input, context), + t.identity +); + export const rangeRt = t.type({ start: isoToEpochRt, end: isoToEpochRt, diff --git a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json index 04d5362bb986..8bec2d8cb1a6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json +++ b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json @@ -55,7 +55,8 @@ "@kbn/server-route-repository-utils", "@kbn/core-analytics-browser", "@kbn/core-lifecycle-browser", - "@kbn/core-notifications-browser" + "@kbn/core-notifications-browser", + "@kbn/rison" ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 86e3e5702baa..f0ef033d6323 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -14608,7 +14608,6 @@ "xpack.dataQuality.Initializing": "Page Initialisation de la qualité de l'ensemble de données", "xpack.dataQuality.name": "Qualité de l’ensemble de données", "xpack.datasetQuality.actionsColumnName": "Actions", - "xpack.datasetQuality.appDescription": "Surveillez la qualité du jeu de données pour les flux de données {logsPattern} suivant le {dsNamingSchemeLink}.", "xpack.datasetQuality.appDescription.dsNamingSchemeLinkText": "Schéma de dénomination du flux de données", "xpack.datasetQuality.appTitle": "Qualité de l’ensemble de données", "xpack.datasetQuality.betaBadgeDescription": "Cette fonctionnalité est actuellement en version bêta. Nous aimerions beaucoup savoir si vous avez des commentaires ou si vous rencontrez des bugs. Veuillez ouvrir un dossier d'assistance et/ou consulter notre forum de discussion.", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 0ab34991b457..b3be85aaa7bd 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14597,7 +14597,6 @@ "xpack.dataQuality.Initializing": "データセット品質ページを初期化中", "xpack.dataQuality.name": "データセット品質", "xpack.datasetQuality.actionsColumnName": "アクション", - "xpack.datasetQuality.appDescription": "{dsNamingSchemeLink}に従う{logsPattern}データストリームのデータセット品質を監視します。", "xpack.datasetQuality.appDescription.dsNamingSchemeLinkText": "データストリーム命名スキーム", "xpack.datasetQuality.appTitle": "データセット品質", "xpack.datasetQuality.betaBadgeDescription": "現在、この機能はベータです。バグが発生した場合やフィードバックがある場合は、お問い合わせください。サポート問題をオープンするか、ディスカッションフォーラムをご覧ください。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9698931c132f..6ffabdf583e7 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14620,7 +14620,6 @@ "xpack.dataQuality.Initializing": "正在初始化“数据集质量”页面", "xpack.dataQuality.name": "数据集质量", "xpack.datasetQuality.actionsColumnName": "操作", - "xpack.datasetQuality.appDescription": "监测跟随 {dsNamingSchemeLink} 的 {logsPattern} 数据流的数据集质量。", "xpack.datasetQuality.appDescription.dsNamingSchemeLinkText": "数据流命名方案", "xpack.datasetQuality.appTitle": "数据集质量", "xpack.datasetQuality.betaBadgeDescription": "此功能当前为公测版。如果遇到任何错误或有任何反馈，我们乐于倾听您的意见。请报告支持问题和/或访问我们的讨论论坛。", diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts index b01dc1f2375a..25a7f419f274 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts @@ -22,8 +22,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'GET /internal/dataset_quality/data_streams/stats', params: { query: { - type: 'logs', - datasetQuery: '-', + types: ['logs'], }, }, }); diff --git a/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts index 13011410a031..2176dbd1fa95 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/integrations/integrations.spec.ts @@ -39,11 +39,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { return await datasetQualityApiClient[user]({ endpoint: 'GET /internal/dataset_quality/integrations', - params: { - query: { - type: 'logs', - }, - }, }); } @@ -53,12 +48,13 @@ export default function ApiTest({ getService }: FtrProviderContext) { await Promise.all(integrationPackages.map((pkg) => installPackage({ supertest, pkg }))); }); - it('returns only log based integrations and its datasets map', async () => { + it('returns all installed integrations and its datasets map', async () => { const resp = await callApiAs(); expect(resp.body.integrations.map((integration) => integration.name)).to.eql([ 'apm', 'endpoint', + 'synthetics', 'system', ]);