Skip to content

Commit

Permalink
[Dataset quality] Enable page for metrics and traces (#190043)
Browse files Browse the repository at this point in the history
Closes elastic/observability-dev#3454.

This PR enables Dataset quality for being used for `traces` and
`metrics`.

### Changes
- Added `KNOWN_TYPES` array containing the types that we allow in
dataset quality page.
- `datasets`, `degradedDocs` and `nonAggregatableDatasets` are now
wrapped in an state. This allow us to retrigger the calls whenever we
need to do it (e.g. when changing timeframe), more importantly now
whenever we select a new type of dataset.
- The `invoke` `degradedDocs` is created dynamically depending on the
types present in `KNOWN_TYPES`.
- `GET /internal/dataset_quality/data_streams/stats` and `GET
/internal/dataset_quality/data_streams/non_aggregatable` now accept an
array of `DataStreamType`. This allow us to query the information for
multiple types of dataStreams at the same time.
- degradedDocs are stored now locally as a nested structure. This nested
structure allow us to update just the needed portion instead of updating
all datasets whenever a change occurs (e.g. a type is deselected).
- redirectLink now takes into account the datastream type, it only
redirects to logs explorer if it's enabled and the type of the
datastream is logs.

### 🎥 Demo


https://github.com/user-attachments/assets/082bd4e9-a8f8-4af9-a425-267adc3b30df

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
yngrdyn and kibanamachine authored Aug 27, 2024
1 parent 8e9b827 commit 2441e9a
Show file tree
Hide file tree
Showing 39 changed files with 610 additions and 313 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export const filtersRT = rt.exact(
inactive: rt.boolean,
fullNames: rt.boolean,
timeRange: timeRangeRT,
types: rt.array(rt.string),
integrations: rt.array(rt.string),
namespaces: rt.array(rt.string),
qualities: rt.array(rt.union([rt.literal('poor'), rt.literal('degraded'), rt.literal('good')])),
Expand Down
14 changes: 14 additions & 0 deletions x-pack/plugins/data_quality/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,20 @@ export class DataQualityPlugin implements Plugin<void, void, any, any> {
['logs-*-*']: ['read'],
},
},
{
ui: [],
requiredClusterPrivileges: [],
requiredIndexPrivileges: {
['traces-*-*']: ['read'],
},
},
{
ui: [],
requiredClusterPrivileges: [],
requiredIndexPrivileges: {
['metrics-*-*']: ['read'],
},
},
],
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
* 2.0.
*/

import { QualityIndicators } from './types';
import { DataStreamType, QualityIndicators } from './types';

export const DATASET_QUALITY_APP_ID = 'dataset_quality';
export const DEFAULT_DATASET_TYPE = 'logs';
export const DEFAULT_DATASET_TYPE: DataStreamType = 'logs';
export const DEFAULT_LOGS_DATA_VIEW = 'logs-*-*';

export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3;
Expand Down Expand Up @@ -41,3 +41,5 @@ export const MAX_DEGRADED_FIELDS = 1000;

export const MASKED_FIELD_PLACEHOLDER = '<custom field>';
export const UNKOWN_FIELD_PLACEHOLDER = '<unkwon>';

export const KNOWN_TYPES: DataStreamType[] = ['logs', 'metrics', 'traces'];
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@
* 2.0.
*/

import { DataStreamType } from '../types';

export interface GetDataStreamIntegrationParams {
type: DataStreamType;
integrationName: string;
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@ export type GetDataStreamsStatsResponse =
export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0];
export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse;

export type GetIntegrationsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/integrations`>['params'];

export type GetDataStreamsDegradedDocsStatsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params'];
export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query'];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,33 @@ import { EuiSuperDatePicker } from '@elastic/eui';
import { UI_SETTINGS } from '@kbn/data-service';
import { TimePickerQuickRange } from '@kbn/observability-shared-plugin/public/hooks/use_quick_time_ranges';
import React, { useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { useDatasetQualityFilters } from '../../../hooks/use_dataset_quality_filters';
import { useKibanaContextForPlugin } from '../../../utils/use_kibana';
import { FilterBar } from './filter_bar';
import { IntegrationsSelector } from './integrations_selector';
import { NamespacesSelector } from './namespaces_selector';
import { QualitiesSelector } from './qualities_selector';
import { Selector } from './selector';

const typesLabel = i18n.translate('xpack.datasetQuality.types.label', {
defaultMessage: 'Types',
});

const typesSearchPlaceholder = i18n.translate(
'xpack.datasetQuality.selector.types.search.placeholder',
{
defaultMessage: 'Filter types',
}
);

const typesNoneMatching = i18n.translate('xpack.datasetQuality.selector.types.noneMatching', {
defaultMessage: 'No types found',
});

const typesNoneAvailable = i18n.translate('xpack.datasetQuality.selector.types.noneAvailable', {
defaultMessage: 'No types available',
});

// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
Expand All @@ -29,9 +50,11 @@ export default function Filters() {
integrations,
namespaces,
qualities,
types,
onIntegrationsChange,
onNamespacesChange,
onQualitiesChange,
onTypesChange,
selectedQuery,
onQueryChange,
} = useDatasetQualityFilters();
Expand Down Expand Up @@ -65,6 +88,14 @@ export default function Filters() {
integrations={integrations}
onIntegrationsChange={onIntegrationsChange}
/>
<Selector
label={typesLabel}
searchPlaceholder={typesSearchPlaceholder}
noneMatchingMessage={typesNoneMatching}
noneAvailableMessage={typesNoneAvailable}
options={types}
onOptionsChange={onTypesChange}
/>
<NamespacesSelector
isLoading={isLoading}
namespaces={namespaces}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { EuiFilterButton, EuiPopover, EuiPopoverTitle, EuiSelectable, EuiText } from '@elastic/eui';
import React, { useState } from 'react';
import type { EuiSelectableOptionCheckedType } from '@elastic/eui/src/components/selectable/selectable_option';
import { i18n } from '@kbn/i18n';

const selectorLoading = i18n.translate('xpack.datasetQuality.selector.loading', {
defaultMessage: 'Loading',
});

interface SelectorProps {
isLoading?: boolean;
options: Item[];
loadingMessage?: string;
label: string;
searchPlaceholder: string;
noneAvailableMessage: string;
noneMatchingMessage: string;
onOptionsChange: (options: Item[]) => void;
}

export interface Item {
label: string;
checked?: EuiSelectableOptionCheckedType;
}

export function Selector({
isLoading,
options,
loadingMessage,
label,
searchPlaceholder,
noneAvailableMessage,
noneMatchingMessage,
onOptionsChange,
}: SelectorProps) {
const [isPopoverOpen, setIsPopoverOpen] = useState(false);

const onButtonClick = () => {
setIsPopoverOpen(!isPopoverOpen);
};

const closePopover = () => {
setIsPopoverOpen(false);
};

const renderOption = (option: Item) => <EuiText size="s">{option.label}</EuiText>;

const button = (
<EuiFilterButton
data-test-subj="datasetQualitySelectableButton"
iconType="arrowDown"
badgeColor="success"
onClick={onButtonClick}
isSelected={isPopoverOpen}
numFilters={options.length}
hasActiveFilters={!!options.find((item) => item.checked === 'on')}
numActiveFilters={options.filter((item) => item.checked === 'on').length}
>
{label}
</EuiFilterButton>
);

return (
<EuiPopover
button={button}
isOpen={isPopoverOpen}
closePopover={closePopover}
panelPaddingSize="none"
>
<EuiSelectable
data-test-subj="datasetQualitySelectableOptions"
searchable
searchProps={{
placeholder: searchPlaceholder,
compressed: true,
}}
aria-label={label}
options={options}
onChange={onOptionsChange}
isLoading={isLoading}
loadingMessage={loadingMessage ?? selectorLoading}
emptyMessage={noneAvailableMessage}
noMatchesMessage={noneMatchingMessage}
renderOption={(option) => renderOption(option)}
>
{(list, search) => (
<div style={{ width: 300 }}>
<EuiPopoverTitle paddingSize="s">{search}</EuiPopoverTitle>
{list}
</div>
)}
</EuiSelectable>
</EuiPopover>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { EuiBetaBadge, EuiLink, EuiPageHeader, EuiCode } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';

import { DEFAULT_LOGS_DATA_VIEW } from '../../../common/constants';
import { KNOWN_TYPES } from '../../../common/constants';
import { datasetQualityAppTitle } from '../../../common/translations';

// Allow for lazy loading
Expand All @@ -33,9 +33,16 @@ export default function Header() {
description={
<FormattedMessage
id="xpack.datasetQuality.appDescription"
defaultMessage="Monitor the data set quality for {logsPattern} data streams that follow the {dsNamingSchemeLink}."
defaultMessage="Monitor the data set quality for {types} data streams that follow the {dsNamingSchemeLink}."
values={{
logsPattern: <EuiCode>{DEFAULT_LOGS_DATA_VIEW}</EuiCode>,
types: KNOWN_TYPES.map((type, index) => {
return (
<>
{index > 0 && ', '}
<EuiCode>{type}</EuiCode>
</>
);
}),
dsNamingSchemeLink: (
<EuiLink
data-test-subj="datasetQualityAppDescriptionDsNamingSchemeLink"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumn
defaultMessage: 'Namespace',
});

const typeColumnName = i18n.translate('xpack.datasetQuality.typeColumnName', {
defaultMessage: 'Type',
});

const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', {
defaultMessage: 'Size',
});
Expand Down Expand Up @@ -213,6 +217,15 @@ export const getDatasetQualityTableColumns = ({
),
width: '160px',
},
{
name: typeColumnName,
field: 'type',
sortable: true,
render: (_, dataStreamStat: DataStreamStat) => (
<EuiBadge color="hollow">{dataStreamStat.type}</EuiBadge>
),
width: '160px',
},
...(isSizeStatsAvailable && canUserMonitorDataset && canUserMonitorAnyDataStream
? [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import React, { useCallback, useEffect, useState } from 'react';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import {
EuiAccordion,
Expand Down Expand Up @@ -133,7 +134,10 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe
<EuiButtonIcon
display="base"
iconType="discoverApp"
aria-label="Discover"
aria-label={i18n.translate(
'xpack.datasetQuality.degradedDocs.euiButtonIcon.discoverLabel',
{ defaultMessage: 'Discover' }
)}
size="s"
data-test-subj="datasetQualityDetailsLinkToDiscover"
{...degradedDocLinkLogsExplorer.linkProps}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@
import { OnRefreshChangeProps } from '@elastic/eui';
import { useSelector } from '@xstate/react';
import { useCallback, useMemo } from 'react';
import { QualityIndicators } from '../../common/types';
import { KNOWN_TYPES } from '../../common/constants';
import { DataStreamType, QualityIndicators } from '../../common/types';
import { Integration } from '../../common/data_streams_stats/integration';
import { useDatasetQualityContext } from '../components/dataset_quality/context';
import { IntegrationItem } from '../components/dataset_quality/filters/integrations_selector';
import { NamespaceItem } from '../components/dataset_quality/filters/namespaces_selector';
import { QualityItem } from '../components/dataset_quality/filters/qualities_selector';
import { Item } from '../components/dataset_quality/filters/selector';

export const useDatasetQualityFilters = () => {
const { service } = useDatasetQualityContext();
Expand All @@ -22,13 +24,14 @@ export const useDatasetQualityFilters = () => {
service,
(state) =>
state.matches('integrations.fetching') &&
(state.matches('datasets.fetching') || state.matches('degradedDocs.fetching'))
(state.matches('stats.datasets.fetching') || state.matches('stats.degradedDocs.fetching'))
);

const {
timeRange,
integrations: selectedIntegrations,
namespaces: selectedNamespaces,
types: selectedTypes,
qualities: selectedQualities,
query: selectedQuery,
} = useSelector(service, (state) => state.context.filters);
Expand Down Expand Up @@ -169,6 +172,25 @@ export const useDatasetQualityFilters = () => {
[service]
);

const typeItems: Item[] = useMemo(() => {
return KNOWN_TYPES.map((type) => ({
label: type,
checked: selectedTypes.includes(type) ? 'on' : undefined,
}));
}, [selectedTypes]);

const onTypesChange = useCallback(
(newTypeItems: Item[]) => {
service.send({
type: 'UPDATE_TYPES',
types: newTypeItems
.filter((quality) => quality.checked === 'on')
.map((type) => type.label as DataStreamType),
});
},
[service]
);

const onQueryChange = useCallback(
(query: string) => {
service.send({
Expand All @@ -187,9 +209,11 @@ export const useDatasetQualityFilters = () => {
integrations: integrationItems,
namespaces: namespaceItems,
qualities: qualityItems,
types: typeItems,
onIntegrationsChange,
onNamespacesChange,
onQualitiesChange,
onTypesChange,
isLoading,
selectedQuery,
onQueryChange,
Expand Down
Loading

0 comments on commit 2441e9a

Please sign in to comment.