From 1598523b4f537c3ee31701dbc073174e3ab2135f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 21:23:12 +0200 Subject: [PATCH] [ML] Explain Log Rate Spikes: Add mini histograms to grouped results table. (#141065) - Adds mini histograms to grouped results table. - Fixes row expansion issue where expanded row could show up under wrong row. --- x-pack/packages/ml/agg_utils/index.ts | 1 + x-pack/packages/ml/agg_utils/src/types.ts | 10 ++ .../api/explain_log_rate_spikes/actions.ts | 23 +++- .../api/explain_log_rate_spikes/index.ts | 1 + .../aiops/common/api/stream_reducer.ts | 9 ++ .../explain_log_rate_spikes_analysis.tsx | 5 +- .../spike_analysis_table_groups.tsx | 50 +++++++- .../server/routes/explain_log_rate_spikes.ts | 116 ++++++++++++++---- .../get_simple_hierarchical_tree.test.ts | 4 + .../queries/get_simple_hierarchical_tree.ts | 9 +- 10 files changed, 196 insertions(+), 32 deletions(-) diff --git a/x-pack/packages/ml/agg_utils/index.ts b/x-pack/packages/ml/agg_utils/index.ts index ac51405d0b8e..cc7a426f9405 100644 --- a/x-pack/packages/ml/agg_utils/index.ts +++ b/x-pack/packages/ml/agg_utils/index.ts @@ -16,6 +16,7 @@ export type { AggCardinality, ChangePoint, ChangePointGroup, + ChangePointGroupHistogram, ChangePointHistogram, ChangePointHistogramItem, HistogramField, diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index de993e4b9f32..a2c0d9f9a156 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair { histogram: ChangePointHistogramItem[]; } +/** + * Change point histogram data for a group of field/value pairs. + */ +export interface ChangePointGroupHistogram { + id: string; + histogram: ChangePointHistogramItem[]; +} + interface ChangePointGroupItem extends FieldValuePair { duplicate?: boolean; } @@ -95,7 +103,9 @@ interface ChangePointGroupItem extends FieldValuePair { * Tree leaves */ export interface ChangePointGroup { + id: string; group: ChangePointGroupItem[]; docCount: number; pValue: number | null; + histogram?: ChangePointHistogramItem[]; } diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts index 938c765d8e0d..e050946a489b 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts @@ -5,12 +5,18 @@ * 2.0. */ -import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils'; +import type { + ChangePoint, + ChangePointHistogram, + ChangePointGroup, + ChangePointGroupHistogram, +} from '@kbn/ml-agg-utils'; export const API_ACTION_NAME = { ADD_CHANGE_POINTS: 'add_change_points', ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram', ADD_CHANGE_POINTS_GROUP: 'add_change_point_group', + ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram', ADD_ERROR: 'add_error', RESET: 'reset', UPDATE_LOADING_STATE: 'update_loading_state', @@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou }; } +interface ApiActionAddChangePointsGroupHistogram { + type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM; + payload: ChangePointGroupHistogram[]; +} + +export function addChangePointsGroupHistogramAction( + payload: ApiActionAddChangePointsGroupHistogram['payload'] +): ApiActionAddChangePointsGroupHistogram { + return { + type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM, + payload, + }; +} + interface ApiActionAddError { type: typeof API_ACTION_NAME.ADD_ERROR; payload: string; @@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction = | ApiActionAddChangePoints | ApiActionAddChangePointsGroup | ApiActionAddChangePointsHistogram + | ApiActionAddChangePointsGroupHistogram | ApiActionAddError | ApiActionReset | ApiActionUpdateLoadingState; diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts index dbc6be43766c..5628b509980a 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts @@ -8,6 +8,7 @@ export { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, addErrorAction, resetAction, diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index ff275d1414e9..690db961f512 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -51,6 +51,15 @@ export function streamReducer( return { ...state, changePoints }; case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP: return { ...state, changePointsGroups: action.payload }; + case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM: + const changePointsGroups = state.changePointsGroups.map((cpg) => { + const cpHistogram = action.payload.find((h) => h.id === cpg.id); + if (cpHistogram) { + cpg.histogram = cpHistogram.histogram; + } + return cpg; + }); + return { ...state, changePointsGroups }; case API_ACTION_NAME.ADD_ERROR: return { ...state, errors: [...state.errors, action.payload] }; case API_ACTION_NAME.RESET: diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx index 350ab9f2e020..b317ac6f2fed 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx @@ -127,7 +127,7 @@ export const ExplainLogRateSpikesAnalysis: FC }, []); const groupTableItems = useMemo(() => { - const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => { + const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => { const sortedGroup = group.sort((a, b) => a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 ); @@ -144,11 +144,12 @@ export const ExplainLogRateSpikesAnalysis: FC }); return { - id: index, + id, docCount, pValue, group: dedupedGroup, repeatedValues, + histogram, }; }); diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index a046250db20b..37563fd2d43a 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -24,7 +24,11 @@ import { import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { ChangePoint } from '@kbn/ml-agg-utils'; + import { useEuiTheme } from '../../hooks/use_eui_theme'; + +import { MiniHistogram } from '../mini_histogram'; + import { SpikeAnalysisTable } from './spike_analysis_table'; const NARROW_COLUMN_WIDTH = '120px'; @@ -36,11 +40,12 @@ const DEFAULT_SORT_FIELD = 'pValue'; const DEFAULT_SORT_DIRECTION = 'asc'; interface GroupTableItem { - id: number; + id: string; docCount: number; pValue: number | null; group: Record; repeatedValues: Record; + histogram: ChangePoint['histogram']; } interface SpikeAnalysisTableProps { @@ -196,6 +201,39 @@ export const SpikeAnalysisGroupsTable: FC = ({ sortable: false, textOnly: true, }, + { + 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnLogRate', + width: NARROW_COLUMN_WIDTH, + field: 'pValue', + name: ( + + <> + + + + + ), + render: (_, { histogram, id }) => ( + + ), + sortable: false, + }, { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue', width: NARROW_COLUMN_WIDTH, @@ -226,9 +264,12 @@ export const SpikeAnalysisGroupsTable: FC = ({ { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount', field: 'docCount', - name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', { - defaultMessage: 'Doc count', - }), + name: i18n.translate( + 'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.docCountLabel', + { + defaultMessage: 'Doc count', + } + ), sortable: true, width: '20%', }, @@ -281,6 +322,7 @@ export const SpikeAnalysisGroupsTable: FC = ({ compressed columns={columns} items={pageOfItems} + itemId="id" itemIdToExpandedRowMap={itemIdToExpandedRowMap} onChange={onChange} pagination={pagination} diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index a6fecc0e1a87..48ea2dbddb1c 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -18,10 +18,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { streamFactory } from '@kbn/aiops-utils'; import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils'; import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, aiopsExplainLogRateSpikesSchema, addErrorAction, @@ -216,6 +218,21 @@ export const defineExplainLogRateSpikesRoute = ( return; } + const histogramFields: [NumericHistogramField] = [ + { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, + ]; + + const [overallTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + { match_all: {} }, + // fields + histogramFields, + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + if (groupingEnabled) { // To optimize the `frequent_items` query, we identify duplicate change points by count attributes. // Note this is a compromise and not 100% accurate because there could be change points that @@ -325,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = ( }); changePointGroups.push( - ...missingChangePoints.map((cp) => { + ...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => { const duplicates = groupedChangePoints.find((d) => - d.group.some( - (dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue - ) + d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue) ); if (duplicates !== undefined) { return { + id: `${stringHash( + JSON.stringify( + duplicates.group.map((d) => ({ + fieldName: d.fieldName, + fieldValue: d.fieldValue, + })) + ) + )}`, group: duplicates.group.map((d) => ({ fieldName: d.fieldName, fieldValue: d.fieldValue, duplicate: false, })), - docCount: cp.doc_count, - pValue: cp.pValue, + docCount, + pValue, }; } else { return { - group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }], - docCount: cp.doc_count, - pValue: cp.pValue, + id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`, + group: [ + { + fieldName, + fieldValue, + duplicate: false, + }, + ], + docCount, + pValue, }; } }) @@ -358,22 +388,62 @@ export const defineExplainLogRateSpikesRoute = ( if (maxItems > 1) { push(addChangePointsGroupAction(changePointGroups)); } - } - const histogramFields: [NumericHistogramField] = [ - { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, - ]; + if (changePointGroups) { + await asyncForEach(changePointGroups, async (cpg, index) => { + const histogramQuery = { + bool: { + filter: cpg.group.map((d) => ({ + term: { [d.fieldName]: d.fieldValue }, + })), + }, + }; - const [overallTimeSeries] = (await fetchHistogramsForFields( - client, - request.body.index, - { match_all: {} }, - // fields - histogramFields, - // samplerShardSize - -1, - undefined - )) as [NumericChartData]; + const [cpgTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + + const histogram = + overallTimeSeries.data.map((o, i) => { + const current = cpgTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_change_point: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + push( + addChangePointsGroupHistogramAction([ + { + id: cpg.id, + histogram, + }, + ]) + ); + }); + } + } // time series filtered by fields if (changePoints) { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts index 1ff92181dee9..5f2125a583db 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts @@ -11,6 +11,7 @@ import { getFieldValuePairCounts, markDuplicates } from './get_simple_hierarchic const changePointGroups: ChangePointGroup[] = [ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -25,6 +26,7 @@ const changePointGroups: ChangePointGroup[] = [ pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', @@ -64,6 +66,7 @@ describe('get_simple_hierarchical_tree', () => { expect(markedDuplicates).toEqual([ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -80,6 +83,7 @@ describe('get_simple_hierarchical_tree', () => { pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts index bebbb1302376..9f39d1eb11f6 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts @@ -8,6 +8,7 @@ // import { omit, uniq } from 'lodash'; import type { ChangePointGroup, FieldValuePair } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import type { ItemsetResult } from './fetch_frequent_items'; @@ -230,9 +231,13 @@ export function getSimpleHierarchicalTreeLeaves( leaves: ChangePointGroup[], level = 1 ) { - // console.log(`${'-'.repeat(level)} ${tree.name} ${tree.children.length}`); if (tree.children.length === 0) { - leaves.push({ group: tree.set, docCount: tree.docCount, pValue: tree.pValue }); + leaves.push({ + id: `${stringHash(JSON.stringify(tree.set))}`, + group: tree.set, + docCount: tree.docCount, + pValue: tree.pValue, + }); } else { for (const child of tree.children) { const newLeaves = getSimpleHierarchicalTreeLeaves(child, [], level + 1);