From f63cb577bb0dff261f415856441d8183b07671ee Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 13:59:57 +0200 Subject: [PATCH 1/5] [aiops] Adds mini histogram to grouped table. --- x-pack/packages/ml/agg_utils/index.ts | 1 + x-pack/packages/ml/agg_utils/src/types.ts | 10 +++ .../api/explain_log_rate_spikes/actions.ts | 23 ++++- .../api/explain_log_rate_spikes/index.ts | 1 + .../aiops/common/api/stream_reducer.ts | 9 ++ .../explain_log_rate_spikes_analysis.tsx | 3 +- .../spike_analysis_table_groups.tsx | 43 ++++++++++ .../server/routes/explain_log_rate_spikes.ts | 84 +++++++++++++++---- .../queries/get_simple_hierarchical_tree.ts | 8 +- 9 files changed, 164 insertions(+), 18 deletions(-) diff --git a/x-pack/packages/ml/agg_utils/index.ts b/x-pack/packages/ml/agg_utils/index.ts index ac51405d0b8e..cc7a426f9405 100644 --- a/x-pack/packages/ml/agg_utils/index.ts +++ b/x-pack/packages/ml/agg_utils/index.ts @@ -16,6 +16,7 @@ export type { AggCardinality, ChangePoint, ChangePointGroup, + ChangePointGroupHistogram, ChangePointHistogram, ChangePointHistogramItem, HistogramField, diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index 0128d5641bf2..44a2496c3a71 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair { histogram: ChangePointHistogramItem[]; } +/** + * Change point histogram data for a group of field/value pairs. + */ +export interface ChangePointGroupHistogram { + id: string; + histogram: ChangePointHistogramItem[]; +} + interface ChangePointGroupItem extends FieldValuePair { duplicate?: boolean; } @@ -95,6 +103,8 @@ interface ChangePointGroupItem extends FieldValuePair { * Tree leaves */ export interface ChangePointGroup { + id: string; group: ChangePointGroupItem[]; docCount: number; + histogram?: ChangePointHistogramItem[]; } diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts index 938c765d8e0d..e050946a489b 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts @@ -5,12 +5,18 @@ * 2.0. */ -import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils'; +import type { + ChangePoint, + ChangePointHistogram, + ChangePointGroup, + ChangePointGroupHistogram, +} from '@kbn/ml-agg-utils'; export const API_ACTION_NAME = { ADD_CHANGE_POINTS: 'add_change_points', ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram', ADD_CHANGE_POINTS_GROUP: 'add_change_point_group', + ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram', ADD_ERROR: 'add_error', RESET: 'reset', UPDATE_LOADING_STATE: 'update_loading_state', @@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou }; } +interface ApiActionAddChangePointsGroupHistogram { + type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM; + payload: ChangePointGroupHistogram[]; +} + +export function addChangePointsGroupHistogramAction( + payload: ApiActionAddChangePointsGroupHistogram['payload'] +): ApiActionAddChangePointsGroupHistogram { + return { + type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM, + payload, + }; +} + interface ApiActionAddError { type: typeof API_ACTION_NAME.ADD_ERROR; payload: string; @@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction = | ApiActionAddChangePoints | ApiActionAddChangePointsGroup | ApiActionAddChangePointsHistogram + | ApiActionAddChangePointsGroupHistogram | ApiActionAddError | ApiActionReset | ApiActionUpdateLoadingState; diff --git a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts index dbc6be43766c..5628b509980a 100644 --- a/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts +++ b/x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts @@ -8,6 +8,7 @@ export { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, addErrorAction, resetAction, diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index ff275d1414e9..690db961f512 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -51,6 +51,15 @@ export function streamReducer( return { ...state, changePoints }; case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP: return { ...state, changePointsGroups: action.payload }; + case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM: + const changePointsGroups = state.changePointsGroups.map((cpg) => { + const cpHistogram = action.payload.find((h) => h.id === cpg.id); + if (cpHistogram) { + cpg.histogram = cpHistogram.histogram; + } + return cpg; + }); + return { ...state, changePointsGroups }; case API_ACTION_NAME.ADD_ERROR: return { ...state, errors: [...state.errors, action.payload] }; case API_ACTION_NAME.RESET: diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx index 98d6053a8b6f..379b7edc960e 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx @@ -124,7 +124,7 @@ export const ExplainLogRateSpikesAnalysis: FC }, []); const groupTableItems = useMemo(() => { - const tableItems = data.changePointsGroups.map(({ group, docCount }, index) => { + const tableItems = data.changePointsGroups.map(({ group, docCount, histogram }, index) => { const sortedGroup = group.sort((a, b) => a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 ); @@ -145,6 +145,7 @@ export const ExplainLogRateSpikesAnalysis: FC docCount, group: dedupedGroup, repeatedValues, + histogram, }; }); diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index 7b61911a9638..a03f7ac171c6 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -13,17 +13,26 @@ import { EuiBasicTable, EuiBasicTableColumn, EuiButtonIcon, + EuiIcon, EuiScreenReaderOnly, EuiSpacer, EuiTableSortingType, + EuiToolTip, RIGHT_ALIGNMENT, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import type { ChangePoint } from '@kbn/ml-agg-utils'; + import { useEuiTheme } from '../../hooks/use_eui_theme'; + +import { MiniHistogram } from '../mini_histogram'; + import { SpikeAnalysisTableExpandedRow } from './spike_analysis_table_expanded_row'; +const NARROW_COLUMN_WIDTH = '120px'; + const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; const DEFAULT_SORT_FIELD = 'docCount'; const DEFAULT_SORT_DIRECTION = 'desc'; @@ -32,6 +41,7 @@ interface GroupTableItem { docCount: number; group: Record; repeatedValues: Record; + histogram: ChangePoint['histogram']; } interface SpikeAnalysisTableProps { @@ -170,6 +180,39 @@ export const SpikeAnalysisGroupsTable: FC = ({ sortable: false, textOnly: true, }, + { + 'data-test-subj': 'aiopsSpikeAnalysisTableColumnLogRate', + width: NARROW_COLUMN_WIDTH, + field: 'pValue', + name: ( + + <> + + + + + ), + render: (_, { histogram, id }) => ( + + ), + sortable: false, + }, { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount', field: 'docCount', diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index f0fadf9476e7..afb98d562ed1 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -22,6 +22,7 @@ import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; import { addChangePointsAction, addChangePointsGroupAction, + addChangePointsGroupHistogramAction, addChangePointsHistogramAction, aiopsExplainLogRateSpikesSchema, addErrorAction, @@ -210,6 +211,21 @@ export const defineExplainLogRateSpikesRoute = ( return; } + const histogramFields: [NumericHistogramField] = [ + { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, + ]; + + const [overallTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + { match_all: {} }, + // fields + histogramFields, + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + if (groupingEnabled) { const { fields, df } = await fetchFrequentItems( client, @@ -236,22 +252,62 @@ export const defineExplainLogRateSpikesRoute = ( const changePointsGroups = getSimpleHierarchicalTreeLeaves(root, []); push(addChangePointsGroupAction(markDuplicates(changePointsGroups))); - } - const histogramFields: [NumericHistogramField] = [ - { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, - ]; + if (changePointsGroups) { + await asyncForEach(changePointsGroups, async (cpg, index) => { + const histogramQuery = { + bool: { + filter: cpg.group.map((d) => ({ + term: { [d.fieldName]: d.fieldValue }, + })), + }, + }; - const [overallTimeSeries] = (await fetchHistogramsForFields( - client, - request.body.index, - { match_all: {} }, - // fields - histogramFields, - // samplerShardSize - -1, - undefined - )) as [NumericChartData]; + const [cpgTimeSeries] = (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined + )) as [NumericChartData]; + + const histogram = + overallTimeSeries.data.map((o, i) => { + const current = cpgTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_change_point: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + push( + addChangePointsGroupHistogramAction([ + { + id: cpg.id, + histogram, + }, + ]) + ); + }); + } + } // time series filtered by fields if (changePoints) { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts index 7d00cc4fc3e0..28985be1e1e2 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts @@ -8,6 +8,7 @@ // import { omit, uniq } from 'lodash'; import type { ChangePointGroup, FieldValuePair } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import type { ItemsetResult } from './fetch_frequent_items'; @@ -224,9 +225,12 @@ export function getSimpleHierarchicalTreeLeaves( leaves: ChangePointGroup[], level = 1 ) { - // console.log(`${'-'.repeat(level)} ${tree.name} ${tree.children.length}`); if (tree.children.length === 0) { - leaves.push({ group: tree.set, docCount: tree.docCount }); + leaves.push({ + id: `${stringHash(JSON.stringify(tree.set))}`, + group: tree.set, + docCount: tree.docCount, + }); } else { for (const child of tree.children) { const newLeaves = getSimpleHierarchicalTreeLeaves(child, [], level + 1); From 87c5a0d4ccc0f882f18f6094803e73d595436e60 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 18:33:35 +0200 Subject: [PATCH 2/5] [aiops] Fix group ids for missing change points. --- .../server/routes/explain_log_rate_spikes.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index ae3fea7ff767..48ea2dbddb1c 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -18,6 +18,7 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { streamFactory } from '@kbn/aiops-utils'; import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils'; import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; +import { stringHash } from '@kbn/ml-string-hash'; import { addChangePointsAction, @@ -341,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = ( }); changePointGroups.push( - ...missingChangePoints.map((cp) => { + ...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => { const duplicates = groupedChangePoints.find((d) => - d.group.some( - (dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue - ) + d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue) ); if (duplicates !== undefined) { return { + id: `${stringHash( + JSON.stringify( + duplicates.group.map((d) => ({ + fieldName: d.fieldName, + fieldValue: d.fieldValue, + })) + ) + )}`, group: duplicates.group.map((d) => ({ fieldName: d.fieldName, fieldValue: d.fieldValue, duplicate: false, })), - docCount: cp.doc_count, - pValue: cp.pValue, + docCount, + pValue, }; } else { return { - group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }], - docCount: cp.doc_count, - pValue: cp.pValue, + id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`, + group: [ + { + fieldName, + fieldValue, + duplicate: false, + }, + ], + docCount, + pValue, }; } }) From 2673f65f3b9a244817f3d4581dd11907202ced68 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 19:11:38 +0200 Subject: [PATCH 3/5] [aiops] Fix jest tests. --- .../routes/queries/get_simple_hierarchical_tree.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts index 1ff92181dee9..5f2125a583db 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts @@ -11,6 +11,7 @@ import { getFieldValuePairCounts, markDuplicates } from './get_simple_hierarchic const changePointGroups: ChangePointGroup[] = [ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -25,6 +26,7 @@ const changePointGroups: ChangePointGroup[] = [ pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', @@ -64,6 +66,7 @@ describe('get_simple_hierarchical_tree', () => { expect(markedDuplicates).toEqual([ { + id: 'group-1', group: [ { fieldName: 'custom_field.keyword', @@ -80,6 +83,7 @@ describe('get_simple_hierarchical_tree', () => { pValue: 0.01, }, { + id: 'group-2', group: [ { fieldName: 'custom_field.keyword', From 03d77917084b7064ba1cea3fbc058cf7c7007037 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 20:10:17 +0200 Subject: [PATCH 4/5] [aiops] Fix table item group id. --- .../explain_log_rate_spikes_analysis.tsx | 48 +++++++++---------- .../spike_analysis_table_groups.tsx | 2 +- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx index 197834138f49..b317ac6f2fed 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx @@ -127,33 +127,31 @@ export const ExplainLogRateSpikesAnalysis: FC }, []); const groupTableItems = useMemo(() => { - const tableItems = data.changePointsGroups.map( - ({ group, docCount, histogram, pValue }, index) => { - const sortedGroup = group.sort((a, b) => - a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 - ); - const dedupedGroup: Record = {}; - const repeatedValues: Record = {}; + const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => { + const sortedGroup = group.sort((a, b) => + a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 + ); + const dedupedGroup: Record = {}; + const repeatedValues: Record = {}; - sortedGroup.forEach((pair) => { - const { fieldName, fieldValue } = pair; - if (pair.duplicate === false) { - dedupedGroup[fieldName] = fieldValue; - } else { - repeatedValues[fieldName] = fieldValue; - } - }); + sortedGroup.forEach((pair) => { + const { fieldName, fieldValue } = pair; + if (pair.duplicate === false) { + dedupedGroup[fieldName] = fieldValue; + } else { + repeatedValues[fieldName] = fieldValue; + } + }); - return { - id: index, - docCount, - pValue, - group: dedupedGroup, - repeatedValues, - histogram, - }; - } - ); + return { + id, + docCount, + pValue, + group: dedupedGroup, + repeatedValues, + histogram, + }; + }); return tableItems; }, [data.changePointsGroups]); diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index 840711928513..2e35ac5723ea 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -40,7 +40,7 @@ const DEFAULT_SORT_FIELD = 'pValue'; const DEFAULT_SORT_DIRECTION = 'asc'; interface GroupTableItem { - id: number; + id: string; docCount: number; pValue: number | null; group: Record; From 731a92e4daea91de8432a59a904c624aed6a094d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 20:16:39 +0200 Subject: [PATCH 5/5] [aiops] Adds missing table id. --- .../spike_analysis_table/spike_analysis_table_groups.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index 4b6d0d154dcc..37563fd2d43a 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -322,6 +322,7 @@ export const SpikeAnalysisGroupsTable: FC = ({ compressed columns={columns} items={pageOfItems} + itemId="id" itemIdToExpandedRowMap={itemIdToExpandedRowMap} onChange={onChange} pagination={pagination}