[ML] Explain Log Rate Spikes: Add mini histograms to grouped results …

…table. (#141065)

- Adds mini histograms to grouped results table.
- Fixes row expansion issue where expanded row could show up under wrong row.

x-pack/packages/ml/agg_utils/index.ts

@@ -16,6 +16,7 @@ export type {
   AggCardinality,
   ChangePoint,
   ChangePointGroup,
+  ChangePointGroupHistogram,
   ChangePointHistogram,
   ChangePointHistogramItem,
   HistogramField,

x-pack/packages/ml/agg_utils/src/types.ts

@@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair {
   histogram: ChangePointHistogramItem[];
 }
 
+/**
+ * Change point histogram data for a group of field/value pairs.
+ */
+export interface ChangePointGroupHistogram {
+  id: string;
+  histogram: ChangePointHistogramItem[];
+}
+
 interface ChangePointGroupItem extends FieldValuePair {
   duplicate?: boolean;
 }
@@ -95,7 +103,9 @@ interface ChangePointGroupItem extends FieldValuePair {
  * Tree leaves
  */
 export interface ChangePointGroup {
+  id: string;
   group: ChangePointGroupItem[];
   docCount: number;
   pValue: number | null;
+  histogram?: ChangePointHistogramItem[];
 }

x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts

@@ -5,12 +5,18 @@
  * 2.0.
  */
 
-import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils';
+import type {
+  ChangePoint,
+  ChangePointHistogram,
+  ChangePointGroup,
+  ChangePointGroupHistogram,
+} from '@kbn/ml-agg-utils';
 
 export const API_ACTION_NAME = {
   ADD_CHANGE_POINTS: 'add_change_points',
   ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram',
   ADD_CHANGE_POINTS_GROUP: 'add_change_point_group',
+  ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram',
   ADD_ERROR: 'add_error',
   RESET: 'reset',
   UPDATE_LOADING_STATE: 'update_loading_state',
@@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou
   };
 }
 
+interface ApiActionAddChangePointsGroupHistogram {
+  type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM;
+  payload: ChangePointGroupHistogram[];
+}
+
+export function addChangePointsGroupHistogramAction(
+  payload: ApiActionAddChangePointsGroupHistogram['payload']
+): ApiActionAddChangePointsGroupHistogram {
+  return {
+    type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM,
+    payload,
+  };
+}
+
 interface ApiActionAddError {
   type: typeof API_ACTION_NAME.ADD_ERROR;
   payload: string;
@@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction =
   | ApiActionAddChangePoints
   | ApiActionAddChangePointsGroup
   | ApiActionAddChangePointsHistogram
+  | ApiActionAddChangePointsGroupHistogram
   | ApiActionAddError
   | ApiActionReset
   | ApiActionUpdateLoadingState;

x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts

@@ -8,6 +8,7 @@
 export {
   addChangePointsAction,
   addChangePointsGroupAction,
+  addChangePointsGroupHistogramAction,
   addChangePointsHistogramAction,
   addErrorAction,
   resetAction,

x-pack/plugins/aiops/common/api/stream_reducer.ts

@@ -51,6 +51,15 @@ export function streamReducer(
       return { ...state, changePoints };
     case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP:
       return { ...state, changePointsGroups: action.payload };
+    case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM:
+      const changePointsGroups = state.changePointsGroups.map((cpg) => {
+        const cpHistogram = action.payload.find((h) => h.id === cpg.id);
+        if (cpHistogram) {
+          cpg.histogram = cpHistogram.histogram;
+        }
+        return cpg;
+      });
+      return { ...state, changePointsGroups };
     case API_ACTION_NAME.ADD_ERROR:
       return { ...state, errors: [...state.errors, action.payload] };
     case API_ACTION_NAME.RESET:

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx

@@ -127,7 +127,7 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
   }, []);
 
   const groupTableItems = useMemo(() => {
-    const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => {
+    const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => {
       const sortedGroup = group.sort((a, b) =>
         a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0
       );
@@ -144,11 +144,12 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
       });
 
       return {
-        id: index,
+        id,
         docCount,
         pValue,
         group: dedupedGroup,
         repeatedValues,
+        histogram,
       };
     });
 

x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx

@@ -24,7 +24,11 @@ import {
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { ChangePoint } from '@kbn/ml-agg-utils';
+
 import { useEuiTheme } from '../../hooks/use_eui_theme';
+
+import { MiniHistogram } from '../mini_histogram';
+
 import { SpikeAnalysisTable } from './spike_analysis_table';
 
 const NARROW_COLUMN_WIDTH = '120px';
@@ -36,11 +40,12 @@ const DEFAULT_SORT_FIELD = 'pValue';
 const DEFAULT_SORT_DIRECTION = 'asc';
 
 interface GroupTableItem {
-  id: number;
+  id: string;
   docCount: number;
   pValue: number | null;
   group: Record<string, any>;
   repeatedValues: Record<string, any>;
+  histogram: ChangePoint['histogram'];
 }
 
 interface SpikeAnalysisTableProps {
@@ -196,6 +201,39 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
       sortable: false,
       textOnly: true,
     },
+    {
+      'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnLogRate',
+      width: NARROW_COLUMN_WIDTH,
+      field: 'pValue',
+      name: (
+        <EuiToolTip
+          position="top"
+          content={i18n.translate(
+            'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateColumnTooltip',
+            {
+              defaultMessage:
+                'A visual representation of the impact of the field on the message rate difference',
+            }
+          )}
+        >
+          <>
+            <FormattedMessage
+              id="xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateLabel"
+              defaultMessage="Log rate"
+            />
+            <EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
+          </>
+        </EuiToolTip>
+      ),
+      render: (_, { histogram, id }) => (
+        <MiniHistogram
+          chartData={histogram}
+          isLoading={loading && histogram === undefined}
+          label="Group x"
+        />
+      ),
+      sortable: false,
+    },
     {
       'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue',
       width: NARROW_COLUMN_WIDTH,
@@ -226,9 +264,12 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
     {
       'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount',
       field: 'docCount',
-      name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', {
-        defaultMessage: 'Doc count',
-      }),
+      name: i18n.translate(
+        'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.docCountLabel',
+        {
+          defaultMessage: 'Doc count',
+        }
+      ),
       sortable: true,
       width: '20%',
     },
@@ -281,6 +322,7 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
       compressed
       columns={columns}
       items={pageOfItems}
+      itemId="id"
       itemIdToExpandedRowMap={itemIdToExpandedRowMap}
       onChange={onChange}
       pagination={pagination}

x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts

@@ -18,10 +18,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server';
 import { streamFactory } from '@kbn/aiops-utils';
 import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils';
 import { fetchHistogramsForFields } from '@kbn/ml-agg-utils';
+import { stringHash } from '@kbn/ml-string-hash';
 
 import {
   addChangePointsAction,
   addChangePointsGroupAction,
+  addChangePointsGroupHistogramAction,
   addChangePointsHistogramAction,
   aiopsExplainLogRateSpikesSchema,
   addErrorAction,
@@ -216,6 +218,21 @@ export const defineExplainLogRateSpikesRoute = (
           return;
         }
 
+        const histogramFields: [NumericHistogramField] = [
+          { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
+        ];
+
+        const [overallTimeSeries] = (await fetchHistogramsForFields(
+          client,
+          request.body.index,
+          { match_all: {} },
+          // fields
+          histogramFields,
+          // samplerShardSize
+          -1,
+          undefined
+        )) as [NumericChartData];
+
         if (groupingEnabled) {
           // To optimize the `frequent_items` query, we identify duplicate change points by count attributes.
           // Note this is a compromise and not 100% accurate because there could be change points that
@@ -325,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = (
           });
 
           changePointGroups.push(
-            ...missingChangePoints.map((cp) => {
+            ...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => {
               const duplicates = groupedChangePoints.find((d) =>
-                d.group.some(
-                  (dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue
-                )
+                d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue)
               );
               if (duplicates !== undefined) {
                 return {
+                  id: `${stringHash(
+                    JSON.stringify(
+                      duplicates.group.map((d) => ({
+                        fieldName: d.fieldName,
+                        fieldValue: d.fieldValue,
+                      }))
+                    )
+                  )}`,
                   group: duplicates.group.map((d) => ({
                     fieldName: d.fieldName,
                     fieldValue: d.fieldValue,
                     duplicate: false,
                   })),
-                  docCount: cp.doc_count,
-                  pValue: cp.pValue,
+                  docCount,
+                  pValue,
                 };
               } else {
                 return {
-                  group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }],
-                  docCount: cp.doc_count,
-                  pValue: cp.pValue,
+                  id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`,
+                  group: [
+                    {
+                      fieldName,
+                      fieldValue,
+                      duplicate: false,
+                    },
+                  ],
+                  docCount,
+                  pValue,
                 };
               }
             })
@@ -358,22 +388,62 @@ export const defineExplainLogRateSpikesRoute = (
           if (maxItems > 1) {
             push(addChangePointsGroupAction(changePointGroups));
           }
-        }
 
-        const histogramFields: [NumericHistogramField] = [
-          { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
-        ];
+          if (changePointGroups) {
+            await asyncForEach(changePointGroups, async (cpg, index) => {
+              const histogramQuery = {
+                bool: {
+                  filter: cpg.group.map((d) => ({
+                    term: { [d.fieldName]: d.fieldValue },
+                  })),
+                },
+              };
 
-        const [overallTimeSeries] = (await fetchHistogramsForFields(
-          client,
-          request.body.index,
-          { match_all: {} },
-          // fields
-          histogramFields,
-          // samplerShardSize
-          -1,
-          undefined
-        )) as [NumericChartData];
+              const [cpgTimeSeries] = (await fetchHistogramsForFields(
+                client,
+                request.body.index,
+                histogramQuery,
+                // fields
+                [
+                  {
+                    fieldName: request.body.timeFieldName,
+                    type: KBN_FIELD_TYPES.DATE,
+                    interval: overallTimeSeries.interval,
+                    min: overallTimeSeries.stats[0],
+                    max: overallTimeSeries.stats[1],
+                  },
+                ],
+                // samplerShardSize
+                -1,
+                undefined
+              )) as [NumericChartData];
+
+              const histogram =
+                overallTimeSeries.data.map((o, i) => {
+                  const current = cpgTimeSeries.data.find(
+                    (d1) => d1.key_as_string === o.key_as_string
+                  ) ?? {
+                    doc_count: 0,
+                  };
+                  return {
+                    key: o.key,
+                    key_as_string: o.key_as_string ?? '',
+                    doc_count_change_point: current.doc_count,
+                    doc_count_overall: Math.max(0, o.doc_count - current.doc_count),
+                  };
+                }) ?? [];
+
+              push(
+                addChangePointsGroupHistogramAction([
+                  {
+                    id: cpg.id,
+                    histogram,
+                  },
+                ])
+              );
+            });
+          }
+        }
 
         // time series filtered by fields
         if (changePoints) {