Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Explain Log Rate Spikes: Add mini histograms to grouped results table. #141065

Merged
merged 7 commits into from
Sep 20, 2022
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/packages/ml/agg_utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export type {
AggCardinality,
ChangePoint,
ChangePointGroup,
ChangePointGroupHistogram,
ChangePointHistogram,
ChangePointHistogramItem,
HistogramField,
Expand Down
10 changes: 10 additions & 0 deletions x-pack/packages/ml/agg_utils/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair {
histogram: ChangePointHistogramItem[];
}

/**
* Change point histogram data for a group of field/value pairs.
*/
export interface ChangePointGroupHistogram {
id: string;
histogram: ChangePointHistogramItem[];
}

interface ChangePointGroupItem extends FieldValuePair {
duplicate?: boolean;
}
Expand All @@ -95,7 +103,9 @@ interface ChangePointGroupItem extends FieldValuePair {
* Tree leaves
*/
export interface ChangePointGroup {
id: string;
group: ChangePointGroupItem[];
docCount: number;
pValue: number | null;
histogram?: ChangePointHistogramItem[];
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@
* 2.0.
*/

import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils';
import type {
ChangePoint,
ChangePointHistogram,
ChangePointGroup,
ChangePointGroupHistogram,
} from '@kbn/ml-agg-utils';

export const API_ACTION_NAME = {
ADD_CHANGE_POINTS: 'add_change_points',
ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram',
ADD_CHANGE_POINTS_GROUP: 'add_change_point_group',
ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram',
ADD_ERROR: 'add_error',
RESET: 'reset',
UPDATE_LOADING_STATE: 'update_loading_state',
Expand Down Expand Up @@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou
};
}

interface ApiActionAddChangePointsGroupHistogram {
type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM;
payload: ChangePointGroupHistogram[];
}

export function addChangePointsGroupHistogramAction(
payload: ApiActionAddChangePointsGroupHistogram['payload']
): ApiActionAddChangePointsGroupHistogram {
return {
type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM,
payload,
};
}

interface ApiActionAddError {
type: typeof API_ACTION_NAME.ADD_ERROR;
payload: string;
Expand Down Expand Up @@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction =
| ApiActionAddChangePoints
| ApiActionAddChangePointsGroup
| ApiActionAddChangePointsHistogram
| ApiActionAddChangePointsGroupHistogram
| ApiActionAddError
| ApiActionReset
| ApiActionUpdateLoadingState;
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
export {
addChangePointsAction,
addChangePointsGroupAction,
addChangePointsGroupHistogramAction,
addChangePointsHistogramAction,
addErrorAction,
resetAction,
Expand Down
9 changes: 9 additions & 0 deletions x-pack/plugins/aiops/common/api/stream_reducer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,15 @@ export function streamReducer(
return { ...state, changePoints };
case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP:
return { ...state, changePointsGroups: action.payload };
case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM:
const changePointsGroups = state.changePointsGroups.map((cpg) => {
const cpHistogram = action.payload.find((h) => h.id === cpg.id);
if (cpHistogram) {
cpg.histogram = cpHistogram.histogram;
}
return cpg;
});
return { ...state, changePointsGroups };
case API_ACTION_NAME.ADD_ERROR:
return { ...state, errors: [...state.errors, action.payload] };
case API_ACTION_NAME.RESET:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,30 +127,33 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
}, []);

const groupTableItems = useMemo(() => {
const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => {
const sortedGroup = group.sort((a, b) =>
a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0
);
const dedupedGroup: Record<string, any> = {};
const repeatedValues: Record<string, any> = {};
const tableItems = data.changePointsGroups.map(
({ group, docCount, histogram, pValue }, index) => {
const sortedGroup = group.sort((a, b) =>
a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0
);
const dedupedGroup: Record<string, any> = {};
const repeatedValues: Record<string, any> = {};

sortedGroup.forEach((pair) => {
const { fieldName, fieldValue } = pair;
if (pair.duplicate === false) {
dedupedGroup[fieldName] = fieldValue;
} else {
repeatedValues[fieldName] = fieldValue;
}
});
sortedGroup.forEach((pair) => {
const { fieldName, fieldValue } = pair;
if (pair.duplicate === false) {
dedupedGroup[fieldName] = fieldValue;
} else {
repeatedValues[fieldName] = fieldValue;
}
});

return {
id: index,
docCount,
pValue,
group: dedupedGroup,
repeatedValues,
};
});
return {
id: index,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like the id here can now be set to the group's id coming on the backend - this should fix the expanded row bug - I can fix this in a follow up so this can just go in but wanted to point it out 👍 Thanks, @walterra

docCount,
pValue,
group: dedupedGroup,
repeatedValues,
histogram,
};
}
);

return tableItems;
}, [data.changePointsGroups]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@ import {
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import type { ChangePoint } from '@kbn/ml-agg-utils';

import { useEuiTheme } from '../../hooks/use_eui_theme';

import { MiniHistogram } from '../mini_histogram';

import { SpikeAnalysisTableExpandedRow } from './spike_analysis_table_expanded_row';

const NARROW_COLUMN_WIDTH = '120px';
Expand All @@ -41,6 +45,7 @@ interface GroupTableItem {
pValue: number | null;
group: Record<string, any>;
repeatedValues: Record<string, any>;
histogram: ChangePoint['histogram'];
}

interface SpikeAnalysisTableProps {
Expand Down Expand Up @@ -182,6 +187,39 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
sortable: false,
textOnly: true,
},
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnLogRate',
width: NARROW_COLUMN_WIDTH,
field: 'pValue',
name: (
<EuiToolTip
position="top"
content={i18n.translate(
'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateColumnTooltip',
{
defaultMessage:
'A visual representation of the impact of the field on the message rate difference',
}
)}
>
<>
<FormattedMessage
id="xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateLabel"
defaultMessage="Log rate"
/>
<EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
</>
</EuiToolTip>
),
render: (_, { histogram, id }) => (
<MiniHistogram
chartData={histogram}
isLoading={loading && histogram === undefined}
label="Group x"
/>
),
sortable: false,
},
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue',
width: NARROW_COLUMN_WIDTH,
Expand Down Expand Up @@ -212,9 +250,12 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount',
field: 'docCount',
name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', {
defaultMessage: 'Doc count',
}),
name: i18n.translate(
'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.docCountLabel',
{
defaultMessage: 'Doc count',
}
),
sortable: true,
width: '20%',
},
Expand Down
116 changes: 93 additions & 23 deletions x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server';
import { streamFactory } from '@kbn/aiops-utils';
import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils';
import { fetchHistogramsForFields } from '@kbn/ml-agg-utils';
import { stringHash } from '@kbn/ml-string-hash';

import {
addChangePointsAction,
addChangePointsGroupAction,
addChangePointsGroupHistogramAction,
addChangePointsHistogramAction,
aiopsExplainLogRateSpikesSchema,
addErrorAction,
Expand Down Expand Up @@ -216,6 +218,21 @@ export const defineExplainLogRateSpikesRoute = (
return;
}

const histogramFields: [NumericHistogramField] = [
{ fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
];

const [overallTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
{ match_all: {} },
// fields
histogramFields,
// samplerShardSize
-1,
undefined
)) as [NumericChartData];

if (groupingEnabled) {
// To optimize the `frequent_items` query, we identify duplicate change points by count attributes.
// Note this is a compromise and not 100% accurate because there could be change points that
Expand Down Expand Up @@ -325,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = (
});

changePointGroups.push(
...missingChangePoints.map((cp) => {
...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => {
const duplicates = groupedChangePoints.find((d) =>
d.group.some(
(dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue
)
d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue)
);
if (duplicates !== undefined) {
return {
id: `${stringHash(
JSON.stringify(
duplicates.group.map((d) => ({
fieldName: d.fieldName,
fieldValue: d.fieldValue,
}))
)
)}`,
group: duplicates.group.map((d) => ({
fieldName: d.fieldName,
fieldValue: d.fieldValue,
duplicate: false,
})),
docCount: cp.doc_count,
pValue: cp.pValue,
docCount,
pValue,
};
} else {
return {
group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }],
docCount: cp.doc_count,
pValue: cp.pValue,
id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`,
group: [
{
fieldName,
fieldValue,
duplicate: false,
},
],
docCount,
pValue,
};
}
})
Expand All @@ -358,22 +388,62 @@ export const defineExplainLogRateSpikesRoute = (
if (maxItems > 1) {
push(addChangePointsGroupAction(changePointGroups));
}
}

const histogramFields: [NumericHistogramField] = [
{ fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
];
if (changePointGroups) {
await asyncForEach(changePointGroups, async (cpg, index) => {
const histogramQuery = {
bool: {
filter: cpg.group.map((d) => ({
term: { [d.fieldName]: d.fieldValue },
})),
},
};

const [overallTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
{ match_all: {} },
// fields
histogramFields,
// samplerShardSize
-1,
undefined
)) as [NumericChartData];
const [cpgTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
histogramQuery,
// fields
[
{
fieldName: request.body.timeFieldName,
type: KBN_FIELD_TYPES.DATE,
interval: overallTimeSeries.interval,
min: overallTimeSeries.stats[0],
max: overallTimeSeries.stats[1],
},
],
// samplerShardSize
-1,
undefined
)) as [NumericChartData];

const histogram =
overallTimeSeries.data.map((o, i) => {
const current = cpgTimeSeries.data.find(
(d1) => d1.key_as_string === o.key_as_string
) ?? {
doc_count: 0,
};
return {
key: o.key,
key_as_string: o.key_as_string ?? '',
doc_count_change_point: current.doc_count,
doc_count_overall: Math.max(0, o.doc_count - current.doc_count),
};
}) ?? [];

push(
addChangePointsGroupHistogramAction([
{
id: cpg.id,
histogram,
},
])
);
});
}
}

// time series filtered by fields
if (changePoints) {
Expand Down
Loading