Skip to content
This repository has been archived by the owner on Mar 31, 2024. It is now read-only.

Commit

Permalink
[ML] Plot chart points for all anomalies where no metric data (elasti…
Browse files Browse the repository at this point in the history 
…c#32645) (elastic#32662)

* [ML] Plot chart points for all anomalies where no metric data

* [ML] Clean up arrow functions following review
  • Loading branch information
@peteharverson
peteharverson authored Mar 7, 2019
1 parent 8aed798 commit 3422d51
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 89 deletions.
99 changes: 31 additions & 68 deletions x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,11 @@ export function explorerChartsContainerServiceFactory(callback) {
return [];
}

// Sort records in ascending time order matching up with chart data
records.sort((recordA, recordB) => {
return recordA[ML_TIME_FIELD_NAME] - recordB[ML_TIME_FIELD_NAME];
});

let chartData;
if (eventDistribution.length > 0 && records.length > 0) {
const filterField = records[0].by_field_value || records[0].over_field_value;
Expand All @@ -210,7 +215,7 @@ export function explorerChartsContainerServiceFactory(callback) {
// For rare chart values we are only interested wether a value is either `0` or not,
// `0` acts like a flag in the chart whether to display the dot/marker.
// All other charts (single metric, population) are metric based and with
// those a value of `null` acts as the flag to hide a datapoint.
// those a value of `null` acts as the flag to hide a data point.
if (
(chartType === CHART_TYPE.EVENT_DISTRIBUTION && value > 0) ||
(chartType !== CHART_TYPE.EVENT_DISTRIBUTION && value !== null)
Expand All @@ -234,51 +239,44 @@ export function explorerChartsContainerServiceFactory(callback) {
const chartDataForPointSearch = getChartDataForPointSearch(chartData, records[0], chartType);
_.each(records, (record) => {
// Look for a chart point with the same time as the record.
// If none found, find closest time in chartData set.
// If none found, insert a point for anomalies due to a gap in the data.
const recordTime = record[ML_TIME_FIELD_NAME];
let chartPoint = findNearestChartPointToTime(chartDataForPointSearch, recordTime);

let chartPoint = findChartPointForTime(chartDataForPointSearch, recordTime);
if (chartPoint === undefined) {
// In case there is a record with a time after that of the last chart point, set the score
// for the last chart point to that of the last record, if that record has a higher score.
const lastChartPoint = chartData[chartData.length - 1];
const lastChartPointScore = lastChartPoint.anomalyScore || 0;
if (record.record_score > lastChartPointScore) {
chartPoint = lastChartPoint;
}
chartPoint = { date: new Date(recordTime), value: null };
chartData.push(chartPoint);
}

if (chartPoint !== undefined) {
chartPoint.anomalyScore = record.record_score;
chartPoint.anomalyScore = record.record_score;

if (record.actual !== undefined) {
chartPoint.actual = record.actual;
chartPoint.typical = record.typical;
} else {
const causes = _.get(record, 'causes', []);
if (causes.length > 0) {
chartPoint.byFieldName = record.by_field_name;
chartPoint.numberOfCauses = causes.length;
if (causes.length === 1) {
// If only a single cause, copy actual and typical values to the top level.
const cause = _.first(record.causes);
chartPoint.actual = cause.actual;
chartPoint.typical = cause.typical;
}
if (record.actual !== undefined) {
chartPoint.actual = record.actual;
chartPoint.typical = record.typical;
} else {
const causes = _.get(record, 'causes', []);
if (causes.length > 0) {
chartPoint.byFieldName = record.by_field_name;
chartPoint.numberOfCauses = causes.length;
if (causes.length === 1) {
// If only a single cause, copy actual and typical values to the top level.
const cause = _.first(record.causes);
chartPoint.actual = cause.actual;
chartPoint.typical = cause.typical;
}
}
}

if (record.multi_bucket_impact !== undefined) {
chartPoint.multiBucketImpact = record.multi_bucket_impact;
}
if (record.multi_bucket_impact !== undefined) {
chartPoint.multiBucketImpact = record.multi_bucket_impact;
}

});

// Add a scheduledEvents property to any points in the chart data set
// which correspond to times of scheduled events for the job.
if (scheduledEvents !== undefined) {
_.each(scheduledEvents, (events, time) => {
const chartPoint = findNearestChartPointToTime(chartDataForPointSearch, Number(time));
const chartPoint = findChartPointForTime(chartDataForPointSearch, Number(time));
if (chartPoint !== undefined) {
// Note if the scheduled event coincides with an absence of the underlying metric data,
// we don't worry about plotting the event.
Expand All @@ -303,43 +301,8 @@ export function explorerChartsContainerServiceFactory(callback) {
return chartData;
}

function findNearestChartPointToTime(chartData, time) {
let chartPoint;
for (let i = 0; i < chartData.length; i++) {
if (chartData[i].date === time) {
chartPoint = chartData[i];
break;
}
}

if (chartPoint === undefined) {
// Find nearest point in time.
// loop through line items until the date is greater than bucketTime
// grab the current and previous items in the and compare the time differences
let foundItem;
for (let i = 0; i < chartData.length; i++) {
const itemTime = chartData[i].date;
if ((itemTime > time) && (i > 0)) {
const item = chartData[i];
const previousItem = (i > 0 ? chartData[i - 1] : null);

const diff1 = Math.abs(time - previousItem.date);
const diff2 = Math.abs(time - itemTime);

// foundItem should be the item with a date closest to bucketTime
if (previousItem === null || diff1 > diff2) {
foundItem = item;
} else {
foundItem = previousItem;
}
break;
}
}

chartPoint = foundItem;
}

return chartPoint;
function findChartPointForTime(chartData, time) {
return chartData.find(point => point.date === time);
}

Promise.all(seriesPromises)
Expand Down
5 changes: 3 additions & 2 deletions x-pack/plugins/ml/public/timeseriesexplorer/components/timeseries_chart/timeseries_chart.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1397,7 +1397,8 @@ const TimeseriesChartIntl = injectI18n(class TimeseriesChart extends React.Compo
// Highlights the anomaly marker in the focus chart corresponding to the specified record.

const {
focusChartData
focusChartData,
focusAggregationInterval
} = this.props;

const focusXScale = this.focusXScale;
Expand All @@ -1408,7 +1409,7 @@ const TimeseriesChartIntl = injectI18n(class TimeseriesChart extends React.Compo
// Depending on the way the chart is aggregated, there may not be
// a point at exactly the same time as the record being highlighted.
const anomalyTime = record.source.timestamp;
const markerToSelect = findChartPointForAnomalyTime(focusChartData, anomalyTime);
const markerToSelect = findChartPointForAnomalyTime(focusChartData, anomalyTime, focusAggregationInterval);

// Render an additional highlighted anomaly marker on the focus chart.
// TODO - plot anomaly markers for cases where there is an anomaly due
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,8 @@ module.controller('MlTimeSeriesExplorerController', function (
refreshFocusData.focusChartData = processDataForFocusAnomalies(
refreshFocusData.focusChartData,
refreshFocusData.anomalyRecords,
$scope.timeFieldName);
$scope.timeFieldName,
$scope.focusAggregationInterval);

refreshFocusData.focusChartData = processScheduledEventsForChart(
refreshFocusData.focusChartData,
Expand Down
54 changes: 36 additions & 18 deletions x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,36 @@ export function processRecordScoreResults(scoreData) {
export function processDataForFocusAnomalies(
chartData,
anomalyRecords,
timeFieldName) {
timeFieldName,
aggregationInterval) {

const timesToAddPointsFor = [];

// Iterate through the anomaly records making sure we have chart points for each anomaly.
const intervalMs = aggregationInterval.asMilliseconds();
let lastChartDataPointTime = undefined;
if (chartData !== undefined && chartData.length > 0) {
lastChartDataPointTime = chartData[chartData.length - 1].date.getTime();
}
anomalyRecords.forEach((record) => {
const recordTime = record[timeFieldName];
const chartPoint = findChartPointForAnomalyTime(chartData, recordTime, aggregationInterval);
if (chartPoint === undefined) {
const timeToAdd = (Math.floor(recordTime / intervalMs)) * intervalMs;
if (timesToAddPointsFor.indexOf(timeToAdd) === -1 && timeToAdd !== lastChartDataPointTime) {
timesToAddPointsFor.push(timeToAdd);
}
}
});

timesToAddPointsFor.sort((a, b) => a - b);

timesToAddPointsFor.forEach((time) => {
chartData.push({
date: new Date(time),
value: null
});
});

// Iterate through the anomaly records adding the
// various properties required for display.
Expand All @@ -106,20 +135,7 @@ export function processDataForFocusAnomalies(
// Look for a chart point with the same time as the record.
// If none found, find closest time in chartData set.
const recordTime = record[timeFieldName];
let chartPoint = findChartPointForAnomalyTime(chartData, recordTime);

// TODO - handle case where there is an anomaly due to the absence of data
// and there is no model plot.
if (chartPoint === undefined && chartData !== undefined && chartData.length) {
// In case there is a record with a time after that of the last chart point, set the score
// for the last chart point to that of the last record, if that record has a higher score.
const lastChartPoint = chartData[chartData.length - 1];
const lastChartPointScore = lastChartPoint.anomalyScore || 0;
if (record.record_score > lastChartPointScore) {
chartPoint = lastChartPoint;
}
}

const chartPoint = findChartPointForAnomalyTime(chartData, recordTime, aggregationInterval);
if (chartPoint !== undefined) {
// If chart aggregation interval > bucket span, there may be more than
// one anomaly record in the interval, so use the properties from
Expand Down Expand Up @@ -222,7 +238,7 @@ export function findNearestChartPointToTime(chartData, time) {

// Finds the chart point which corresponds to an anomaly with the
// specified time.
export function findChartPointForAnomalyTime(chartData, anomalyTime) {
export function findChartPointForAnomalyTime(chartData, anomalyTime, aggregationInterval) {
let chartPoint;
if(chartData === undefined) {
return chartPoint;
Expand All @@ -240,10 +256,12 @@ export function findChartPointForAnomalyTime(chartData, anomalyTime) {
// time of the anomaly. This is the start of the chart 'bucket'
// which contains the anomalous bucket.
let foundItem;
const intervalMs = aggregationInterval.asMilliseconds();
const anomalyTimeForAggInt = (Math.floor(anomalyTime / intervalMs)) * intervalMs;
for (let i = 0; i < chartData.length; i++) {
const itemTime = chartData[i].date.getTime();
if (itemTime > anomalyTime) {
foundItem = (i > 0) ? chartData[i - 1] : chartData[0];
if (itemTime === anomalyTimeForAggInt) {
foundItem = chartData[i];
break;
}
}
Expand Down

0 comments on commit 3422d51

Please sign in to comment.