From 16722d9a28c9bc2495a8642d8d60b9fbe302505e Mon Sep 17 00:00:00 2001 From: Zacqary Adam Xeper Date: Mon, 6 Jul 2020 15:33:27 -0500 Subject: [PATCH] [Metrics UI] Fix a bug in Metric Threshold query filter construction (#70672) Co-authored-by: Elastic Machine # Conflicts: # x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts --- .../metric_threshold/lib/metric_query.test.ts | 59 ++++++++ .../metric_threshold/lib/metric_query.ts | 139 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts create mode 100644 x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts new file mode 100644 index 000000000000..3ad1031f574e --- /dev/null +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { MetricExpressionParams } from '../types'; +import { getElasticsearchMetricQuery } from './metric_query'; + +describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { + const expressionParams = { + metric: 'system.is.a.good.puppy.dog', + aggType: 'avg', + timeUnit: 'm', + timeSize: 1, + } as MetricExpressionParams; + + const timefield = '@timestamp'; + const groupBy = 'host.doggoname'; + + describe('when passed no filterQuery', () => { + const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, groupBy); + test('includes a range filter', () => { + expect( + searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) + ).toBeTruthy(); + }); + + test('includes a metric field filter', () => { + expect(searchBody.query.bool.filter).toMatchObject( + expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }]) + ); + }); + }); + + describe('when passed a filterQuery', () => { + const filterQuery = + // This is adapted from a real-world query that previously broke alerts + // We want to make sure it doesn't override any existing filters + '{"bool":{"filter":[{"bool":{"filter":[{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"bark*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}},{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"woof*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}'; + + const searchBody = getElasticsearchMetricQuery( + expressionParams, + timefield, + groupBy, + filterQuery + ); + test('includes a range filter', () => { + expect( + searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) + ).toBeTruthy(); + }); + + test('includes a metric field filter', () => { + expect(searchBody.query.bool.filter).toMatchObject( + expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }]) + ); + }); + }); +}); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts new file mode 100644 index 000000000000..15506a30529c --- /dev/null +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { networkTraffic } from '../../../../../common/inventory_models/shared/metrics/snapshot/network_traffic'; +import { MetricExpressionParams, Aggregators } from '../types'; +import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds'; +import { getDateHistogramOffset } from '../../../snapshot/query_helpers'; +import { createPercentileAggregation } from './create_percentile_aggregation'; + +const MINIMUM_BUCKETS = 5; + +const getParsedFilterQuery: (filterQuery: string | undefined) => Record | null = ( + filterQuery +) => { + if (!filterQuery) return null; + return JSON.parse(filterQuery); +}; + +export const getElasticsearchMetricQuery = ( + { metric, aggType, timeUnit, timeSize }: MetricExpressionParams, + timefield: string, + groupBy?: string | string[], + filterQuery?: string, + timeframe?: { start: number; end: number } +) => { + if (aggType === Aggregators.COUNT && metric) { + throw new Error('Cannot aggregate document count with a metric'); + } + if (aggType !== Aggregators.COUNT && !metric) { + throw new Error('Can only aggregate without a metric if using the document count aggregator'); + } + const interval = `${timeSize}${timeUnit}`; + const intervalAsSeconds = getIntervalInSeconds(interval); + + const to = timeframe ? timeframe.end : Date.now(); + // We need enough data for 5 buckets worth of data. We also need + // to convert the intervalAsSeconds to milliseconds. + const minimumFrom = to - intervalAsSeconds * 1000 * MINIMUM_BUCKETS; + + const from = timeframe && timeframe.start <= minimumFrom ? timeframe.start : minimumFrom; + + const offset = getDateHistogramOffset(from, interval); + + const aggregations = + aggType === Aggregators.COUNT + ? {} + : aggType === Aggregators.RATE + ? networkTraffic('aggregatedValue', metric) + : aggType === Aggregators.P95 || aggType === Aggregators.P99 + ? createPercentileAggregation(aggType, metric) + : { + aggregatedValue: { + [aggType]: { + field: metric, + }, + }, + }; + + const baseAggs = { + aggregatedIntervals: { + date_histogram: { + field: timefield, + fixed_interval: interval, + offset, + extended_bounds: { + min: from, + max: to, + }, + }, + aggregations, + }, + }; + + const aggs = groupBy + ? { + groupings: { + composite: { + size: 10, + sources: Array.isArray(groupBy) + ? groupBy.map((field, index) => ({ + [`groupBy${index}`]: { + terms: { field }, + }, + })) + : [ + { + groupBy0: { + terms: { + field: groupBy, + }, + }, + }, + ], + }, + aggs: baseAggs, + }, + } + : baseAggs; + + const rangeFilters = [ + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'epoch_millis', + }, + }, + }, + ]; + + const metricFieldFilters = metric + ? [ + { + exists: { + field: metric, + }, + }, + ] + : []; + + const parsedFilterQuery = getParsedFilterQuery(filterQuery); + + return { + query: { + bool: { + filter: [ + ...rangeFilters, + ...metricFieldFilters, + ...(parsedFilterQuery ? [parsedFilterQuery] : []), + ], + }, + }, + size: 0, + aggs, + }; +};