From 008421f170214648336bd9d5afb5cc794efbd82c Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 15 Oct 2021 09:33:06 +0200 Subject: [PATCH] [APM] Revert multi-metric ML job. (#114961) --- .../plugins/apm/common/anomaly_detection.ts | 2 - .../apm/common/utils/apm_ml_anomaly_query.ts | 25 ----- .../create_anomaly_detection_jobs.ts | 20 ++-- .../lib/service_map/get_service_anomalies.ts | 9 +- .../transactions/get_anomaly_data/fetcher.ts | 4 +- .../modules/apm_transaction/manifest.json | 18 ++-- .../apm_transaction/ml/apm_metrics.json | 53 ----------- .../ml/datafeed_apm_metrics.json | 95 ------------------- ...tafeed_high_mean_transaction_duration.json | 14 +++ .../ml/high_mean_transaction_duration.json | 35 +++++++ .../apis/ml/modules/recognize_module.ts | 2 +- .../apis/ml/modules/setup_module.ts | 10 +- 12 files changed, 76 insertions(+), 211 deletions(-) delete mode 100644 x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json diff --git a/x-pack/plugins/apm/common/anomaly_detection.ts b/x-pack/plugins/apm/common/anomaly_detection.ts index eb7c74a4540be..43a779407d2a4 100644 --- a/x-pack/plugins/apm/common/anomaly_detection.ts +++ b/x-pack/plugins/apm/common/anomaly_detection.ts @@ -33,8 +33,6 @@ export function getSeverityColor(score: number) { return mlGetSeverityColor(score); } -export const ML_TRANSACTION_LATENCY_DETECTOR_INDEX = 0; - export const ML_ERRORS = { INVALID_LICENSE: i18n.translate( 'xpack.apm.anomaly_detection.error.invalid_license', diff --git a/x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts b/x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts deleted file mode 100644 index 26b859d37cf7f..0000000000000 --- a/x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export function apmMlAnomalyQuery(detectorIndex: 0 | 1 | 2) { - return [ - { - bool: { - filter: [ - { - terms: { - result_type: ['model_plot', 'record'], - }, - }, - { - term: { detector_index: detectorIndex }, - }, - ], - }, - }, - ]; -} diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts index 10758b6d90cdc..4d4bc8dc185ab 100644 --- a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts @@ -5,21 +5,21 @@ * 2.0. */ -import Boom from '@hapi/boom'; import { Logger } from 'kibana/server'; +import uuid from 'uuid/v4'; import { snakeCase } from 'lodash'; +import Boom from '@hapi/boom'; import moment from 'moment'; -import uuid from 'uuid/v4'; import { ML_ERRORS } from '../../../common/anomaly_detection'; -import { - METRICSET_NAME, - PROCESSOR_EVENT, -} from '../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../common/processor_event'; import { environmentQuery } from '../../../common/utils/environment_query'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup } from '../helpers/setup_request'; +import { + TRANSACTION_DURATION, + PROCESSOR_EVENT, +} from '../../../common/elasticsearch_fieldnames'; import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants'; +import { withApmSpan } from '../../utils/with_apm_span'; import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs'; export async function createAnomalyDetectionJobs( @@ -92,8 +92,8 @@ async function createAnomalyDetectionJob({ query: { bool: { filter: [ - { term: { [PROCESSOR_EVENT]: ProcessorEvent.metric } }, - { term: { [METRICSET_NAME]: 'transaction' } }, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { exists: { field: TRANSACTION_DURATION } }, ...environmentQuery(environment), ], }, @@ -105,7 +105,7 @@ async function createAnomalyDetectionJob({ job_tags: { environment, // identifies this as an APM ML job & facilitates future migrations - apm_ml_version: 3, + apm_ml_version: 2, }, }, }, diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts index 97c95e4e40045..9b2d79dc726ee 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts @@ -11,11 +11,7 @@ import { estypes } from '@elastic/elasticsearch'; import { ESSearchResponse } from '../../../../../../src/core/types/elasticsearch'; import { MlPluginSetup } from '../../../../ml/server'; import { PromiseReturnType } from '../../../../observability/typings/common'; -import { - getSeverity, - ML_ERRORS, - ML_TRANSACTION_LATENCY_DETECTOR_INDEX, -} from '../../../common/anomaly_detection'; +import { getSeverity, ML_ERRORS } from '../../../common/anomaly_detection'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; import { getServiceHealthStatus } from '../../../common/service_health_status'; import { @@ -26,7 +22,6 @@ import { rangeQuery } from '../../../../observability/server'; import { withApmSpan } from '../../utils/with_apm_span'; import { getMlJobsWithAPMGroup } from '../anomaly_detection/get_ml_jobs_with_apm_group'; import { Setup } from '../helpers/setup_request'; -import { apmMlAnomalyQuery } from '../../../common/utils/apm_ml_anomaly_query'; export const DEFAULT_ANOMALIES: ServiceAnomaliesResponse = { mlJobIds: [], @@ -61,7 +56,7 @@ export async function getServiceAnomalies({ query: { bool: { filter: [ - ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX), + { terms: { result_type: ['model_plot', 'record'] } }, ...rangeQuery( Math.min(end - 30 * 60 * 1000, start), end, diff --git a/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts index a7357bbc1dd34..a61e0614f5b1a 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts @@ -12,8 +12,6 @@ import { rangeQuery } from '../../../../../observability/server'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; -import { apmMlAnomalyQuery } from '../../../../common/utils/apm_ml_anomaly_query'; -import { ML_TRANSACTION_LATENCY_DETECTOR_INDEX } from '../../../../common/anomaly_detection'; export type ESResponse = Exclude< PromiseReturnType, @@ -42,7 +40,7 @@ export function anomalySeriesFetcher({ query: { bool: { filter: [ - ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX), + { terms: { result_type: ['model_plot', 'record'] } }, { term: { partition_field_value: serviceName } }, { term: { by_field_value: transactionType } }, ...rangeQuery(start, end, 'timestamp'), diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json index 123232935df05..f8feaef3be5f8 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json @@ -1,29 +1,29 @@ { "id": "apm_transaction", "title": "APM", - "description": "Detect anomalies in transactions from your APM services for metric data.", + "description": "Detect anomalies in transactions from your APM services.", "type": "Transaction data", "logoFile": "logo.json", - "defaultIndexPattern": "apm-*-metric,metrics-apm*", + "defaultIndexPattern": "apm-*-transaction", "query": { "bool": { "filter": [ - { "term": { "processor.event": "metric" } }, - { "term": { "metricset.name": "transaction" } } + { "term": { "processor.event": "transaction" } }, + { "exists": { "field": "transaction.duration" } } ] } }, "jobs": [ { - "id": "apm_metrics", - "file": "apm_metrics.json" + "id": "high_mean_transaction_duration", + "file": "high_mean_transaction_duration.json" } ], "datafeeds": [ { - "id": "datafeed-apm_metrics", - "file": "datafeed_apm_metrics.json", - "job_id": "apm_metrics" + "id": "datafeed-high_mean_transaction_duration", + "file": "datafeed_high_mean_transaction_duration.json", + "job_id": "high_mean_transaction_duration" } ] } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json deleted file mode 100644 index d5092f3ffc553..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "job_type": "anomaly_detector", - "groups": [ - "apm" - ], - "description": "Detects anomalies in transaction duration, throughput and error percentage for metric data.", - "analysis_config": { - "bucket_span": "15m", - "summary_count_field_name" : "doc_count", - "detectors" : [ - { - "detector_description" : "high duration by transaction type for an APM service", - "function" : "high_mean", - "field_name" : "transaction_duration", - "by_field_name" : "transaction.type", - "partition_field_name" : "service.name" - }, - { - "detector_description" : "transactions per minute for an APM service", - "function" : "mean", - "field_name" : "transactions_per_min", - "by_field_name" : "transaction.type", - "partition_field_name" : "service.name" - }, - { - "detector_description" : "percent failed for an APM service", - "function" : "high_mean", - "field_name" : "transaction_failure_percentage", - "by_field_name" : "transaction.type", - "partition_field_name" : "service.name" - } - ], - "influencers" : [ - "transaction.type", - "service.name" - ] - }, - "analysis_limits": { - "model_memory_limit": "32mb" - }, - "data_description": { - "time_field" : "@timestamp", - "time_format" : "epoch_ms" - }, - "model_plot_config": { - "enabled" : true, - "annotations_enabled" : true - }, - "results_index_name" : "custom-apm", - "custom_settings": { - "created_by": "ml-module-apm-transaction" - } -} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json deleted file mode 100644 index ba45582252cd7..0000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "job_id": "JOB_ID", - "indices": [ - "INDEX_PATTERN_NAME" - ], - "chunking_config" : { - "mode" : "off" - }, - "query": { - "bool": { - "filter": [ - { "term": { "processor.event": "metric" } }, - { "term": { "metricset.name": "transaction" } } - ] - } - }, - "aggregations" : { - "buckets" : { - "composite" : { - "size" : 5000, - "sources" : [ - { - "date" : { - "date_histogram" : { - "field" : "@timestamp", - "fixed_interval" : "90s" - } - } - }, - { - "transaction.type" : { - "terms" : { - "field" : "transaction.type" - } - } - }, - { - "service.name" : { - "terms" : { - "field" : "service.name" - } - } - } - ] - }, - "aggs" : { - "@timestamp" : { - "max" : { - "field" : "@timestamp" - } - }, - "transactions_per_min" : { - "rate" : { - "unit" : "minute" - } - }, - "transaction_duration" : { - "avg" : { - "field" : "transaction.duration.histogram" - } - }, - "error_count" : { - "filter" : { - "term" : { - "event.outcome" : "failure" - } - }, - "aggs" : { - "actual_error_count" : { - "value_count" : { - "field" : "event.outcome" - } - } - } - }, - "success_count" : { - "filter" : { - "term" : { - "event.outcome" : "success" - } - } - }, - "transaction_failure_percentage" : { - "bucket_script" : { - "buckets_path" : { - "failure_count" : "error_count>_count", - "success_count" : "success_count>_count" - }, - "script" : "if ((params.failure_count + params.success_count)==0){return 0;}else{return params.failure_count/(params.failure_count + params.success_count);}" - } - } - } - } - } -} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json new file mode 100644 index 0000000000000..d312577902f51 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json @@ -0,0 +1,14 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "processor.event": "transaction" } }, + { "exists": { "field": "transaction.duration.us" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json new file mode 100644 index 0000000000000..77284cb275cd8 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json @@ -0,0 +1,35 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "Detect transaction duration anomalies across transaction types for your APM services.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high duration by transaction type for an APM service", + "function": "high_mean", + "field_name": "transaction.duration.us", + "by_field_name": "transaction.type", + "partition_field_name": "service.name" + } + ], + "influencers": [ + "transaction.type", + "service.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apm-transaction" + } +} diff --git a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts index 00b820a025c8b..2742fbff294c0 100644 --- a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts @@ -44,7 +44,7 @@ export default ({ getService }: FtrProviderContext) => { user: USER.ML_POWERUSER, expected: { responseCode: 200, - moduleIds: ['apm_jsbase', 'apm_nodejs'], + moduleIds: ['apm_jsbase', 'apm_transaction', 'apm_nodejs'], }, }, { diff --git a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts index 6ff6b8113cb1a..c4dd529ac14f5 100644 --- a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts @@ -187,11 +187,9 @@ export default ({ getService }: FtrProviderContext) => { dashboards: [] as string[], }, }, - // Set startDatafeed and estimateModelMemory to false for the APM transaction test - // until there is a new data set available with metric data. { testTitleSuffix: - 'for apm_transaction with prefix, startDatafeed false and estimateModelMemory false', + 'for apm_transaction with prefix, startDatafeed true and estimateModelMemory true', sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_apm', indexPattern: { name: 'ft_module_apm', timeField: '@timestamp' }, module: 'apm_transaction', @@ -199,14 +197,14 @@ export default ({ getService }: FtrProviderContext) => { requestBody: { prefix: 'pf5_', indexPatternName: 'ft_module_apm', - startDatafeed: false, - estimateModelMemory: false, + startDatafeed: true, + end: Date.now(), }, expected: { responseCode: 200, jobs: [ { - jobId: 'pf5_apm_metrics', + jobId: 'pf5_high_mean_transaction_duration', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, },