Merge remote-tracking branch 'upstream/master' into kbn-113734-skip-c…

…onfig-deprecations

x-pack/plugins/apm/common/anomaly_detection.ts

@@ -33,8 +33,6 @@ export function getSeverityColor(score: number) {
   return mlGetSeverityColor(score);
 }
 
-export const ML_TRANSACTION_LATENCY_DETECTOR_INDEX = 0;
-
 export const ML_ERRORS = {
   INVALID_LICENSE: i18n.translate(
     'xpack.apm.anomaly_detection.error.invalid_license',

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -5,21 +5,21 @@
  * 2.0.
  */
 
-import Boom from '@hapi/boom';
 import { Logger } from 'kibana/server';
+import uuid from 'uuid/v4';
 import { snakeCase } from 'lodash';
+import Boom from '@hapi/boom';
 import moment from 'moment';
-import uuid from 'uuid/v4';
 import { ML_ERRORS } from '../../../common/anomaly_detection';
-import {
-  METRICSET_NAME,
-  PROCESSOR_EVENT,
-} from '../../../common/elasticsearch_fieldnames';
 import { ProcessorEvent } from '../../../common/processor_event';
 import { environmentQuery } from '../../../common/utils/environment_query';
-import { withApmSpan } from '../../utils/with_apm_span';
 import { Setup } from '../helpers/setup_request';
+import {
+  TRANSACTION_DURATION,
+  PROCESSOR_EVENT,
+} from '../../../common/elasticsearch_fieldnames';
 import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
+import { withApmSpan } from '../../utils/with_apm_span';
 import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs';
 
 export async function createAnomalyDetectionJobs(
@@ -92,8 +92,8 @@ async function createAnomalyDetectionJob({
       query: {
         bool: {
           filter: [
-            { term: { [PROCESSOR_EVENT]: ProcessorEvent.metric } },
-            { term: { [METRICSET_NAME]: 'transaction' } },
+            { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } },
+            { exists: { field: TRANSACTION_DURATION } },
             ...environmentQuery(environment),
           ],
         },
@@ -105,7 +105,7 @@ async function createAnomalyDetectionJob({
             job_tags: {
               environment,
               // identifies this as an APM ML job & facilitates future migrations
-              apm_ml_version: 3,
+              apm_ml_version: 2,
             },
           },
         },

x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts

@@ -11,11 +11,7 @@ import { estypes } from '@elastic/elasticsearch';
 import { ESSearchResponse } from '../../../../../../src/core/types/elasticsearch';
 import { MlPluginSetup } from '../../../../ml/server';
 import { PromiseReturnType } from '../../../../observability/typings/common';
-import {
-  getSeverity,
-  ML_ERRORS,
-  ML_TRANSACTION_LATENCY_DETECTOR_INDEX,
-} from '../../../common/anomaly_detection';
+import { getSeverity, ML_ERRORS } from '../../../common/anomaly_detection';
 import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
 import { getServiceHealthStatus } from '../../../common/service_health_status';
 import {
@@ -26,7 +22,6 @@ import { rangeQuery } from '../../../../observability/server';
 import { withApmSpan } from '../../utils/with_apm_span';
 import { getMlJobsWithAPMGroup } from '../anomaly_detection/get_ml_jobs_with_apm_group';
 import { Setup } from '../helpers/setup_request';
-import { apmMlAnomalyQuery } from '../../../common/utils/apm_ml_anomaly_query';
 
 export const DEFAULT_ANOMALIES: ServiceAnomaliesResponse = {
   mlJobIds: [],
@@ -61,7 +56,7 @@ export async function getServiceAnomalies({
         query: {
           bool: {
             filter: [
-              ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
+              { terms: { result_type: ['model_plot', 'record'] } },
               ...rangeQuery(
                 Math.min(end - 30 * 60 * 1000, start),
                 end,

x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts

@@ -12,8 +12,6 @@ import { rangeQuery } from '../../../../../observability/server';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { withApmSpan } from '../../../utils/with_apm_span';
 import { Setup } from '../../helpers/setup_request';
-import { apmMlAnomalyQuery } from '../../../../common/utils/apm_ml_anomaly_query';
-import { ML_TRANSACTION_LATENCY_DETECTOR_INDEX } from '../../../../common/anomaly_detection';
 
 export type ESResponse = Exclude<
   PromiseReturnType<typeof anomalySeriesFetcher>,
@@ -42,7 +40,7 @@ export function anomalySeriesFetcher({
         query: {
           bool: {
             filter: [
-              ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
+              { terms: { result_type: ['model_plot', 'record'] } },
               { term: { partition_field_value: serviceName } },
               { term: { by_field_value: transactionType } },
               ...rangeQuery(start, end, 'timestamp'),

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json

@@ -1,29 +1,29 @@
 {
   "id": "apm_transaction",
   "title": "APM",
-  "description": "Detect anomalies in transactions from your APM services for metric data.",
+  "description": "Detect anomalies in transactions from your APM services.",
   "type": "Transaction data",
   "logoFile": "logo.json",
-  "defaultIndexPattern": "apm-*-metric,metrics-apm*",
+  "defaultIndexPattern": "apm-*-transaction",
   "query": {
     "bool": {
       "filter": [
-        { "term": { "processor.event": "metric" } },
-        { "term": { "metricset.name": "transaction" } }
+        { "term": { "processor.event": "transaction" } },
+        { "exists": { "field": "transaction.duration" } }
       ]
     }
   },
   "jobs": [
     {
-      "id": "apm_metrics",
-      "file": "apm_metrics.json"
+      "id": "high_mean_transaction_duration",
+      "file": "high_mean_transaction_duration.json"
     }
   ],
   "datafeeds": [
     {
-      "id": "datafeed-apm_metrics",
-      "file": "datafeed_apm_metrics.json",
-      "job_id": "apm_metrics"
+      "id": "datafeed-high_mean_transaction_duration",
+      "file": "datafeed_high_mean_transaction_duration.json",
+      "job_id": "high_mean_transaction_duration"
     }
   ]
 }

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json

@@ -0,0 +1,14 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "query": {
+    "bool": {
+      "filter": [
+        { "term": { "processor.event": "transaction" } },
+        { "exists": { "field": "transaction.duration.us" } }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json

@@ -0,0 +1,35 @@
+{
+  "job_type": "anomaly_detector",
+  "groups": [
+    "apm"
+  ],
+  "description": "Detect transaction duration anomalies across transaction types for your APM services.",
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high duration by transaction type for an APM service",
+        "function": "high_mean",
+        "field_name": "transaction.duration.us",
+        "by_field_name": "transaction.type",
+        "partition_field_name": "service.name"
+      }
+    ],
+    "influencers": [
+      "transaction.type",
+      "service.name"
+    ]
+  },
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "model_plot_config": {
+    "enabled": true
+  },
+  "custom_settings": {
+    "created_by": "ml-module-apm-transaction"
+  }
+}