From 662507b91e4960484a0fda749a340627465da62b Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 3 Apr 2023 15:47:09 +0100 Subject: [PATCH] [ML] Removing counter fields from wizards (#153639) Further improvements to the anomaly detection and data frame analysts wizards to ensure counter fields cannot be selected when configuring a new job. Adds a `counter` flag to the `new_job_caps` response which is used to remove counter fields from dropdowns in the DFA and advanced AD wizards. The outlier detection wizard still has an issue caused by the call to the `_explain` endpoint. This needs to be fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/94691 --- x-pack/plugins/ml/common/types/fields.ts | 4 +- .../new_job/common/job_creator/job_creator.ts | 2 + .../common/job_creator/util/general.ts | 1 + .../cloudwatch_job_caps_response.json | 12 + .../nested_field_index_response.json | 1212 +++++++++-------- .../new_job_capabilities.ts | 4 + .../new_job_capabilities_service.ts | 2 + .../new_job_capabilities_service_analytics.ts | 1 + .../results/cloudwatch_rollup_job_caps.json | 89 +- .../__mocks__/results/farequote_job_caps.json | 3 + .../job_service/new_job_caps/field_service.ts | 6 +- 11 files changed, 717 insertions(+), 619 deletions(-) diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts index 7dea8c1386aca..0862d1aa66ead 100644 --- a/x-pack/plugins/ml/common/types/fields.ts +++ b/x-pack/plugins/ml/common/types/fields.ts @@ -25,7 +25,8 @@ export interface Field { id: FieldId; name: string; type: ES_FIELD_TYPES; - aggregatable?: boolean; + aggregatable: boolean; + counter: boolean; aggIds?: AggId[]; aggs?: Aggregation[]; runtimeField?: estypes.MappingRuntimeField; @@ -95,6 +96,7 @@ export const mlCategory: Field = { name: MLCATEGORY, type: ES_FIELD_TYPES.KEYWORD, aggregatable: false, + counter: false, }; export type RollupFields = Record]>; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index 607031e6e70d8..f445afe634b90 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -775,6 +775,7 @@ export class JobCreator { name: id, type: Array.isArray(runtimeField) ? runtimeField[0].type : runtimeField.type, aggregatable: true, + counter: false, aggs: [], runtimeField, } as Field) @@ -793,6 +794,7 @@ export class JobCreator { name: f, type: ES_FIELD_TYPES.KEYWORD, aggregatable: true, + counter: false, })); } } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index c3b74fdfad5fe..a933fdc0af5d5 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -348,6 +348,7 @@ export function collectAggs(o: any, aggFields: Field[]) { name: k, type: ES_FIELD_TYPES.KEYWORD, aggregatable: true, + counter: false, }); } }); diff --git a/x-pack/plugins/ml/public/application/services/__mocks__/cloudwatch_job_caps_response.json b/x-pack/plugins/ml/public/application/services/__mocks__/cloudwatch_job_caps_response.json index 783b2d4e081bc..324a94b6ecaa3 100644 --- a/x-pack/plugins/ml/public/application/services/__mocks__/cloudwatch_job_caps_response.json +++ b/x-pack/plugins/ml/public/application/services/__mocks__/cloudwatch_job_caps_response.json @@ -627,6 +627,7 @@ "name": "@timestamp", "type": "date", "aggregatable": true, + "counter": false, "aggIds": [] }, { @@ -634,6 +635,7 @@ "name": "CPUUtilization", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -667,6 +669,7 @@ "name": "DiskReadBytes", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -700,6 +703,7 @@ "name": "DiskReadOps", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -733,6 +737,7 @@ "name": "DiskWriteBytes", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -766,6 +771,7 @@ "name": "DiskWriteOps", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -799,6 +805,7 @@ "name": "instance", "type": "keyword", "aggregatable": true, + "counter": false, "aggIds": [ "distinct_count", "high_distinct_count", @@ -810,6 +817,7 @@ "name": "NetworkIn", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -843,6 +851,7 @@ "name": "NetworkOut", "type": "double", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", @@ -876,6 +885,7 @@ "name": "region", "type": "keyword", "aggregatable": true, + "counter": false, "aggIds": [ "distinct_count", "high_distinct_count", @@ -887,6 +897,7 @@ "name": "sourcetype", "type": "text", "aggregatable": false, + "counter": false, "aggIds": [] }, { @@ -894,6 +905,7 @@ "name": "sourcetype.keyword", "type": "keyword", "aggregatable": true, + "counter": false, "aggIds": [ "distinct_count", "high_distinct_count", diff --git a/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json b/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json index 49baa9fb7e427..4f37c3cd3a45b 100644 --- a/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json +++ b/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json @@ -1,611 +1,615 @@ { - "nested-field-index":{ - "aggs":[ - { - "id":"count", - "title":"Count", - "kibanaName":"count", - "dslName":"count", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"high_count", - "title":"High count", - "kibanaName":"count", - "dslName":"count", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"low_count", - "title":"Low count", - "kibanaName":"count", - "dslName":"count", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"mean", - "title":"Mean", - "kibanaName":"avg", - "dslName":"avg", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"avg", - "min":"avg" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"high_mean", - "title":"High mean", - "kibanaName":"avg", - "dslName":"avg", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"avg", - "min":"avg" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"low_mean", - "title":"Low mean", - "kibanaName":"avg", - "dslName":"avg", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"avg", - "min":"avg" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"sum", - "title":"Sum", - "kibanaName":"sum", - "dslName":"sum", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"sum", - "min":"sum" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"high_sum", - "title":"High sum", - "kibanaName":"sum", - "dslName":"sum", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"sum", - "min":"sum" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"low_sum", - "title":"Low sum", - "kibanaName":"sum", - "dslName":"sum", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"sum", - "min":"sum" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"median", - "title":"Median", - "kibanaName":"median", - "dslName":"percentiles", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds": [ - "time_spent" - ] - }, - { - "id":"high_median", - "title":"High median", - "kibanaName":"median", - "dslName":"percentiles", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"low_median", - "title":"Low median", - "kibanaName":"median", - "dslName":"percentiles", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"min", - "title":"Min", - "kibanaName":"min", - "dslName":"min", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"min", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"max", - "title":"Max", - "kibanaName":"max", - "dslName":"max", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"max" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"distinct_count", - "title":"Distinct count", - "kibanaName":"cardinality", - "dslName":"cardinality", - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group.keyword", - "user.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"non_zero_count", - "title":"Non zero count", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"high_non_zero_count", - "title":"High non zero count", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"low_non_zero_count", - "title":"Low non zero count", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"high_distinct_count", - "title":"High distinct count", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group.keyword", - "user.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"low_distinct_count", - "title":"Low distinct count", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group.keyword", - "user.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"metric", - "title":"Metric", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"varp", - "title":"varp", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"high_varp", - "title":"High varp", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"low_varp", - "title":"Low varp", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"non_null_sum", - "title":"Non null sum", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"high_non_null_sum", - "title":"High non null sum", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"low_non_null_sum", - "title":"Low non null sum", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "time_spent" - ] - }, - { - "id":"rare", - "title":"Rare", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"freq_rare", - "title":"Freq rare", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"info_content", - "title":"Info content", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group", - "user.first", - "user.last", - "group.keyword", - "user.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"high_info_content", - "title":"High info content", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group", - "user.first", - "user.last", - "group.keyword", - "usr.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"low_info_content", - "title":"Low info content", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - "group", - "user.first", - "user.last", - "group.keyword", - "user.first.keyword", - "user.last.keyword", - "time_spent" - ] - }, - { - "id":"time_of_day", - "title":"Time of day", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"time_of_week", - "title":"Time of week", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - } - }, - { - "id":"lat_long", - "title":"Lat long", - "kibanaName":null, - "dslName":null, - "type":"metrics", - "mlModelPlotAgg":{ - "max":"max", - "min":"min" - }, - "fieldIds":[ - - ] + "nested-field-index": { + "aggs": [ + { + "id": "count", + "title": "Count", + "kibanaName": "count", + "dslName": "count", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" } - ], - "fields":[ - { - "id":"group", - "name":"group", - "type":"text", - "aggregatable":false, - "aggIds":[ - "info_content", - "high_info_content", - "low_info_content" - ] - }, - { - "id":"group.keyword", - "name":"group.keyword", - "type":"keyword", - "aggregatable":true, - "aggIds":[ - "distinct_count", - "high_distinct_count", - "low_distinct_count", - "info_content", - "high_info_content", - "low_info_content" - ] - }, - { - "id":"time_spent", - "name":"time_spent", - "type":"long", - "aggregatable":true, - "aggIds":[ - "mean", - "high_mean", - "low_mean", - "sum", - "high_sum", - "low_sum", - "median", - "high_median", - "low_median", - "min", - "max", - "distinct_count", - "high_distinct_count", - "low_distinct_count", - "metric", - "varp", - "high_varp", - "low_varp", - "non_null_sum", - "high_non_null_sum", - "low_non_null_sum", - "info_content", - "high_info_content", - "low_info_content" + }, + { + "id": "high_count", + "title": "High count", + "kibanaName": "count", + "dslName": "count", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "low_count", + "title": "Low count", + "kibanaName": "count", + "dslName": "count", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "mean", + "title": "Mean", + "kibanaName": "avg", + "dslName": "avg", + "type": "metrics", + "mlModelPlotAgg": { + "max": "avg", + "min": "avg" + }, + "fieldIds": [ + "time_spent" ] - }, - { - "id":"user", - "name":"user", - "type":"nested", - "aggregatable":false, - "aggIds":[ - - ] - }, - { - "id":"user.first", - "name":"user.first", - "type":"text", - "aggregatable":false, - "aggIds":[ - "info_content", - "high_info_content", - "low_info_content" - ] - }, - { - "id":"user.first.keyword", - "name":"user.first.keyword", - "type":"keyword", - "aggregatable":true, - "aggIds":[ - "distinct_count", - "high_distinct_count", - "low_distinct_count", - "info_content", - "high_info_content", - "low_info_content" - ] - }, - { - "id":"user.last", - "name":"user.last", - "type":"text", - "aggregatable":false, - "aggIds":[ - "info_content", - "high_info_content", - "low_info_content" - ] - }, - { - "id":"user.last.keyword", - "name":"user.last.keyword", - "type":"keyword", - "aggregatable":true, - "aggIds":[ - "distinct_count", - "high_distinct_count", - "low_distinct_count", - "info_content", - "high_info_content", - "low_info_content" - ] + }, + { + "id": "high_mean", + "title": "High mean", + "kibanaName": "avg", + "dslName": "avg", + "type": "metrics", + "mlModelPlotAgg": { + "max": "avg", + "min": "avg" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "low_mean", + "title": "Low mean", + "kibanaName": "avg", + "dslName": "avg", + "type": "metrics", + "mlModelPlotAgg": { + "max": "avg", + "min": "avg" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "sum", + "title": "Sum", + "kibanaName": "sum", + "dslName": "sum", + "type": "metrics", + "mlModelPlotAgg": { + "max": "sum", + "min": "sum" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "high_sum", + "title": "High sum", + "kibanaName": "sum", + "dslName": "sum", + "type": "metrics", + "mlModelPlotAgg": { + "max": "sum", + "min": "sum" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "low_sum", + "title": "Low sum", + "kibanaName": "sum", + "dslName": "sum", + "type": "metrics", + "mlModelPlotAgg": { + "max": "sum", + "min": "sum" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "median", + "title": "Median", + "kibanaName": "median", + "dslName": "percentiles", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "high_median", + "title": "High median", + "kibanaName": "median", + "dslName": "percentiles", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "low_median", + "title": "Low median", + "kibanaName": "median", + "dslName": "percentiles", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "min", + "title": "Min", + "kibanaName": "min", + "dslName": "min", + "type": "metrics", + "mlModelPlotAgg": { + "max": "min", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "max", + "title": "Max", + "kibanaName": "max", + "dslName": "max", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "max" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "distinct_count", + "title": "Distinct count", + "kibanaName": "cardinality", + "dslName": "cardinality", + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "non_zero_count", + "title": "Non zero count", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "high_non_zero_count", + "title": "High non zero count", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "low_non_zero_count", + "title": "Low non zero count", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "high_distinct_count", + "title": "High distinct count", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "low_distinct_count", + "title": "Low distinct count", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "metric", + "title": "Metric", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "varp", + "title": "varp", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "high_varp", + "title": "High varp", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "low_varp", + "title": "Low varp", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "non_null_sum", + "title": "Non null sum", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "high_non_null_sum", + "title": "High non null sum", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "low_non_null_sum", + "title": "Low non null sum", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id": "rare", + "title": "Rare", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "freq_rare", + "title": "Freq rare", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "info_content", + "title": "Info content", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group", + "user.first", + "user.last", + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "high_info_content", + "title": "High info content", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group", + "user.first", + "user.last", + "group.keyword", + "usr.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "low_info_content", + "title": "Low info content", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [ + "group", + "user.first", + "user.last", + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id": "time_of_day", + "title": "Time of day", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + } + }, + { + "id": "time_of_week", + "title": "Time of week", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" } + }, + { + "id": "lat_long", + "title": "Lat long", + "kibanaName": null, + "dslName": null, + "type": "metrics", + "mlModelPlotAgg": { + "max": "max", + "min": "min" + }, + "fieldIds": [] + } + ], + "fields": [ + { + "id": "group", + "name": "group", + "type": "text", + "aggregatable": false, + "counter": false, + "aggIds": [ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "group.keyword", + "name": "group.keyword", + "type": "keyword", + "aggregatable": true, + "counter": false, + "aggIds": [ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "time_spent", + "name": "time_spent", + "type": "long", + "aggregatable": true, + "counter": false, + "aggIds": [ + "mean", + "high_mean", + "low_mean", + "sum", + "high_sum", + "low_sum", + "median", + "high_median", + "low_median", + "min", + "max", + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "metric", + "varp", + "high_varp", + "low_varp", + "non_null_sum", + "high_non_null_sum", + "low_non_null_sum", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "user", + "name": "user", + "type": "nested", + "aggregatable": false, + "counter": false, + "aggIds": [] + }, + { + "id": "user.first", + "name": "user.first", + "type": "text", + "aggregatable": false, + "counter": false, + "aggIds": [ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "user.first.keyword", + "name": "user.first.keyword", + "type": "keyword", + "aggregatable": true, + "counter": false, + "aggIds": [ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "user.last", + "name": "user.last", + "type": "text", + "aggregatable": false, + "counter": false, + "aggIds": [ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id": "user.last.keyword", + "name": "user.last.keyword", + "type": "keyword", + "aggregatable": true, + "counter": false, + "aggIds": [ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + } ] } } diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts index 2bc51a10ab9d2..041f508d23fa3 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts @@ -62,4 +62,8 @@ export class NewJobCapabilitiesServiceBase { const agg = this._aggs.find((f) => f.id === id); return agg === undefined ? null : agg; } + + protected removeCounterFields() { + this._fields = this._fields.filter((f) => f.counter === false); + } } diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts index 87983326f7226..0628b23ed96cf 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts @@ -64,6 +64,7 @@ class NewJobCapsService extends NewJobCapabilitiesServiceBase { // keyword fields over text fields. // e.g. if foo.keyword and foo exist, don't add foo to the list. this._fields = fields; + this.removeCounterFields(); // set the category fields to contain fields which have been filtered to prefer text fields. this._catFields = catFields; this._dateFields = dateFields; @@ -154,6 +155,7 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) { name: 'Event rate', type: ES_FIELD_TYPES.INTEGER, aggregatable: true, + counter: false, aggs: [], }; diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts index 6796db5424993..56e9342b19c7e 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts @@ -58,6 +58,7 @@ class NewJobCapsServiceAnalytics extends NewJobCapabilitiesServiceBase { // keyword fields over text fields. // e.g. if foo.keyword and foo exist, don't add foo to the list. this._fields = fieldsPreferringKeyword; + this.removeCounterFields(); } catch (error) { console.error('Unable to load analytics index fields', error); // eslint-disable-line no-console } diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/cloudwatch_rollup_job_caps.json b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/cloudwatch_rollup_job_caps.json index ae940ffbf4828..dff97f5d6085e 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/cloudwatch_rollup_job_caps.json +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/cloudwatch_rollup_job_caps.json @@ -44,7 +44,12 @@ "max": "avg", "min": "avg" }, - "fieldIds": ["CPUUtilization", "DiskReadBytes", "NetworkIn", "NetworkOut"] + "fieldIds": [ + "CPUUtilization", + "DiskReadBytes", + "NetworkIn", + "NetworkOut" + ] }, { "id": "high_mean", @@ -56,7 +61,12 @@ "max": "avg", "min": "avg" }, - "fieldIds": ["CPUUtilization", "DiskReadBytes", "NetworkIn", "NetworkOut"] + "fieldIds": [ + "CPUUtilization", + "DiskReadBytes", + "NetworkIn", + "NetworkOut" + ] }, { "id": "low_mean", @@ -68,7 +78,12 @@ "max": "avg", "min": "avg" }, - "fieldIds": ["CPUUtilization", "DiskReadBytes", "NetworkIn", "NetworkOut"] + "fieldIds": [ + "CPUUtilization", + "DiskReadBytes", + "NetworkIn", + "NetworkOut" + ] }, { "id": "sum", @@ -80,7 +95,10 @@ "max": "sum", "min": "sum" }, - "fieldIds": ["DiskReadBytes", "DiskWriteOps"] + "fieldIds": [ + "DiskReadBytes", + "DiskWriteOps" + ] }, { "id": "high_sum", @@ -92,7 +110,10 @@ "max": "sum", "min": "sum" }, - "fieldIds": ["DiskReadBytes", "DiskWriteOps"] + "fieldIds": [ + "DiskReadBytes", + "DiskWriteOps" + ] }, { "id": "low_sum", @@ -104,7 +125,10 @@ "max": "sum", "min": "sum" }, - "fieldIds": ["DiskReadBytes", "DiskWriteOps"] + "fieldIds": [ + "DiskReadBytes", + "DiskWriteOps" + ] }, { "dslName": "percentiles", @@ -152,7 +176,10 @@ "max": "min", "min": "min" }, - "fieldIds": ["NetworkIn", "NetworkOut"] + "fieldIds": [ + "NetworkIn", + "NetworkOut" + ] }, { "id": "max", @@ -164,7 +191,10 @@ "max": "max", "min": "max" }, - "fieldIds": ["CPUUtilization", "DiskReadBytes"] + "fieldIds": [ + "CPUUtilization", + "DiskReadBytes" + ] }, { "dslName": "cardinality", @@ -418,6 +448,7 @@ "name": "@timestamp", "type": "date", "aggregatable": true, + "counter": false, "aggIds": [] }, { @@ -425,35 +456,67 @@ "name": "CPUUtilization", "type": "double", "aggregatable": true, - "aggIds": ["mean", "high_mean", "low_mean", "max"] + "counter": false, + "aggIds": [ + "mean", + "high_mean", + "low_mean", + "max" + ] }, { "id": "DiskReadBytes", "name": "DiskReadBytes", "type": "double", "aggregatable": true, - "aggIds": ["mean", "high_mean", "low_mean", "sum", "high_sum", "low_sum", "max"] + "counter": false, + "aggIds": [ + "mean", + "high_mean", + "low_mean", + "sum", + "high_sum", + "low_sum", + "max" + ] }, { "id": "DiskWriteOps", "name": "DiskWriteOps", "type": "double", "aggregatable": true, - "aggIds": ["sum", "high_sum", "low_sum"] + "counter": false, + "aggIds": [ + "sum", + "high_sum", + "low_sum" + ] }, { "id": "NetworkIn", "name": "NetworkIn", "type": "double", "aggregatable": true, - "aggIds": ["mean", "high_mean", "low_mean", "min"] + "counter": false, + "aggIds": [ + "mean", + "high_mean", + "low_mean", + "min" + ] }, { "id": "NetworkOut", "name": "NetworkOut", "type": "double", "aggregatable": true, - "aggIds": ["mean", "high_mean", "low_mean", "min"] + "counter": false, + "aggIds": [ + "mean", + "high_mean", + "low_mean", + "min" + ] } ] } diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/farequote_job_caps.json b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/farequote_job_caps.json index 9c10f9083be5e..bb5e21f8a9da6 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/farequote_job_caps.json +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/__mocks__/results/farequote_job_caps.json @@ -472,6 +472,7 @@ "name": "@timestamp", "type": "date", "aggregatable": true, + "counter": false, "aggIds": [] }, { @@ -479,6 +480,7 @@ "name": "airline", "type": "keyword", "aggregatable": true, + "counter": false, "aggIds": [ "distinct_count", "high_distinct_count", @@ -493,6 +495,7 @@ "name": "responsetime", "type": "float", "aggregatable": true, + "counter": false, "aggIds": [ "mean", "high_mean", diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts index 83144d20b39bc..ba3c0c9c0f759 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts @@ -92,6 +92,7 @@ class FieldsService { name: k, type: field.type as ES_FIELD_TYPES, aggregatable: this.isFieldAggregatable(field), + counter: this.isCounterField(field), aggs: [], }); } @@ -101,11 +102,14 @@ class FieldsService { return fields.sort((a, b) => a.id.localeCompare(b.id)); } + private isCounterField(field: estypes.FieldCapsFieldCapability) { + return field.time_series_metric === 'counter'; + } // check to see whether the field is aggregatable // If it is a counter field from a time series data stream, we cannot currently // support any aggregations and so it cannot be used as a field_name in a detector. private isFieldAggregatable(field: estypes.FieldCapsFieldCapability) { - return field.time_series_metric !== 'counter' ?? field.aggregatable; + return field.aggregatable && this.isCounterField(field) === false; } // public function to load fields from _field_caps and create a list