[ML] AIOps: Fix missing field caps filters for log rate analysis. (#1…

…81109) ## Summary Part of #172981. Field caps requests can be heavy calls in larger clusters. For all other queries for log rate analysis we were applying filters based on the time range selection. This was missing from the field caps call. The following parameters were added to improve the call: - `index_filter`: Adds a range filter to only get field caps from indices spanning the deviation time range. - `filters`: `-metadata` was added to not return fields like `_id` and esp. `_tier`. We previously had a manually check for `_tier` which is now unnecessary using this option. - `types`: Previously we fetched all field types and then filtered out the ones we don't support. This option allows us to pass in the supported fields right away and not return unsupported ones in the first place. ---- Here are examples that show how `index_filter` get applied correctly: Here the deviation selection spans only 1 month and that is reflected in the response from the field caps call: <img width="1026" alt="image" src="https://github.com/elastic/kibana/assets/230104/50a00e5a-2b59-4ae5-9d50-e2ed766f68f5"> ``` { indices: [ 'gallery-2021-11' ], fields: { ... } } ``` Now the deviation selection covers more months: <img width="1010" alt="image" src="https://github.com/elastic/kibana/assets/230104/0034a19e-b136-4261-9761-6b48fdf45989"> ``` { indices: [ 'gallery-2021-09', 'gallery-2021-10', 'gallery-2021-11', 'gallery-2021-12', 'gallery-2022-01' ], fields: { ... } } ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
elastic · Apr 22, 2024 · 078dd22 · 078dd22
1 parent e18d19f
commit 078dd22
Show file tree

Hide file tree

Showing 5 changed files with 562 additions and 261 deletions.
diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_ecommerce.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_ecommerce.ts
@@ -6,228 +6,100 @@
  */
 
 export const fieldCapsEcommerceMock = {
-  indices: ['ft_ecommerce'],
+  indices: ['kibana_sample_data_ecommerce'],
   fields: {
     'products.manufacturer': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'products.discount_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products.base_unit_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    type: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.discount_percentage': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products._id.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    day_of_week_i: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    total_quantity: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    total_unique_products: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    taxless_total_price: {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'geoip.continent_name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    sku: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _version: {
-      _version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
-    },
-    'customer_full_name.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'category.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.taxless_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products.quantity': {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    customer_first_name: {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
-    customer_phone: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'geoip.region_name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _tier: {
-      keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _seq_no: {
-      _seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    customer_full_name: {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
-    'geoip.country_iso_code': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _source: {
-      _source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
-    },
-    _id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
-    order_id: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'products._id': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     'products.product_name.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    _index: {
-      _index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    'products.product_id': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'products.category': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     'products.manufacturer.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    type: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     manufacturer: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     products: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    'products.unit_discount_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     customer_last_name: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'geoip.location': {
-      geo_point: { type: 'geo_point', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.tax_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
+    'products._id.keyword': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'products.product_name': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'products.min_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     'manufacturer.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.taxful_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     currency: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.base_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
+    'geoip.continent_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    event: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    sku: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     email: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'customer_full_name.keyword': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     day_of_week: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'customer_last_name.keyword': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'products.sku': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'customer_last_name.keyword': {
+    'category.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     geoip: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    customer_first_name: {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
+    customer_phone: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'products.category.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'geoip.city_name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    order_date: {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+    'geoip.region_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'customer_first_name.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.created_on': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+    customer_full_name: {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
+    'geoip.country_iso_code': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     category: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
@@ -238,16 +110,14 @@ export const fieldCapsEcommerceMock = {
     user: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    customer_gender: {
+    order_id: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'event.dataset': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    taxful_total_price: {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
+    customer_gender: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
   },
 };
diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_large_arrays.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_large_arrays.ts
@@ -6,27 +6,8 @@
  */
 
 export const fieldCapsLargeArraysMock = {
-  indices: ['large_arrays'],
+  indices: ['large_array'],
   fields: {
-    _tier: {
-      keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _seq_no: {
-      _seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    '@timestamp': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _index: {
-      _index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _source: {
-      _source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
-    },
-    _id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
-    _version: {
-      _version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
-    },
     items: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },