Skip to content

Commit

Permalink
[ML] AIOps: Fix missing field caps filters for log rate analysis. (el…
Browse files Browse the repository at this point in the history
…astic#181109)

## Summary

Part of elastic#172981.

Field caps requests can be heavy calls in larger clusters. For all other
queries for log rate analysis we were applying filters based on the time
range selection. This was missing from the field caps call. The
following parameters were added to improve the call:

- `index_filter`: Adds a range filter to only get field caps from
indices spanning the deviation time range.
- `filters`: `-metadata` was added to not return fields like `_id` and
esp. `_tier`. We previously had a manually check for `_tier` which is
now unnecessary using this option.
- `types`: Previously we fetched all field types and then filtered out
the ones we don't support. This option allows us to pass in the
supported fields right away and not return unsupported ones in the first
place.

----

Here are examples that show how `index_filter` get applied correctly:

Here the deviation selection spans only 1 month and that is reflected in
the response from the field caps call:

<img width="1026" alt="image"
src="https://github.com/elastic/kibana/assets/230104/50a00e5a-2b59-4ae5-9d50-e2ed766f68f5">

```
{
  indices: [ 'gallery-2021-11' ],
  fields: {
   ...
  }
}
```

Now the deviation selection covers more months:

<img width="1010" alt="image"
src="https://github.com/elastic/kibana/assets/230104/0034a19e-b136-4261-9761-6b48fdf45989">

```
{
  indices: [
    'gallery-2021-09',
    'gallery-2021-10',
    'gallery-2021-11',
    'gallery-2021-12',
    'gallery-2022-01'
  ],
  fields: {
   ...
  }
}
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
  • Loading branch information
walterra authored Apr 22, 2024
1 parent e18d19f commit 078dd22
Show file tree
Hide file tree
Showing 5 changed files with 562 additions and 261 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,228 +6,100 @@
*/

export const fieldCapsEcommerceMock = {
indices: ['ft_ecommerce'],
indices: ['kibana_sample_data_ecommerce'],
fields: {
'products.manufacturer': {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'products.discount_amount': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
'products.base_unit_price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
type: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.discount_percentage': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
'products._id.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
day_of_week_i: {
integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
},
total_quantity: {
integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
},
total_unique_products: {
integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
},
taxless_total_price: {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
'geoip.continent_name': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
sku: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
_version: {
_version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
},
'customer_full_name.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'category.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.taxless_price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
'products.quantity': {
integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
},
'products.price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
customer_first_name: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
customer_phone: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'geoip.region_name': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
_tier: {
keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
},
_seq_no: {
_seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
},
customer_full_name: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'geoip.country_iso_code': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
_source: {
_source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
},
_id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
order_id: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products._id': {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'products.product_name.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
_index: {
_index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
},
'products.product_id': {
long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
},
'products.category': {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'products.manufacturer.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
type: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
manufacturer: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
products: {
object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
},
'products.unit_discount_amount': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
customer_last_name: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'geoip.location': {
geo_point: { type: 'geo_point', metadata_field: false, searchable: true, aggregatable: true },
},
'products.tax_amount': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
'products._id.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.product_name': {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'products.min_price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
'manufacturer.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.taxful_price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
},
currency: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.base_price': {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
'geoip.continent_name': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
event: {
object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
},
sku: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
email: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'customer_full_name.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
day_of_week: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'customer_last_name.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.sku': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'customer_last_name.keyword': {
'category.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
geoip: {
object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
},
customer_first_name: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
customer_phone: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.category.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'geoip.city_name': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
order_date: {
date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
'geoip.region_name': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'customer_first_name.keyword': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'products.created_on': {
date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
customer_full_name: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
},
'geoip.country_iso_code': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
category: {
text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
Expand All @@ -238,16 +110,14 @@ export const fieldCapsEcommerceMock = {
user: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
customer_gender: {
order_id: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
'event.dataset': {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
taxful_total_price: {
half_float: {
type: 'half_float',
metadata_field: false,
searchable: true,
aggregatable: true,
},
customer_gender: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
},
};
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,8 @@
*/

export const fieldCapsLargeArraysMock = {
indices: ['large_arrays'],
indices: ['large_array'],
fields: {
_tier: {
keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
},
_seq_no: {
_seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
},
'@timestamp': {
date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
},
_index: {
_index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
},
_source: {
_source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
},
_id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
_version: {
_version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
},
items: {
keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
},
Expand Down
Loading

0 comments on commit 078dd22

Please sign in to comment.