Support nested fields in existing filter types (#49537)

* Add automatic support for nested fields in existing filter types

* Index pattern could be undefined

* add test for handleNestedFilter function

* remove console.log

* add tests for all "getFilterField" functions

* update migrateFilters to work on full filter objects so that it doesn't
have to worry about queries that have been wrapped with `nested`

* add test to ensure fromFilters auto wraps filters on nested fields

* Add smoke test for nested filter and move filter editor tests into
their own suite for easier running and debugging

* fix bad type change

* dedupe filterToQuery logic

* fix helper that wasn't doing what it said it did

* Convert test from pre-merge to jest

* Use new time range style

src/plugins/data/common/es_query/__fixtures__/index_pattern_response.ts

@@ -0,0 +1,322 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const indexPatternResponse = {
+  id: 'logstash-*',
+  title: 'logstash-*',
+  fields: [
+    {
+      name: 'bytes',
+      type: 'number',
+      esTypes: ['long'],
+      count: 10,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'ssl',
+      type: 'boolean',
+      esTypes: ['boolean'],
+      count: 20,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: '@timestamp',
+      type: 'date',
+      esTypes: ['date'],
+      count: 30,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'time',
+      type: 'date',
+      esTypes: ['date'],
+      count: 30,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: '@tags',
+      type: 'string',
+      esTypes: ['keyword'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'utc_time',
+      type: 'date',
+      esTypes: ['date'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'phpmemory',
+      type: 'number',
+      esTypes: ['integer'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'ip',
+      type: 'ip',
+      esTypes: ['ip'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'request_body',
+      type: 'attachment',
+      esTypes: ['attachment'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'point',
+      type: 'geo_point',
+      esTypes: ['geo_point'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'area',
+      type: 'geo_shape',
+      esTypes: ['geo_shape'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'hashed',
+      type: 'murmur3',
+      esTypes: ['murmur3'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: false,
+      readFromDocValues: false,
+    },
+    {
+      name: 'geo.coordinates',
+      type: 'geo_point',
+      esTypes: ['geo_point'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'extension',
+      type: 'string',
+      esTypes: ['keyword'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'machine.os',
+      type: 'string',
+      esTypes: ['text'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'machine.os.raw',
+      type: 'string',
+      esTypes: ['keyword'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+      subType: { multi: { parent: 'machine.os' } },
+    },
+    {
+      name: 'geo.src',
+      type: 'string',
+      esTypes: ['keyword'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: '_id',
+      type: 'string',
+      esTypes: ['_id'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: '_type',
+      type: 'string',
+      esTypes: ['_type'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: '_source',
+      type: '_source',
+      esTypes: ['_source'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'non-filterable',
+      type: 'string',
+      esTypes: ['text'],
+      count: 0,
+      scripted: false,
+      searchable: false,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'non-sortable',
+      type: 'string',
+      esTypes: ['text'],
+      count: 0,
+      scripted: false,
+      searchable: false,
+      aggregatable: false,
+      readFromDocValues: false,
+    },
+    {
+      name: 'custom_user_field',
+      type: 'conflict',
+      esTypes: ['long', 'text'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: true,
+    },
+    {
+      name: 'script string',
+      type: 'string',
+      count: 0,
+      scripted: true,
+      script: "'i am a string'",
+      lang: 'expression',
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'script number',
+      type: 'number',
+      count: 0,
+      scripted: true,
+      script: '1234',
+      lang: 'expression',
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'script date',
+      type: 'date',
+      count: 0,
+      scripted: true,
+      script: '1234',
+      lang: 'painless',
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'script murmur3',
+      type: 'murmur3',
+      count: 0,
+      scripted: true,
+      script: '1234',
+      lang: 'expression',
+      searchable: true,
+      aggregatable: true,
+      readFromDocValues: false,
+    },
+    {
+      name: 'nestedField.child',
+      type: 'string',
+      esTypes: ['text'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: false,
+      readFromDocValues: false,
+      subType: { nested: { path: 'nestedField' } },
+    },
+    {
+      name: 'nestedField.nestedChild.doublyNestedChild',
+      type: 'string',
+      esTypes: ['text'],
+      count: 0,
+      scripted: false,
+      searchable: true,
+      aggregatable: false,
+      readFromDocValues: false,
+      subType: { nested: { path: 'nestedField.nestedChild' } },
+    },
+  ],
+};

src/plugins/data/common/es_query/es_query/from_filters.test.ts

@@ -144,5 +144,30 @@ describe('build query', () => {
 
       expect(result.filter).toEqual(expectedESQueries);
     });
+
+    test('should wrap filters targeting nested fields in a nested query', () => {
+      const filters = [
+        {
+          exists: { field: 'nestedField.child' },
+          meta: { type: 'exists', alias: '', disabled: false, negate: false },
+        },
+      ];
+
+      const expectedESQueries = [
+        {
+          nested: {
+            path: 'nestedField',
+            query: {
+              exists: {
+                field: 'nestedField.child',
+              },
+            },
+          },
+        },
+      ];
+
+      const result = buildQueryFromFilters(filters, indexPattern);
+      expect(result.filter).toEqual(expectedESQueries);
+    });
   });
 });

src/plugins/data/common/es_query/es_query/from_filters.ts

@@ -21,6 +21,7 @@ import { migrateFilter } from './migrate_filter';
 import { filterMatchesIndex } from './filter_matches_index';
 import { Filter, cleanFilter, isFilterDisabled } from '../filters';
 import { IIndexPattern } from '../../index_patterns';
+import { handleNestedFilter } from './handle_nested_filter';
 
 /**
  * Create a filter that can be reversed for filters with negate set
@@ -59,20 +60,22 @@ export const buildQueryFromFilters = (
 ) => {
   filters = filters.filter(filter => filter && !isFilterDisabled(filter));
 
-  return {
-    must: [],
-    filter: filters
-      .filter(filterNegate(false))
+  const filtersToESQueries = (negate: boolean) => {
+    return filters
+      .filter(filterNegate(negate))
       .filter(filter => !ignoreFilterIfFieldNotInIndex || filterMatchesIndex(filter, indexPattern))
+      .map(filter => {
+        return migrateFilter(filter, indexPattern);
+      })
+      .map(filter => handleNestedFilter(filter, indexPattern))
       .map(translateToQuery)
-      .map(cleanFilter)
-      .map(filter => migrateFilter(filter, indexPattern)),
+      .map(cleanFilter);
+  };
+
+  return {
+    must: [],
+    filter: filtersToESQueries(false),
     should: [],
-    must_not: filters
-      .filter(filterNegate(true))
-      .filter(filter => !ignoreFilterIfFieldNotInIndex || filterMatchesIndex(filter, indexPattern))
-      .map(translateToQuery)
-      .map(cleanFilter)
-      .map(filter => migrateFilter(filter, indexPattern)),
+    must_not: filtersToESQueries(true),
   };
 };