[8.x] [ES|QL] Add autocomplete and validation to support MATCH and QS…

…RT (#199032) (#200127) # Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] Add autocomplete and validation to support MATCH and QSRT (#199032)](#199032)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <[email protected]>
elastic · Nov 14, 2024 · 8085b63 · 8085b63
1 parent d298669
commit 8085b63
Show file tree

Hide file tree

Showing 14 changed files with 422 additions and 58 deletions.
diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
@@ -13,7 +13,7 @@ import { join } from 'path';
 import _ from 'lodash';
 import type { RecursivePartial } from '@kbn/utility-types';
 import { FunctionDefinition } from '../src/definitions/types';
-
+import { FULL_TEXT_SEARCH_FUNCTIONS } from '../src/shared/constants';
 const aliasTable: Record<string, string[]> = {
   to_version: ['to_ver'],
   to_unsigned_long: ['to_ul', 'to_ulong'],
@@ -246,23 +246,40 @@ const convertDateTime = (s: string) => (s === 'datetime' ? 'date' : s);
  * @returns
  */
 function getFunctionDefinition(ESFunctionDefinition: Record<string, any>): FunctionDefinition {
+  let supportedCommandsAndOptions: Pick<
+    FunctionDefinition,
+    'supportedCommands' | 'supportedOptions'
+  > =
+    ESFunctionDefinition.type === 'eval'
+      ? scalarSupportedCommandsAndOptions
+      : aggregationSupportedCommandsAndOptions;
+
+  // MATCH and QSRT has limited supported for where commands only
+  if (FULL_TEXT_SEARCH_FUNCTIONS.includes(ESFunctionDefinition.name)) {
+    supportedCommandsAndOptions = {
+      supportedCommands: ['where'],
+      supportedOptions: [],
+    };
+  }
   const ret = {
     type: ESFunctionDefinition.type,
     name: ESFunctionDefinition.name,
-    ...(ESFunctionDefinition.type === 'eval'
-      ? scalarSupportedCommandsAndOptions
-      : aggregationSupportedCommandsAndOptions),
+    ...supportedCommandsAndOptions,
     description: ESFunctionDefinition.description,
     alias: aliasTable[ESFunctionDefinition.name],
     ignoreAsSuggestion: ESFunctionDefinition.snapshot_only,
     preview: ESFunctionDefinition.preview,
     signatures: _.uniqBy(
       ESFunctionDefinition.signatures.map((signature: any) => ({
         ...signature,
-        params: signature.params.map((param: any) => ({
+        params: signature.params.map((param: any, idx: number) => ({
           ...param,
           type: convertDateTime(param.type),
           description: undefined,
+          ...(idx === 0 && FULL_TEXT_SEARCH_FUNCTIONS.includes(ESFunctionDefinition.name)
+            ? // Default to false. If set to true, this parameter does not accept a function or literal, only fields.
+              { fieldsOnly: true }
+            : {}),
         })),
         returnType: convertDateTime(signature.returnType),
         variadic: undefined, // we don't support variadic property

diff --git a/...sql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts b/...sql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts
@@ -39,7 +39,7 @@ describe('WHERE <expression>', () => {
           .map((name) => `${name} `)
           .map(attachTriggerCommand),
         attachTriggerCommand('var0 '),
-        ...allEvalFns,
+        ...allEvalFns.filter((fn) => fn.label !== 'QSTR'),
       ],
       {
         callbacks: {

diff --git a/...esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts b/...esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts
@@ -371,7 +371,7 @@ describe('autocomplete.suggest', () => {
       for (const fn of scalarFunctionDefinitions) {
         // skip this fn for the moment as it's quite hard to test
         // Add match in the text when the autocomplete is ready https://github.com/elastic/kibana/issues/196995
-        if (!['bucket', 'date_extract', 'date_diff', 'case', 'match'].includes(fn.name)) {
+        if (!['bucket', 'date_extract', 'date_diff', 'case', 'match', 'qstr'].includes(fn.name)) {
           test(`${fn.name}`, async () => {
             const testedCases = new Set<string>();
 

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts
@@ -121,7 +121,7 @@ export const policies = [
  * @returns
  */
 export function getFunctionSignaturesByReturnType(
-  command: string,
+  command: string | string[],
   _expectedReturnType: Readonly<FunctionReturnType | 'any' | Array<FunctionReturnType | 'any'>>,
   {
     agg,
@@ -165,12 +165,16 @@ export function getFunctionSignaturesByReturnType(
 
   const deduped = Array.from(new Set(list));
 
+  const commands = Array.isArray(command) ? command : [command];
   return deduped
     .filter(({ signatures, ignoreAsSuggestion, supportedCommands, supportedOptions, name }) => {
       if (ignoreAsSuggestion) {
         return false;
       }
-      if (!supportedCommands.includes(command) && !supportedOptions?.includes(option || '')) {
+      if (
+        !commands.some((c) => supportedCommands.includes(c)) &&
+        !supportedOptions?.includes(option || '')
+      ) {
         return false;
       }
       const filteredByReturnType = signatures.filter(

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
@@ -10,6 +10,7 @@
 import { uniq, uniqBy } from 'lodash';
 import type {
   AstProviderFn,
+  ESQLAst,
   ESQLAstItem,
   ESQLCommand,
   ESQLCommandOption,
@@ -151,14 +152,16 @@ export async function suggest(
   astProvider: AstProviderFn,
   resourceRetriever?: ESQLCallbacks
 ): Promise<SuggestionRawDefinition[]> {
+  // Partition out to inner ast / ast context for the latest command
   const innerText = fullText.substring(0, offset);
-
   const correctedQuery = correctQuerySyntax(innerText, context);
-
   const { ast } = await astProvider(correctedQuery);
-
   const astContext = getAstContext(innerText, ast, offset);
 
+  // But we also need the full ast for the full query
+  const correctedFullQuery = correctQuerySyntax(fullText, context);
+  const { ast: fullAst } = await astProvider(correctedFullQuery);
+
   if (astContext.type === 'comment') {
     return [];
   }
@@ -216,7 +219,8 @@ export async function suggest(
       getFieldsMap,
       getPolicies,
       getPolicyMetadata,
-      resourceRetriever?.getPreferences
+      resourceRetriever?.getPreferences,
+      fullAst
     );
   }
   if (astContext.type === 'setting') {
@@ -394,7 +398,8 @@ async function getSuggestionsWithinCommandExpression(
   getFieldsMap: GetFieldsMapFn,
   getPolicies: GetPoliciesFn,
   getPolicyMetadata: GetPolicyMetadataFn,
-  getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>
+  getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
+  fullAst?: ESQLAst
 ) {
   const commandDef = getCommandDefinition(command.name);
 
@@ -413,7 +418,8 @@ async function getSuggestionsWithinCommandExpression(
       () => findNewVariable(anyVariables),
       (expression: ESQLAstItem | undefined) =>
         getExpressionType(expression, references.fields, references.variables),
-      getPreferences
+      getPreferences,
+      fullAst
     );
   } else {
     // The deprecated path.
@@ -1173,19 +1179,21 @@ async function getFunctionArgsSuggestions(
     );
 
     // Functions
-    suggestions.push(
-      ...getFunctionSuggestions({
-        command: command.name,
-        option: option?.name,
-        returnTypes: canBeBooleanCondition
-          ? ['any']
-          : (getTypesFromParamDefs(typesToSuggestNext) as string[]),
-        ignored: fnToIgnore,
-      }).map((suggestion) => ({
-        ...suggestion,
-        text: addCommaIf(shouldAddComma, suggestion.text),
-      }))
-    );
+    if (typesToSuggestNext.every((d) => !d.fieldsOnly)) {
+      suggestions.push(
+        ...getFunctionSuggestions({
+          command: command.name,
+          option: option?.name,
+          returnTypes: canBeBooleanCondition
+            ? ['any']
+            : (getTypesFromParamDefs(typesToSuggestNext) as string[]),
+          ignored: fnToIgnore,
+        }).map((suggestion) => ({
+          ...suggestion,
+          text: addCommaIf(shouldAddComma, suggestion.text),
+        }))
+      );
+    }
     // could also be in stats (bucket) but our autocomplete is not great yet
     if (
       (getTypesFromParamDefs(typesToSuggestNext).includes('date') &&

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts
@@ -13,6 +13,7 @@ import {
   type ESQLCommand,
   type ESQLSingleAstItem,
   type ESQLFunction,
+  ESQLAst,
 } from '@kbn/esql-ast';
 import { logicalOperators } from '../../../definitions/builtin';
 import { isParameterType, type SupportedDataType } from '../../../definitions/types';
@@ -27,6 +28,10 @@ import {
 import { getOverlapRange, getSuggestionsToRightOfOperatorExpression } from '../../helper';
 import { getPosition } from './util';
 import { pipeCompleteItem } from '../../complete_items';
+import {
+  UNSUPPORTED_COMMANDS_BEFORE_MATCH,
+  UNSUPPORTED_COMMANDS_BEFORE_QSTR,
+} from '../../../shared/constants';
 
 export async function suggest(
   innerText: string,
@@ -35,7 +40,8 @@ export async function suggest(
   _columnExists: (column: string) => boolean,
   _getSuggestedVariableName: () => string,
   getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-  _getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>
+  _getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
+  fullTextAst?: ESQLAst
 ): Promise<SuggestionRawDefinition[]> {
   const suggestions: SuggestionRawDefinition[] = [];
 
@@ -154,11 +160,25 @@ export async function suggest(
       break;
 
     case 'empty_expression':
+      // Don't suggest MATCH or QSTR after unsupported commands
+      const priorCommands = fullTextAst?.map((a) => a.name) ?? [];
+      const ignored = [];
+      if (priorCommands.some((c) => UNSUPPORTED_COMMANDS_BEFORE_MATCH.has(c))) {
+        ignored.push('match');
+      }
+      if (priorCommands.some((c) => UNSUPPORTED_COMMANDS_BEFORE_QSTR.has(c))) {
+        ignored.push('qstr');
+      }
+
       const columnSuggestions = await getColumnsByType('any', [], {
         advanceCursor: true,
         openSuggestions: true,
       });
-      suggestions.push(...columnSuggestions, ...getFunctionSuggestions({ command: 'where' }));
+
+      suggestions.push(
+        ...columnSuggestions,
+        ...getFunctionSuggestions({ command: 'where', ignored })
+      );
 
       break;
   }

diff --git a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts
@@ -15,6 +15,7 @@ import type { CodeActionOptions } from './types';
 import type { ESQLRealField } from '../validation/types';
 import type { FieldType } from '../definitions/types';
 import type { ESQLCallbacks, PartialFieldsMetadataClient } from '../shared/types';
+import { FULL_TEXT_SEARCH_FUNCTIONS } from '../shared/constants';
 
 function getCallbackMocks(): jest.Mocked<ESQLCallbacks> {
   return {
@@ -285,6 +286,16 @@ describe('quick fixes logic', () => {
       { relaxOnMissingCallbacks: false },
     ]) {
       for (const fn of getAllFunctions({ type: 'eval' })) {
+        if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) {
+          testQuickFixes(
+            `FROM index | WHERE ${BROKEN_PREFIX}${fn.name}()`,
+            [fn.name].map(toFunctionSignature),
+            { equalityCheck: 'include', ...options }
+          );
+        }
+      }
+      for (const fn of getAllFunctions({ type: 'eval' })) {
+        if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) continue;
         // add an A to the function name to make it invalid
         testQuickFixes(
           `FROM index | EVAL ${BROKEN_PREFIX}${fn.name}()`,
@@ -313,6 +324,8 @@ describe('quick fixes logic', () => {
         );
       }
       for (const fn of getAllFunctions({ type: 'agg' })) {
+        if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) continue;
+
         // add an A to the function name to make it invalid
         testQuickFixes(
           `FROM index | STATS ${BROKEN_PREFIX}${fn.name}()`,

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts
@@ -3259,6 +3259,7 @@ const matchDefinition: FunctionDefinition = {
           name: 'field',
           type: 'keyword',
           optional: false,
+          fieldsOnly: true,
         },
         {
           name: 'query',
@@ -3274,6 +3275,7 @@ const matchDefinition: FunctionDefinition = {
           name: 'field',
           type: 'keyword',
           optional: false,
+          fieldsOnly: true,
         },
         {
           name: 'query',
@@ -3289,6 +3291,7 @@ const matchDefinition: FunctionDefinition = {
           name: 'field',
           type: 'text',
           optional: false,
+          fieldsOnly: true,
         },
         {
           name: 'query',
@@ -3304,6 +3307,7 @@ const matchDefinition: FunctionDefinition = {
           name: 'field',
           type: 'text',
           optional: false,
+          fieldsOnly: true,
         },
         {
           name: 'query',
@@ -3314,8 +3318,8 @@ const matchDefinition: FunctionDefinition = {
       returnType: 'boolean',
     },
   ],
-  supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'],
-  supportedOptions: ['by'],
+  supportedCommands: ['where'],
+  supportedOptions: [],
   validate: undefined,
   examples: [
     'from books \n| where match(author, "Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;',
@@ -5912,6 +5916,7 @@ const qstrDefinition: FunctionDefinition = {
           name: 'query',
           type: 'keyword',
           optional: false,
+          fieldsOnly: true,
         },
       ],
       returnType: 'boolean',
@@ -5922,13 +5927,14 @@ const qstrDefinition: FunctionDefinition = {
           name: 'query',
           type: 'text',
           optional: false,
+          fieldsOnly: true,
         },
       ],
       returnType: 'boolean',
     },
   ],
-  supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'],
-  supportedOptions: ['by'],
+  supportedCommands: ['where'],
+  supportedOptions: [],
   validate: undefined,
   examples: [
     'from books \n| where qstr("author: Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;',

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts
@@ -8,6 +8,7 @@
  */
 
 import type {
+  ESQLAst,
   ESQLAstItem,
   ESQLCommand,
   ESQLCommandOption,
@@ -136,6 +137,10 @@ export interface FunctionDefinition {
        * though a function can be used to create the value. (e.g. now() for dates or concat() for strings)
        */
       constantOnly?: boolean;
+      /**
+       * Default to false. If set to true, this parameter does not accept a function or literal, only fields.
+       */
+      fieldsOnly?: boolean;
       /**
        * if provided this means that the value must be one
        * of the options in the array iff the value is a literal.
@@ -181,7 +186,8 @@ export interface CommandBaseDefinition<CommandName extends string> {
     columnExists: (column: string) => boolean,
     getSuggestedVariableName: () => string,
     getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-    getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>
+    getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
+    fullTextAst?: ESQLAst
   ) => Promise<SuggestionRawDefinition[]>;
   /** @deprecated this property will disappear in the future */
   signature: {

diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts b/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts
@@ -16,3 +16,19 @@ export const DOUBLE_BACKTICK = '``';
 export const SINGLE_BACKTICK = '`';
 
 export const METADATA_FIELDS = ['_version', '_id', '_index', '_source', '_ignored', '_index_mode'];
+
+export const FULL_TEXT_SEARCH_FUNCTIONS = ['match', 'qstr'];
+export const UNSUPPORTED_COMMANDS_BEFORE_QSTR = new Set([
+  'show',
+  'row',
+  'dissect',
+  'enrich',
+  'eval',
+  'grok',
+  'keep',
+  'mv_expand',
+  'rename',
+  'stats',
+  'limit',
+]);
+export const UNSUPPORTED_COMMANDS_BEFORE_MATCH = new Set(['limit']);