Skip to content

Commit

Permalink
[ES|QL] Advanced validation and autocomplete (#170071)
Browse files Browse the repository at this point in the history 
## Summary

Fixes #166242 , #166876, #166173
, part of #166092, #166084

List of tasks:

* [x] AST work ( #166185 )
* [x] Basic validate work ( #166185 )
* [x] Hover feature ( #166185 )
* [x] Initial autocomplete work with new AST ( #166185 )
* Complete validation feature for MVP
  * [x] wildcards support ( #170014 )
  * [x] remote index validation support ( #171996 )
  * [x] wildcard support as `count` argument  ( #172054 )
  * [x] Aggressive caching for field queries:
* cache as much as possible the `FROM` queries - possible clear the
cache every 10/15 minutes?
    * do not fire a query when code == submitted code
* cache as much as possible the custom `FROM` built from `ENRICH`
policies - same clear policy as above
  * [x] Add unsupported fields warning messages
  * [x] Notify usage of `project` command with deprecation `warning`
* Complete autocomplete work ( #171664 )
  * [x] `stats`
  * [x] `where`
  * [x] `eval`
    * `math syntax`
  * [x] Aggressive cache for fields queries? ( #171866 )
  * [x] Fix when cursor is not at the end position ( #172060 ) 
* [x] Revisit copy messages
  * Label Kibana-only messages
* [x] Extend hover feature ( #171940 )
* [x] Disable editor query highlight for warnings ( #171968 )
* Fix editor highlight with new grammar
  * [x] on multi-line ( #172080 )
  * [x] for functions ( #172287 )

## Release notes

Enhance ES|QL query editing experience with client side validation.
Enhance ES|QL suggestions experience with more in context suggestions
leveraging field and variable types.
Show meta informations on ES|QL query hover on policy names.
Show function signature on ES|QL query hover on function text.


### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Stratoula Kalafateli <[email protected]>
Co-authored-by: Abdon Pijpelink <[email protected]>
  • Loading branch information
@dej611 @kibanamachine
@stratoula @abdonpijpelink
4 people authored Dec 7, 2023
1 parent ef5305d commit ff49526
Show file tree
Hide file tree
Showing 78 changed files with 12,342 additions and 7,125 deletions.
2 changes: 1 addition & 1 deletion packages/kbn-monaco/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export { monaco } from './src/monaco_imports';
export { XJsonLang } from './src/xjson';
export { SQLLang } from './src/sql';
export { ESQL_LANG_ID, ESQL_THEME_ID, ESQLLang } from './src/esql';
export type { ESQLCustomAutocompleteCallbacks } from './src/esql';
export type { ESQLCallbacks } from './src/esql';

export * from './src/painless';
/* eslint-disable-next-line @kbn/eslint/module_migration */
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-monaco/src/common/error_listener.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export class ANTLREErrorListener implements ANTLRErrorListener<any> {
startColumn: column,
endColumn,
message,
severity: 8,
});
}

Expand Down
244 changes: 50 additions & 194 deletions packages/kbn-monaco/src/esql/antlr/esql_lexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,27 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

lexer grammar esql_lexer;
options { }

DISSECT : D I S S E C T -> pushMode(EXPRESSION);
GROK : G R O K -> pushMode(EXPRESSION);
DROP : D R O P -> pushMode(SOURCE_IDENTIFIERS);
ENRICH : E N R I C H -> pushMode(SOURCE_IDENTIFIERS);
EVAL : E V A L -> pushMode(EXPRESSION);
EXPLAIN : E X P L A I N -> pushMode(EXPLAIN_MODE);
FROM : F R O M -> pushMode(SOURCE_IDENTIFIERS);
GROK : G R O K -> pushMode(EXPRESSION);
KEEP : K E E P -> pushMode(SOURCE_IDENTIFIERS);
LIMIT : L I M I T -> pushMode(EXPRESSION);
MV_EXPAND : M V UNDERSCORE E X P A N D -> pushMode(SOURCE_IDENTIFIERS);
PROJECT : P R O J E C T -> pushMode(SOURCE_IDENTIFIERS);
RENAME : R E N A M E -> pushMode(SOURCE_IDENTIFIERS);
ROW : R O W -> pushMode(EXPRESSION);
SHOW : S H O W -> pushMode(EXPRESSION);
SORT : S O R T -> pushMode(EXPRESSION);
STATS : S T A T S -> pushMode(EXPRESSION);
WHERE : W H E R E -> pushMode(EXPRESSION);
SORT : S O R T -> pushMode(EXPRESSION);
MV_EXPAND : M V UNDERSCORE E X P A N D -> pushMode(EXPRESSION);
LIMIT : L I M I T -> pushMode(EXPRESSION);
PROJECT : P R O J E C T -> pushMode(EXPRESSION);
DROP : D R O P -> pushMode(EXPRESSION);
RENAME : R E N A M E -> pushMode(EXPRESSION);
SHOW : S H O W -> pushMode(EXPRESSION);
ENRICH : E N R I C H -> pushMode(ENRICH_IDENTIFIERS);
KEEP : K E E P -> pushMode(EXPRESSION);
UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION);

LINE_COMMENT
: '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
Expand All @@ -36,12 +37,7 @@ MULTILINE_COMMENT
WS
: [ \r\n\t]+ -> channel(HIDDEN)
;
mode EXPLAIN_MODE;
EXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
EXPLAIN_PIPE : '|' -> type(PIPE), popMode;
EXPLAIN_WS : WS -> channel(HIDDEN);
EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);

mode EXPRESSION;

PIPE : '|' -> popMode;
Expand Down Expand Up @@ -82,169 +78,60 @@ DECIMAL_LITERAL
| DOT DIGIT+ EXPONENT
;

BY : 'by';

DATE_LITERAL
: 'year'
| 'month'
| 'day'
| 'second'
| 'minute'
| 'hour'
| 'week'
| 'millisecond'
| 'years'
| 'months'
| 'days'
| 'seconds'
| 'minutes'
| 'hours'
| 'weeks'
| 'milliseconds'
;
BY : B Y;

AND : 'and';
AND : A N D;
ASC : A S C;
ASSIGN : '=';
COMMA : ',';
DESC : D E S C;
DOT : '.';
FALSE : F A L S E;
FIRST : F I R S T;
LAST : L A S T;
LP : '(';
OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);
CLOSING_BRACKET : ']' -> popMode, popMode;
NOT : N O T;
LIKE: L I K E;
RLIKE: R L I K E;
IN: I N;
IS: I S;
AS: A S;
LIKE: L I K E;
NOT : N O T;
NULL : N U L L;
OR : 'or';
NULLS : N U L L S;
OR : O R;
PARAM: '?';
RLIKE: R L I K E;
RP : ')';
TRUE : T R U E;
INFO : I N F O;
FUNCTIONS : F U N C T I O N S;
UNDERSCORE: '_';
INFO : 'info';
FUNCTIONS : 'functions';

BOOLEAN_VALUE
: 'true'
| 'false'
;

COMPARISON_OPERATOR
: '=='
|'!='
| '<'
| '<='
| '>'
| '>='
;

EQ : '==';
NEQ : '!=';
LT : '<';
LTE : '<=';
GT : '>';
GTE : '>=';

PLUS : '+';
MINUS : '-';
ASTERISK : '*';
SLASH : '/';
PERCENT : '%';
TEN: '10';

ORDERING
: 'asc'
| 'desc'
;

NULLS_ORDERING: 'nulls';
NULLS_ORDERING_DIRECTION
: 'first'
| 'last'
;

MATH_FUNCTION
: R O U N D
| A B S
| P O W
| L O G TEN
| P I
| T A U
| E
| S U B S T R I N G
| T R I M
| C O N C A T
| C O A L E S C E
| G R E A T E S T
| L E F T
| N O W
| R I G H T
| S T A R T S UNDERSCORE W I T H
| D A T E UNDERSCORE F O R M A T
| D A T E UNDERSCORE T R U N C
| D A T E UNDERSCORE P A R S E
| A U T O UNDERSCORE B U C K E T
| D A T E UNDERSCORE E X T R A C T
| I S UNDERSCORE F I N I T E
| I S UNDERSCORE I N F I N I T E
| C A S E
| L E N G T H
| M V UNDERSCORE M A X
| M V UNDERSCORE M I N
| M V UNDERSCORE A V G
| M V UNDERSCORE S U M
| M V UNDERSCORE C O U N T
| M V UNDERSCORE C O N C A T
| M V UNDERSCORE J O I N
| M V UNDERSCORE M E D I A N
| M V UNDERSCORE D E D U P E
| M E T A D A T A
| S P L I T
| T O UNDERSCORE S T R I N G
| T O UNDERSCORE S T R
| T O UNDERSCORE B O O L
| T O UNDERSCORE B O O L E A N
| T O UNDERSCORE D A T E T I M E
| T O UNDERSCORE D T
| T O UNDERSCORE D B L
| T O UNDERSCORE D O U B L E
| T O UNDERSCORE D E G R E E S
| T O UNDERSCORE I N T
| T O UNDERSCORE I N T E G E R
| T O UNDERSCORE I P
| T O UNDERSCORE L O N G
| T O UNDERSCORE R A D I A N S
| T O UNDERSCORE V E R S I O N
| T O UNDERSCORE U N S I G N E D UNDERSCORE L O N G
;

UNARY_FUNCTION
: A V G
| M I N
| M A X
| S U M
| C O U N T
| C O U N T UNDERSCORE D I S T I N C T
| P E R C E N T I L E
| M E D I A N
| M E D I A N UNDERSCORE A B S O L U T E UNDERSCORE D E V I A T I O N
| A C O S
| A S I N
| A T A N
| A T A N '2'
| C E I L
| C O S
| C O S H
| F L O O R
| L T R I M
| S I N
| S I N H
| S Q R T
| T A N
| T A N H
;
// Brackets are funny. We can happen upon a CLOSING_BRACKET in two ways - one
// way is to start in an explain command which then shifts us to expression
// mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as
// the start of a multivalued field constant. To line up with the double pop
// the explain mode needs, we double push when we see that.
OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);
CLOSING_BRACKET : ']' -> popMode, popMode;

WHERE_FUNCTIONS
: C I D R UNDERSCORE M A T C H
;

UNQUOTED_IDENTIFIER
: LETTER (LETTER | DIGIT | '_' | ASTERISK)*
: LETTER (LETTER | DIGIT | '_')*
// only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future
// also, single `_` and `@` characters are not valid identifiers
| ('_' | '@') (LETTER | DIGIT | '_' | ASTERISK)+
| ('_' | '@') (LETTER | DIGIT | '_')+
;

QUOTED_IDENTIFIER
Expand All @@ -264,14 +151,18 @@ EXPR_WS
;



mode SOURCE_IDENTIFIERS;

SRC_PIPE : '|' -> type(PIPE), popMode;
SRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS);
SRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
SRC_COMMA : ',' -> type(COMMA);
SRC_ASSIGN : '=' -> type(ASSIGN);
AS : A S;
METADATA: M E T A D A T A;
ON : O N;
WITH : W I T H;

SRC_UNQUOTED_IDENTIFIER
: SRC_UNQUOTED_IDENTIFIER_PART+
Expand All @@ -298,41 +189,6 @@ SRC_WS
: WS -> channel(HIDDEN)
;

mode ENRICH_IDENTIFIERS;

ON : O N;
WITH : W I T H;

ENR_PIPE : '|' -> type(PIPE), popMode;
ENR_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
ENR_COMMA : ',' -> type(COMMA);
ENR_ASSIGN : '=' -> type(ASSIGN);

ENR_UNQUOTED_IDENTIFIER
: ENR_UNQUOTED_IDENTIFIER_PART+
;

fragment ENR_UNQUOTED_IDENTIFIER_PART
: ~[=`|,[\]/ \t\r\n]+
| '/' ~[*/] // allow single / but not followed by another / or * which would start a comment
;

ENR_QUOTED_IDENTIFIER
: QUOTED_IDENTIFIER
;

ENR_LINE_COMMENT
: LINE_COMMENT -> channel(HIDDEN)
;

ENR_MULTILINE_COMMENT
: MULTILINE_COMMENT -> channel(HIDDEN)
;

ENR_WS
: WS -> channel(HIDDEN)
;

fragment A : [aA]; // match either an 'a' or 'A'
fragment B : [bB];
fragment C : [cC];
Expand Down
Loading

0 comments on commit ff49526

Please sign in to comment.