Skip to content

Commit

Permalink
[ES|QL] use lexer from elasticsearch (#178257)
Browse files Browse the repository at this point in the history 
## Summary

Up till now, we had to define our own lexer rules for our client-side
ES|QL validation. This was because we were using an unofficial ANTLR
package (before the official ANTLR had typescript support).

Now that we are using the official ANTLR library (as of
#177211), we no longer have to
encode case insensitivity into the lexer rules themselves because the
[`caseInsensitive` option](antlr/antlr4#3399) is
now available to us.

This means we can adopt the very [same
definitions](https://github.com/elastic/elasticsearch/blob/343b1ae1ba74fbf2e75c29adddb2790312dd680b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4)
that Elasticsearch uses as long as we set `caseInsensitive`
(Elasticsearch handles case insensitivity at runtime).

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
  • Loading branch information
@drewdaemon
drewdaemon authored Mar 8, 2024
1 parent b0170e5 commit 05dfb28
Show file tree
Hide file tree
Showing 8 changed files with 904 additions and 971 deletions.
651 changes: 286 additions & 365 deletions packages/kbn-monaco/src/esql/antlr/esql_lexer.g4
View file

Large diffs are not rendered by default.

106 changes: 40 additions & 66 deletions packages/kbn-monaco/src/esql/antlr/esql_lexer.interp
View file

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions packages/kbn-monaco/src/esql/antlr/esql_lexer.tokens
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,46 @@ SETTING=101
SETTING_LINE_COMMENT=102
SETTTING_MULTILINE_COMMENT=103
SETTING_WS=104
'dissect'=1
'drop'=2
'enrich'=3
'eval'=4
'explain'=5
'from'=6
'grok'=7
'inlinestats'=8
'keep'=9
'limit'=10
'mv_expand'=11
'rename'=12
'row'=13
'show'=14
'sort'=15
'stats'=16
'where'=17
'|'=25
'by'=29
'and'=30
'asc'=31
'='=32
','=33
'desc'=34
'.'=35
'false'=36
'first'=37
'last'=38
'('=39
'in'=40
'is'=41
'like'=42
'not'=43
'null'=44
'nulls'=45
'or'=46
'?'=47
'rlike'=48
')'=49
'true'=50
'=='=51
'=~'=52
'!='=53
Expand All @@ -122,4 +155,10 @@ SETTING_WS=104
'/'=61
'%'=62
']'=64
'metadata'=70
'as'=79
'on'=83
'with'=84
'info'=95
'functions'=96
':'=100
841 changes: 400 additions & 441 deletions packages/kbn-monaco/src/esql/antlr/esql_lexer.ts
View file

Large diffs are not rendered by default.

78 changes: 39 additions & 39 deletions packages/kbn-monaco/src/esql/antlr/esql_parser.interp
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
token literal names:
null
null
null
null
null
null
null
null
null
null
null
null
null
null
null
null
null
null
'dissect'
'drop'
'enrich'
'eval'
'explain'
'from'
'grok'
'inlinestats'
'keep'
'limit'
'mv_expand'
'rename'
'row'
'show'
'sort'
'stats'
'where'
null
null
null
Expand All @@ -28,28 +28,28 @@ null
null
null
null
null
null
null
'by'
'and'
'asc'
'='
','
null
'desc'
'.'
null
null
null
'false'
'first'
'last'
'('
null
null
null
null
null
null
null
'in'
'is'
'like'
'not'
'null'
'nulls'
'or'
'?'
null
'rlike'
')'
null
'true'
'=='
'=~'
'!='
Expand All @@ -69,6 +69,7 @@ null
null
null
null
'metadata'
null
null
null
Expand All @@ -77,9 +78,12 @@ null
null
null
null
'as'
null
null
null
'on'
'with'
null
null
null
Expand All @@ -90,12 +94,8 @@ null
null
null
null
null
null
null
null
null
null
'info'
'functions'
null
null
null
Expand Down
39 changes: 39 additions & 0 deletions packages/kbn-monaco/src/esql/antlr/esql_parser.tokens
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,46 @@ SETTING_LINE_COMMENT=102
SETTTING_MULTILINE_COMMENT=103
SETTING_WS=104
UNQUOTED_ID_PATTERN=105
'dissect'=1
'drop'=2
'enrich'=3
'eval'=4
'explain'=5
'from'=6
'grok'=7
'inlinestats'=8
'keep'=9
'limit'=10
'mv_expand'=11
'rename'=12
'row'=13
'show'=14
'sort'=15
'stats'=16
'where'=17
'|'=25
'by'=29
'and'=30
'asc'=31
'='=32
','=33
'desc'=34
'.'=35
'false'=36
'first'=37
'last'=38
'('=39
'in'=40
'is'=41
'like'=42
'not'=43
'null'=44
'nulls'=45
'or'=46
'?'=47
'rlike'=48
')'=49
'true'=50
'=='=51
'=~'=52
'!='=53
Expand All @@ -123,4 +156,10 @@ UNQUOTED_ID_PATTERN=105
'/'=61
'%'=62
']'=64
'metadata'=70
'as'=79
'on'=83
'with'=84
'info'=95
'functions'=96
':'=100
69 changes: 35 additions & 34 deletions packages/kbn-monaco/src/esql/antlr/esql_parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,57 +175,58 @@ export default class esql_parser extends Parser {
public static readonly RULE_showCommand = 48;
public static readonly RULE_enrichCommand = 49;
public static readonly RULE_enrichWithClause = 50;
public static readonly literalNames: (string | null)[] = [ null, null,
public static readonly literalNames: (string | null)[] = [ null, "'dissect'",
"'drop'", "'enrich'",
"'eval'", "'explain'",
"'from'", "'grok'",
"'inlinestats'",
"'keep'", "'limit'",
"'mv_expand'",
"'rename'",
"'row'", "'show'",
"'sort'", "'stats'",
"'where'", null,
null, null,
null, null,
null, null,
"'|'", null,
null, null,
"'by'", "'and'",
"'asc'", "'='",
"','", "'desc'",
"'.'", "'false'",
"'first'", "'last'",
"'('", "'in'",
"'is'", "'like'",
"'not'", "'null'",
"'nulls'", "'or'",
"'?'", "'rlike'",
"')'", "'true'",
"'=='", "'=~'",
"'!='", "'<'",
"'<='", "'>'",
"'>='", "'+'",
"'-'", "'*'",
"'/'", "'%'",
null, "']'",
null, null,
null, null,
null, "'metadata'",
null, null,
null, null,
null, null,
null, null,
"'as'", null,
null, null,
null, "'|'",
"'on'", "'with'",
null, null,
null, null,
null, null,
"'='", "','",
null, "'.'",
null, null,
null, "'('",
null, null,
"'info'", "'functions'",
null, null,
null, null,
null, "'?'",
null, "')'",
null, "'=='",
"'=~'", "'!='",
"'<'", "'<='",
"'>'", "'>='",
"'+'", "'-'",
"'*'", "'/'",
"'%'", null,
"']'", null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
null, null,
"':'" ];
null, "':'" ];
public static readonly symbolicNames: (string | null)[] = [ null, "DISSECT",
"DROP", "ENRICH",
"EVAL", "EXPLAIN",
Expand Down
Loading

0 comments on commit 05dfb28

Please sign in to comment.