Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ES|QL] Update grammars #186013

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .buildkite/pipelines/esql_grammar_sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,12 @@ steps:
provider: gcp
machineType: n2-standard-2
preemptible: true
- command: .buildkite/scripts/steps/esql_generate_function_docs.sh
label: Generate Function Definitions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
label: Generate Function Definitions
label: Generate Function Docs

timeout_in_minutes: 10
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can bump these timeouts a bit higher. Generate Function Definitions is at 8-9 minutes runtime.

agents:
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
provider: gcp
machineType: n2-standard-2
preemptible: true
72 changes: 72 additions & 0 deletions .buildkite/scripts/steps/esql_generate_function_docs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
set -euo pipefail

report_main_step () {
echo "--- $1"
}

main () {
cd "$PARENT_DIR"

report_main_step "Cloning Elasticsearch repository"

rm -rf elasticsearch
git clone https://github.com/elastic/elasticsearch --depth 1

report_main_step "Bootstrapping Kibana"

cd "$KIBANA_DIR"

.buildkite/scripts/bootstrap.sh

cd "$KIBANA_DIR/packages/kbn-text-based-editor"

report_main_step "Generate function docs"

yarn make:docs $PARENT_DIR/elasticsearch

# Check for differences
set +e
git diff --exit-code --quiet .
if [ $? -eq 0 ]; then
echo "No differences found. Our work is done here."
exit
fi
set -e

report_main_step "Differences found. Checking for an existing pull request."

KIBANA_MACHINE_USERNAME="kibanamachine"
git config --global user.name "$KIBANA_MACHINE_USERNAME"
git config --global user.email '[email protected]'

PR_TITLE='[ES|QL] Update function docs'
PR_BODY='This PR updates the function docs based on the latest metadata from Elasticsearch.'

# Check if a PR already exists
pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME" --limit 1 --json title -q ".[].title")

if [ "$pr_search_result" == "$PR_TITLE" ]; then
echo "PR already exists. Exiting."
exit
fi

echo "No existing PR found. Committing changes."

# Make a commit
BRANCH_NAME="esql_generate_function_docs_$(date +%s)"

git checkout -b "$BRANCH_NAME"

git add ./**/*
git commit -m "Update function docs"

report_main_step "Changes committed. Creating pull request."

git push origin "$BRANCH_NAME"

# Create a PR
gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL'
}

main
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -1225,6 +1225,7 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
/.buildkite/ @elastic/kibana-operations
/.buildkite/scripts/steps/esql_grammar_sync.sh @elastic/kibana-esql
/.buildkite/scripts/steps/esql_generate_function_definitions.sh @elastic/kibana-esql
/.buildkite/scripts/steps/esql_generate_function_docs.sh @elastic/kibana-esql
/.buildkite/pipelines/esql_grammar_sync.yml @elastic/kibana-esql
/kbn_pm/ @elastic/kibana-operations
/x-pack/dev-tools @elastic/kibana-operations
Expand Down
5 changes: 5 additions & 0 deletions packages/kbn-esql-ast/src/antlr/esql_lexer.g4
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,11 @@ ASTERISK : '*';
SLASH : '/';
PERCENT : '%';

NAMED_OR_POSITIONAL_PARAM
: PARAM LETTER UNQUOTED_ID_BODY*
| PARAM DIGIT+
;

// Brackets are funny. We can happen upon a CLOSING_BRACKET in two ways - one
// way is to start in an explain command which then shifts us to expression
// mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as
Expand Down
5 changes: 4 additions & 1 deletion packages/kbn-esql-ast/src/antlr/esql_lexer.interp

Large diffs are not rendered by default.

129 changes: 65 additions & 64 deletions packages/kbn-esql-ast/src/antlr/esql_lexer.tokens
Original file line number Diff line number Diff line change
Expand Up @@ -65,62 +65,63 @@ MINUS=64
ASTERISK=65
SLASH=66
PERCENT=67
OPENING_BRACKET=68
CLOSING_BRACKET=69
UNQUOTED_IDENTIFIER=70
QUOTED_IDENTIFIER=71
EXPR_LINE_COMMENT=72
EXPR_MULTILINE_COMMENT=73
EXPR_WS=74
METADATA=75
FROM_LINE_COMMENT=76
FROM_MULTILINE_COMMENT=77
FROM_WS=78
ID_PATTERN=79
PROJECT_LINE_COMMENT=80
PROJECT_MULTILINE_COMMENT=81
PROJECT_WS=82
AS=83
RENAME_LINE_COMMENT=84
RENAME_MULTILINE_COMMENT=85
RENAME_WS=86
ON=87
WITH=88
ENRICH_POLICY_NAME=89
ENRICH_LINE_COMMENT=90
ENRICH_MULTILINE_COMMENT=91
ENRICH_WS=92
ENRICH_FIELD_LINE_COMMENT=93
ENRICH_FIELD_MULTILINE_COMMENT=94
ENRICH_FIELD_WS=95
LOOKUP_LINE_COMMENT=96
LOOKUP_MULTILINE_COMMENT=97
LOOKUP_WS=98
LOOKUP_FIELD_LINE_COMMENT=99
LOOKUP_FIELD_MULTILINE_COMMENT=100
LOOKUP_FIELD_WS=101
MVEXPAND_LINE_COMMENT=102
MVEXPAND_MULTILINE_COMMENT=103
MVEXPAND_WS=104
INFO=105
SHOW_LINE_COMMENT=106
SHOW_MULTILINE_COMMENT=107
SHOW_WS=108
FUNCTIONS=109
META_LINE_COMMENT=110
META_MULTILINE_COMMENT=111
META_WS=112
COLON=113
SETTING=114
SETTING_LINE_COMMENT=115
SETTTING_MULTILINE_COMMENT=116
SETTING_WS=117
METRICS_LINE_COMMENT=118
METRICS_MULTILINE_COMMENT=119
METRICS_WS=120
CLOSING_METRICS_LINE_COMMENT=121
CLOSING_METRICS_MULTILINE_COMMENT=122
CLOSING_METRICS_WS=123
NAMED_OR_POSITIONAL_PARAM=68
OPENING_BRACKET=69
CLOSING_BRACKET=70
UNQUOTED_IDENTIFIER=71
QUOTED_IDENTIFIER=72
EXPR_LINE_COMMENT=73
EXPR_MULTILINE_COMMENT=74
EXPR_WS=75
METADATA=76
FROM_LINE_COMMENT=77
FROM_MULTILINE_COMMENT=78
FROM_WS=79
ID_PATTERN=80
PROJECT_LINE_COMMENT=81
PROJECT_MULTILINE_COMMENT=82
PROJECT_WS=83
AS=84
RENAME_LINE_COMMENT=85
RENAME_MULTILINE_COMMENT=86
RENAME_WS=87
ON=88
WITH=89
ENRICH_POLICY_NAME=90
ENRICH_LINE_COMMENT=91
ENRICH_MULTILINE_COMMENT=92
ENRICH_WS=93
ENRICH_FIELD_LINE_COMMENT=94
ENRICH_FIELD_MULTILINE_COMMENT=95
ENRICH_FIELD_WS=96
LOOKUP_LINE_COMMENT=97
LOOKUP_MULTILINE_COMMENT=98
LOOKUP_WS=99
LOOKUP_FIELD_LINE_COMMENT=100
LOOKUP_FIELD_MULTILINE_COMMENT=101
LOOKUP_FIELD_WS=102
MVEXPAND_LINE_COMMENT=103
MVEXPAND_MULTILINE_COMMENT=104
MVEXPAND_WS=105
INFO=106
SHOW_LINE_COMMENT=107
SHOW_MULTILINE_COMMENT=108
SHOW_WS=109
FUNCTIONS=110
META_LINE_COMMENT=111
META_MULTILINE_COMMENT=112
META_WS=113
COLON=114
SETTING=115
SETTING_LINE_COMMENT=116
SETTTING_MULTILINE_COMMENT=117
SETTING_WS=118
METRICS_LINE_COMMENT=119
METRICS_MULTILINE_COMMENT=120
METRICS_WS=121
CLOSING_METRICS_LINE_COMMENT=122
CLOSING_METRICS_MULTILINE_COMMENT=123
CLOSING_METRICS_WS=124
'dissect'=1
'drop'=2
'enrich'=3
Expand Down Expand Up @@ -177,11 +178,11 @@ CLOSING_METRICS_WS=123
'*'=65
'/'=66
'%'=67
']'=69
'metadata'=75
'as'=83
'on'=87
'with'=88
'info'=105
'functions'=109
':'=113
']'=70
'metadata'=76
'as'=84
'on'=88
'with'=89
'info'=106
'functions'=110
':'=114
Loading