[Auto Suggest] OpenSearch SQL autosuggest with ANTLR

.eslintignore

@@ -30,3 +30,7 @@ target
 /packages/osd-test/src/functional_test_runner/lib/config/__tests__/fixtures/
 /packages/osd-ui-framework/dist
 /packages/osd-ui-shared-deps/flot_charts
+
+# antlr overrides
+/src/plugins/data/public/antlr/opensearch_sql/.generated/*
+/src/plugins/data/public/antlr/opensearch_sql/grammar/**/*

.eslintrc.js

@@ -751,5 +751,14 @@ module.exports = {
         'no-undef': 'off',
       },
     },
+    {
+      files: [
+        'src/plugins/data/public/antlr/opensearch_sql/.generated/*',
+        'src/plugins/data/public/antlr/opensearch_sql/grammar/**/*',
+      ],
+      rules: {
+        'filenames/match-regex': 'off',
+      },
+    },
   ],
 };

changelogs/fragments/7336.yml

@@ -0,0 +1,2 @@
+feat:
+- [Auto Suggest] OpenSearch SQL autosuggest with ANTLR ([#7336](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7336))

config/opensearch_dashboards.yml

@@ -362,4 +362,4 @@
 # Set the backend roles in groups or users, whoever has the backend roles or exactly match the user ids defined in this config will be regard as dashboard admin.
 # Dashboard admin will have the access to all the workspaces(workspace.enabled: true) and objects inside OpenSearch Dashboards.
 # opensearchDashboards.dashboardAdmin.groups: ["dashboard_admin"]
-# opensearchDashboards.dashboardAdmin.users: ["dashboard_admin"]
+# opensearchDashboards.dashboardAdmin.users: ["dashboard_admin"]

package.json

@@ -84,7 +84,8 @@
     "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false",
     "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500",
     "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"",
-    "osd:ciGroup11": "echo \"apps/vis_builder/*.js\""
+    "osd:ciGroup11": "echo \"apps/vis_builder/*.js\"",
+    "generate:opensearchsqlantlr": "./node_modules/antlr4ng-cli/index.js -Dlanguage=TypeScript -o ./src/plugins/data/public/antlr/opensearch_sql/.generated -visitor -no-listener -Xexact-output-dir ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLLexer.g4 ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLParser.g4"
   },
   "repository": {
     "type": "git",
@@ -114,7 +115,7 @@
     "**/nth-check": "^2.0.1",
     "**/semver": "^7.5.3",
     "**/set-value": "^4.1.0",
-    "**/tar":"^6.2.1",
+    "**/tar": "^6.2.1",
     "**/topo/hoek": "npm:@amoo-miki/[email protected]",
     "**/trim": "^0.0.3",
     "**/typescript": "4.6.4",
@@ -179,6 +180,8 @@
     "JSONStream": "1.3.5",
     "abortcontroller-polyfill": "^1.4.0",
     "ajv": "^8.11.0",
+    "antlr4-c3": "^3.4.1",
+    "antlr4ng": "^3.0.4",
     "bluebird": "3.5.5",
     "chalk": "^4.1.0",
     "chokidar": "^3.4.2",
@@ -245,11 +248,11 @@
     "whatwg-fetch": "^3.0.0",
     "yauzl": "^2.10.0",
     "@opensearch-dashboards-test/opensearch-dashboards-test-library": "https://github.com/opensearch-project/opensearch-dashboards-test-library/archive/refs/tags/1.0.6.tar.gz"
-
   },
   "devDependencies": {
     "@babel/core": "^7.22.9",
     "@babel/parser": "^7.22.9",
+    "@babel/plugin-transform-class-static-block": "^7.24.4",
     "@babel/register": "^7.22.9",
     "@babel/types": "^7.22.9",
     "@elastic/apm-rum": "^5.6.1",
@@ -364,6 +367,7 @@
     "@types/zen-observable": "^0.8.0",
     "@typescript-eslint/eslint-plugin": "^3.10.0",
     "@typescript-eslint/parser": "^3.10.0",
+    "antlr4ng-cli": "^2.0.0",
     "archiver": "^5.3.0",
     "axe-core": "^4.0.2",
     "babel-eslint": "^10.0.3",
@@ -465,6 +469,7 @@
     "regenerate": "^1.4.0",
     "reselect": "^4.0.0",
     "resize-observer-polyfill": "^1.5.1",
+    "rimraf": "^5.0.7",
     "selenium-webdriver": "^4.0.0-alpha.7",
     "simple-git": "^3.16.0",
     "sinon": "^7.4.2",

packages/osd-babel-preset/webpack_preset.js

@@ -55,6 +55,12 @@ module.exports = () => {
           fileName: false,
         },
       ],
+      [
+        require.resolve('@babel/plugin-transform-class-static-block'),
+        {
+          fileName: false,
+        },
+      ],
     ],
     env: {
       production: {

packages/osd-monaco/src/xjson/lexer_rules/index.ts

@@ -33,6 +33,7 @@ import * as monaco from 'monaco-editor/esm/vs/editor/editor.api';
 import * as xJson from './xjson';
 import * as opensearchql from './opensearchql';
 import * as painless from './painless';
+import * as opensearchsql from './opensearchsql';
 
 export const registerLexerRules = (m: typeof monaco) => {
   m.languages.register({ id: xJson.ID });
@@ -41,4 +42,6 @@ export const registerLexerRules = (m: typeof monaco) => {
   m.languages.setMonarchTokensProvider(painless.ID, painless.lexerRules);
   m.languages.register({ id: opensearchql.ID });
   m.languages.setMonarchTokensProvider(opensearchql.ID, opensearchql.lexerRules);
+  m.languages.register({ id: opensearchsql.ID });
+  m.languages.setMonarchTokensProvider(opensearchsql.ID, opensearchsql.lexerRules);
 };

packages/osd-monaco/src/xjson/lexer_rules/opensearchsql.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { monaco } from '../../monaco';
+
+export const ID = 'SQL';
+
+const keywords = [
+  'ALL',
+  'AND',
+  'AS',
+  'ASC',
+  'BOOLEAN',
+  'BETWEEN',
+  'BY',
+  'CASE',
+  'CAST',
+  'CROSS',
+  'COLUMNS',
+  'DATETIME',
+  'DELETE',
+  'DESC',
+  'DESCRIBE',
+  'DISTINCT',
+  'DOUBLE',
+  'ELSE',
+  'EXISTS',
+  'FALSE',
+  'FLOAT',
+  'FIRST',
+  'FROM',
+  'GROUP',
+  'HAVING',
+  'IN',
+  'INNER',
+  'INT',
+  'INTEGER',
+  'IS',
+  'JOIN',
+  'LAST',
+  'LEFT',
+  'LIKE',
+  'LIMIT',
+  'LONG',
+  'MATCH',
+  'NATURAL',
+  'NOT',
+  'NULL',
+  'NULLS',
+  'ON',
+  'OR',
+  'ORDER',
+  'OUTER',
+  'OVER',
+  'PARTITION',
+  'REGEXP',
+  'RIGHT',
+  'SELECT',
+  'SHOW',
+  'STRING',
+  'THEN',
+  'TRUE',
+  'UNION',
+  'USING',
+  'WHEN',
+  'WHERE',
+  'EXCEPT',
+];
+
+const functions = [
+  'AVG',
+  'COUNT',
+  'MAX',
+  'MIN',
+  'SUM',
+  'VAR_POP',
+  'VAR_SAMP',
+  'VARIANCE',
+  'STD',
+  'STDDEV',
+  'STDDEV_POP',
+  'STDDEV_SAMP',
+  'SUBSTRING',
+  'TRIM',
+];
+
+const operators = [
+  '=',
+  '>',
+  '<',
+  '!',
+  '~',
+  '\\|',
+  '&',
+  '\\^',
+  '\\*',
+  '/',
+  '%',
+  '\\+',
+  '-',
+  'DIV',
+  'MOD',
+];
+
+const brackets = [
+  { open: '(', close: ')', token: 'delimiter.parenthesis' },
+  { open: '[', close: ']', token: 'delimiter.square' },
+];
+
+export const lexerRules = {
+  defaultToken: 'invalid',
+  ignoreCase: true,
+  tokenPostfix: '',
+  keywords,
+  functions,
+  operators,
+  brackets,
+  tokenizer: {
+    root: [
+      [
+        /[a-zA-Z_$][a-zA-Z0-9_$]*/,
+        {
+          cases: {
+            '@keywords': 'keyword',
+            '@functions': 'function',
+            '@default': 'identifier',
+          },
+        },
+      ],
+      { include: '@whitespace' },
+      [/[()]/, '@brackets'],
+      [new RegExp(operators.join('|')), 'operator'],
+      [/[0-9]+(\.[0-9]+)?/, 'number'],
+      [/'([^'\\]|\\.)*$/, 'string.invalid'], // non-terminated string
+      [/'/, 'string', '@string'],
+      [/"/, 'string', '@string'],
+    ],
+    whitespace: [
+      [/[ \t\r\n]+/, 'white'],
+      [/\/\*/, 'comment', '@comment'],
+      [/--.*$/, 'comment'],
+    ],
+    string: [
+      [/[^'\\]+/, 'string'],
+      [/\\./, 'string.escape'],
+      [/'/, 'string', '@pop'],
+      [/"/, 'string', '@pop'],
+    ],
+    comment: [
+      [/[^/*]+/, 'comment'],
+      [/\*\//, 'comment', '@pop'],
+      [/[\/*]/, 'comment'],
+    ],
+  },
+} as monaco.languages.IMonarchLanguage;

packages/osd-optimizer/src/worker/webpack.config.ts

@@ -256,6 +256,16 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
             loader: 'raw-loader',
           },
         },
+        {
+          test: /\.cjs$/,
+          include: /node_modules/,
+          use: {
+            loader: 'babel-loader',
+            options: {
+              presets: [BABEL_PRESET_PATH],
+            },
+          },
+        },
       ],
     },