Merge branch 'master' of github.com:elastic/kibana into sessions/retr…

…y-logic

.eslintignore

@@ -44,3 +44,4 @@ snapshots.js
 /packages/kbn-ui-framework/doc_site/build
 /packages/kbn-ui-framework/generator-kui/*/templates/
 /packages/kbn-ui-shared-deps/flot_charts
+/packages/kbn-monaco/src/painless/antlr

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -101,6 +101,12 @@ readonly links: {
             readonly dateMath: string;
         };
         readonly management: Record<string, string>;
+        readonly ml: {
+            readonly guide: string;
+            readonly anomalyDetection: string;
+            readonly anomalyDetectionJobs: string;
+            readonly dataFrameAnalytics: string;
+        };
         readonly visualize: Record<string, string>;
     };
 ```

docs/development/core/public/kibana-plugin-core-public.doclinksstart.md

@@ -17,5 +17,5 @@ export interface DocLinksStart
 |  --- | --- | --- |
 |  [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | <code>string</code> |  |
 |  [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | <code>string</code> |  |
-|  [links](./kibana-plugin-core-public.doclinksstart.links.md) | <code>{</code><br/><code>        readonly dashboard: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly drilldowns: string;</code><br/><code>            readonly drilldownsTriggerPicker: string;</code><br/><code>            readonly urlDrilldownTemplateSyntax: string;</code><br/><code>            readonly urlDrilldownVariables: string;</code><br/><code>        };</code><br/><code>        readonly filebeat: {</code><br/><code>            readonly base: string;</code><br/><code>            readonly installation: string;</code><br/><code>            readonly configuration: string;</code><br/><code>            readonly elasticsearchOutput: string;</code><br/><code>            readonly startup: string;</code><br/><code>            readonly exportedFields: string;</code><br/><code>        };</code><br/><code>        readonly auditbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly metricbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly heartbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly logstash: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly functionbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly winlogbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly aggs: {</code><br/><code>            readonly date_histogram: string;</code><br/><code>            readonly date_range: string;</code><br/><code>            readonly filter: string;</code><br/><code>            readonly filters: string;</code><br/><code>            readonly geohash_grid: string;</code><br/><code>            readonly histogram: string;</code><br/><code>            readonly ip_range: string;</code><br/><code>            readonly range: string;</code><br/><code>            readonly significant_terms: string;</code><br/><code>            readonly terms: string;</code><br/><code>            readonly avg: string;</code><br/><code>            readonly avg_bucket: string;</code><br/><code>            readonly max_bucket: string;</code><br/><code>            readonly min_bucket: string;</code><br/><code>            readonly sum_bucket: string;</code><br/><code>            readonly cardinality: string;</code><br/><code>            readonly count: string;</code><br/><code>            readonly cumulative_sum: string;</code><br/><code>            readonly derivative: string;</code><br/><code>            readonly geo_bounds: string;</code><br/><code>            readonly geo_centroid: string;</code><br/><code>            readonly max: string;</code><br/><code>            readonly median: string;</code><br/><code>            readonly min: string;</code><br/><code>            readonly moving_avg: string;</code><br/><code>            readonly percentile_ranks: string;</code><br/><code>            readonly serial_diff: string;</code><br/><code>            readonly std_dev: string;</code><br/><code>            readonly sum: string;</code><br/><code>            readonly top_hits: string;</code><br/><code>        };</code><br/><code>        readonly scriptedFields: {</code><br/><code>            readonly scriptFields: string;</code><br/><code>            readonly scriptAggs: string;</code><br/><code>            readonly painless: string;</code><br/><code>            readonly painlessApi: string;</code><br/><code>            readonly painlessSyntax: string;</code><br/><code>            readonly luceneExpressions: string;</code><br/><code>        };</code><br/><code>        readonly indexPatterns: {</code><br/><code>            readonly loadingData: string;</code><br/><code>            readonly introduction: string;</code><br/><code>        };</code><br/><code>        readonly addData: string;</code><br/><code>        readonly kibana: string;</code><br/><code>        readonly siem: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly gettingStarted: string;</code><br/><code>        };</code><br/><code>        readonly query: {</code><br/><code>            readonly eql: string;</code><br/><code>            readonly luceneQuerySyntax: string;</code><br/><code>            readonly queryDsl: string;</code><br/><code>            readonly kueryQuerySyntax: string;</code><br/><code>        };</code><br/><code>        readonly date: {</code><br/><code>            readonly dateMath: string;</code><br/><code>        };</code><br/><code>        readonly management: Record&lt;string, string&gt;;</code><br/><code>        readonly visualize: Record&lt;string, string&gt;;</code><br/><code>    }</code> |  |
+|  [links](./kibana-plugin-core-public.doclinksstart.links.md) | <code>{</code><br/><code>        readonly dashboard: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly drilldowns: string;</code><br/><code>            readonly drilldownsTriggerPicker: string;</code><br/><code>            readonly urlDrilldownTemplateSyntax: string;</code><br/><code>            readonly urlDrilldownVariables: string;</code><br/><code>        };</code><br/><code>        readonly filebeat: {</code><br/><code>            readonly base: string;</code><br/><code>            readonly installation: string;</code><br/><code>            readonly configuration: string;</code><br/><code>            readonly elasticsearchOutput: string;</code><br/><code>            readonly startup: string;</code><br/><code>            readonly exportedFields: string;</code><br/><code>        };</code><br/><code>        readonly auditbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly metricbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly heartbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly logstash: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly functionbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly winlogbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly aggs: {</code><br/><code>            readonly date_histogram: string;</code><br/><code>            readonly date_range: string;</code><br/><code>            readonly filter: string;</code><br/><code>            readonly filters: string;</code><br/><code>            readonly geohash_grid: string;</code><br/><code>            readonly histogram: string;</code><br/><code>            readonly ip_range: string;</code><br/><code>            readonly range: string;</code><br/><code>            readonly significant_terms: string;</code><br/><code>            readonly terms: string;</code><br/><code>            readonly avg: string;</code><br/><code>            readonly avg_bucket: string;</code><br/><code>            readonly max_bucket: string;</code><br/><code>            readonly min_bucket: string;</code><br/><code>            readonly sum_bucket: string;</code><br/><code>            readonly cardinality: string;</code><br/><code>            readonly count: string;</code><br/><code>            readonly cumulative_sum: string;</code><br/><code>            readonly derivative: string;</code><br/><code>            readonly geo_bounds: string;</code><br/><code>            readonly geo_centroid: string;</code><br/><code>            readonly max: string;</code><br/><code>            readonly median: string;</code><br/><code>            readonly min: string;</code><br/><code>            readonly moving_avg: string;</code><br/><code>            readonly percentile_ranks: string;</code><br/><code>            readonly serial_diff: string;</code><br/><code>            readonly std_dev: string;</code><br/><code>            readonly sum: string;</code><br/><code>            readonly top_hits: string;</code><br/><code>        };</code><br/><code>        readonly scriptedFields: {</code><br/><code>            readonly scriptFields: string;</code><br/><code>            readonly scriptAggs: string;</code><br/><code>            readonly painless: string;</code><br/><code>            readonly painlessApi: string;</code><br/><code>            readonly painlessSyntax: string;</code><br/><code>            readonly luceneExpressions: string;</code><br/><code>        };</code><br/><code>        readonly indexPatterns: {</code><br/><code>            readonly loadingData: string;</code><br/><code>            readonly introduction: string;</code><br/><code>        };</code><br/><code>        readonly addData: string;</code><br/><code>        readonly kibana: string;</code><br/><code>        readonly siem: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly gettingStarted: string;</code><br/><code>        };</code><br/><code>        readonly query: {</code><br/><code>            readonly eql: string;</code><br/><code>            readonly luceneQuerySyntax: string;</code><br/><code>            readonly queryDsl: string;</code><br/><code>            readonly kueryQuerySyntax: string;</code><br/><code>        };</code><br/><code>        readonly date: {</code><br/><code>            readonly dateMath: string;</code><br/><code>        };</code><br/><code>        readonly management: Record&lt;string, string&gt;;</code><br/><code>        readonly ml: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly anomalyDetection: string;</code><br/><code>            readonly anomalyDetectionJobs: string;</code><br/><code>            readonly dataFrameAnalytics: string;</code><br/><code>        };</code><br/><code>        readonly visualize: Record&lt;string, string&gt;;</code><br/><code>    }</code> |  |
 

docs/redirects.asciidoc

@@ -168,6 +168,11 @@ This content has moved. See <<discover, **Discover**>>.
 
 This content has moved. See <<discover, **Discover**>>.
 
+[role="exclude",id="lens"]
+== Lens
+
+This content has moved. See <<create-panels-with-lens>>.
+
 [role="exclude",id="known-plugins"]
 == Known plugins
 

docs/user/dashboard/dashboard.asciidoc

@@ -185,7 +185,7 @@ image:images/Dashboard_add_new_visualization.png[Example add new visualization t
 {kib} provides you with several editors that help you create panels.
 
 [float]
-[[lens]]
+[[create-panels-with-lens]]
 === Create panels with Lens
 
 *Lens* is the simplest and fastest way to create powerful visualizations of your data. To use *Lens*, you drag and drop as many data fields

docs/user/security/reporting.asciidoc

@@ -101,7 +101,7 @@ If you are using an external identity provider, such as
 LDAP or Active Directory, you can either assign
 roles on a per user basis, or assign roles to groups of users. By default, role
 mappings are configured in
-{ref}/mapping-roles.html[`config/shield/role_mapping.yml`].
+{ref}/mapping-roles.html[`config/role_mapping.yml`].
 For example, the following snippet assigns the user named Bill Murray the
 `kibana_admin` and `reporting_user` roles:
 

package.json

@@ -157,6 +157,7 @@
     "angular-resource": "1.8.0",
     "angular-sanitize": "^1.8.0",
     "angular-ui-ace": "0.2.3",
+    "antlr4ts": "^0.5.0-alpha.3",
     "apollo-cache-inmemory": "1.6.2",
     "apollo-client": "^2.3.8",
     "apollo-link-http": "^1.5.16",
@@ -348,7 +349,7 @@
     "@babel/traverse": "^7.11.5",
     "@babel/types": "^7.11.0",
     "@cypress/snapshot": "^2.1.7",
-    "@cypress/webpack-preprocessor": "^5.4.10",
+    "@cypress/webpack-preprocessor": "^5.4.11",
     "@elastic/apm-rum": "^5.6.1",
     "@elastic/apm-rum-react": "^1.2.5",
     "@elastic/charts": "24.3.0",
@@ -576,6 +577,7 @@
     "angular-recursion": "^1.0.5",
     "angular-route": "^1.8.0",
     "angular-sortable-view": "^0.0.17",
+    "antlr4ts-cli": "^0.5.0-alpha.3",
     "apidoc": "^0.25.0",
     "apidoc-markdown": "^5.1.8",
     "apollo-link": "^1.2.3",
@@ -608,7 +610,7 @@
     "cpy": "^8.1.1",
     "cronstrue": "^1.51.0",
     "css-loader": "^3.4.2",
-    "cypress": "^5.5.0",
+    "cypress": "^6.0.1",
     "cypress-cucumber-preprocessor": "^2.5.2",
     "cypress-multi-reporters": "^1.4.0",
     "d3": "3.5.17",

packages/kbn-monaco/package.json

@@ -6,7 +6,8 @@
   "license": "Apache-2.0",
   "scripts": {
     "build": "node ./scripts/build.js",
-    "kbn:bootstrap": "yarn build --dev"
+    "kbn:bootstrap": "yarn build --dev",
+    "build:antlr4ts": "../../node_modules/antlr4ts-cli/antlr4ts ./src/painless/antlr/painless_lexer.g4 ./src/painless/antlr/painless_parser.g4 && node ./scripts/fix_generated_antlr.js"
   },
   "devDependencies": {
     "@kbn/babel-preset": "link:../kbn-babel-preset",
@@ -15,4 +16,4 @@
   "dependencies": {
     "@kbn/i18n": "link:../kbn-i18n"
   }
-}
+}

packages/kbn-monaco/scripts/fix_generated_antlr.js

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { join } = require('path');
+const { readdirSync, readFileSync, writeFileSync, renameSync } = require('fs');
+const ora = require('ora');
+
+const generatedAntlrFolder = join(__dirname, '..', 'src', 'painless', 'antlr');
+
+const generatedAntlrFolderContents = readdirSync(generatedAntlrFolder);
+
+const log = ora('Updating generated antlr grammar').start();
+
+// The generated TS produces some TS linting errors
+// This script adds a //@ts-nocheck comment at the top of each generated file
+// so that the errors can be ignored for now
+generatedAntlrFolderContents
+  .filter((file) => {
+    const fileExtension = file.split('.')[1];
+    return fileExtension.includes('ts');
+  })
+  .forEach((file) => {
+    try {
+      const fileContentRows = readFileSync(join(generatedAntlrFolder, file), 'utf8')
+        .toString()
+        .split('\n');
+
+      fileContentRows.unshift('// @ts-nocheck');
+
+      const filePath = join(generatedAntlrFolder, file);
+      const fileContent = fileContentRows.join('\n');
+
+      writeFileSync(filePath, fileContent, { encoding: 'utf8' });
+    } catch (err) {
+      return log.fail(err.message);
+    }
+  });
+
+// Rename generated parserListener file to snakecase to satisfy file casing check
+// There doesn't appear to be a way to fix this OOTB with antlr4ts-cli
+try {
+  renameSync(
+    join(generatedAntlrFolder, 'painless_parserListener.ts'),
+    join(generatedAntlrFolder, 'painless_parser_listener.ts')
+  );
+} catch (err) {
+  log.warn(`Unable to rename parserListener file to snakecase: ${err.message}`);
+}
+
+log.succeed('Updated generated antlr grammar successfully');

packages/kbn-monaco/scripts/utils/clone_es.js

@@ -21,7 +21,7 @@ const { accessSync, mkdirSync } = require('fs');
 const { join } = require('path');
 const simpleGit = require('simple-git');
 
-// Note: The generated whitelists have not yet been merged to master
+// Note: The generated allowlists have not yet been merged to ES
 // so this script may fail until code in this branch has been merged:
 // https://github.com/stu-elastic/elasticsearch/tree/scripting/whitelists
 const esRepo = 'https://github.com/elastic/elasticsearch.git';

packages/kbn-monaco/src/painless/README.md

@@ -8,7 +8,7 @@ This folder contains the language definitions for Painless used by the Monaco ed
 
 Initializes the worker proxy service when the Painless language is first needed. It also exports the [suggestion provider](https://microsoft.github.io/monaco-editor/api/interfaces/monaco.languages.completionitemprovider.html) needed for autocompletion.
 
-### ./services
+### ./lib
 This directory exports two services:
 
 1. Worker proxy: Responsible for holding a reference to the Monaco-provided proxy getter.
@@ -32,12 +32,15 @@ Contains the Monarch-specific language tokenization rules for Painless.
 
 ### ./worker
 
-The worker proxy and worker instantiation code used in both the main thread and the worker thread. The logic for providing autocomplete suggestions resides here.
+The worker proxy and worker instantiation code used in both the main thread and the worker thread. The logic for providing autocomplete suggestions and error reporting resides here.
 
 ### ./autocomplete_definitions
 
 This directory is generated by a script and should not be changed manually. Read [Updating autocomplete definitions](#updating-autocomplete-definitions) for more information.
 
+### ./antlr
+This directory contains the Painless lexer and grammar rules, as well as the generated Typescript code. Read [Compiling ANTLR](#compiling-ANTLR) for more information.
+
 ## Example usage
 
 ```
@@ -102,4 +105,20 @@ node scripts/generate_autocomplete --branch <branch_name>
   - `score`
   - `string_script_field_script_field`
 
-To add additional contexts, edit the `supportedContexts` constant in `kbn-monaco/scripts/constants.js`.
+To add additional contexts, edit the `supportedContexts` constant in `kbn-monaco/scripts/constants.js`.
+
+## Compiling ANTLR
+
+[ANTLR](https://www.antlr.org/) generates lexical and syntax errors out of the box, which we can use to set error markers in monaco. 
+
+Elasticsearch has defined [lexer and parser grammar](https://github.com/elastic/elasticsearch/tree/master/modules/lang-painless/src/main/antlr) for the Painless language. For now, these rules have been largely copied from ES to Kibana and reside in the `antlr` directory with the `.g4` file extension. We then use [antlr4ts](https://github.com/tunnelvisionlabs/antlr4ts) to generate a lexer and a parser in Typescript.
+
+To regenerate the lexer and parser, run the following script:
+
+```
+npm run build:antlr4ts
+```
+
+*Note:* This script should only need to be run if a change has been made to `painless_lexer.g4` or `painless_parser.g4`.
+
+*Note:* There is a manual change made to the `sempred()` method in the generated `painless_lexer.ts`. This needs further investigation, but it appears there is an offset between the rule index and the token value. Without this manual change, ANTLR incorrectly reports an error when using a `/` or regex in a script. There is a comment in the generated code to this effect.