Merge branch 'master' into 65035-make-asset-ids-unique

.ci/Jenkinsfile_coverage

@@ -23,15 +23,22 @@ kibanaPipeline(timeoutMinutes: 240) {
 }
 
 def handleIngestion(timestamp) {
+  def previousSha = handlePreviousSha()
   kibanaPipeline.downloadCoverageArtifacts()
   kibanaCoverage.prokLinks("### Process HTML Links")
   kibanaCoverage.collectVcsInfo("### Collect VCS Info")
   kibanaCoverage.generateReports("### Merge coverage reports")
   kibanaCoverage.uploadCombinedReports()
-  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, '### Ingest && Upload')
+  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, '### Ingest && Upload')
   kibanaCoverage.uploadCoverageStaticSite(timestamp)
 }
 
+def handlePreviousSha() {
+  def previous = kibanaCoverage.downloadPrevious('### Download OLD Previous')
+  kibanaCoverage.uploadPrevious('### Upload NEW Previous')
+  return previous
+}
+
 def handleFail() {
   def buildStatus = buildUtils.getBuildStatus()
   if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED' && buildStatus != 'UNSTABLE') {

.ci/end2end.groovy

@@ -14,6 +14,7 @@ pipeline {
     HOME = "${env.WORKSPACE}"
     E2E_DIR = 'x-pack/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
+    KBN_OPTIMIZER_THEMES = 'v7light'
   }
   options {
     timeout(time: 1, unit: 'HOURS')

.ci/es-snapshots/Jenkinsfile_build_es

@@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
 timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
-      node(workers.label('s')) {
+      node(workers.label('l')) {
         catchErrors {
           def VERSION
           def SNAPSHOT_ID
@@ -154,9 +154,10 @@ def buildArchives(destination) {
     "NODE_NAME=",
   ]) {
     sh """
-      ./gradlew -p distribution/archives assemble --parallel
+      ./gradlew -Dbuild.docker=true assemble --parallel
       mkdir -p ${destination}
-      find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
+      find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
+      docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
     """
   }
 }

.fossa.yml

@@ -0,0 +1,15 @@
+# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
+# Visit https://fossa.com to learn more
+
+version: 2
+cli:
+  server: https://app.fossa.com
+  fetcher: custom
+  project: kibana
+analyze:
+  modules:
+  - name: kibana
+    type: nodejs
+    strategy: yarn.lock
+    target: .
+    path: .

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,7 +11,7 @@ Delete any items that are not applicable to this PR.
 - [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
 - [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)
 - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)
-- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)
+- [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
 
 ### For maintainers
 

CONTRIBUTING.md

@@ -436,7 +436,7 @@ We are still to develop a proper process to accept any contributed translations.
 
 When writing a new component, create a sibling SASS file of the same name and import directly into the JS/TS component file. Doing so ensures the styles are never separated or lost on import and allows for better modularization (smaller individual plugin asset footprint).
 
-Any JavaScript (or TypeScript) file that imports SASS (.scss) files will automatically build with the [EUI](https://elastic.github.io/eui/#/guidelines/sass) & Kibana invisibles (SASS variables, mixins, functions) from the [`styling_constants.scss` file](https://github.com/elastic/kibana/blob/master/src/legacy/ui/public/styles/_styling_constants.scss). However, any Legacy (file path includes `/legacy`) files will not.
+All SASS (.scss) files will automatically build with the [EUI](https://elastic.github.io/eui/#/guidelines/sass) & Kibana invisibles (SASS variables, mixins, functions) from the [`globals_[theme].scss` file](src/legacy/ui/public/styles/_globals_v7light.scss).
 
 **Example:**
 
@@ -679,15 +679,15 @@ Part of this process only applies to maintainers, since it requires access to Gi
 Kibana publishes [Release Notes](https://www.elastic.co/guide/en/kibana/current/release-notes.html) for major and minor releases. The Release Notes summarize what the PRs accomplish in language that is meaningful to users. To generate the Release Notes, the team runs a script against this repo to collect the merged PRs against the release.
 
 #### Create the Release Notes text
-The text that appears in the Release Notes is pulled directly from your PR title, or a single paragraph of text that you specify in the PR description. 
+The text that appears in the Release Notes is pulled directly from your PR title, or a single paragraph of text that you specify in the PR description.
 
 To use a single paragraph of text, enter `Release note:` or a `## Release note` header in the PR description, followed by your text. For example, refer to this [PR](https://github.com/elastic/kibana/pull/65796) that uses the `## Release note` header.
 
 When you create the Release Notes text, use the following best practices:
-* Use present tense. 
+* Use present tense.
 * Use sentence case.
 * When you create a feature PR, start with `Adds`.
-* When you create an enhancement PR, start with `Improves`. 
+* When you create an enhancement PR, start with `Improves`.
 * When you create a bug fix PR, start with `Fixes`.
 * When you create a deprecation PR, start with `Deprecates`.
 

docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) &gt; [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md)
+
+## SavedObjectsComplexFieldMapping.doc\_values property
+
+<b>Signature:</b>
+
+```typescript
+doc_values?: boolean;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md

@@ -18,6 +18,7 @@ export interface SavedObjectsComplexFieldMapping
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md) | <code>boolean</code> |  |
 |  [properties](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.properties.md) | <code>SavedObjectsMappingProperties</code> |  |
 |  [type](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.type.md) | <code>string</code> |  |
 

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) &gt; [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md)
+
+## SavedObjectsCoreFieldMapping.doc\_values property
+
+<b>Signature:</b>
+
+```typescript
+doc_values?: boolean;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md

@@ -16,6 +16,7 @@ export interface SavedObjectsCoreFieldMapping
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md) | <code>boolean</code> |  |
 |  [enabled](./kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md) | <code>boolean</code> |  |
 |  [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | <code>{</code><br/><code>        [subfield: string]: {</code><br/><code>            type: string;</code><br/><code>            ignore_above?: number;</code><br/><code>        };</code><br/><code>    }</code> |  |
 |  [index](./kibana-plugin-core-server.savedobjectscorefieldmapping.index.md) | <code>boolean</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.fieldformat.tojson.md

@@ -12,14 +12,14 @@ Serialize this format to a simple POJO, with only the params that are not defaul
 
 ```typescript
 toJSON(): {
-        id: unknown;
-        params: _.Dictionary<unknown> | undefined;
+        id: any;
+        params: any;
     };
 ```
 <b>Returns:</b>
 
 `{
-        id: unknown;
-        params: _.Dictionary<unknown> | undefined;
+        id: any;
+        params: any;
     }`
 

docs/plugins/known-plugins.asciidoc

@@ -59,6 +59,7 @@ This list of plugins is not guaranteed to work on your version of Kibana. Instea
 * https://github.com/sbeyn/kibana-plugin-traffic-sg[Traffic] (sbeyn)
 * https://github.com/PhaedrusTheGreek/transform_vis[Transform Visualization] (PhaedrusTheGreek)
 * https://github.com/nyurik/kibana-vega-vis[Vega-based visualizations] (nyurik) - Support for user-defined graphs, external data sources, maps, images, and user-defined interactivity.
+* https://github.com/Camichan/kbn_aframe[VR Graph Visualizations] (Camichan)
 
 [float]
 === Other

docs/settings/reporting-settings.asciidoc

@@ -210,6 +210,25 @@ When `xpack.reporting.capture.browser.type` is set to `chromium` (default) you c
   large exports from causing performance and storage issues.
   Defaults to `10485760` (10mB).
 
+| `xpack.reporting.csv.scroll.size`
+  | Number of documents retrieved from {es} for each scroll iteration during a CSV 
+  export.
+  Defaults to `500`.
+
+| `xpack.reporting.csv.scroll.duration`
+  | Amount of time allowed before {kib} cleans the scroll context during a CSV export.
+  Defaults to `30s`.
+
+| `xpack.reporting.csv.checkForFormulas`
+  | Enables a check that warns you when there's a potential formula involved in the output (=, -, +, and @ chars).
+  See OWASP: https://www.owasp.org/index.php/CSV_Injection
+  Defaults to `true`.
+
+| `xpack.reporting.csv.enablePanelActionDownload`
+  | Enables CSV export from a saved search on a dashboard. This action is available in the dashboard 
+  panel menu for the saved search.
+  Defaults to `true`.
+
 |===
 
 [float]

package.json

@@ -86,8 +86,9 @@
     "**/@types/angular": "^1.6.56",
     "**/@types/hoist-non-react-statics": "^3.3.1",
     "**/@types/chai": "^4.2.11",
+    "**/cypress/@types/lodash": "^4.14.155",
     "**/typescript": "3.9.5",
-    "**/graphql-toolkit/lodash": "^4.17.13",
+    "**/graphql-toolkit/lodash": "^4.17.15",
     "**/hoist-non-react-statics": "^3.3.2",
     "**/isomorphic-git/**/base64-js": "^1.2.1",
     "**/image-diff/gm/debug": "^2.6.9",
@@ -122,7 +123,7 @@
     "@babel/plugin-transform-modules-commonjs": "^7.10.1",
     "@babel/register": "^7.10.1",
     "@elastic/apm-rum": "^5.2.0",
-    "@elastic/charts": "19.5.2",
+    "@elastic/charts": "19.7.0",
     "@elastic/datemath": "5.0.3",
     "@elastic/ems-client": "7.9.3",
     "@elastic/eui": "24.1.0",
@@ -213,8 +214,7 @@
     "leaflet.heat": "0.2.0",
     "less": "npm:@elastic/[email protected]",
     "less-loader": "5.0.0",
-    "lodash": "npm:@elastic/[email protected]",
-    "lodash.clonedeep": "^4.5.0",
+    "lodash": "^4.17.15",
     "lru-cache": "4.1.5",
     "markdown-it": "^10.0.0",
     "mini-css-extract-plugin": "0.8.0",
@@ -355,8 +355,7 @@
     "@types/json5": "^0.0.30",
     "@types/license-checker": "15.0.0",
     "@types/listr": "^0.14.0",
-    "@types/lodash": "^3.10.1",
-    "@types/lodash.clonedeep": "^4.5.4",
+    "@types/lodash": "^4.14.155",
     "@types/lru-cache": "^5.1.0",
     "@types/markdown-it": "^0.0.7",
     "@types/minimatch": "^2.0.29",

packages/kbn-config-schema/package.json

@@ -14,6 +14,7 @@
     "tsd": "^0.7.4"
   },
   "peerDependencies": {
+    "lodash": "^4.17.15",
     "joi": "^13.5.2",
     "moment": "^2.24.0",
     "type-detect": "^4.0.8"

packages/kbn-interpreter/package.json

@@ -11,8 +11,7 @@
   "dependencies": {
     "@babel/runtime": "^7.10.2",
     "@kbn/i18n": "1.0.0",
-    "lodash": "npm:@elastic/[email protected]",
-    "lodash.clone": "^4.5.0",
+    "lodash": "^4.17.15",
     "uuid": "3.3.2"
   },
   "devDependencies": {

packages/kbn-interpreter/src/common/lib/registry.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import clone from 'lodash.clone';
+import { clone } from 'lodash';
 
 export class Registry {
   constructor(prop = 'name') {

packages/kbn-optimizer/README.md

@@ -42,6 +42,26 @@ When a directory is listed in the "extraPublicDirs" it will always be included i
 
 Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed.
 
+## Themes
+
+SASS imports in bundles are automatically converted to CSS for one or more themes. In development we build the `v7light` and `v7dark` themes by default to improve build performance. When producing distributable bundles the default shifts to `*` so that the distributable bundles will include all themes, preventing the bundles from needing to be rebuilt when users change the active theme in Kibana's advanced settings.
+
+To customize the themes that are built for development you can specify the `KBN_OPTIMIZER_THEMES` environment variable to one or more theme tags, or use `*` to build styles for all themes. Unfortunately building more than one theme significantly impacts build performance, so try to be strategic about which themes you build.
+
+Currently supported theme tags: `v7light`, `v7dark`, `v8light`, `v8dark`
+
+Examples:
+```sh
+# start Kibana with only a single theme
+KBN_OPTIMIZER_THEMES=v7light yarn start
+
+# start Kibana with dark themes for version 7 and 8
+KBN_OPTIMIZER_THEMES=v7dark,v8dark yarn start
+
+# start Kibana with all the themes
+KBN_OPTIMIZER_THEMES=* yarn start
+```
+
 ## API
 
 To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used.

packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_globals_v7light.scss

@@ -0,0 +1 @@
+$globalStyleConstant: 11;

packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_globals_v8dark.scss

@@ -0,0 +1 @@
+$globalStyleConstant: 12;

packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_globals_v8light.scss

@@ -0,0 +1 @@
+$globalStyleConstant: 13;

packages/kbn-optimizer/src/common/index.ts

@@ -29,3 +29,4 @@ export * from './array_helpers';
 export * from './event_stream_helpers';
 export * from './disallowed_syntax_plugin';
 export * from './parse_path';
+export * from './theme_tags';

packages/kbn-optimizer/src/common/theme_tags.test.ts

@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { parseThemeTags } from './theme_tags';
+
+it('returns default tags when passed undefined', () => {
+  expect(parseThemeTags()).toMatchInlineSnapshot(`
+    Array [
+      "v7dark",
+      "v7light",
+    ]
+  `);
+});
+
+it('returns all tags when passed *', () => {
+  expect(parseThemeTags('*')).toMatchInlineSnapshot(`
+    Array [
+      "v7dark",
+      "v7light",
+      "v8dark",
+      "v8light",
+    ]
+  `);
+});
+
+it('returns specific tag when passed a single value', () => {
+  expect(parseThemeTags('v8light')).toMatchInlineSnapshot(`
+    Array [
+      "v8light",
+    ]
+  `);
+});
+
+it('returns specific tags when passed a comma separated list', () => {
+  expect(parseThemeTags('v8light, v7dark,v7light')).toMatchInlineSnapshot(`
+    Array [
+      "v7dark",
+      "v7light",
+      "v8light",
+    ]
+  `);
+});
+
+it('returns specific tags when passed an array', () => {
+  expect(parseThemeTags(['v8light', 'v7light'])).toMatchInlineSnapshot(`
+    Array [
+      "v7light",
+      "v8light",
+    ]
+  `);
+});
+
+it('throws when an invalid tag is in the array', () => {
+  expect(() => parseThemeTags(['v8light', 'v7light', 'bar'])).toThrowErrorMatchingInlineSnapshot(
+    `"Invalid theme tags [bar], options: [v7dark, v7light, v8dark, v8light]"`
+  );
+});
+
+it('throws when an invalid tags in comma separated list', () => {
+  expect(() => parseThemeTags('v8light ,v7light,bar,box ')).toThrowErrorMatchingInlineSnapshot(
+    `"Invalid theme tags [bar, box], options: [v7dark, v7light, v8dark, v8light]"`
+  );
+});
+
+it('returns tags in alphabetical order', () => {
+  const tags = parseThemeTags(['v7light', 'v8light']);
+  expect(tags).toEqual(tags.slice().sort((a, b) => a.localeCompare(b)));
+});
+
+it('returns an immutable array', () => {
+  expect(() => {
+    const tags = parseThemeTags('v8light');
+    // @ts-expect-error
+    tags.push('foo');
+  }).toThrowErrorMatchingInlineSnapshot(`"Cannot add property 1, object is not extensible"`);
+});