Merge branch '7.x' into backport/7.x/pr-70031

.ci/es-snapshots/Jenkinsfile_build_es

@@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
 timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
-      node(workers.label('s')) {
+      node(workers.label('l')) {
         catchErrors {
           def VERSION
           def SNAPSHOT_ID
@@ -154,9 +154,10 @@ def buildArchives(destination) {
     "NODE_NAME=",
   ]) {
     sh """
-      ./gradlew -p distribution/archives assemble --parallel
+      ./gradlew -Dbuild.docker=true assemble --parallel
       mkdir -p ${destination}
-      find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
+      find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
+      docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
     """
   }
 }

docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) &gt; [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md)
+
+## SavedObjectsComplexFieldMapping.doc\_values property
+
+<b>Signature:</b>
+
+```typescript
+doc_values?: boolean;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md

@@ -18,6 +18,7 @@ export interface SavedObjectsComplexFieldMapping
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md) | <code>boolean</code> |  |
 |  [properties](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.properties.md) | <code>SavedObjectsMappingProperties</code> |  |
 |  [type](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.type.md) | <code>string</code> |  |
 

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) &gt; [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md)
+
+## SavedObjectsCoreFieldMapping.doc\_values property
+
+<b>Signature:</b>
+
+```typescript
+doc_values?: boolean;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md

@@ -16,6 +16,7 @@ export interface SavedObjectsCoreFieldMapping
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md) | <code>boolean</code> |  |
 |  [enabled](./kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md) | <code>boolean</code> |  |
 |  [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | <code>{</code><br/><code>        [subfield: string]: {</code><br/><code>            type: string;</code><br/><code>            ignore_above?: number;</code><br/><code>        };</code><br/><code>    }</code> |  |
 |  [index](./kibana-plugin-core-server.savedobjectscorefieldmapping.index.md) | <code>boolean</code> |  |

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -63,7 +63,8 @@ afterAll(async () => {
   await del(TMP_DIR);
 });
 
-it('builds expected bundles, saves bundle counts to metadata', async () => {
+// FLAKY: https://github.com/elastic/kibana/issues/70762
+it.skip('builds expected bundles, saves bundle counts to metadata', async () => {
   const config = OptimizerConfig.create({
     repoRoot: MOCK_REPO_DIR,
     pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')],

src/core/server/saved_objects/mappings/types.ts

@@ -133,6 +133,7 @@ export interface SavedObjectsCoreFieldMapping {
   type: string;
   null_value?: number | boolean | string;
   index?: boolean;
+  doc_values?: boolean;
   enabled?: boolean;
   fields?: {
     [subfield: string]: {
@@ -153,6 +154,7 @@ export interface SavedObjectsCoreFieldMapping {
  * @public
  */
 export interface SavedObjectsComplexFieldMapping {
+  doc_values?: boolean;
   type?: string;
   properties: SavedObjectsMappingProperties;
 }

src/core/server/server.api.md

@@ -1978,6 +1978,8 @@ export interface SavedObjectsClientWrapperOptions {
 
 // @public
 export interface SavedObjectsComplexFieldMapping {
+    // (undocumented)
+    doc_values?: boolean;
     // (undocumented)
     properties: SavedObjectsMappingProperties;
     // (undocumented)
@@ -1986,6 +1988,8 @@ export interface SavedObjectsComplexFieldMapping {
 
 // @public
 export interface SavedObjectsCoreFieldMapping {
+    // (undocumented)
+    doc_values?: boolean;
     // (undocumented)
     enabled?: boolean;
     // (undocumented)

src/plugins/es_ui_shared/public/forms/form_wizard/form_wizard_context.tsx

@@ -23,7 +23,7 @@ import { WithMultiContent, useMultiContentContext, HookProps } from '../multi_co
 
 export interface Props<T extends object> {
   onSave: (data: T) => void | Promise<void>;
-  children: JSX.Element | JSX.Element[];
+  children: JSX.Element | Array<JSX.Element | null | false>;
   isEditing?: boolean;
   defaultActiveStep?: number;
   defaultValue?: HookProps<T>['defaultValue'];

src/plugins/es_ui_shared/public/forms/multi_content/multi_content_context.tsx

@@ -54,7 +54,7 @@ export function useMultiContentContext<T extends object = { [key: string]: any }
  *
  * @param contentId The content id to be added to the "contents" map
  */
-export function useContent<T extends object = { [key: string]: any }>(contentId: keyof T) {
+export function useContent<T extends object, K extends keyof T>(contentId: K) {
   const { updateContentAt, saveSnapshotAndRemoveContent, getData } = useMultiContentContext<T>();
 
   const updateContent = useCallback(
@@ -71,8 +71,11 @@ export function useContent<T extends object = { [key: string]: any }>(contentId:
     };
   }, [contentId, saveSnapshotAndRemoveContent]);
 
+  const data = getData();
+  const defaultValue = data[contentId];
+
   return {
-    defaultValue: getData()[contentId]!,
+    defaultValue,
     updateContent,
     getData,
   };

src/plugins/es_ui_shared/public/forms/multi_content/use_multi_content.ts

@@ -150,6 +150,10 @@ export function useMultiContent<T extends object>({
    * Validate the multi-content active content(s) in the DOM
    */
   const validate = useCallback(async () => {
+    if (Object.keys(contents.current).length === 0) {
+      return Boolean(validation.isValid);
+    }
+
     const updatedValidation = {} as { [key in keyof T]?: boolean | undefined };
 
     for (const [id, _content] of Object.entries(contents.current)) {

src/plugins/es_ui_shared/public/index.ts

@@ -22,6 +22,7 @@
  * In the future, each top level folder should be exported like that to avoid naming collision
  */
 import * as Forms from './forms';
+import * as Monaco from './monaco';
 
 export { JsonEditor, OnJsonEditorUpdateHandler } from './components/json_editor';
 
@@ -53,10 +54,6 @@ export {
   expandLiteralStrings,
 } from './console_lang';
 
-import * as Monaco from './monaco';
-
-export { Monaco };
-
 export {
   AuthorizationContext,
   AuthorizationProvider,
@@ -69,7 +66,7 @@ export {
   useAuthorizationContext,
 } from './authorization';
 
-export { Forms };
+export { Monaco, Forms };
 
 /** dummy plugin, we just want esUiShared to have its own bundle */
 export function plugin() {

src/plugins/es_ui_shared/static/forms/helpers/field_validators/is_json.ts

@@ -21,12 +21,13 @@ import { ValidationFunc } from '../../hook_form_lib';
 import { isJSON } from '../../../validators/string';
 import { ERROR_CODE } from './types';
 
-export const isJsonField = (message: string) => (
-  ...args: Parameters<ValidationFunc>
-): ReturnType<ValidationFunc<any, ERROR_CODE>> => {
+export const isJsonField = (
+  message: string,
+  { allowEmptyString = false }: { allowEmptyString?: boolean } = {}
+) => (...args: Parameters<ValidationFunc>): ReturnType<ValidationFunc<any, ERROR_CODE>> => {
   const [{ value }] = args;
 
-  if (typeof value !== 'string') {
+  if (typeof value !== 'string' || (allowEmptyString && value.trim() === '')) {
     return;
   }
 

src/plugins/kibana_usage_collection/README.md

@@ -2,7 +2,7 @@
 
 This plugin registers the basic usage collectors from Kibana:
 
-- Application Usage
+- [Application Usage](./server/collectors/application_usage/README.md)
 - UI Metrics
 - Ops stats
 - Number of Saved Objects per type

src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md

@@ -0,0 +1,37 @@
+# Application Usage
+
+This collector reports the number of general clicks and minutes on screen for each registered application in Kibana.
+
+The final payload matches the following contract:
+
+```JSON
+{
+  "application_usage": {
+    "application_ID": {
+        "clicks_7_days": 10,
+        "clicks_30_days": 100,
+        "clicks_90_days": 300,
+        "clicks_total": 600,
+        "minutes_on_screen_7_days": 10.40,
+        "minutes_on_screen_30_days": 20.0,
+        "minutes_on_screen_90_days": 110.1,
+        "minutes_on_screen_total": 112.5
+    }
+  }
+}
+```
+
+Where `application_ID` matches the `id` registered when calling the method `core.application.register`.
+This collection occurs by default for every application registered via the mentioned method and there is no need to do anything else to enable it or _opt-in_ for your plugin.
+
+**Note to maintainers in the Kibana repo:** At the moment of writing, the `usageCollector.schema` is not updated automatically ([#70622](https://github.com/elastic/kibana/issues/70622)) so, if you are adding a new app to Kibana, you'll need to give the Kibana Telemetry team a heads up to update the mappings in the Telemetry Cluster accordingly.
+
+## Developer notes
+
+In order to keep the count of the events, this collector uses 2 Saved Objects:
+
+1. `application_usage_transactional`: It stores each individually reported event (up to 90 days old). Grouped by `timestamp` and `appId`.
+2. `application_usage_totals`: It stores the sum of all the events older than 90 days old per `appId`.
+
+Both of them use the shared fields `appId: 'keyword'`, `numberOfClicks: 'long'` and `minutesOnScreen: 'float'`. `application_usage_transactional` also stores `timestamp: { type: 'date' }`.
+but they are currently not added in the mappings because we don't use them for search purposes, and we need to be thoughtful with the number of mapped fields in the SavedObjects index ([#43673](https://github.com/elastic/kibana/issues/43673)).

src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts

@@ -35,13 +35,10 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
     hidden: false,
     namespaceType: 'agnostic',
     mappings: {
+      // Not indexing any of its contents because we use them "as-is" and don't search by these fields
+      // for more info, see the README.md for application_usage
       dynamic: false,
-      properties: {
-        // Disabled the mapping of these fields since they are not searched and we need to reduce the amount of indexed fields (#43673)
-        // appId: { type: 'keyword' },
-        // numberOfClicks: { type: 'long' },
-        // minutesOnScreen: { type: 'float' },
-      },
+      properties: {},
     },
   });
 
@@ -53,10 +50,6 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
       dynamic: false,
       properties: {
         timestamp: { type: 'date' },
-        // Disabled the mapping of these fields since they are not searched and we need to reduce the amount of indexed fields (#43673)
-        // appId: { type: 'keyword' },
-        // numberOfClicks: { type: 'long' },
-        // minutesOnScreen: { type: 'float' },
       },
     },
   });

test/functional/apps/dashboard/index.js

@@ -49,6 +49,7 @@ export default function ({ getService, loadTestFile }) {
       after(unloadCurrentData);
 
       loadTestFile(require.resolve('./empty_dashboard'));
+      loadTestFile(require.resolve('./url_field_formatter'));
       loadTestFile(require.resolve('./embeddable_rendering'));
       loadTestFile(require.resolve('./create_and_add_embeddables'));
       loadTestFile(require.resolve('./edit_embeddable_redirects'));

test/functional/apps/dashboard/url_field_formatter.ts

@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+import { WebElementWrapper } from 'test/functional/services/lib/web_element_wrapper';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const { common, dashboard, settings, timePicker, visChart } = getPageObjects([
+    'common',
+    'dashboard',
+    'settings',
+    'timePicker',
+    'visChart',
+  ]);
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const testSubjects = getService('testSubjects');
+  const browser = getService('browser');
+  const fieldName = 'clientip';
+
+  const clickFieldAndCheckUrl = async (fieldLink: WebElementWrapper) => {
+    const fieldValue = await fieldLink.getVisibleText();
+    await fieldLink.click();
+    const windowHandlers = await browser.getAllWindowHandles();
+    expect(windowHandlers.length).to.equal(2);
+    await browser.switchToWindow(windowHandlers[1]);
+    const currentUrl = await browser.getCurrentUrl();
+    const fieldUrl = common.getHostPort() + '/app/' + fieldValue;
+    expect(currentUrl).to.equal(fieldUrl);
+  };
+
+  describe('Changing field formatter to Url', () => {
+    before(async function () {
+      await esArchiver.load('dashboard/current/kibana');
+      await kibanaServer.uiSettings.replace({
+        defaultIndex: '0bf35f60-3dc9-11e8-8660-4d65aa086b3c',
+      });
+      await common.navigateToApp('settings');
+      await settings.clickKibanaIndexPatterns();
+      await settings.clickIndexPatternLogstash();
+      await settings.filterField(fieldName);
+      await settings.openControlsByName(fieldName);
+      await settings.setFieldFormat('url');
+      await settings.controlChangeSave();
+    });
+
+    it('applied on dashboard', async () => {
+      await common.navigateToApp('dashboard');
+      await dashboard.loadSavedDashboard('dashboard with everything');
+      await dashboard.waitForRenderComplete();
+      const fieldLink = await visChart.getFieldLinkInVisTable(`${fieldName}: Descending`, 1);
+      await clickFieldAndCheckUrl(fieldLink);
+    });
+
+    it('applied on discover', async () => {
+      await common.navigateToApp('discover');
+      await timePicker.setAbsoluteRange(
+        'Sep 19, 2017 @ 06:31:44.000',
+        'Sep 23, 2018 @ 18:31:44.000'
+      );
+      await testSubjects.click('docTableExpandToggleColumn');
+      const fieldLink = await testSubjects.find(`tableDocViewRow-${fieldName}-value`);
+      await clickFieldAndCheckUrl(fieldLink);
+    });
+
+    afterEach(async function () {
+      const windowHandlers = await browser.getAllWindowHandles();
+      if (windowHandlers.length > 1) {
+        await browser.closeCurrentWindow();
+        await browser.switchToWindow(windowHandlers[0]);
+      }
+    });
+  });
+}

test/functional/page_objects/visualize_chart_page.ts

@@ -302,6 +302,20 @@ export function VisualizeChartPageProvider({ getService, getPageObjects }: FtrPr
       return element.getVisibleText();
     }
 
+    public async getFieldLinkInVisTable(fieldName: string, rowIndex: number = 1) {
+      const tableVis = await testSubjects.find('tableVis');
+      const $ = await tableVis.parseDomContent();
+      const headers = $('span[ng-bind="::col.title"]')
+        .toArray()
+        .map((header: any) => $(header).text());
+      const fieldColumnIndex = headers.indexOf(fieldName);
+      return await find.byCssSelector(
+        `[data-test-subj="paginated-table-body"] tr:nth-of-type(${rowIndex}) td:nth-of-type(${
+          fieldColumnIndex + 1
+        }) a`
+      );
+    }
+
     /**
      * If you are writing new tests, you should rather look into getTableVisContent method instead.
      * @deprecated Use getTableVisContent instead.

test/scripts/jenkins_visual_regression.sh

@@ -11,7 +11,7 @@ mkdir -p "$installDir"
 tar -xzf "$linuxBuild" -C "$installDir" --strip=1
 
 echo " -> running visual regression tests from kibana directory"
-yarn percy exec -t 500 -- -- \
+yarn percy exec -t 10000 -- -- \
   node scripts/functional_tests \
     --debug --bail \
     --kibana-install-dir "$installDir" \