Merge branch 'master' of github.com:elastic/kibana into security-rule…

…-type-threshold

docs/setup/access.asciidoc

@@ -37,4 +37,21 @@ image::images/kibana-status-page-7_14_0.png[Kibana server status page]
 
 For JSON-formatted server status details, use the `localhost:5601/api/status` API endpoint. 
 
+[float]
+[[not-ready]]
+=== {kib} not ready
+
+If you receive an error that the {kib} `server is not ready`, check the following:
+
+* The {es} connectivity: 
++
+[source,sh]
+----
+`curl -XGET elasticsearch_ip_or_hostname:9200/`
+----
+* The {kib} logs:
+** Linux, DEB or RPM package: `/var/log/kibana/kibana.log`
+** Linux, tar.gz package: `$KIBANA_HOME/log/kibana.log`
+** Windows: `$KIBANA_HOME\log\kibana.log`
+* The health status of `.kibana*` indices
 

docs/setup/docker.asciidoc

@@ -93,6 +93,16 @@ services:
       - ./kibana.yml:/usr/share/kibana/config/kibana.yml
 --------------------------------------------
 
+==== Persist the {kib} keystore
+
+By default, {kib] auto-generates a keystore file for secure settings at startup. To persist your {kibana-ref}/secure-settings.html[secure settings], use the `kibana-keystore` utility to bind-mount the parent directory of the keystore to the container. For example:
+
+[source,sh]
+----
+docker run -it --rm -v full_path_to/config:/usr/share/kibana/config -v full_path_to/data:/usr/share/kibana/data docker.elastic.co/kibana/kibana:7.14.0 bin/kibana-keystore create
+docker run -it --rm -v full_path_to/config:/usr/share/kibana/config -v full_path_to/data:/usr/share/kibana/data docker.elastic.co/kibana/kibana:7.14.0 bin/kibana-keystore add test_keystore_setting
+----
+
 [float]
 [[environment-variable-config]]
 ==== Environment variable configuration

packages/kbn-pm/dist/index.js

@@ -27495,16 +27495,23 @@ function isIdentifierChar(code) {
 function isIdentifierName(name) {
   let isFirst = true;
 
-  for (let _i = 0, _Array$from = Array.from(name); _i < _Array$from.length; _i++) {
-    const char = _Array$from[_i];
-    const cp = char.codePointAt(0);
+  for (let i = 0; i < name.length; i++) {
+    let cp = name.charCodeAt(i);
+
+    if ((cp & 0xfc00) === 0xd800 && i + 1 < name.length) {
+      const trail = name.charCodeAt(++i);
+
+      if ((trail & 0xfc00) === 0xdc00) {
+        cp = 0x10000 + ((cp & 0x3ff) << 10) + (trail & 0x3ff);
+      }
+    }
 
     if (isFirst) {
+      isFirst = false;
+
       if (!isIdentifierStart(cp)) {
         return false;
       }
-
-      isFirst = false;
     } else if (!isIdentifierChar(cp)) {
       return false;
     }

renovate.json5

@@ -74,6 +74,21 @@
         labels: ['Feature:Vega', 'Team:VisEditors'],
         enabled: true,
       },
+      {
+        groupName: 'platform security modules',
+        packageNames: [
+          'broadcast-channel', 
+          'jsonwebtoken', '@types/jsonwebtoken', 
+          'node-forge', '@types/node-forge', 
+          'require-in-the-middle', 
+          'tough-cookie', '@types/tough-cookie',
+          'xml-crypto', '@types/xml-crypto'
+        ],
+        reviewers: ['team:kibana-security'],
+        matchBaseBranches: ['master', '7.x'],
+        labels: ['Team:Security'],
+        enabled: true,
+      },
     ],
   },
 }

src/core/public/_variables.scss

@@ -1,7 +1,7 @@
 @import '@elastic/eui/src/global_styling/variables/header';
 
 // height of the header banner
-$kbnHeaderBannerHeight: $euiSizeXL;
+$kbnHeaderBannerHeight: $euiSizeXL;  // This value is also declared in `/x-pack/plugins/canvas/common/lib/constants.ts`
 // total height of the header (when the banner is *not* present)
 $kbnHeaderOffset: $euiHeaderHeightCompensation * 2;
 // total height of the header when the banner is present

src/core/public/rendering/_base.scss

@@ -49,6 +49,12 @@
 
   &.kbnBody--hasHeaderBanner {
     @include kbnAffordForHeader($kbnHeaderOffsetWithBanner);
+
+    // Prevents banners from covering full screen data grids
+    .euiDataGrid--fullScreen {
+      height: calc(100vh - #{$kbnHeaderBannerHeight});
+      top: $kbnHeaderBannerHeight;
+    }
   }
   &.kbnBody--chromeHidden {
     @include kbnAffordForHeader(0);

src/plugins/data/common/index_patterns/fields/index_pattern_field.test.ts

@@ -153,14 +153,14 @@ describe('Field', function () {
   it('spec snapshot', () => {
     const field = new IndexPatternField(fieldValues);
     const getFormatterForField = () =>
-      ({
+      (({
         toJSON: () => ({
           id: 'number',
           params: {
             pattern: '$0,0.[00]',
           },
         }),
-      } as FieldFormat);
+      } as unknown) as FieldFormat);
     expect(field.toSpec({ getFormatterForField })).toMatchSnapshot();
   });
 });

src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts

@@ -57,9 +57,10 @@ describe('flattenHit', () => {
       fields: {
         date: ['1'],
         zzz: ['z'],
+        _abc: ['a'],
       },
     });
-    const expectedOrder = ['date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
+    const expectedOrder = ['_abc', 'date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
     expect(Object.keys(response)).toEqual(expectedOrder);
     expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
   });

src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts

@@ -66,7 +66,6 @@ function decorateFlattenedWrapper(hit: Record<string, any>, metaFields: Record<s
 
     // unwrap computed fields
     _.forOwn(hit.fields, function (val, key: any) {
-      if (key[0] === '_' && !_.includes(metaFields, key)) return;
       // Flatten an array with 0 or 1 elements to a single value.
       if (Array.isArray(val) && val.length <= 1) {
         flattened[key] = val[0];

src/plugins/data/common/index_patterns/index_patterns/index_pattern.test.ts

@@ -289,17 +289,17 @@ describe('IndexPattern', () => {
 
   describe('toSpec', () => {
     test('should match snapshot', () => {
-      const formatter = {
+      const formatter = ({
         toJSON: () => ({ id: 'number', params: { pattern: '$0,0.[00]' } }),
-      } as FieldFormat;
+      } as unknown) as FieldFormat;
       indexPattern.getFormatterForField = () => formatter;
       expect(indexPattern.toSpec()).toMatchSnapshot();
     });
 
     test('can restore from spec', async () => {
-      const formatter = {
+      const formatter = ({
         toJSON: () => ({ id: 'number', params: { pattern: '$0,0.[00]' } }),
-      } as FieldFormat;
+      } as unknown) as FieldFormat;
       indexPattern.getFormatterForField = () => formatter;
       const spec = indexPattern.toSpec();
       const restoredPattern = new IndexPattern({

src/plugins/data/common/search/search_source/search_source.ts

@@ -828,7 +828,7 @@ export class SearchSource {
     body.query = buildEsQuery(index, query, filters, esQueryConfigs);
 
     if (highlightAll && body.query) {
-      body.highlight = getHighlightRequest(body.query, getConfig(UI_SETTINGS.DOC_HIGHLIGHT));
+      body.highlight = getHighlightRequest(getConfig(UI_SETTINGS.DOC_HIGHLIGHT));
       delete searchRequest.highlightAll;
     }
 

src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts

@@ -25,6 +25,7 @@ export interface FieldDescriptor {
   type: string;
   esTypes: string[];
   subType?: FieldSubType;
+  metadata_field?: boolean;
 }
 
 interface FieldSubType {

src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js

@@ -69,6 +69,17 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   });
 
   describe('response order', () => {
+    it('supports fields that start with an underscore', async () => {
+      const fields = ['_field_a', '_field_b'];
+
+      stubDeps({
+        fieldsFromFieldCaps: fields.map((name) => ({ name })),
+      });
+
+      const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
+      expect(fieldNames).toEqual(fields);
+    });
+
     it('always returns fields in alphabetical order', async () => {
       const letters = 'ambcdfjopngihkel'.split('');
       const sortedLetters = sortBy(letters);

src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -34,20 +34,21 @@ export async function getFieldCapabilities(
   const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name');
 
   const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName)
-    .filter((name) => !name.startsWith('_'))
+    // not all meta fields are provided, so remove and manually add
+    .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field)
     .concat(metaFields)
-    .reduce<{ names: string[]; hash: Record<string, string> }>(
+    .reduce<{ names: string[]; map: Map<string, string> }>(
       (agg, value) => {
-        // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
-        if (agg.hash[value] != null) {
+        // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data
+        if (agg.map.get(value) != null) {
           return agg;
         } else {
-          agg.hash[value] = value;
+          agg.map.set(value, value);
           agg.names.push(value);
           return agg;
         }
       },
-      { names: [], hash: {} }
+      { names: [], map: new Map<string, string>() }
     )
     .names.map<FieldDescriptor>((name) =>
       defaults({}, fieldsFromFieldCapsByName[name], {
@@ -56,6 +57,7 @@ export async function getFieldCapabilities(
         searchable: false,
         aggregatable: false,
         readFromDocValues: false,
+        metadata_field: metaFields.includes(name),
       })
     )
     .map(mergeOverrides);

src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js

@@ -48,6 +48,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
               'searchable',
               'aggregatable',
               'readFromDocValues',
+              'metadata_field',
             ]);
           });
         }

src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts

@@ -116,6 +116,8 @@ export function readFieldCapsResponse(
             }),
             {}
           ),
+          // @ts-expect-error
+          metadata_field: capsByType[types[0]].metadata_field,
         };
         // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
         agg.array.push(field);
@@ -131,6 +133,8 @@ export function readFieldCapsResponse(
         searchable: isSearchable,
         aggregatable: isAggregatable,
         readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
+        // @ts-expect-error
+        metadata_field: capsByType[types[0]].metadata_field,
       };
       // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
       agg.array.push(field);

src/plugins/discover/public/application/apps/main/components/doc_table/lib/row_formatter.tsx

@@ -73,7 +73,7 @@ export const formatTopLevelObject = (
     const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined;
     const formatter = field
       ? indexPattern.getFormatterForField(field)
-      : { convert: (v: string, ...rest: unknown[]) => String(v) };
+      : { convert: (v: unknown, ...rest: unknown[]) => String(v) };
     if (!values.map) return;
     const formatted = values
       .map((val: unknown) =>

src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx

@@ -89,7 +89,7 @@ export const getRenderCellValueFn = (
         : undefined;
       const formatter = subField
         ? indexPattern.getFormatterForField(subField)
-        : { convert: (v: string, ...rest: unknown[]) => String(v) };
+        : { convert: (v: unknown, ...rest: unknown[]) => String(v) };
       const formatted = (values as unknown[])
         .map((val: unknown) =>
           formatter.convert(val, 'html', {

src/plugins/field_formats/common/content_types/html_content_type.ts

@@ -43,7 +43,7 @@ export const setup = (
       return convert.call(format, value, options);
     }
 
-    const subValues = value.map((v: any) => recurse(v, options));
+    const subValues = value.map((v: unknown) => recurse(v, options));
     const useMultiLine = subValues.some((sub: string) => sub.indexOf('\n') > -1);
 
     return subValues.join(',' + (useMultiLine ? '\n' : ' '));

src/plugins/field_formats/common/converters/boolean.test.ts

@@ -9,7 +9,7 @@
 import { BoolFormat } from './boolean';
 
 describe('Boolean Format', () => {
-  let boolean: Record<string, any>;
+  let boolean: BoolFormat;
 
   beforeEach(() => {
     boolean = new BoolFormat({}, jest.fn());

src/plugins/field_formats/common/converters/boolean.ts

@@ -20,7 +20,7 @@ export class BoolFormat extends FieldFormat {
   });
   static fieldType = [KBN_FIELD_TYPES.BOOLEAN, KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.STRING];
 
-  textConvert: TextContextTypeConvert = (value) => {
+  textConvert: TextContextTypeConvert = (value: string | number | boolean) => {
     if (typeof value === 'string') {
       value = value.trim().toLowerCase();
     }

src/plugins/field_formats/common/converters/bytes.test.ts

@@ -8,13 +8,14 @@
 
 import { BytesFormat } from './bytes';
 import { FORMATS_UI_SETTINGS } from '../constants/ui_settings';
+import { FieldFormatsGetConfigFn } from '../types';
 
 describe('BytesFormat', () => {
-  const config: Record<string, any> = {};
+  const config: { [key: string]: string } = {
+    [FORMATS_UI_SETTINGS.FORMAT_BYTES_DEFAULT_PATTERN]: '0,0.[000]b',
+  };
 
-  config[FORMATS_UI_SETTINGS.FORMAT_BYTES_DEFAULT_PATTERN] = '0,0.[000]b';
-
-  const getConfig = (key: string) => config[key];
+  const getConfig: FieldFormatsGetConfigFn = (key: string) => config[key];
 
   test('default pattern', () => {
     const formatter = new BytesFormat({}, getConfig);

src/plugins/field_formats/common/converters/color.tsx

@@ -31,11 +31,11 @@ export class ColorFormat extends FieldFormat {
     };
   }
 
-  findColorRuleForVal(val: any) {
+  findColorRuleForVal(val: string | number) {
     switch (this.param('fieldType')) {
       case 'string':
         return findLast(this.param('colors'), (colorParam: typeof DEFAULT_CONVERTER_COLOR) => {
-          return new RegExp(colorParam.regex).test(val);
+          return new RegExp(colorParam.regex).test(val as string);
         });
 
       case 'number':
@@ -50,7 +50,7 @@ export class ColorFormat extends FieldFormat {
     }
   }
 
-  htmlConvert: HtmlContextTypeConvert = (val) => {
+  htmlConvert: HtmlContextTypeConvert = (val: string | number) => {
     const color = this.findColorRuleForVal(val) as typeof DEFAULT_CONVERTER_COLOR;
 
     const displayVal = escape(asPrettyString(val));

src/plugins/field_formats/common/converters/date_nanos_shared.test.ts

@@ -8,17 +8,23 @@
 
 import moment from 'moment-timezone';
 import { DateNanosFormat, analysePatternForFract, formatWithNanos } from './date_nanos_shared';
+import { FieldFormatsGetConfigFn } from '../types';
 
 describe('Date Nanos Format', () => {
   let convert: Function;
-  let mockConfig: Record<string, any>;
+  let mockConfig: {
+    dateNanosFormat: string;
+    'dateFormat:tz': string;
+    [other: string]: string;
+  };
 
   beforeEach(() => {
-    mockConfig = {};
-    mockConfig.dateNanosFormat = 'MMMM Do YYYY, HH:mm:ss.SSSSSSSSS';
-    mockConfig['dateFormat:tz'] = 'Browser';
+    mockConfig = {
+      dateNanosFormat: 'MMMM Do YYYY, HH:mm:ss.SSSSSSSSS',
+      'dateFormat:tz': 'Browser',
+    };
 
-    const getConfig = (key: string) => mockConfig[key];
+    const getConfig: FieldFormatsGetConfigFn = (key: string) => mockConfig[key];
     const date = new DateNanosFormat({}, getConfig);
 
     convert = date.convert.bind(date);