Merge branch 'master' into fix/92712

docs/canvas/canvas-function-reference.asciidoc

@@ -1697,6 +1697,16 @@ Aliases: `column`, `name`
 Aliases: `exp`, `fn`, `function`
 |`boolean`, `number`, `string`, `null`
 |A Canvas expression that is passed to each row as a single row `datatable`.
+
+|`id`
+
+|`string`, `null`
+|An optional id of the resulting column. When not specified or `null` the name argument is used as id.
+
+|`copyMetaFrom`
+
+|`string`, `null`
+|If set, the meta object from the specified column id is copied over to the specified target column. Throws an exception if the column doesn't exist
 |===
 
 *Returns:* `datatable`
@@ -1755,9 +1765,16 @@ Interprets a `TinyMath` math expression using a `number` or `datatable` as _cont
 Alias: `expression`
 |`string`
 |An evaluated `TinyMath` expression. See https://www.elastic.co/guide/en/kibana/current/canvas-tinymath-functions.html.
+
+|`onError` 
+
+|`string`
+|In case the `TinyMath` evaluation fails or returns NaN, the return value is specified by onError. For example, `"null"`, `"zero"`, `"false"`, `"throw"`. When `"throw"`, it will throw an exception, terminating expression execution.
+
+Default: `"throw"`
 |===
 
-*Returns:* `number`
+*Returns:* `number` | `boolean` | `null`
 
 
 [float]

packages/kbn-docs-utils/package.json

@@ -13,6 +13,7 @@
   },
   "dependencies": {
     "@kbn/utils": "link:../kbn-utils",
+    "@kbn/config": "link:../kbn-config",
     "@kbn/dev-utils": "link:../kbn-dev-utils"
   }
 }

src/core/server/elasticsearch/default_headers.test.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getReservedHeaders, PRODUCT_ORIGIN_HEADER } from './default_headers';
+
+describe('getReservedHeaders', () => {
+  it('returns the list of reserved headers contained in a list', () => {
+    expect(getReservedHeaders(['foo', 'bar', PRODUCT_ORIGIN_HEADER])).toEqual([
+      PRODUCT_ORIGIN_HEADER,
+    ]);
+  });
+
+  it('ignores the case when identifying headers', () => {
+    expect(getReservedHeaders(['foo', 'bar', PRODUCT_ORIGIN_HEADER.toUpperCase()])).toEqual([
+      PRODUCT_ORIGIN_HEADER.toUpperCase(),
+    ]);
+  });
+});

src/core/server/elasticsearch/default_headers.ts

@@ -8,9 +8,22 @@
 
 import { deepFreeze } from '@kbn/std';
 
+export const PRODUCT_ORIGIN_HEADER = 'x-elastic-product-origin';
+
+export const RESERVED_HEADERS = deepFreeze([PRODUCT_ORIGIN_HEADER]);
+
 export const DEFAULT_HEADERS = deepFreeze({
   // Elasticsearch uses this to identify when a request is coming from Kibana, to allow Kibana to
-  // access system indices using the standard ES APIs without logging a warning. After migrating to
-  // use the new system index APIs, this header can be removed.
-  'x-elastic-product-origin': 'kibana',
+  // access system indices using the standard ES APIs.
+  [PRODUCT_ORIGIN_HEADER]: 'kibana',
 });
+
+export const getReservedHeaders = (headerNames: string[]): string[] => {
+  const reservedHeaders = [];
+  for (const headerName of headerNames) {
+    if (RESERVED_HEADERS.includes(headerName.toLowerCase())) {
+      reservedHeaders.push(headerName);
+    }
+  }
+  return reservedHeaders;
+};

src/core/server/elasticsearch/elasticsearch_config.test.ts

@@ -108,6 +108,35 @@ test('#requestHeadersWhitelist accepts both string and array of strings', () =>
   expect(configValue.requestHeadersWhitelist).toEqual(['token', 'X-Forwarded-Proto']);
 });
 
+describe('reserved headers', () => {
+  test('throws if customHeaders contains reserved headers', () => {
+    expect(() => {
+      config.schema.validate({
+        customHeaders: { foo: 'bar', 'x-elastic-product-origin': 'beats' },
+      });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[customHeaders]: cannot use reserved headers: [x-elastic-product-origin]"`
+    );
+  });
+
+  test('throws if requestHeadersWhitelist contains reserved headers', () => {
+    expect(() => {
+      config.schema.validate({ requestHeadersWhitelist: ['foo', 'x-elastic-product-origin'] });
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[requestHeadersWhitelist]: types that failed validation:
+      - [requestHeadersWhitelist.0]: expected value of type [string] but got [Array]
+      - [requestHeadersWhitelist.1]: cannot use reserved headers: [x-elastic-product-origin]"
+    `);
+    expect(() => {
+      config.schema.validate({ requestHeadersWhitelist: 'x-elastic-product-origin' });
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[requestHeadersWhitelist]: types that failed validation:
+      - [requestHeadersWhitelist.0]: cannot use reserved headers: [x-elastic-product-origin]
+      - [requestHeadersWhitelist.1]: could not parse array value from json input"
+    `);
+  });
+});
+
 describe('reads files', () => {
   beforeEach(() => {
     mockReadFileSync.mockReset();

src/core/server/elasticsearch/elasticsearch_config.ts

@@ -12,6 +12,7 @@ import { readFileSync } from 'fs';
 import { ConfigDeprecationProvider } from 'src/core/server';
 import { readPkcs12Keystore, readPkcs12Truststore } from '../utils';
 import { ServiceConfigDescriptor } from '../internal_types';
+import { getReservedHeaders } from './default_headers';
 
 const hostURISchema = schema.uri({ scheme: ['http', 'https'] });
 
@@ -52,10 +53,42 @@ export const configSchema = schema.object({
     )
   ),
   password: schema.maybe(schema.string()),
-  requestHeadersWhitelist: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], {
-    defaultValue: ['authorization'],
+  requestHeadersWhitelist: schema.oneOf(
+    [
+      schema.string({
+        // can't use `validate` option on union types, forced to validate each individual subtypes
+        // see https://github.com/elastic/kibana/issues/64906
+        validate: (headersWhitelist) => {
+          const reservedHeaders = getReservedHeaders([headersWhitelist]);
+          if (reservedHeaders.length) {
+            return `cannot use reserved headers: [${reservedHeaders.join(', ')}]`;
+          }
+        },
+      }),
+      schema.arrayOf(schema.string(), {
+        // can't use `validate` option on union types, forced to validate each individual subtypes
+        // see https://github.com/elastic/kibana/issues/64906
+        validate: (headersWhitelist) => {
+          const reservedHeaders = getReservedHeaders(headersWhitelist);
+          if (reservedHeaders.length) {
+            return `cannot use reserved headers: [${reservedHeaders.join(', ')}]`;
+          }
+        },
+      }),
+    ],
+    {
+      defaultValue: ['authorization'],
+    }
+  ),
+  customHeaders: schema.recordOf(schema.string(), schema.string(), {
+    defaultValue: {},
+    validate: (customHeaders) => {
+      const reservedHeaders = getReservedHeaders(Object.keys(customHeaders));
+      if (reservedHeaders.length) {
+        return `cannot use reserved headers: [${reservedHeaders.join(', ')}]`;
+      }
+    },
   }),
-  customHeaders: schema.recordOf(schema.string(), schema.string(), { defaultValue: {} }),
   shardTimeout: schema.duration({ defaultValue: '30s' }),
   requestTimeout: schema.duration({ defaultValue: '30s' }),
   pingTimeout: schema.duration({ defaultValue: schema.siblingRef('requestTimeout') }),

src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts

@@ -96,8 +96,7 @@ describe('getSearchDsl', () => {
         mappings,
         opts.type,
         opts.sortField,
-        opts.sortOrder,
-        opts.pit
+        opts.sortOrder
       );
     });
 

src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts

@@ -78,7 +78,7 @@ export function getSearchDsl(
       hasReferenceOperator,
       kueryNode,
     }),
-    ...getSortingParams(mappings, type, sortField, sortOrder, pit),
+    ...getSortingParams(mappings, type, sortField, sortOrder),
     ...(pit ? getPitParams(pit) : {}),
     ...(searchAfter ? { search_after: searchAfter } : {}),
   };

src/core/server/saved_objects/service/lib/search_dsl/sorting_params.test.ts

@@ -79,11 +79,6 @@ describe('searchDsl/getSortParams', () => {
           ],
         });
       });
-      it('appends tiebreaker when PIT is provided', () => {
-        expect(getSortingParams(MAPPINGS, 'saved', 'title', undefined, { id: 'abc' }).sort).toEqual(
-          expect.arrayContaining([{ _shard_doc: 'asc' }])
-        );
-      });
     });
     describe('sortField is simple root property with multiple types', () => {
       it('returns correct params', () => {
@@ -98,11 +93,6 @@ describe('searchDsl/getSortParams', () => {
           ],
         });
       });
-      it('appends tiebreaker when PIT is provided', () => {
-        expect(
-          getSortingParams(MAPPINGS, ['saved', 'pending'], 'type', undefined, { id: 'abc' }).sort
-        ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }]));
-      });
     });
     describe('sortField is simple non-root property with multiple types', () => {
       it('returns correct params', () => {
@@ -124,11 +114,6 @@ describe('searchDsl/getSortParams', () => {
           ],
         });
       });
-      it('appends tiebreaker when PIT is provided', () => {
-        expect(
-          getSortingParams(MAPPINGS, 'saved', 'title.raw', undefined, { id: 'abc' }).sort
-        ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }]));
-      });
     });
     describe('sortField is multi-field with single type as array', () => {
       it('returns correct params', () => {
@@ -143,11 +128,6 @@ describe('searchDsl/getSortParams', () => {
           ],
         });
       });
-      it('appends tiebreaker when PIT is provided', () => {
-        expect(
-          getSortingParams(MAPPINGS, ['saved'], 'title.raw', undefined, { id: 'abc' }).sort
-        ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }]));
-      });
     });
     describe('sortField is root multi-field with multiple types', () => {
       it('returns correct params', () => {
@@ -162,12 +142,6 @@ describe('searchDsl/getSortParams', () => {
           ],
         });
       });
-      it('appends tiebreaker when PIT is provided', () => {
-        expect(
-          getSortingParams(MAPPINGS, ['saved', 'pending'], 'type.raw', undefined, { id: 'abc' })
-            .sort
-        ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }]));
-      });
     });
     describe('sortField is not-root multi-field with multiple types', () => {
       it('returns correct params', () => {

src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts

@@ -8,21 +8,14 @@
 
 import Boom from '@hapi/boom';
 import { getProperty, IndexMapping } from '../../../mappings';
-import { SavedObjectsPitParams } from '../../../types';
-
-// TODO: The plan is for ES to automatically add this tiebreaker when
-// using PIT. We should remove this logic once that is resolved.
-// https://github.com/elastic/elasticsearch/issues/56828
-const ES_PROVIDED_TIEBREAKER = { _shard_doc: 'asc' };
 
 const TOP_LEVEL_FIELDS = ['_id', '_score'];
 
 export function getSortingParams(
   mappings: IndexMapping,
   type: string | string[],
   sortField?: string,
-  sortOrder?: string,
-  pit?: SavedObjectsPitParams
+  sortOrder?: string
 ) {
   if (!sortField) {
     return {};
@@ -38,7 +31,6 @@ export function getSortingParams(
             order: sortOrder,
           },
         },
-        ...(pit ? [ES_PROVIDED_TIEBREAKER] : []),
       ],
     };
   }
@@ -59,7 +51,6 @@ export function getSortingParams(
             unmapped_type: rootField.type,
           },
         },
-        ...(pit ? [ES_PROVIDED_TIEBREAKER] : []),
       ],
     };
   }
@@ -84,7 +75,6 @@ export function getSortingParams(
           unmapped_type: field.type,
         },
       },
-      ...(pit ? [ES_PROVIDED_TIEBREAKER] : []),
     ],
   };
 }

src/dev/typescript/ref_output_cache/ref_output_cache.ts

@@ -76,28 +76,28 @@ export class RefOutputCache {
    * written to the directory.
    */
   async initCaches() {
-    const archive =
-      this.archives.get(this.mergeBase) ??
-      (await this.archives.getFirstAvailable([
-        this.mergeBase,
-        ...(await this.repo.getRecentShasFrom(this.mergeBase, 5)),
-      ]));
-
-    if (!archive) {
-      return;
-    }
-
     const outdatedOutDirs = (
       await concurrentMap(100, this.outDirs, async (outDir) => ({
         path: outDir,
-        outdated: !(await matchMergeBase(outDir, archive.sha)),
+        outdated: !(await matchMergeBase(outDir, this.mergeBase)),
       }))
     )
       .filter((o) => o.outdated)
       .map((o) => o.path);
 
     if (!outdatedOutDirs.length) {
-      this.log.debug('all outDirs have the most recent cache');
+      this.log.debug('all outDirs have a recent cache');
+      return;
+    }
+
+    const archive =
+      this.archives.get(this.mergeBase) ??
+      (await this.archives.getFirstAvailable([
+        this.mergeBase,
+        ...(await this.repo.getRecentShasFrom(this.mergeBase, 5)),
+      ]));
+
+    if (!archive) {
       return;
     }
 

src/plugins/discover/public/application/components/sidebar/lib/field_filter.ts

@@ -59,6 +59,7 @@ export function isFieldFiltered(
   const scriptedOrMissing =
     !filterState.missing ||
     field.type === '_source' ||
+    field.type === 'unknown_selected' ||
     field.scripted ||
     fieldCounts[field.name] > 0;
   const needle = filterState.name ? filterState.name.toLowerCase() : '';

src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts

@@ -217,6 +217,20 @@ describe('group_fields', function () {
     ]);
   });
 
+  it('should filter fields by a given name', function () {
+    const fieldFilterState = { ...getDefaultFieldFilter(), ...{ name: 'curr' } };
+
+    const actual1 = groupFields(
+      fields as IndexPatternField[],
+      ['customer_birth_date', 'currency', 'unknown'],
+      5,
+      fieldCounts,
+      fieldFilterState,
+      false
+    );
+    expect(actual1.selected.map((field) => field.name)).toEqual(['currency']);
+  });
+
   it('excludes unmapped fields if showUnmappedFields set to false', function () {
     const fieldFilterState = getDefaultFieldFilter();
     const fieldsWithUnmappedField = [...fields];