Merge branch 'main' into fix-artifact-entries-list-ftr

packages/kbn-config/src/__fixtures__/unsplittable_3.yml

@@ -0,0 +1,7 @@
+
+'[foo.bar]': "foobar"
+list:
+  - id: "id1"
+    '[a.b]': ['foo', 'bar']
+    test.this.out: ['foo', 'bar']
+

packages/kbn-config/src/raw/read_config.test.ts

@@ -131,6 +131,33 @@ test('supports unsplittable key syntax on nested list', () => {
   `);
 });
 
+test('supports unsplittable key syntax on nested list with splittable subkeys', () => {
+  const config = getConfigFromFiles([fixtureFile('/unsplittable_3.yml')]);
+
+  expect(config).toMatchInlineSnapshot(`
+    Object {
+      "foo.bar": "foobar",
+      "list": Array [
+        Object {
+          "a.b": Array [
+            "foo",
+            "bar",
+          ],
+          "id": "id1",
+          "test": Object {
+            "this": Object {
+              "out": Array [
+                "foo",
+                "bar",
+              ],
+            },
+          },
+        },
+      ],
+    }
+  `);
+});
+
 test('supports var:default syntax', () => {
   process.env.KBN_ENV_VAR1 = 'val1';
 

packages/kbn-config/src/raw/read_config.ts

@@ -59,7 +59,9 @@ function processEntryValue(value: any) {
         delete value[subKey];
         set(value, [unsplitKey], processEntryValue(subVal));
       } else {
-        set(value, subKey, processEntryValue(subVal));
+        const subKeySplits = splitKey(subKey);
+        if (subKeySplits.length > 1) delete value[subKey];
+        set(value, subKeySplits, processEntryValue(subVal));
       }
     }
   } else if (typeof value === 'string') {

packages/kbn-es/src/cluster.ts

@@ -18,6 +18,7 @@ import { promisify } from 'util';
 import { CA_CERT_PATH, ES_NOPASSWORD_P12_PATH, extract } from '@kbn/dev-utils';
 import { ToolingLog } from '@kbn/tooling-log';
 import treeKill from 'tree-kill';
+import { MOCK_IDP_REALM_NAME, ensureSAMLRoleMapping } from '@kbn/mock-idp-utils';
 import { downloadSnapshot, installSnapshot, installSource, installArchive } from './install';
 import { ES_BIN, ES_PLUGIN_BIN, ES_KEYSTORE_BIN } from './paths';
 import {
@@ -312,7 +313,7 @@ export class Cluster {
    */
   private exec(installPath: string, opts: EsClusterExecOptions) {
     const {
-      skipNativeRealmSetup = false,
+      skipSecuritySetup = false,
       reportTime = () => {},
       startTime,
       skipReadyCheck,
@@ -437,17 +438,21 @@ export class Cluster {
         });
       }
 
-      // once the cluster is ready setup the native realm
-      if (!skipNativeRealmSetup) {
+      // once the cluster is ready setup the realm
+      if (!skipSecuritySetup) {
         const nativeRealm = new NativeRealm({
           log: this.log,
           elasticPassword: options.password,
           client,
         });
 
         await nativeRealm.setPasswords(options);
-      }
 
+        const samlRealmConfigPrefix = `authc.realms.saml.${MOCK_IDP_REALM_NAME}.`;
+        if (args.some((arg) => arg.includes(samlRealmConfigPrefix))) {
+          await ensureSAMLRoleMapping(client);
+        }
+      }
       this.log.success('kbn/es setup complete');
     });
 

packages/kbn-es/src/cluster_exec_options.ts

@@ -7,7 +7,7 @@
  */
 
 export interface EsClusterExecOptions {
-  skipNativeRealmSetup?: boolean;
+  skipSecuritySetup?: boolean;
   reportTime?: (...args: any[]) => void;
   startTime?: number;
   esArgs?: string[] | string;

packages/kbn-es/src/install/install_archive.ts

@@ -18,7 +18,7 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { BASE_PATH, ES_CONFIG, ES_KEYSTORE_BIN } from '../paths';
 import { Artifact } from '../artifact';
 import { parseSettings, SettingsFilter } from '../settings';
-import { log as defaultLog } from '../utils/log';
+import { log as defaultLog, isFile, copyFileSync } from '../utils';
 import { InstallArchiveOptions } from './types';
 
 const isHttpUrl = (str: string) => {
@@ -41,6 +41,7 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
     log = defaultLog,
     esArgs = [],
     disableEsTmpDir = process.env.FTR_DISABLE_ES_TMPDIR?.toLowerCase() === 'true',
+    resources,
   } = options || {};
 
   let dest = archive;
@@ -84,6 +85,23 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
     ...parseSettings(esArgs, { filter: SettingsFilter.SecureOnly }),
   ]);
 
+  // copy resources to ES config directory
+  if (resources) {
+    resources.forEach((resource) => {
+      if (!isFile(resource)) {
+        throw new Error(
+          `Invalid resource: '${resource}'.\nOnly valid files can be copied to ES config directory`
+        );
+      }
+
+      const filename = path.basename(resource);
+      const destPath = path.resolve(installPath, 'config', filename);
+
+      copyFileSync(resource, destPath);
+      log.info('moved %s in config to %s', resource, destPath);
+    });
+  }
+
   return { installPath, disableEsTmpDir };
 }
 

packages/kbn-es/src/install/install_snapshot.ts

@@ -51,6 +51,7 @@ export async function installSnapshot({
   log = defaultLog,
   esArgs,
   useCached = false,
+  resources,
 }: InstallSnapshotOptions) {
   const { downloadPath } = await downloadSnapshot({
     license,
@@ -68,5 +69,6 @@ export async function installSnapshot({
     installPath,
     log,
     esArgs,
+    resources,
   });
 }

packages/kbn-es/src/install/install_source.ts

@@ -33,6 +33,7 @@ export async function installSource({
   installPath = path.resolve(basePath, 'source'),
   log = defaultLog,
   esArgs,
+  resources,
 }: InstallSourceOptions) {
   log.info('source path: %s', chalk.bold(sourcePath));
   log.info('install path: %s', chalk.bold(installPath));
@@ -59,6 +60,7 @@ export async function installSource({
     installPath,
     log,
     esArgs,
+    resources,
   });
 }
 

packages/kbn-es/src/install/types.ts

@@ -17,6 +17,7 @@ export interface InstallSourceOptions {
   installPath?: string;
   log?: ToolingLog;
   esArgs?: string[];
+  resources?: string[];
 }
 
 export interface DownloadSnapshotOptions {
@@ -26,6 +27,7 @@ export interface DownloadSnapshotOptions {
   installPath?: string;
   log?: ToolingLog;
   useCached?: boolean;
+  resources?: string[];
 }
 
 export interface InstallSnapshotOptions extends DownloadSnapshotOptions {
@@ -42,4 +44,5 @@ export interface InstallArchiveOptions {
   esArgs?: string[];
   /** Disable creating a temp directory, allowing ES to write to OS's /tmp directory */
   disableEsTmpDir?: boolean;
+  resources?: string[];
 }

packages/kbn-es/src/integration_tests/cluster.test.ts

@@ -311,6 +311,7 @@ describe('#installArchive()', () => {
       esArgs: ['foo=true'],
       log,
       disableEsTmpDir: true,
+      resources: ['path/to/resource'],
     };
     const cluster = new Cluster({ log });
     await cluster.installArchive('bar', options);

packages/kbn-es/src/utils/extract_config_files.ts

@@ -43,11 +43,11 @@ export function extractConfigFiles(
   return localConfig;
 }
 
-function isFile(dest = '') {
+export function isFile(dest = '') {
   return fs.existsSync(dest) && fs.statSync(dest).isFile();
 }
 
-function copyFileSync(src: string, dest: string) {
+export function copyFileSync(src: string, dest: string) {
   const destPath = path.dirname(dest);
 
   if (!fs.existsSync(destPath)) {

packages/kbn-es/src/utils/index.ts

@@ -10,7 +10,7 @@ export { cache } from './cache';
 export { log } from './log';
 export { parseEsLog } from './parse_es_log';
 export { findMostRecentlyChanged } from './find_most_recently_changed';
-export { extractConfigFiles } from './extract_config_files';
+export { extractConfigFiles, isFile, copyFileSync } from './extract_config_files';
 // @ts-expect-error not typed yet
 export { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm';
 export { buildSnapshot } from './build_snapshot';

packages/kbn-esql-validation-autocomplete/scripts/generate_function_validation_tests.ts

@@ -21,8 +21,8 @@ import {
   SupportedDataType,
   FunctionDefinition,
   dataTypes,
-  isSupportedDataType,
   fieldTypes,
+  isFieldType,
 } from '../src/definitions/types';
 import { FUNCTION_DESCRIBE_BLOCK_NAME } from '../src/validation/function_describe_block_name';
 import { getMaxMinNumberOfParams } from '../src/validation/helpers';
@@ -1110,7 +1110,7 @@ function getFieldMapping(
 
   return params.map(({ name: _name, type, constantOnly, literalOptions, ...rest }) => {
     const typeString: string = type as string;
-    if (isSupportedDataType(typeString)) {
+    if (isFieldType(typeString)) {
       if (useLiterals && literalOptions) {
         return {
           name: `"${literalOptions[0]}"`,