Merge branch 'main' into feature/show-underlying-data-2

.buildkite/scripts/steps/functional/performance_sub_playwright.sh

@@ -23,7 +23,7 @@ cd "$XPACK_DIR"
 jobId=$(npx uuid)
 export TEST_JOB_ID="$jobId"
 
-journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard")
+journeys=("ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard")
 
 for i in "${journeys[@]}"; do
     echo "JOURNEY[${i}] is running"

docs/development/core/public/kibana-plugin-core-public.simplesavedobject._constructor_.md

@@ -9,13 +9,13 @@ Constructs a new instance of the `SimpleSavedObject` class
 <b>Signature:</b>
 
 ```typescript
-constructor(client: SavedObjectsClientContract, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, }: SavedObjectType<T>);
+constructor(client: SavedObjectsClientContract, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, updated_at: updatedAt, }: SavedObjectType<T>);
 ```
 
 ## Parameters
 
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  client | SavedObjectsClientContract |  |
-|  { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, } | SavedObjectType&lt;T&gt; |  |
+|  { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, updated\_at: updatedAt, } | SavedObjectType&lt;T&gt; |  |
 

docs/development/core/public/kibana-plugin-core-public.simplesavedobject.md

@@ -18,7 +18,7 @@ export declare class SimpleSavedObject<T = unknown>
 
 |  Constructor | Modifiers | Description |
 |  --- | --- | --- |
-|  [(constructor)(client, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, })](./kibana-plugin-core-public.simplesavedobject._constructor_.md) |  | Constructs a new instance of the <code>SimpleSavedObject</code> class |
+|  [(constructor)(client, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, updated\_at: updatedAt, })](./kibana-plugin-core-public.simplesavedobject._constructor_.md) |  | Constructs a new instance of the <code>SimpleSavedObject</code> class |
 
 ## Properties
 
@@ -33,6 +33,7 @@ export declare class SimpleSavedObject<T = unknown>
 |  [namespaces](./kibana-plugin-core-public.simplesavedobject.namespaces.md) |  | SavedObjectType&lt;T&gt;\['namespaces'\] | Space(s) that this saved object exists in. This attribute is not used for "global" saved object types which are registered with <code>namespaceType: 'agnostic'</code>. |
 |  [references](./kibana-plugin-core-public.simplesavedobject.references.md) |  | SavedObjectType&lt;T&gt;\['references'\] |  |
 |  [type](./kibana-plugin-core-public.simplesavedobject.type.md) |  | SavedObjectType&lt;T&gt;\['type'\] |  |
+|  [updatedAt](./kibana-plugin-core-public.simplesavedobject.updatedat.md) |  | SavedObjectType&lt;T&gt;\['updated\_at'\] |  |
 
 ## Methods
 

docs/development/core/public/kibana-plugin-core-public.simplesavedobject.updatedat.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-public](./kibana-plugin-core-public.md) &gt; [SimpleSavedObject](./kibana-plugin-core-public.simplesavedobject.md) &gt; [updatedAt](./kibana-plugin-core-public.simplesavedobject.updatedat.md)
+
+## SimpleSavedObject.updatedAt property
+
+<b>Signature:</b>
+
+```typescript
+updatedAt: SavedObjectType<T>['updated_at'];
+```

docs/development/core/server/kibana-plugin-core-server.elasticsearchservicesetup.legacy.md

@@ -6,7 +6,6 @@
 
 > Warning: This API is now obsolete.
 > 
-> Use [ElasticsearchServiceStart.legacy](./kibana-plugin-core-server.elasticsearchservicestart.legacy.md) instead.
 > 
 
 <b>Signature:</b>

docs/development/core/server/kibana-plugin-core-server.elasticsearchservicestart.md

@@ -17,5 +17,4 @@ export interface ElasticsearchServiceStart
 |  --- | --- | --- |
 |  [client](./kibana-plugin-core-server.elasticsearchservicestart.client.md) | IClusterClient | A pre-configured [Elasticsearch client](./kibana-plugin-core-server.iclusterclient.md) |
 |  [createClient](./kibana-plugin-core-server.elasticsearchservicestart.createclient.md) | (type: string, clientConfig?: Partial&lt;ElasticsearchClientConfig&gt;) =&gt; ICustomClusterClient | Create application specific Elasticsearch cluster API client with customized config. See [IClusterClient](./kibana-plugin-core-server.iclusterclient.md)<!-- -->. |
-|  [legacy](./kibana-plugin-core-server.elasticsearchservicestart.legacy.md) | { readonly config$: Observable&lt;ElasticsearchConfig&gt;; } |  |
 

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.md

@@ -77,7 +77,6 @@ async (context, request, response) => {
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [auth](./kibana-plugin-core-server.httpservicesetup.auth.md) | HttpAuth | Auth status. See [HttpAuth](./kibana-plugin-core-server.httpauth.md) |
 |  [basePath](./kibana-plugin-core-server.httpservicesetup.basepath.md) | IBasePath | Access or manipulate the Kibana base path See [IBasePath](./kibana-plugin-core-server.ibasepath.md)<!-- -->. |
 |  [createCookieSessionStorageFactory](./kibana-plugin-core-server.httpservicesetup.createcookiesessionstoragefactory.md) | &lt;T&gt;(cookieOptions: SessionStorageCookieOptions&lt;T&gt;) =&gt; Promise&lt;SessionStorageFactory&lt;T&gt;&gt; | Creates cookie based session storage factory [SessionStorageFactory](./kibana-plugin-core-server.sessionstoragefactory.md) |
 |  [createRouter](./kibana-plugin-core-server.httpservicesetup.createrouter.md) | &lt;Context extends RequestHandlerContext = RequestHandlerContext&gt;() =&gt; IRouter&lt;Context&gt; | Provides ability to declare a handler function for a particular path and HTTP request method. |

packages/kbn-es/src/cli_commands/snapshot.js

@@ -33,6 +33,8 @@ exports.help = (defaults = {}) => {
       --use-cached      Skips cache verification and use cached ES snapshot.
       --skip-ready-check  Disable the ready check,
       --ready-timeout   Customize the ready check timeout, in seconds or "Xm" format, defaults to 1m
+      --plugins         Comma seperated list of Elasticsearch plugins to install
+      --secure-files     Comma seperated list of secure_setting_name=/path pairs
 
     Example:
 
@@ -58,6 +60,7 @@ exports.run = async (defaults = {}) => {
       useCached: 'use-cached',
       skipReadyCheck: 'skip-ready-check',
       readyTimeout: 'ready-timeout',
+      secureFiles: 'secure-files',
     },
 
     string: ['version', 'ready-timeout'],
@@ -76,6 +79,13 @@ exports.run = async (defaults = {}) => {
     if (options.dataArchive) {
       await cluster.extractDataDirectory(installPath, options.dataArchive);
     }
+    if (options.plugins) {
+      await cluster.installPlugins(installPath, options.plugins, options);
+    }
+    if (options.secureFiles) {
+      const pairs = options.secureFiles.split(',').map((kv) => kv.split('=').map((v) => v.trim()));
+      await cluster.configureKeystoreWithSecureSettingsFiles(installPath, pairs);
+    }
 
     reportTime(installStartTime, 'installed', {
       success: true,

packages/kbn-es/src/cli_commands/source.js

@@ -27,6 +27,8 @@ exports.help = (defaults = {}) => {
       --password        Sets password for elastic user [default: ${password}]
       --password.[user] Sets password for native realm user [default: ${password}]
       --ssl             Sets up SSL on Elasticsearch
+      --plugins         Comma seperated list of Elasticsearch plugins to install
+      --secure-files     Comma seperated list of secure_setting_name=/path pairs
       -E                Additional key=value settings to pass to Elasticsearch
       --skip-ready-check  Disable the ready check,
       --ready-timeout   Customize the ready check timeout, in seconds or "Xm" format, defaults to 1m
@@ -47,6 +49,7 @@ exports.run = async (defaults = {}) => {
       dataArchive: 'data-archive',
       skipReadyCheck: 'skip-ready-check',
       readyTimeout: 'ready-timeout',
+      secureFiles: 'secure-files',
       esArgs: 'E',
     },
 
@@ -62,6 +65,13 @@ exports.run = async (defaults = {}) => {
   if (options.dataArchive) {
     await cluster.extractDataDirectory(installPath, options.dataArchive);
   }
+  if (options.plugins) {
+    await cluster.installPlugins(installPath, options.plugins, options);
+  }
+  if (options.secureFiles) {
+    const pairs = options.secureFiles.split(',').map((kv) => kv.split('=').map((v) => v.trim()));
+    await cluster.configureKeystoreWithSecureSettingsFiles(installPath, pairs);
+  }
 
   await cluster.run(installPath, {
     ...options,

packages/kbn-es/src/cluster.js

@@ -12,7 +12,7 @@ const chalk = require('chalk');
 const path = require('path');
 const { Client } = require('@elastic/elasticsearch');
 const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install');
-const { ES_BIN } = require('./paths');
+const { ES_BIN, ES_PLUGIN_BIN, ES_KEYSTORE_BIN } = require('./paths');
 const {
   log: defaultLog,
   parseEsLog,
@@ -150,6 +150,42 @@ exports.Cluster = class Cluster {
     });
   }
 
+  /**
+   * Starts ES and returns resolved promise once started
+   *
+   * @param {String} installPath
+   * @param {String} plugins - comma separated list of plugins to install
+   * @param {Object} options
+   * @returns {Promise}
+   */
+  async installPlugins(installPath, plugins, options) {
+    const esJavaOpts = this.javaOptions(options);
+    for (const plugin of plugins.split(',')) {
+      await execa(ES_PLUGIN_BIN, ['install', plugin.trim()], {
+        cwd: installPath,
+        env: {
+          JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
+          ES_JAVA_OPTS: esJavaOpts.trim(),
+        },
+      });
+    }
+  }
+
+  async configureKeystoreWithSecureSettingsFiles(installPath, secureSettingsFiles) {
+    const env = { JAVA_HOME: '' };
+    for (const [secureSettingName, secureSettingFile] of secureSettingsFiles) {
+      this._log.info(
+        `setting secure setting %s to %s`,
+        chalk.bold(secureSettingName),
+        chalk.bold(secureSettingFile)
+      );
+      await execa(ES_KEYSTORE_BIN, ['add-file', secureSettingName, secureSettingFile], {
+        cwd: installPath,
+        env,
+      });
+    }
+  }
+
   /**
    * Starts ES and returns resolved promise once started
    *
@@ -280,27 +316,17 @@ exports.Cluster = class Cluster {
     );
 
     this._log.info('%s %s', ES_BIN, args.join(' '));
+    const esJavaOpts = this.javaOptions(options);
 
-    let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`;
-
-    // ES now automatically sets heap size to 50% of the machine's available memory
-    // so we need to set it to a smaller size for local dev and CI
-    // especially because we currently run many instances of ES on the same machine during CI
-    // inital and max must be the same, so we only need to check the max
-    if (!esJavaOpts.includes('Xmx')) {
-      // 1536m === 1.5g
-      esJavaOpts += ' -Xms1536m -Xmx1536m';
-    }
-
-    this._log.info('ES_JAVA_OPTS: %s', esJavaOpts.trim());
+    this._log.info('ES_JAVA_OPTS: %s', esJavaOpts);
 
     this._process = execa(ES_BIN, args, {
       cwd: installPath,
       env: {
         ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}),
         ...process.env,
         JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
-        ES_JAVA_OPTS: esJavaOpts.trim(),
+        ES_JAVA_OPTS: esJavaOpts,
       },
       stdio: ['ignore', 'pipe', 'pipe'],
     });
@@ -429,4 +455,18 @@ exports.Cluster = class Cluster {
       }
     }
   }
+
+  javaOptions(options) {
+    let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`;
+
+    // ES now automatically sets heap size to 50% of the machine's available memory
+    // so we need to set it to a smaller size for local dev and CI
+    // especially because we currently run many instances of ES on the same machine during CI
+    // inital and max must be the same, so we only need to check the max
+    if (!esJavaOpts.includes('Xmx')) {
+      // 1536m === 1.5g
+      esJavaOpts += ' -Xms1536m -Xmx1536m';
+    }
+    return esJavaOpts.trim();
+  }
 };

packages/kbn-es/src/paths.ts

@@ -19,6 +19,7 @@ export const BASE_PATH = Path.resolve(tempDir, 'kbn-es');
 
 export const GRADLE_BIN = maybeUseBat('./gradlew');
 export const ES_BIN = maybeUseBat('bin/elasticsearch');
+export const ES_PLUGIN_BIN = maybeUseBat('bin/elasticsearch-plugin');
 export const ES_CONFIG = 'config/elasticsearch.yml';
 
 export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore');

packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts

@@ -219,6 +219,25 @@ export class KbnClientSavedObjects {
     this.log.success('deleted', deleted, 'objects');
   }
 
+  public async cleanStandardList(options?: { space?: string }) {
+    // add types here
+    const types = [
+      'search',
+      'index-pattern',
+      'visualization',
+      'dashboard',
+      'lens',
+      'map',
+      'graph-workspace',
+      'query',
+      'tag',
+      'url',
+      'canvas-workpad',
+    ];
+    const newOptions = { types, space: options?.space };
+    await this.clean(newOptions);
+  }
+
   public async bulkDelete(options: DeleteObjectsOptions) {
     let deleted = 0;
     let missing = 0;

src/core/public/public.api.md

@@ -1389,7 +1389,7 @@ export class ScopedHistory<HistoryLocationState = unknown> implements History_2<
 
 // @public
 export class SimpleSavedObject<T = unknown> {
-    constructor(client: SavedObjectsClientContract, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, }: SavedObject<T>);
+    constructor(client: SavedObjectsClientContract, { id, type, version, attributes, error, references, migrationVersion, coreMigrationVersion, namespaces, updated_at: updatedAt, }: SavedObject<T>);
     // (undocumented)
     attributes: T;
     // (undocumented)
@@ -1416,6 +1416,8 @@ export class SimpleSavedObject<T = unknown> {
     // (undocumented)
     type: SavedObject<T>['type'];
     // (undocumented)
+    updatedAt: SavedObject<T>['updated_at'];
+    // (undocumented)
     _version?: SavedObject<T>['version'];
 }
 

src/core/public/saved_objects/simple_saved_object.test.ts

@@ -45,4 +45,67 @@ describe('SimpleSavedObject', () => {
     const savedObject = new SimpleSavedObject(client, { version } as SavedObject);
     expect(savedObject._version).toEqual(version);
   });
+
+  it('save() changes updatedAt field on existing SimpleSavedObject with an id', async function () {
+    const date = new Date();
+    const initialDate = date.toISOString();
+    date.setDate(date.getDate() + 1);
+    const secondDate = date.toISOString();
+
+    const config = {
+      attributes: {},
+      id: 'id',
+      type: 'type',
+    };
+
+    const initialSavedObject = new SimpleSavedObject(client, {
+      ...config,
+      updated_at: initialDate,
+    } as SavedObject);
+
+    const updatedSavedObject = new SimpleSavedObject(client, {
+      ...config,
+      updated_at: secondDate,
+    } as SavedObject);
+
+    (client.update as jest.Mock).mockReturnValue(Promise.resolve(updatedSavedObject));
+
+    const initialValue = initialSavedObject.updatedAt;
+    await initialSavedObject.save();
+    const updatedValue = updatedSavedObject.updatedAt;
+
+    expect(initialValue).not.toEqual(updatedValue);
+    expect(initialSavedObject.updatedAt).toEqual(updatedValue);
+  });
+
+  it('save() changes updatedAt field on existing SimpleSavedObject without an id', async () => {
+    const date = new Date();
+    const initialDate = date.toISOString();
+    date.setDate(date.getDate() + 1);
+    const secondDate = date.toISOString();
+
+    const config = {
+      attributes: {},
+      type: 'type',
+    };
+
+    const initialSavedObject = new SimpleSavedObject(client, {
+      ...config,
+      updated_at: initialDate,
+    } as SavedObject);
+
+    const updatedSavedObject = new SimpleSavedObject(client, {
+      ...config,
+      updated_at: secondDate,
+    } as SavedObject);
+
+    (client.create as jest.Mock).mockReturnValue(Promise.resolve(updatedSavedObject));
+
+    const initialValue = initialSavedObject.updatedAt;
+    await initialSavedObject.save();
+    const updatedValue = updatedSavedObject.updatedAt;
+
+    expect(initialValue).not.toEqual(updatedValue);
+    expect(initialSavedObject.updatedAt).toEqual(updatedValue);
+  });
 });