Merge branch 'main' into ml-transform-fix-transform-list-reload

.buildkite/scripts/lifecycle/pre_command.sh

@@ -2,6 +2,12 @@
 
 set -euo pipefail
 
+echo "BUILDKITE_COMMAND: $BUILDKITE_COMMAND"
+if [[ "$BUILDKITE_COMMAND" =~ .*"upload".* ]]; then
+  echo "Skipped pre-command when running the Upload pipeline"
+  exit 0
+fi
+
 source .buildkite/scripts/common/util.sh
 
 echo '--- Setup environment vars'

.github/workflows/create-deploy-tag.yml

@@ -77,6 +77,10 @@ jobs:
                       "type": "mrkdwn",
                       "text": "*Workflow run:*\n<https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|${{ github.run_id }}>"
                     },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit:*\n<https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|${{ env.COMMIT }}>"
+                    },
                     {
                       "type": "mrkdwn",
                       "text": "*Git tag:*\n<https://github.com/elastic/kibana/releases/tag/${{ env.TAG_NAME }}|${{ env.TAG_NAME }}>"
@@ -103,7 +107,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>\n • <https://buildkite.com/elastic/kibana-tests/builds?branch=run-kibana-quality-gate-suites|QA Quality Gate pipeline>"
+                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>\n • <https://example.com|QA Quality Gate pipeline>"
                   }
                 },
                 {
@@ -162,6 +166,10 @@ jobs:
                     {
                       "type": "mrkdwn",
                       "text": "*Workflow run:*\n<https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|${{ github.run_id }}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit:*\n<https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|${{ env.COMMIT }}>"
                     }
                   ]
                 },

catalog-info.yaml

@@ -151,10 +151,12 @@ spec:
         build_pull_request_forks: false
         build_tags: true
         # https://regex101.com/r/tY52jo/1
-        filter_condition: 'build.tag =~ "/^deploy@\d+\$/"'
+        filter_condition: 'build.tag =~ /^deploy@\d+$/'
         filter_enabled: true
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY

packages/core/test-helpers/core-test-helpers-kbn-server/src/create_serverless_root.ts

@@ -78,9 +78,9 @@ function createServerlessES() {
         teardown: true,
         background: true,
         clean: true,
+        kill: true,
+        waitForReady: true,
       });
-      // runServerless doesn't wait until the nodes are up
-      await waitUntilClusterReady(getServerlessESClient());
       return {
         getClient: getServerlessESClient,
         stop: async () => {
@@ -91,22 +91,6 @@ function createServerlessES() {
   };
 }
 
-const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-const waitUntilClusterReady = async (client: Client, timeoutMs = 60 * 1000) => {
-  const started = Date.now();
-
-  while (started + timeoutMs > Date.now()) {
-    try {
-      await client.info();
-      break;
-    } catch (e) {
-      await delay(1000);
-      /* trap to continue */
-    }
-  }
-};
-
 const getServerlessESClient = () => {
   return new Client({
     // node ports not configurable from

packages/kbn-es/src/cli_commands/serverless.ts

@@ -11,7 +11,7 @@ import getopts from 'getopts';
 import { ToolingLog } from '@kbn/tooling-log';
 import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
-import { Cluster } from '../cluster';
+import { Cluster, type ServerlessOptions } from '../cluster';
 import { SERVERLESS_REPO, SERVERLESS_TAG, SERVERLESS_IMG, DEFAULT_PORT } from '../utils';
 import { Command } from './types';
 
@@ -58,7 +58,7 @@ export const serverless: Command = {
       boolean: ['clean', 'ssl', 'kill', 'background'],
 
       default: defaults,
-    });
+    }) as unknown as ServerlessOptions;
 
     const cluster = new Cluster();
     await cluster.runServerless({

packages/kbn-es/src/cluster.js

@@ -36,7 +36,7 @@ const DEFAULT_READY_TIMEOUT = parseTimeoutToMs('1m');
 
 /** @typedef {import('./cluster_exec_options').EsClusterExecOptions} ExecOptions */
 /** @typedef {import('./utils').DockerOptions} DockerOptions */
-/** @typedef {import('./utils').ServerlessOptions}ServerlessrOptions */
+/** @typedef {import('./utils').ServerlessOptions}ServerlessOptions */
 
 // listen to data on stream until map returns anything but undefined
 const first = (stream, map) =>
@@ -579,8 +579,6 @@ exports.Cluster = class Cluster {
    * @param {ServerlessOptions} options
    */
   async runServerless(options = {}) {
-    // Ensure serverless ES nodes are not running
-    teardownServerlessClusterSync(this._log, options);
     if (this._process || this._outcome) {
       throw new Error('ES has already been started');
     }

packages/kbn-es/src/utils/docker.test.ts

@@ -34,6 +34,11 @@ import { ESS_RESOURCES_PATHS } from '../paths';
 
 jest.mock('execa');
 const execa = jest.requireMock('execa');
+jest.mock('@elastic/elasticsearch', () => {
+  return {
+    Client: jest.fn(),
+  };
+});
 
 const log = new ToolingLog();
 const logWriter = new ToolingLogCollectingWriter();
@@ -465,6 +470,22 @@ describe('runServerlessCluster()', () => {
     // setupDocker execa calls then run three nodes and attach logger
     expect(execa.mock.calls).toHaveLength(8);
   });
+  describe('waitForReady', () => {
+    test('should wait for serverless nodes to be ready to serve requests', async () => {
+      mockFs({
+        [baseEsPath]: {},
+      });
+      execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
+      const info = jest.fn();
+      jest.requireMock('@elastic/elasticsearch').Client.mockImplementation(() => ({ info }));
+
+      info.mockImplementationOnce(() => Promise.reject()); // first call fails
+      info.mockImplementationOnce(() => Promise.resolve()); // then succeeds
+
+      await runServerlessCluster(log, { basePath: baseEsPath, waitForReady: true });
+      expect(info).toHaveBeenCalledTimes(2);
+    });
+  });
 });
 
 describe('stopServerlessCluster()', () => {

packages/kbn-es/src/utils/docker.ts

@@ -10,6 +10,7 @@ import execa from 'execa';
 import fs from 'fs';
 import Fsp from 'fs/promises';
 import { resolve, basename, join } from 'path';
+import { Client, HttpConnection } from '@elastic/elasticsearch';
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { kibanaPackageJson as pkg, REPO_ROOT } from '@kbn/repo-info';
@@ -35,6 +36,7 @@ interface BaseOptions {
   image?: string;
   port?: number;
   ssl?: boolean;
+  /** Kill running cluster before starting a new cluster  */
   kill?: boolean;
   files?: string | string[];
 }
@@ -44,10 +46,16 @@ export interface DockerOptions extends EsClusterExecOptions, BaseOptions {
 }
 
 export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
+  /** Clean (or delete) all data created by the ES cluster after it is stopped */
   clean?: boolean;
+  /** Path to the directory where the ES cluster will store data */
   basePath: string;
+  /** If this process exits, teardown the ES cluster as well */
   teardown?: boolean;
+  /** Start the ES cluster in the background instead of remaining attached: useful for running tests */
   background?: boolean;
+  /** Wait for the ES cluster to be ready to serve requests */
+  waitForReady?: boolean;
 }
 
 interface ServerlessEsNodeArgs {
@@ -539,6 +547,30 @@ export async function runServerlessEsNode(
   );
 }
 
+function getESClient(
+  { node }: { node: string } = { node: `http://localhost:${DEFAULT_PORT}` }
+): Client {
+  return new Client({
+    node,
+    Connection: HttpConnection,
+  });
+}
+
+const delay = (ms: number) => new Promise((res) => setTimeout(res, ms));
+async function waitUntilClusterReady(timeoutMs = 60 * 1000): Promise<void> {
+  const started = Date.now();
+  const client = getESClient();
+  while (started + timeoutMs > Date.now()) {
+    try {
+      await client.info();
+      break;
+    } catch (e) {
+      await delay(1000);
+      /* trap to continue */
+    }
+  }
+}
+
 /**
  * Runs an ES Serverless Cluster through Docker
  */
@@ -583,10 +615,16 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
     `);
 
     log.warning(`Kibana should be started with the SSL flag so that it can authenticate with ES.
-      See packages/kbn-es/src/ess_resources/README.md for additional information on authentication.    
+      See packages/kbn-es/src/ess_resources/README.md for additional information on authentication.
     `);
   }
 
+  if (options.waitForReady) {
+    log.info('Waiting until ES is ready to serve requests...');
+    await waitUntilClusterReady();
+    log.success('ES is ready');
+  }
+
   if (!options.background) {
     // The ESS cluster has to be started detached, so we attach a logger afterwards for output
     await execa('docker', ['logs', '-f', SERVERLESS_NODES[0].name], {

x-pack/plugins/fleet/server/services/files/client_from_host.ts

@@ -81,6 +81,9 @@ export class FleetFromHostFilesClient implements FleetFromHostFileClientInterfac
       if (error instanceof FleetFilesClientError) {
         throw error;
       }
+      if (error.message.includes('index_not_found')) {
+        throw new FleetFileNotFound(error.message, error);
+      }
 
       throw new FleetFilesClientError(error.message, error);
     }

x-pack/plugins/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx

@@ -230,7 +230,7 @@ export const LogRateAnalysis: FC<AlertDetailsLogRateAnalysisSectionProps> = ({ r
 
       Do not mention indidivual p-values from the analysis results. Do not guess, just say what you are sure of. Do not repeat the given instructions in your output.`;
 
-    const now = new Date().toString();
+    const now = new Date().toISOString();
 
     return [
       {

x-pack/plugins/observability_ai_assistant/public/components/buttons/ask_assistant_button.tsx

@@ -83,6 +83,12 @@ export function AskAssistantButton({
           )}
         >
           <EuiButtonIcon
+            aria-label={i18n.translate(
+              'xpack.observabilityAiAssistant.askAssistantButton.popoverTitle',
+              {
+                defaultMessage: 'Elastic Assistant',
+              }
+            )}
             iconType="sparkles"
             display={fill ? 'fill' : 'base'}
             size={size}